From d0d4f32bbb2ba3c91d0f923cb47c0cc7827bb778 Mon Sep 17 00:00:00 2001 From: Cameron Meissner Date: Thu, 12 Dec 2024 14:34:54 -0800 Subject: [PATCH 1/4] chore: periodically garbage collect linux VHD blobs --- vhdbuilder/scripts/gc.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vhdbuilder/scripts/gc.sh b/vhdbuilder/scripts/gc.sh index 0c367afe2aa..775ce45daa2 100644 --- a/vhdbuilder/scripts/gc.sh +++ b/vhdbuilder/scripts/gc.sh @@ -2,6 +2,8 @@ set -euxo pipefail [ -z "${SUBSCRIPTION_ID:-}" ] && echo "SUBSCRIPTION_ID must be set" && exit 1 +[ -z "${VHD_BLOB_STORAGE_ACCOUNT_NAME}" ] && echo "VHD_BLOB_STORAGE_ACCOUNT_NAME must be set" && exit 1 +[ -z "${VHD_BLOB_STORAGE_CONTAINER_NAME}" ] && echo "VHD_BLOB_STORAGE_CONTAINER_NAME must be set" && exit 1 SKIP_TAG_NAME="gc.skip" SKIP_TAG_VALUE="true" @@ -10,6 +12,7 @@ DRY_RUN="${DRY_RUN:-}" DAY_AGO=$(( $(date +%s) - 86400 )) # 24 hours ago WEEK_AGO=$(( $(date +%s) - 604800 )) # 7 days ago +WEEK_AGO_ISO=$(date @${WEEK_AGO} -Iseconds) # 7 days ago ISO format function main() { az login --identity # relies on an appropriately permissioned identity being attached to the build agent @@ -47,6 +50,23 @@ function cleanup_rgs() { done } +function cleanup_storage_blobs() { + blobs=$(az storage blob list --account-name $VHD_BLOB_STORAGE_ACCOUNT_NAME --container-name $VHD_BLOB_STORAGE_CONTAINER_NAME --auth-mode login --query "[?properties.creationTime<='$WEEK_AGO_ISO'].{name:name}" -o tsv || "") + for blob in $blobs; do + echo "will delete VHD blob $blob if unmodified since $WEEK_AGO_ISO..." + if [ "${DRY_RUN,,}" == "true" ]; then + echo "DRY_RUN: az storage blob delete --account-name $VHD_BLOB_STORAGE_ACCOUNT_NAME --container-name $VHD_BLOB_STORAGE_CONTAINER_NAME --name $blob --if-unmodified-since $WEEK_AGO_ISO --auth-mode login" + continue + fi + az storage blob delete \ + --account-name $VHD_BLOB_STORAGE_ACCOUNT_NAME \ + --container-name $VHD_BLOB_STORAGE_CONTAINER_NAME \ + --name $blob \ + --if-unmodified-since "$WEEK_AGO_ISO" \ + --auth-mode login || return $? + done +} + function delete_group() { local group=$1 From cf92eda30b75683dd46911f20d849a7801d960ef Mon Sep 17 00:00:00 2001 From: Cameron Meissner Date: Thu, 12 Dec 2024 14:36:40 -0800 Subject: [PATCH 2/4] chore: vars --- vhdbuilder/scripts/gc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/scripts/gc.sh b/vhdbuilder/scripts/gc.sh index 775ce45daa2..f79d21c4aa4 100644 --- a/vhdbuilder/scripts/gc.sh +++ b/vhdbuilder/scripts/gc.sh @@ -51,7 +51,7 @@ function cleanup_rgs() { } function cleanup_storage_blobs() { - blobs=$(az storage blob list --account-name $VHD_BLOB_STORAGE_ACCOUNT_NAME --container-name $VHD_BLOB_STORAGE_CONTAINER_NAME --auth-mode login --query "[?properties.creationTime<='$WEEK_AGO_ISO'].{name:name}" -o tsv || "") + blobs=$(az storage blob list --account-name $VHD_BLOB_STORAGE_ACCOUNT_NAME --container-name $VHD_BLOB_STORAGE_CONTAINER_NAME --auth-mode login --query "[?properties.creationTime<='${WEEK_AGO_ISO}'].{name:name}" -o tsv || "") for blob in $blobs; do echo "will delete VHD blob $blob if unmodified since $WEEK_AGO_ISO..." if [ "${DRY_RUN,,}" == "true" ]; then From 4e2eb59b11e0db6a68a2741555cac4e6e9d6c857 Mon Sep 17 00:00:00 2001 From: Cameron Meissner Date: Thu, 12 Dec 2024 14:39:38 -0800 Subject: [PATCH 3/4] chore: pipeline vars --- .pipelines/.vsts-garabge-collection.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/.vsts-garabge-collection.yaml b/.pipelines/.vsts-garabge-collection.yaml index 2d04f3fae35..56ebfacddc3 100644 --- a/.pipelines/.vsts-garabge-collection.yaml +++ b/.pipelines/.vsts-garabge-collection.yaml @@ -20,5 +20,7 @@ jobs: ./vhdbuilder/scripts/gc.sh env: SUBSCRIPTION_ID: $(SUBSCRIPTION_ID) + VHD_BLOB_STORAGE_ACCOUNT_NAME: $(VHD_BLOB_STORAGE_ACCOUNT_NAME) + VHD_BLOB_STORAGE_CONTAINER_NAME: $(VHD_BLOB_STORAGE_CONTAINER_NAME) DRY_RUN: ${{ parameters.DRY_RUN }} displayName: Garbage collect resource groups From 4ef661331e86a6a679c0df631dd8905f78392750 Mon Sep 17 00:00:00 2001 From: Cameron Meissner Date: Thu, 12 Dec 2024 14:40:07 -0800 Subject: [PATCH 4/4] chore: naming --- .pipelines/.vsts-garabge-collection.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/.vsts-garabge-collection.yaml b/.pipelines/.vsts-garabge-collection.yaml index 56ebfacddc3..07d9ccc13af 100644 --- a/.pipelines/.vsts-garabge-collection.yaml +++ b/.pipelines/.vsts-garabge-collection.yaml @@ -23,4 +23,4 @@ jobs: VHD_BLOB_STORAGE_ACCOUNT_NAME: $(VHD_BLOB_STORAGE_ACCOUNT_NAME) VHD_BLOB_STORAGE_CONTAINER_NAME: $(VHD_BLOB_STORAGE_CONTAINER_NAME) DRY_RUN: ${{ parameters.DRY_RUN }} - displayName: Garbage collect resource groups + displayName: Garbage VHD build resources