From 12a88ed20a5c68c6706d1e41cfa8f083c98522d6 Mon Sep 17 00:00:00 2001 From: Sergiy Matusevych Date: Mon, 8 Jul 2024 19:29:37 -0700 Subject: [PATCH] remove the storageAccountKey parameter from all JSON configs and schemas --- mlos_bench/README.md | 3 +-- .../environments/os/linux/boot/boot-ubuntu.jsonc | 2 -- .../environments/os/linux/runtime/setup-ubuntu.jsonc | 2 -- .../azure/azure-fileshare-service-subschema.json | 4 ---- .../services/remote/azure/service-fileshare.jsonc | 3 +-- .../config/experiments/experiment_test_config.jsonc | 1 - .../azure_fileshare_service-config-missing.jsonc | 5 ++--- .../azure_fileshare_service-config-extras.jsonc | 1 - .../bad/unhandled/azure_flex_service-extras.jsonc | 1 - .../good/full/azure_fileshare_service-full.jsonc | 1 - .../partial/azure_fileshare_service-partial.jsonc | 3 +-- scripts/generate-azure-credentials-config.ps1 | 12 +----------- scripts/generate-azure-credentials-config.sh | 8 +------- 13 files changed, 7 insertions(+), 39 deletions(-) diff --git a/mlos_bench/README.md b/mlos_bench/README.md index 2a18d6bbad7..a12aa5c560d 100644 --- a/mlos_bench/README.md +++ b/mlos_bench/README.md @@ -104,8 +104,7 @@ This data produced in the `global_config_azure.jsonc` file is in the format that ```jsonc { "subscription": "some-guid", - "tenant": "some-other-guid", - "storageAccountKey": "some-base-64-encoded-key", + "tenant": "some-other-guid" } ``` diff --git a/mlos_bench/mlos_bench/config/environments/os/linux/boot/boot-ubuntu.jsonc b/mlos_bench/mlos_bench/config/environments/os/linux/boot/boot-ubuntu.jsonc index 51a5e378fc0..43865143c5d 100644 --- a/mlos_bench/mlos_bench/config/environments/os/linux/boot/boot-ubuntu.jsonc +++ b/mlos_bench/mlos_bench/config/environments/os/linux/boot/boot-ubuntu.jsonc @@ -54,7 +54,6 @@ "vmName", "storageAccountName", "storageFileShareName", - "storageAccountKey", "mountPoint", "experiment_id", "trial_id" @@ -63,7 +62,6 @@ "shell_env_params": [ "storageAccountName", "storageFileShareName", - "storageAccountKey", "mountPoint", "experiment_id", "trial_id" diff --git a/mlos_bench/mlos_bench/config/environments/os/linux/runtime/setup-ubuntu.jsonc b/mlos_bench/mlos_bench/config/environments/os/linux/runtime/setup-ubuntu.jsonc index 64eec5e1c77..aaeb06fd0f5 100644 --- a/mlos_bench/mlos_bench/config/environments/os/linux/runtime/setup-ubuntu.jsonc +++ b/mlos_bench/mlos_bench/config/environments/os/linux/runtime/setup-ubuntu.jsonc @@ -56,7 +56,6 @@ "vmName", "storageAccountName", "storageFileShareName", - "storageAccountKey", "mountPoint", "experiment_id", "trial_id" @@ -65,7 +64,6 @@ "shell_env_params": [ "storageAccountName", "storageFileShareName", - "storageAccountKey", "mountPoint", "experiment_id", "trial_id" diff --git a/mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-fileshare-service-subschema.json b/mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-fileshare-service-subschema.json index 283dc75ee51..9f0a498a67a 100644 --- a/mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-fileshare-service-subschema.json +++ b/mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-fileshare-service-subschema.json @@ -27,10 +27,6 @@ "storageFileShareName": { "description": "Azure storage file share name.", "type": "string" - }, - "storageAccountKey": { - "description": "Azure storage account key (typically provided in the global config in order to omit from source control).", - "type": "string" } }, "required": [ diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/service-fileshare.jsonc b/mlos_bench/mlos_bench/config/services/remote/azure/service-fileshare.jsonc index d800494c4c7..374201b7261 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/service-fileshare.jsonc +++ b/mlos_bench/mlos_bench/config/services/remote/azure/service-fileshare.jsonc @@ -2,7 +2,6 @@ "class": "mlos_bench.services.remote.azure.AzureFileShareService", "config": { "storageAccountName": "PLACEHOLDER; e.g.: osatsharedstorage", - "storageFileShareName": "PLACEHOLDER; e.g.: os-autotune-file-share", - "storageAccountKey": "PLACEHOLDER; comes from global config" + "storageFileShareName": "PLACEHOLDER; e.g.: os-autotune-file-share" } } diff --git a/mlos_bench/mlos_bench/tests/config/experiments/experiment_test_config.jsonc b/mlos_bench/mlos_bench/tests/config/experiments/experiment_test_config.jsonc index 2ca87c6f215..a5df029b116 100644 --- a/mlos_bench/mlos_bench/tests/config/experiments/experiment_test_config.jsonc +++ b/mlos_bench/mlos_bench/tests/config/experiments/experiment_test_config.jsonc @@ -8,7 +8,6 @@ // FIXME: The setup ubuntu configs currently use these values in their mounting scripts. // We should abstract that out so those details are only needed when a service that uses those is used. "storageAccountName": "foo", - "storageAccountKey": "bar", "storageFileShareName": "baz", // Assign some values to variadic tunables and required parameters present in the config examples. diff --git a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/invalid/azure_fileshare_service-config-missing.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/invalid/azure_fileshare_service-config-missing.jsonc index a197af783da..14abac43d98 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/invalid/azure_fileshare_service-config-missing.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/invalid/azure_fileshare_service-config-missing.jsonc @@ -1,8 +1,7 @@ { "class": "mlos_bench.services.remote.azure.AzureFileShareService", "config": { - "storageAccountName": "storage-account-name", - //"storageFileShareName": "required param missing", - "storageAccountKey": "required param missing" + // "storageAccountName": "storage-account-name", + "storageFileShareName": "required param missing" } } diff --git a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/unhandled/azure_fileshare_service-config-extras.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/unhandled/azure_fileshare_service-config-extras.jsonc index d2f576e98e3..77f796831d9 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/unhandled/azure_fileshare_service-config-extras.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/unhandled/azure_fileshare_service-config-extras.jsonc @@ -3,7 +3,6 @@ "config": { "storageAccountName": "storage-account-name", "storageFileShareName": "file-share-name", - "storageAccountKey": "storage-account-key-blob", "extra": "invalid" } diff --git a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/unhandled/azure_flex_service-extras.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/unhandled/azure_flex_service-extras.jsonc index 45a57e7cd27..80d1abbb33f 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/unhandled/azure_flex_service-extras.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/bad/unhandled/azure_flex_service-extras.jsonc @@ -16,7 +16,6 @@ // 3 unhandled config parameters: "storageAccountName": "storage-account-name", "storageFileShareName": "file-share-name", - "storageAccountKey": "storage-account-key-blob", "requestTimeout": 20 } diff --git a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/full/azure_fileshare_service-full.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/full/azure_fileshare_service-full.jsonc index 5b3da47149e..8aaad6d8636 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/full/azure_fileshare_service-full.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/full/azure_fileshare_service-full.jsonc @@ -6,7 +6,6 @@ "config": { "storageAccountName": "storage-account-name", "storageFileShareName": "file-share-name", - "storageAccountKey": "storage-account-key-blob", "pollInterval": 1, "pollTimeout": 10, diff --git a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/partial/azure_fileshare_service-partial.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/partial/azure_fileshare_service-partial.jsonc index cc1b423eea1..bc8af3de12e 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/partial/azure_fileshare_service-partial.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/partial/azure_fileshare_service-partial.jsonc @@ -1,8 +1,7 @@ { "class": "mlos_bench.services.remote.azure.AzureFileShareService", "config": { - //"storageAccountName": "storage-account-name", - //"storageAccountKey": "storage-account-key-blob", + // "storageAccountName": "storage-account-name", "storageFileShareName": "file-share-name" } } diff --git a/scripts/generate-azure-credentials-config.ps1 b/scripts/generate-azure-credentials-config.ps1 index 991420fee60..553e2e4cc47 100644 --- a/scripts/generate-azure-credentials-config.ps1 +++ b/scripts/generate-azure-credentials-config.ps1 @@ -17,14 +17,4 @@ if (-not $AZURE_STORAGE_ACCOUNT_NAME) { Write-Error "Missing default az storage account name config." } -az account get-access-token ` - --query "{tenant:tenant,subscription:subscription}" | - ConvertFrom-Json | - Add-Member "storageAccountKey" ( - az storage account keys list ` - --resource-group $AZURE_DEFAULTS_GROUP ` - --account-name $AZURE_STORAGE_ACCOUNT_NAME ` - --query "[0].value" ` - --output tsv ` - ) -PassThru | - ConvertTo-Json +az account get-access-token --query "{tenant:tenant,subscription:subscription}" diff --git a/scripts/generate-azure-credentials-config.sh b/scripts/generate-azure-credentials-config.sh index 890d9100289..2dd60ad71c3 100755 --- a/scripts/generate-azure-credentials-config.sh +++ b/scripts/generate-azure-credentials-config.sh @@ -11,10 +11,4 @@ AZURE_DEFAULTS_GROUP=${AZURE_DEFAULTS_GROUP:-$(az config get --local defaults.gr AZURE_STORAGE_ACCOUNT_RG=${AZURE_STORAGE_ACCOUNT_RG:-$AZURE_DEFAULTS_GROUP} AZURE_STORAGE_ACCOUNT_NAME=${AZURE_STORAGE_ACCOUNT_NAME:-$(az config get --local storage.account --query value -o tsv)} -az account get-access-token \ - --query "{tenant:tenant,subscription:subscription}" | - jq ".storageAccountKey = ` - az storage account keys list \ - --resource-group $AZURE_STORAGE_ACCOUNT_RG \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --query '[0].value'`" +az account get-access-token --query "{tenant:tenant,subscription:subscription}"