From 7076c9c5ec2af77b08adca42e8238abb6ed2d2ab Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 21 Aug 2023 14:21:01 -0700 Subject: [PATCH 1/5] backport fix for case-sensitive org names in vault S3 credential generation (#117) --- features/src/utils/devcontainer-feature.json | 2 +- .../opt/devcontainer/bin/vault/auth/github.sh | 27 ++++++++++++------- .../bin/vault/s3/creds/generate.sh | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index e15790ec..39129a58 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "23.8.12", + "version": "23.8.13", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/utils/opt/devcontainer/bin/vault/auth/github.sh b/features/src/utils/opt/devcontainer/bin/vault/auth/github.sh index c07b2cda..73cb4133 100755 --- a/features/src/utils/opt/devcontainer/bin/vault/auth/github.sh +++ b/features/src/utils/opt/devcontainer/bin/vault/auth/github.sh @@ -8,19 +8,24 @@ get_vault_token() { local VAULT_HOST="$1"; local user_orgs="${@:2}"; local gh_token="$(gh auth token)"; - local org local vault_token=null; + local org; for org in ${user_orgs}; do - vault_token="$( \ - curl -s \ - -X POST \ - -H "Content-Type: application/json" \ - -d "{\"token\": \"$gh_token\"}" \ - "$VAULT_HOST/v1/auth/github-${org}/login" \ - | jq -r '.auth.client_token' \ - )"; + for org in $(echo -e "${org}\n${org,,}\n${org^^}" | sort -su); do + vault_token="$( \ + curl -s \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"token\": \"$gh_token\"}" \ + "$VAULT_HOST/v1/auth/github-${org}/login" \ + | jq -r '.auth.client_token' \ + )"; + if [[ "${vault_token:-null}" != null ]]; then + break; + fi + done if [[ "${vault_token:-null}" != null ]]; then break; fi @@ -29,4 +34,8 @@ get_vault_token() { echo "vault_token='$vault_token'"; } +if test -n "${devcontainer_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + (get_vault_token "$@"); diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/generate.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/generate.sh index 1907a123..0294c471 100755 --- a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/generate.sh +++ b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/generate.sh @@ -37,7 +37,7 @@ generate_s3_creds() { local user_orgs="$( \ gh api user/orgs --jq '.[].login' \ -H "Accept: application/vnd.github+json" \ - | grep --color=never -E "(${allowed_orgs})" \ + | grep --color=never -iE "(${allowed_orgs})" \ )"; if test -z "${user_orgs:-}"; then From 71d515e597009d64ce7029a0826a798b2eb6401b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Aug 2023 18:23:21 -0700 Subject: [PATCH 2/5] use docker import instead of docker load, print docker info --- .github/workflows/build-and-push-image.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 7749ea1b..a8a37dd9 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -65,7 +65,8 @@ jobs: with: persist-credentials: false - - name: Free up disk space + - if: contains(runner.name, 'rapidsai') != true + name: Free up disk space uses: ./.github/actions/free-disk-space with: tool_cache: "${{ runner.tool_cache }}" @@ -88,6 +89,8 @@ jobs: name: "${{ needs.build-and-upload.outputs.repo }}:${{ needs.build-and-upload.outputs.tag }}" run: | + docker info; + find "${{ runner.temp }}/" -type f -name '*.tar'; docker manifest rm ${name} || true; @@ -95,10 +98,8 @@ jobs: hashes=(); for arch in "amd64" "arm64"; do - # Load and push the linux/$arch image and read its local hash - hash="$(docker load --input "${{ runner.temp }}/${arch}.tar" | tail -n1 | cut -d':' -f3)"; - # Tag the loaded image with the final name - docker image tag ${hash} ${name}; + # Load the linux/$arch image with the final name + docker import "${{ runner.temp }}/${arch}.tar" ${name}; # Push the local image to the registry to get its remote hash hash="$(docker push ${name} | tail -n1 | cut -d' ' -f3)"; # Save the remote hash for use below From ce32eac1c931a30d1e528e096417fe8600e78761 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Aug 2023 19:09:18 -0700 Subject: [PATCH 3/5] add platform tag to imported image --- .github/workflows/build-and-push-image.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index a8a37dd9..15bb9ae5 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -65,7 +65,6 @@ jobs: with: persist-credentials: false - - if: contains(runner.name, 'rapidsai') != true name: Free up disk space uses: ./.github/actions/free-disk-space with: @@ -89,8 +88,6 @@ jobs: name: "${{ needs.build-and-upload.outputs.repo }}:${{ needs.build-and-upload.outputs.tag }}" run: | - docker info; - find "${{ runner.temp }}/" -type f -name '*.tar'; docker manifest rm ${name} || true; @@ -99,7 +96,7 @@ jobs: for arch in "amd64" "arm64"; do # Load the linux/$arch image with the final name - docker import "${{ runner.temp }}/${arch}.tar" ${name}; + docker import --platform "linux/${arch}" "${{ runner.temp }}/${arch}.tar" ${name}; # Push the local image to the registry to get its remote hash hash="$(docker push ${name} | tail -n1 | cut -d' ' -f3)"; # Save the remote hash for use below From 92f5eb2fe0021b3e67161e91244bece54cee3a66 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Aug 2023 19:21:31 -0700 Subject: [PATCH 4/5] use output=type=docker --- .github/actions/build-image/action.yml | 2 +- .github/workflows/build-and-push-image.yml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/build-image/action.yml b/.github/actions/build-image/action.yml index 173e6de1..5ce36c2f 100644 --- a/.github/actions/build-image/action.yml +++ b/.github/actions/build-image/action.yml @@ -47,5 +47,5 @@ runs: --cache-from "${repo}:${{ inputs.tag }}" \ --image-name "${repo}:${{ inputs.tag }}" \ --workspace-folder "$(realpath -m ./image)" \ - --output "type=oci,dest=${{ runner.temp }}/${{ inputs.arch }}.tar" \ + --output "type=docker,dest=${{ runner.temp }}/${{ inputs.arch }}.tar" \ ; diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 15bb9ae5..38199b42 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -65,7 +65,7 @@ jobs: with: persist-credentials: false - name: Free up disk space + - name: Free up disk space uses: ./.github/actions/free-disk-space with: tool_cache: "${{ runner.tool_cache }}" @@ -95,12 +95,12 @@ jobs: hashes=(); for arch in "amd64" "arm64"; do - # Load the linux/$arch image with the final name - docker import --platform "linux/${arch}" "${{ runner.temp }}/${arch}.tar" ${name}; - # Push the local image to the registry to get its remote hash - hash="$(docker push ${name} | tail -n1 | cut -d' ' -f3)"; - # Save the remote hash for use below - hashes+=("$hash"); + # Load the linux/$arch and tag the loaded image with the final name + docker image tag \ + $(docker load --input "${{ runner.temp }}/${arch}.tar" | tail -n1 | cut -d' ' -f3) \ + ${name}; + # Push the local image to the registry to get its remote hash and save the remote hash for use below + hashes+=("$(docker push ${name} | tail -n1 | cut -d' ' -f3)"); done # Create and push the multiarch manifest From aac4600862e49434c0e9deb472b35da248c6e0d3 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Tue, 9 Jul 2024 13:12:36 -0700 Subject: [PATCH 5/5] Pin to rapids-dependency-file-generator<1.14 (23.08) (#349) * pin to rapids-dependency-file-generator<1.14 --- features/src/rapids-build-utils/devcontainer-feature.json | 2 +- features/src/rapids-build-utils/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index e4491ab7..0e5f016a 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "23.8.13", + "version": "23.8.14", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/install.sh b/features/src/rapids-build-utils/install.sh index 92e145dd..65bb3c8b 100644 --- a/features/src/rapids-build-utils/install.sh +++ b/features/src/rapids-build-utils/install.sh @@ -24,7 +24,7 @@ fi # Install the rapids dependency file generator and conda-merge if type python >/dev/null 2>&1; then - python -m pip install rapids-dependency-file-generator conda-merge toml; + python -m pip install 'rapids-dependency-file-generator<1.14' conda-merge toml; fi # Install RAPIDS build utility scripts to /opt/