From e2fdab90de673bdf225f18193b59e77b2c8e4405 Mon Sep 17 00:00:00 2001
From: Dawud <7688823+technowhizz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 09:14:32 +0000
Subject: [PATCH 01/17] Update smartmon-tools.yml

Fix grammar
---
 etc/kayobe/ansible/smartmon-tools.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/smartmon-tools.yml b/etc/kayobe/ansible/smartmon-tools.yml
index 1893421c6..9499cb339 100644
--- a/etc/kayobe/ansible/smartmon-tools.yml
+++ b/etc/kayobe/ansible/smartmon-tools.yml
@@ -2,7 +2,7 @@
 - hosts: overcloud
 
   tasks:
-    - name: Ensure smartmon-tools, jq, nvme-cli and cron/cronie is installed
+    - name: Ensure smartmon-tools, jq, nvme-cli and cron/cronie are installed
       package:
         name:
           - smartmontools

From e7191191ef4e430725fc7c2f1cfb9c5b68418299 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 9 Feb 2024 10:34:47 +0000
Subject: [PATCH 02/17] Fix permissions on reset-bls-entries playbook

---
 etc/kayobe/ansible/reset-bls-entries.yml | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 etc/kayobe/ansible/reset-bls-entries.yml

diff --git a/etc/kayobe/ansible/reset-bls-entries.yml b/etc/kayobe/ansible/reset-bls-entries.yml
old mode 100755
new mode 100644

From a76dc7e26358f7fe858a5822b56c24b03cc7f6de Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 9 Feb 2024 13:45:51 +0000
Subject: [PATCH 03/17] CI: Fix check container image tags job for Ark

The stackhpc-dev container repositories on Ark are container-push
repositories, so we must specify is_push=true. This requires
stackhpc.pulp 0.5.5.
---
 etc/kayobe/ansible/check-tags.yml   | 5 +----
 etc/kayobe/ansible/requirements.yml | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/etc/kayobe/ansible/check-tags.yml b/etc/kayobe/ansible/check-tags.yml
index 3f0d22a68..dc429a7cd 100644
--- a/etc/kayobe/ansible/check-tags.yml
+++ b/etc/kayobe/ansible/check-tags.yml
@@ -16,10 +16,6 @@
       set_fact:
         kolla_images: "{{ kolla_images_result.stdout | from_yaml }}"
 
-    - name: Set a fact about the Pulp URL
-      set_fact:
-        pulp_url: "{{ stackhpc_repo_mirror_url }}"
-
     # Use state=read and allow_missing=false to check for missing tags in test pulp.
     - import_role:
         name: stackhpc.pulp.pulp_container_content
@@ -30,6 +26,7 @@
           {%- set repository = kolla_docker_namespace ~ "/" ~ image -%}
           {%- set content = {
             "allow_missing": False,
+            "is_push": pulp_url == stackhpc_release_pulp_url,
             "repository": repository,
             "state": "read",
             "tags": tags,
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index 0bae52204..97086fd7e 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -7,7 +7,7 @@ collections:
   - name: pulp.squeezer
     version: 0.0.13
   - name: stackhpc.pulp
-    version: 0.5.4
+    version: 0.5.5
   - name: stackhpc.hashicorp
     version: 2.4.0
   - name: stackhpc.kayobe_workflows

From a56a95ceac8aa3d9a7c96caa582c1f4190a6052a Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 9 Feb 2024 11:26:48 +0000
Subject: [PATCH 04/17] CI: Add retries to TF apply in aio job

We sometimes hit temporary failures during VM creation. Adding a retry
may help to keep things running.

(cherry picked from commit 6329f1c83b18f1db7a0ceb875126249c62d24a8e)
---
 .github/workflows/stackhpc-all-in-one.yml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index ba6fa26cb..6df546799 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -154,7 +154,17 @@ jobs:
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
       - name: Terraform Apply
-        run: terraform apply -auto-approve
+        run: |
+          for attempt in $(seq 5); do
+              if terraform apply -auto-approve; then
+                  echo "Created infrastructure on attempt $attempt"
+                  break
+              fi
+              echo "Failed to create infrastructure on attempt $attempt"
+              sleep 10
+              terraform destroy -auto-approve
+              sleep 60
+          done
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
           OS_CLOUD: ${{ inputs.OS_CLOUD }}

From f849d26b9f527c8244a630b234e773d9dc7df898 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 9 Feb 2024 09:46:17 +0000
Subject: [PATCH 05/17] CI: Trim down package dependencies

(cherry picked from commit 8d9e620717217745b41726cd7f7100bc8d6bd53c)
---
 .github/workflows/stackhpc-all-in-one.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 6df546799..26de98f66 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -80,7 +80,7 @@ jobs:
       - name: Install Package
         uses: ConorMacBride/install-package@main
         with:
-          apt: build-essential git unzip nodejs python3 python3-pip python3-wheel python-is-python3
+          apt: git unzip nodejs
 
       - uses: actions/checkout@v4
         with:

From 3543c9fbff4369c6721b7262cb5850ce53d21450 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Fri, 9 Feb 2024 16:02:07 +0000
Subject: [PATCH 06/17] Correct OpenSearch migration instructions

Because we set ``kolla_enable_central_logging: true``,
``kolla_enable_elsticsearch`` is also set to true, as per the Kayobe
defaults:

```
kolla_enable_elasticsearch: "{{ not kolla_base_distro == 'rocky' and (kolla_enable_central_logging | bool or kolla_enable_osprofiler | bool or kolla_enable_skydive | bool or kolla_enable_monasca | bool) }}"
```

Update the instructions to disbale Elasticsearch explicitly.
---
 doc/source/operations/rocky-linux-9.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/operations/rocky-linux-9.rst b/doc/source/operations/rocky-linux-9.rst
index 123534e59..5b13c2807 100644
--- a/doc/source/operations/rocky-linux-9.rst
+++ b/doc/source/operations/rocky-linux-9.rst
@@ -161,7 +161,7 @@ Elasticsearch/Kibana should be migrated to OpenSearch.
 - Read the `Kolla Ansible OpenSearch migration
   docs <https://docs.openstack.org/kolla-ansible/yoga/reference/logging-and-monitoring/central-logging-guide-opensearch.html#migration>`__
 - If necessary, take a backup of the Elasticsearch data.
-- Ensure ``kolla_enable_elasticsearch`` is unset in ``etc/kayobe/kolla.yml``
+- Ensure ``kolla_enable_elasticsearch`` is set to false in ``etc/kayobe/kolla.yml``
 - If you have a custom Kolla Ansible inventory, ensure that it contains the ``opensearch`` and ``opensearch-dashboards`` groups. Otherwise, sync with the inventory in Kayobe.
 - Set ``kolla_enable_opensearch: true`` in ``etc/kayobe/kolla.yml``
 - ``kayobe overcloud service configuration generate --node-config-dir '/tmp/ignore' --kolla-tags none``

From 5bb34318d1ceb18f26f6e151b13b6c19e99b62f7 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 9 Feb 2024 16:12:10 +0000
Subject: [PATCH 07/17] Bump docker build and push action to v5

---
 .github/workflows/stackhpc-build-kayobe-image.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml
index ace9a544c..8c8ce4d2f 100644
--- a/.github/workflows/stackhpc-build-kayobe-image.yml
+++ b/.github/workflows/stackhpc-build-kayobe-image.yml
@@ -81,7 +81,7 @@ jobs:
       # Setting KAYOBE_USER_UID and KAYOBE_USER_GID to 1001 to match docker's defaults
       # so that docker can run as a privileged user within the Kayobe image.
       - name: Build and push Docker image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           file: ./.automation/docker/kayobe/Dockerfile
           context: .

From 8522950b57c666db30496b828664b8229d951c8d Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 9 Feb 2024 19:14:30 +0000
Subject: [PATCH 08/17] CI: Conditional container tag check job

Use the same technique as the aio job to allow it to be a required status check.
---
 .github/path-filters.yml                    | 3 +++
 .github/workflows/stackhpc-check-tags.yml   | 6 +++++-
 .github/workflows/stackhpc-pull-request.yml | 5 ++++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/path-filters.yml b/.github/path-filters.yml
index 337b1f9a7..b9815d9d6 100644
--- a/.github/path-filters.yml
+++ b/.github/path-filters.yml
@@ -20,3 +20,6 @@ aio:
   - 'kayobe-env'
   - 'requirements.txt'
   - 'terraform/aio/**'
+check-tags:
+  - '.github/workflows/stackhpc-check-tags.yml'
+  - 'etc/kayobe/kolla-image-tags.yml'
diff --git a/.github/workflows/stackhpc-check-tags.yml b/.github/workflows/stackhpc-check-tags.yml
index 0ef1526ae..f5a12a714 100644
--- a/.github/workflows/stackhpc-check-tags.yml
+++ b/.github/workflows/stackhpc-check-tags.yml
@@ -10,6 +10,10 @@ on:
         description: Kayobe container image
         type: string
         required: true
+      if:
+        description: Whether to run the workflow (workaround for required status checks issue)
+        type: boolean
+        default: true
     secrets:
       KAYOBE_VAULT_PASSWORD:
         required: true
@@ -19,7 +23,7 @@ env:
 jobs:
   check-tags:
     name: Check container image tags
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    if: inputs.if
     runs-on: arc-skc-aio-runner
     permissions: {}
     env:
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index dcc0c7c34..fbb6e378a 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -20,6 +20,7 @@ jobs:
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     outputs:
       aio: ${{ steps.changes.outputs.aio }}
+      check-tags: ${{ steps.changes.outputs.check-tags }}
     steps:
       - name: GitHub Checkout
         uses: actions/checkout@v4
@@ -79,12 +80,14 @@ jobs:
   check-tags:
     name: Check container image tags
     needs:
+      - check-changes
       - build-kayobe-image
     uses: ./.github/workflows/stackhpc-check-tags.yml
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      if: ${{ needs.check-changes.outputs.check-tags == 'true' }}
     secrets: inherit
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   all-in-one-ubuntu-jammy-ovs:
     name: aio (Ubuntu Jammy OVS)

From 936f195fcf9295e0ecfbdf37ef8b85b3f8e7f47a Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Fri, 9 Feb 2024 16:15:53 +0000
Subject: [PATCH 09/17] Bump stackhpc.kayobe_workflows to version 1.0.3

We hit this error in a control host bootstrap otherwise.

```
ERROR! Failed to resolve the requested dependencies map. Could not satisfy the following requirements:
* stackhpc.kayobe_workflows:1.0.2 (direct request)
* stackhpc.kayobe_workflows:1.0.3 (dependency of "virtual collection Git repo")
```
---
 etc/kayobe/ansible/requirements.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index b8444c22a..aa28f43f6 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -11,7 +11,7 @@ collections:
   - name: stackhpc.hashicorp
     version: 2.4.0
   - name: stackhpc.kayobe_workflows
-    version: 1.0.2
+    version: 1.0.3
 roles:
   - src: stackhpc.vxlan
   - name: ansible-lockdown.rhel8_cis

From 707d65b8bd6794e8000511eb6d65f28031140a01 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 12:01:58 +0000
Subject: [PATCH 10/17] Revert "Don't verify Apt repo CA initially when using
 HTTPS in container build"

This reverts commit f8947a99318d73827de2653d8027f69a6df31f3e.

This approach is not secure and leaves credentials in images.
---
 etc/kayobe/kolla.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 15407f00c..45c9b573c 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -328,10 +328,6 @@ kolla_build_blocks:
     RUN \
         rm /etc/apt/sources.list && \
         rm -f /etc/apt/auth.conf && \
-    {% if stackhpc_repo_mirror_url | urlsplit('scheme') == 'https' %}
-    {# We lack the ca-certificates package at this stage, so don't verify the CA #}
-        echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \
-    {% endif %}
     {% if stackhpc_repo_mirror_username is truthy %}
         echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
         echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
@@ -369,7 +365,6 @@ kolla_build_blocks:
     RUN \
         rm /etc/apt/sources.list && \
         rm -f /etc/apt/auth.conf && \
-        rm -f /etc/apt/apt.conf.d/90no-verify-peer && \
     {% if stackhpc_repo_mirror_username is truthy %}
         echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
         echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \

From 5d505223ef3524751f3aa2cedf096bac495d3f9c Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 12:02:20 +0000
Subject: [PATCH 11/17] Revert "Add package repository credentials to container
 image build"

This reverts commit 05e1995f6553e617f6bbc0712e9e1e7be18639c9.

This approach is not secure and leaves credentials in images.
---
 etc/kayobe/kolla.yml | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 45c9b573c..082e9243b 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -317,22 +317,12 @@ kolla_build_blocks:
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
-    {% if stackhpc_repo_mirror_username is truthy %}
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \
-    {% endif %}
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
-        rm -f /etc/apt/auth.conf && \
-    {% if stackhpc_repo_mirror_username is truthy %}
-        echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
-        echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
-        echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \
-    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_base_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}
@@ -350,10 +340,6 @@ kolla_build_blocks:
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
-    {% if stackhpc_repo_mirror_username is truthy %}
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \
-    {% endif %}
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
     {% endif %}
@@ -364,12 +350,6 @@ kolla_build_blocks:
     {% endif %}
     RUN \
         rm /etc/apt/sources.list && \
-        rm -f /etc/apt/auth.conf && \
-    {% if stackhpc_repo_mirror_username is truthy %}
-        echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
-        echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
-        echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \
-    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}

From c794383d7b40a15ab5f009f69ba2891c89a82937 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 11:46:32 +0000
Subject: [PATCH 12/17] Add Ansible Galaxy roles and collections to .gitignore

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index d83d0ce41..5891d3fdd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -58,3 +58,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring
 
 # Tempest logs
 tempest-artifacts
+
+# Ansible Galaxy roles & collections
+etc/kayobe/ansible/roles/*\.*/
+etc/kayobe/ansible/collections/

From 88133608abff538ce721ee44a9215622bdebed80 Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Tue, 13 Feb 2024 12:26:44 +0000
Subject: [PATCH 13/17] Update etc/kayobe/ansible/smartmon-tools.yml

Co-authored-by: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com>
---
 etc/kayobe/ansible/smartmon-tools.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/smartmon-tools.yml b/etc/kayobe/ansible/smartmon-tools.yml
index 9499cb339..bb5cf5dca 100644
--- a/etc/kayobe/ansible/smartmon-tools.yml
+++ b/etc/kayobe/ansible/smartmon-tools.yml
@@ -2,7 +2,7 @@
 - hosts: overcloud
 
   tasks:
-    - name: Ensure smartmon-tools, jq, nvme-cli and cron/cronie are installed
+    - name: Ensure smartmontools, jq, nvme-cli and cron/cronie are installed
       package:
         name:
           - smartmontools

From 6dd2e7f5fb94188659572d5b2c44a4742966189a Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 11:44:55 +0000
Subject: [PATCH 14/17] Add a playbook to deploy an authenticating Pulp proxy

There is currently no practical, secure way to provide credentials for
accessing Ark's authenticated package repositories from within a Kolla
build.  Docker provides build secrets, but these must be explicitly
requested for each RUN statement, making them challenging to use in
Kolla.

This change adds a playbook that deploys an Nginx container that runs as
a reverse proxy, injecting an HTTP basic authentication header into
requests.

Because this proxy bypasses Pulp's authentication, it must not be
exposed to any untrusted environment.

[1] https://docs.docker.com/build/building/secrets/
---
 etc/kayobe/ansible/pulp-auth-proxy.yml        | 14 ++++++++++
 .../ansible/roles/pulp_auth_proxy/README.md   | 26 +++++++++++++++++++
 .../roles/pulp_auth_proxy/defaults/main.yml   |  7 +++++
 .../roles/pulp_auth_proxy/tasks/main.yml      | 26 +++++++++++++++++++
 .../templates/pulp_proxy.conf.j2              | 17 ++++++++++++
 5 files changed, 90 insertions(+)
 create mode 100644 etc/kayobe/ansible/pulp-auth-proxy.yml
 create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/README.md
 create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml
 create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml
 create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2

diff --git a/etc/kayobe/ansible/pulp-auth-proxy.yml b/etc/kayobe/ansible/pulp-auth-proxy.yml
new file mode 100644
index 000000000..4cebbd386
--- /dev/null
+++ b/etc/kayobe/ansible/pulp-auth-proxy.yml
@@ -0,0 +1,14 @@
+---
+# See roles/pulp_auth_proxy/README.md for details.
+
+- name: Deploy Pulp auth proxy
+  hosts: container-image-builders
+  gather_facts: false
+  tasks:
+    - import_role:
+        name: pulp_auth_proxy
+      vars:
+        pulp_auth_proxy_url: "{{ stackhpc_repo_mirror_url }}"
+        pulp_auth_proxy_username: "{{ stackhpc_repo_mirror_username }}"
+        pulp_auth_proxy_password: "{{ stackhpc_repo_mirror_password }}"
+        pulp_auth_proxy_conf_path: "{{ base_path }}/containers/pulp_proxy"
diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md
new file mode 100644
index 000000000..f14a5b2e8
--- /dev/null
+++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md
@@ -0,0 +1,26 @@
+# Pulp Auth Proxy
+
+There is currently no practical, secure way to provide credentials for
+accessing Ark's authenticated package repositories from within a Kolla build.
+Docker provides [build
+secrets](https://docs.docker.com/build/building/secrets/), but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+This role deploys an Nginx container that runs as a reverse proxy, injecting an
+HTTP basic authentication header into requests.
+
+Because this proxy bypasses Pulp's authentication, it must not be exposed to
+any untrusted environment.
+
+## Role variables
+
+* `pulp_auth_proxy_pulp_url`: URL of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_username`: Username of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_password`: Password of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_conf_path`: Path to a directory in which to write Nginx
+  configuration.
+* `pulp_auth_proxy_listen_ip`: IP address on the Docker host on which to
+  listen. Default is `127.0.0.1`.
+* `pulp_auth_proxy_listen_port`: Port on the Docker host on which to listen.
+  Default is 80.
diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml
new file mode 100644
index 000000000..ae723565d
--- /dev/null
+++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml
@@ -0,0 +1,7 @@
+---
+pulp_auth_proxy_url:
+pulp_auth_proxy_username:
+pulp_auth_proxy_password:
+pulp_auth_proxy_conf_path:
+pulp_auth_proxy_listen_ip: 127.0.0.1
+pulp_auth_proxy_listen_port: 80
diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml
new file mode 100644
index 000000000..c15421510
--- /dev/null
+++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml
@@ -0,0 +1,26 @@
+---
+- name: "Ensure {{ pulp_auth_proxy_conf_path }} exists"
+  ansible.builtin.file:
+    path: "{{ pulp_auth_proxy_conf_path }}"
+    state: directory
+    mode: 0700
+  become: true
+
+- name: Ensure pulp_proxy.conf is templated
+  ansible.builtin.template:
+    src: pulp_proxy.conf.j2
+    dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf"
+    mode: 0600
+  become: true
+  register: pulp_proxy_conf
+
+- name: Ensure pulp_proxy container is running
+  community.docker.docker_container:
+    name: pulp_proxy
+    image: nginx:stable-alpine
+    ports:
+      - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80"
+    restart_policy: "no"
+    restart: "{{ pulp_proxy_conf is changed }}"
+    volumes:
+      - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro"
diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2
new file mode 100644
index 000000000..3d5a87ae7
--- /dev/null
+++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2
@@ -0,0 +1,17 @@
+server {
+    listen {{ pulp_auth_proxy_listen_port }};
+    server_name pulp_proxy;
+    location / {
+        proxy_pass {{ pulp_auth_proxy_url }};
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host {{ pulp_auth_proxy_url | urlsplit('hostname') }};
+        # The important part: add basic auth header
+        proxy_set_header Authorization "Basic {{ (pulp_auth_proxy_username ~ ':' ~ pulp_auth_proxy_password) | b64encode }}";
+        proxy_pass_header Authorization;
+        # See https://stackoverflow.com/questions/25329941/nginx-caching-proxy-fails-with-ssl23-get-server-hellosslv3-alert-handshake-fail/25330027#25330027
+        proxy_ssl_server_name on;
+        proxy_ssl_protocols TLSv1.2;
+    }
+}

From 9f940b7c82540863bbb957424df95978158a6d07 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 11:51:07 +0000
Subject: [PATCH 15/17] Use authenticating Pulp proxy during container image
 builds

This change uses the authenticating Pulp proxy added in
6320be9d01c38a8bf0c2b5950aebea8d4906ac64 to provide container image
builds access to Ark's authenticated package repositories without
injecting the credentials into the built images or their metadata.
---
 .../stackhpc-container-image-build.yml        | 18 +++++++++--
 .../contributor/environments/ci-builder.rst   | 31 +++++++++++++++++++
 .../environments/ci-builder/stackhpc-ci.yml   |  2 +-
 etc/kayobe/kolla.yml                          | 20 +++++++++---
 etc/kayobe/stackhpc.yml                       | 11 +++++++
 .../pulp-auth-proxy-24f0b31a4498441b.yaml     |  7 +++++
 6 files changed, 81 insertions(+), 8 deletions(-)
 create mode 100644 releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 296637775..717d6508f 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -136,10 +136,11 @@ jobs:
           pip install -U pip &&
           pip install ../src/kayobe
 
-      # Required for Docker registry login. Normally installed during host configure.
+      # Required for Pulp auth proxy deployment and Docker registry login.
+      # Normally installed during host configure.
       - name: Install Docker Python SDK
         run: |
-          pip install --user docker
+          sudo pip install docker
 
       - name: Configure localhost as a seed
         run: |
@@ -150,11 +151,23 @@ jobs:
           localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
           EOF
 
+      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
+      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+      # runner and dind containers.
+      - name: Deploy an authenticating package repository mirror proxy
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
       - name: Build and push kolla overcloud images
         run: |
           args="${{ github.event.inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
@@ -169,6 +182,7 @@ jobs:
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst
index 9df7426b5..ec14a9f1d 100644
--- a/doc/source/contributor/environments/ci-builder.rst
+++ b/doc/source/contributor/environments/ci-builder.rst
@@ -95,6 +95,34 @@ Next, configure the host OS & services.
 
    kayobe seed host configure
 
+.. _authenticating-pulp-proxy:
+
+Authenticating Pulp proxy
+-------------------------
+
+If you are building against authenticated package repositories such as those in
+`Ark <https://ark.stackhpc.com>`_, you will need to provide secure access to
+the repositories without leaking credentials into the built images or their
+metadata.  This is typically not the case for a client-local Pulp, which
+provides unauthenticated read-only access to the repositories on a trusted
+network.
+
+Docker provides `build
+secrets <https://docs.docker.com/build/building/secrets/>`_, but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+StackHPC Kayobe Configuration provides support for deploying an authenticating
+Pulp proxy that injects an HTTP basic auth header into requests that it
+proxies. Because this proxy bypasses Pulp's authentication, it must not be
+exposed to any untrusted environment.
+
+To deploy the proxy:
+
+.. parsed-literal::
+
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml
+
 Building images
 ===============
 
@@ -105,6 +133,9 @@ At this point you are ready to build and push some container images.
    kayobe seed container image build --push
    kayobe overcloud container image build --push
 
+If using an :ref:`authenticating Pulp proxy <authenticating-pulp-proxy>`,
+append ``-e stackhpc_repo_mirror_auth_proxy_enabled=true`` to these commands.
+
 The container images are tagged as |current_release|-<datetime>.
 
 To use the new images, edit
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index 946759718..f31629357 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -43,7 +43,7 @@ resolv_is_managed: false
 # Host and port of a package repository mirror.
 # Build against the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
-stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
+stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}"
 stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
 stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"
 
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 082e9243b..1a047f33c 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -287,16 +287,21 @@ base_centos_repo_overrides_post_yum_rocky_list: "{{ stackhpc_rocky_9_repos + sta
 base_centos_repo_overrides_post_yum_list: "{{ base_centos_repo_overrides_post_yum_centos_list if kolla_base_distro == 'centos' else base_centos_repo_overrides_post_yum_rocky_list }}"
 stackhpc_yum_repos: "{{ stackhpc_centos_stream_repos if kolla_base_distro == 'centos' else stackhpc_rocky_9_repos }}"
 
+# Apt sources.list entry prefix.
+# If using an authenticating Pulp proxy we need to trust the repository because
+# the certificate provided by the upstream repo will not match the proxy's IP.
+stackhpc_ubuntu_repo_prefix: "deb {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}[trusted=yes] {% endif %}"
+
 # List of base repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_base_repos:
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe"
 
 # List of UCA repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_uca_repos:
-  - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main"
 
 # List of repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_repos: "{{ stackhpc_ubuntu_focal_base_repos + stackhpc_ubuntu_focal_uca_repos }}"
@@ -323,6 +328,10 @@ kolla_build_blocks:
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
+    {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}
+    {# We lack the ca-certificates package at this stage, so don't verify the CA initially #}
+        echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \
+    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_base_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}
@@ -350,6 +359,7 @@ kolla_build_blocks:
     {% endif %}
     RUN \
         rm /etc/apt/sources.list && \
+        rm -f /etc/apt/apt.conf.d/90no-verify-peer && \
     {% for repo in stackhpc_ubuntu_focal_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 21233ab34..513dd56d8 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -8,6 +8,17 @@ stackhpc_repo_mirror_username:
 # Password of a package repository mirror.
 stackhpc_repo_mirror_password:
 
+# Whether to use an authenticating reverse proxy to access the package
+# repository mirror.  This may be used when building container images, to avoid
+# injecting package repository mirror credentials into the built images.  See
+# ansible/roles/pulp_auth_proxy/README.md for details.
+stackhpc_repo_mirror_auth_proxy_enabled: false
+
+# URL of an authenticating reverse proxy used to access the package repository
+# mirror. Used during container image builds when
+# stackhpc_repo_mirror_auth_proxy_enabled is true.
+stackhpc_repo_mirror_auth_proxy_url: "http://localhost"
+
 # Distribution name. Either 'development' or 'production'.
 stackhpc_repo_distribution: "development"
 
diff --git a/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml
new file mode 100644
index 000000000..e9d54f989
--- /dev/null
+++ b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds a custom playbook (``pulp-auth-proxy.yml``) for deploying an
+    authenticating proxy for Pulp. This can be used when building container
+    images to avoid leaking credentials for package repositories into the built
+    images or their metadata.

From 74e87c51652aed8dc0f51bde92e22fa7c4224201 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 14:22:13 +0000
Subject: [PATCH 16/17] CI: Avoid hitting Dockerhub rate limits

Running the hello-world container from Dockerhub at the beginning of
workflows can cause us to hit rate limits. Switch to a simple docker ps.
---
 .github/workflows/stackhpc-all-in-one.yml            | 4 ++--
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 26de98f66..704009412 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -86,9 +86,9 @@ jobs:
         with:
            submodules: true
 
-      - name: Make sure dockerd is running and test Docker.
+      - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Output image tag
         id: image_tag
diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 717d6508f..b8afea93e 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -125,7 +125,7 @@ jobs:
 
       - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Install Kayobe
         run: |

From f557aedb34cd51460012dfd44db8d4cbbaaea19b Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 15:45:15 +0000
Subject: [PATCH 17/17] CI: Replace cat line when writing inventory for
 container image build

I don't know where this was removed, possibly in a bad merge.
---
 .github/workflows/stackhpc-container-image-build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 349a9b91d..6a8055ded 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -135,6 +135,7 @@ jobs:
 
       - name: Configure localhost as a seed
         run: |
+          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
           # A 'seed' host used for building images.
           # Use localhost for container image builds.
           [seed]