Merge pull request #939 from stackhpc/2023.1-zed-merge

2023.1: zed merge

.github/path-filters.yml

@@ -20,3 +20,6 @@ aio:
   - 'kayobe-env'
   - 'requirements.txt'
   - 'terraform/aio/**'
+check-tags:
+  - '.github/workflows/stackhpc-check-tags.yml'
+  - 'etc/kayobe/kolla-image-tags.yml'

.github/workflows/stackhpc-all-in-one.yml

@@ -107,9 +107,9 @@ jobs:
           fi
           echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT
 
-      - name: Make sure dockerd is running and test Docker.
+      - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Output image tag
         id: image_tag

.github/workflows/stackhpc-build-kayobe-image.yml

@@ -85,7 +85,7 @@ jobs:
       # Setting KAYOBE_USER_UID and KAYOBE_USER_GID to 1001 to match docker's defaults
       # so that docker can run as a privileged user within the Kayobe image.
       - name: Build and push Docker image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           file: ./.automation/docker/kayobe/Dockerfile
           context: .

.github/workflows/stackhpc-check-tags.yml

@@ -10,6 +10,10 @@ on:
         description: Kayobe container image
         type: string
         required: true
+      if:
+        description: Whether to run the workflow (workaround for required status checks issue)
+        type: boolean
+        default: true
     secrets:
       KAYOBE_VAULT_PASSWORD:
         required: true
@@ -19,7 +23,7 @@ env:
 jobs:
   check-tags:
     name: Check container image tags
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    if: inputs.if
     runs-on: arc-skc-aio-runner
     permissions: {}
     env:

.github/workflows/stackhpc-container-image-build.yml

@@ -116,7 +116,7 @@ jobs:
 
       - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Install Kayobe
         run: |
@@ -127,10 +127,11 @@ jobs:
           pip install -U pip &&
           pip install ../src/kayobe
 
-      # Required for Docker registry login. Normally installed during host configure.
+      # Required for Pulp auth proxy deployment and Docker registry login.
+      # Normally installed during host configure.
       - name: Install Docker Python SDK
         run: |
-          pip install --user docker
+          sudo pip install docker
 
       - name: Configure localhost as a seed
         run: |
@@ -141,11 +142,23 @@ jobs:
           localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
           EOF
 
+      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
+      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+      # runner and dind containers.
+      - name: Deploy an authenticating package repository mirror proxy
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
       - name: Build and push kolla overcloud images
         run: |
           args="${{ github.event.inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=$KOLLA_TAG"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
@@ -161,6 +174,7 @@ jobs:
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=$KOLLA_TAG"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi

.github/workflows/stackhpc-pull-request.yml

@@ -20,6 +20,7 @@ jobs:
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     outputs:
       aio: ${{ steps.changes.outputs.aio }}
+      check-tags: ${{ steps.changes.outputs.check-tags }}
     steps:
       - name: GitHub Checkout
         uses: actions/checkout@v4
@@ -79,12 +80,14 @@ jobs:
   check-tags:
     name: Check container image tags
     needs:
+      - check-changes
       - build-kayobe-image
     uses: ./.github/workflows/stackhpc-check-tags.yml
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      if: ${{ needs.check-changes.outputs.check-tags == 'true' }}
     secrets: inherit
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
 
   all-in-one-ubuntu-jammy-ovs:
     name: aio (Ubuntu Jammy OVS)

.gitignore

@@ -56,3 +56,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring
 
 # Tempest logs
 tempest-artifacts
+
+# Ansible Galaxy roles & collections
+etc/kayobe/ansible/roles/*\.*/
+etc/kayobe/ansible/collections/

doc/source/contributor/environments/ci-builder.rst

@@ -101,6 +101,34 @@ Next, configure the host OS & services.
 
    kayobe seed host configure
 
+.. _authenticating-pulp-proxy:
+
+Authenticating Pulp proxy
+-------------------------
+
+If you are building against authenticated package repositories such as those in
+`Ark <https://ark.stackhpc.com>`_, you will need to provide secure access to
+the repositories without leaking credentials into the built images or their
+metadata.  This is typically not the case for a client-local Pulp, which
+provides unauthenticated read-only access to the repositories on a trusted
+network.
+
+Docker provides `build
+secrets <https://docs.docker.com/build/building/secrets/>`_, but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+StackHPC Kayobe Configuration provides support for deploying an authenticating
+Pulp proxy that injects an HTTP basic auth header into requests that it
+proxies. Because this proxy bypasses Pulp's authentication, it must not be
+exposed to any untrusted environment.
+
+To deploy the proxy:
+
+.. parsed-literal::
+
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml
+
 Building images
 ===============
 
@@ -111,6 +139,9 @@ At this point you are ready to build and push some container images.
    kayobe seed container image build --push
    kayobe overcloud container image build --push
 
+If using an :ref:`authenticating Pulp proxy <authenticating-pulp-proxy>`,
+append ``-e stackhpc_repo_mirror_auth_proxy_enabled=true`` to these commands.
+
 The container images are tagged as |current_release|-<datetime>.
 
 To use the new images, edit

etc/kayobe/ansible/check-tags.yml

@@ -16,10 +16,6 @@
       set_fact:
         kolla_images: "{{ kolla_images_result.stdout | from_yaml }}"
 
-    - name: Set a fact about the Pulp URL
-      set_fact:
-        pulp_url: "{{ stackhpc_repo_mirror_url }}"
-
     # Use state=read and allow_missing=false to check for missing tags in test pulp.
     - import_role:
         name: stackhpc.pulp.pulp_container_content
@@ -30,6 +26,7 @@
           {%- set repository = kolla_docker_namespace ~ "/" ~ image -%}
           {%- set content = {
             "allow_missing": False,
+            "is_push": pulp_url == stackhpc_release_pulp_url,
             "repository": repository,
             "state": "read",
             "tags": tags,

etc/kayobe/ansible/pulp-auth-proxy.yml

@@ -0,0 +1,14 @@
+---
+# See roles/pulp_auth_proxy/README.md for details.
+
+- name: Deploy Pulp auth proxy
+  hosts: container-image-builders
+  gather_facts: false
+  tasks:
+    - import_role:
+        name: pulp_auth_proxy
+      vars:
+        pulp_auth_proxy_url: "{{ stackhpc_repo_mirror_url }}"
+        pulp_auth_proxy_username: "{{ stackhpc_repo_mirror_username }}"
+        pulp_auth_proxy_password: "{{ stackhpc_repo_mirror_password }}"
+        pulp_auth_proxy_conf_path: "{{ base_path }}/containers/pulp_proxy"

etc/kayobe/ansible/requirements.yml

@@ -7,11 +7,11 @@ collections:
   - name: pulp.squeezer
     version: 0.0.13
   - name: stackhpc.pulp
-    version: 0.5.4
+    version: 0.5.5
   - name: stackhpc.hashicorp
     version: 2.4.0
   - name: stackhpc.kayobe_workflows
-    version: 1.0.2
+    version: 1.0.3
 roles:
   - src: stackhpc.vxlan
   - name: ansible-lockdown.ubuntu22_cis

etc/kayobe/ansible/roles/pulp_auth_proxy/README.md

@@ -0,0 +1,26 @@
+# Pulp Auth Proxy
+
+There is currently no practical, secure way to provide credentials for
+accessing Ark's authenticated package repositories from within a Kolla build.
+Docker provides [build
+secrets](https://docs.docker.com/build/building/secrets/), but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+This role deploys an Nginx container that runs as a reverse proxy, injecting an
+HTTP basic authentication header into requests.
+
+Because this proxy bypasses Pulp's authentication, it must not be exposed to
+any untrusted environment.
+
+## Role variables
+
+* `pulp_auth_proxy_pulp_url`: URL of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_username`: Username of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_password`: Password of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_conf_path`: Path to a directory in which to write Nginx
+  configuration.
+* `pulp_auth_proxy_listen_ip`: IP address on the Docker host on which to
+  listen. Default is `127.0.0.1`.
+* `pulp_auth_proxy_listen_port`: Port on the Docker host on which to listen.
+  Default is 80.

etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+pulp_auth_proxy_url:
+pulp_auth_proxy_username:
+pulp_auth_proxy_password:
+pulp_auth_proxy_conf_path:
+pulp_auth_proxy_listen_ip: 127.0.0.1
+pulp_auth_proxy_listen_port: 80

etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: "Ensure {{ pulp_auth_proxy_conf_path }} exists"
+  ansible.builtin.file:
+    path: "{{ pulp_auth_proxy_conf_path }}"
+    state: directory
+    mode: 0700
+  become: true
+
+- name: Ensure pulp_proxy.conf is templated
+  ansible.builtin.template:
+    src: pulp_proxy.conf.j2
+    dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf"
+    mode: 0600
+  become: true
+  register: pulp_proxy_conf
+
+- name: Ensure pulp_proxy container is running
+  community.docker.docker_container:
+    name: pulp_proxy
+    image: nginx:stable-alpine
+    ports:
+      - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80"
+    restart_policy: "no"
+    restart: "{{ pulp_proxy_conf is changed }}"
+    volumes:
+      - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro"

etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2

@@ -0,0 +1,17 @@
+server {
+    listen {{ pulp_auth_proxy_listen_port }};
+    server_name pulp_proxy;
+    location / {
+        proxy_pass {{ pulp_auth_proxy_url }};
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host {{ pulp_auth_proxy_url | urlsplit('hostname') }};
+        # The important part: add basic auth header
+        proxy_set_header Authorization "Basic {{ (pulp_auth_proxy_username ~ ':' ~ pulp_auth_proxy_password) | b64encode }}";
+        proxy_pass_header Authorization;
+        # See https://stackoverflow.com/questions/25329941/nginx-caching-proxy-fails-with-ssl23-get-server-hellosslv3-alert-handshake-fail/25330027#25330027
+        proxy_ssl_server_name on;
+        proxy_ssl_protocols TLSv1.2;
+    }
+}

etc/kayobe/ansible/smartmon-tools.yml

@@ -2,7 +2,7 @@
 - hosts: overcloud
 
   tasks:
-    - name: Ensure smartmon-tools, jq, nvme-cli and cron/cronie is installed
+    - name: Ensure smartmontools, jq, nvme-cli and cron/cronie are installed
       package:
         name:
           - smartmontools

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -40,7 +40,7 @@ resolv_is_managed: false
 # Host and port of a package repository mirror.
 # Build against the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
-stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
+stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}"
 stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
 stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"