diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 26de98f66..704009412 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -86,9 +86,9 @@ jobs: with: submodules: true - - name: Make sure dockerd is running and test Docker. + - name: Make sure dockerd is running and test Docker run: | - docker run --rm hello-world + docker ps - name: Output image tag id: image_tag diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 296637775..b8afea93e 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -125,7 +125,7 @@ jobs: - name: Make sure dockerd is running and test Docker run: | - docker run --rm hello-world + docker ps - name: Install Kayobe run: | @@ -136,10 +136,11 @@ jobs: pip install -U pip && pip install ../src/kayobe - # Required for Docker registry login. Normally installed during host configure. + # Required for Pulp auth proxy deployment and Docker registry login. + # Normally installed during host configure. - name: Install Docker Python SDK run: | - pip install --user docker + sudo pip install docker - name: Configure localhost as a seed run: | @@ -150,11 +151,23 @@ jobs: localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 EOF + # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details. + # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the + # runner and dind containers. + - name: Deploy an authenticating package repository mirror proxy + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + - name: Build and push kolla overcloud images run: | args="${{ github.event.inputs.regexes }}" args="$args -e kolla_base_distro=${{ matrix.distro }}" args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}" + args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" if ${{ inputs.push }} == 'true'; then args="$args --push" fi @@ -169,6 +182,7 @@ jobs: run: | args="-e kolla_base_distro=${{ matrix.distro }}" args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}" + args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" if ${{ inputs.push }} == 'true'; then args="$args --push" fi diff --git a/.gitignore b/.gitignore index d83d0ce41..5891d3fdd 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring # Tempest logs tempest-artifacts + +# Ansible Galaxy roles & collections +etc/kayobe/ansible/roles/*\.*/ +etc/kayobe/ansible/collections/ diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst index 9df7426b5..ec14a9f1d 100644 --- a/doc/source/contributor/environments/ci-builder.rst +++ b/doc/source/contributor/environments/ci-builder.rst @@ -95,6 +95,34 @@ Next, configure the host OS & services. kayobe seed host configure +.. _authenticating-pulp-proxy: + +Authenticating Pulp proxy +------------------------- + +If you are building against authenticated package repositories such as those in +`Ark `_, you will need to provide secure access to +the repositories without leaking credentials into the built images or their +metadata. This is typically not the case for a client-local Pulp, which +provides unauthenticated read-only access to the repositories on a trusted +network. + +Docker provides `build +secrets `_, but these must be +explicitly requested for each RUN statement, making them challenging to use in +Kolla. + +StackHPC Kayobe Configuration provides support for deploying an authenticating +Pulp proxy that injects an HTTP basic auth header into requests that it +proxies. Because this proxy bypasses Pulp's authentication, it must not be +exposed to any untrusted environment. + +To deploy the proxy: + +.. parsed-literal:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml + Building images =============== @@ -105,6 +133,9 @@ At this point you are ready to build and push some container images. kayobe seed container image build --push kayobe overcloud container image build --push +If using an :ref:`authenticating Pulp proxy `, +append ``-e stackhpc_repo_mirror_auth_proxy_enabled=true`` to these commands. + The container images are tagged as |current_release|-. To use the new images, edit diff --git a/etc/kayobe/ansible/pulp-auth-proxy.yml b/etc/kayobe/ansible/pulp-auth-proxy.yml new file mode 100644 index 000000000..4cebbd386 --- /dev/null +++ b/etc/kayobe/ansible/pulp-auth-proxy.yml @@ -0,0 +1,14 @@ +--- +# See roles/pulp_auth_proxy/README.md for details. + +- name: Deploy Pulp auth proxy + hosts: container-image-builders + gather_facts: false + tasks: + - import_role: + name: pulp_auth_proxy + vars: + pulp_auth_proxy_url: "{{ stackhpc_repo_mirror_url }}" + pulp_auth_proxy_username: "{{ stackhpc_repo_mirror_username }}" + pulp_auth_proxy_password: "{{ stackhpc_repo_mirror_password }}" + pulp_auth_proxy_conf_path: "{{ base_path }}/containers/pulp_proxy" diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md new file mode 100644 index 000000000..f14a5b2e8 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md @@ -0,0 +1,26 @@ +# Pulp Auth Proxy + +There is currently no practical, secure way to provide credentials for +accessing Ark's authenticated package repositories from within a Kolla build. +Docker provides [build +secrets](https://docs.docker.com/build/building/secrets/), but these must be +explicitly requested for each RUN statement, making them challenging to use in +Kolla. + +This role deploys an Nginx container that runs as a reverse proxy, injecting an +HTTP basic authentication header into requests. + +Because this proxy bypasses Pulp's authentication, it must not be exposed to +any untrusted environment. + +## Role variables + +* `pulp_auth_proxy_pulp_url`: URL of the Pulp server to proxy requests to. +* `pulp_auth_proxy_username`: Username of the Pulp server to proxy requests to. +* `pulp_auth_proxy_password`: Password of the Pulp server to proxy requests to. +* `pulp_auth_proxy_conf_path`: Path to a directory in which to write Nginx + configuration. +* `pulp_auth_proxy_listen_ip`: IP address on the Docker host on which to + listen. Default is `127.0.0.1`. +* `pulp_auth_proxy_listen_port`: Port on the Docker host on which to listen. + Default is 80. diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml new file mode 100644 index 000000000..ae723565d --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml @@ -0,0 +1,7 @@ +--- +pulp_auth_proxy_url: +pulp_auth_proxy_username: +pulp_auth_proxy_password: +pulp_auth_proxy_conf_path: +pulp_auth_proxy_listen_ip: 127.0.0.1 +pulp_auth_proxy_listen_port: 80 diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml new file mode 100644 index 000000000..c15421510 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: "Ensure {{ pulp_auth_proxy_conf_path }} exists" + ansible.builtin.file: + path: "{{ pulp_auth_proxy_conf_path }}" + state: directory + mode: 0700 + become: true + +- name: Ensure pulp_proxy.conf is templated + ansible.builtin.template: + src: pulp_proxy.conf.j2 + dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf" + mode: 0600 + become: true + register: pulp_proxy_conf + +- name: Ensure pulp_proxy container is running + community.docker.docker_container: + name: pulp_proxy + image: nginx:stable-alpine + ports: + - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80" + restart_policy: "no" + restart: "{{ pulp_proxy_conf is changed }}" + volumes: + - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro" diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 new file mode 100644 index 000000000..3d5a87ae7 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 @@ -0,0 +1,17 @@ +server { + listen {{ pulp_auth_proxy_listen_port }}; + server_name pulp_proxy; + location / { + proxy_pass {{ pulp_auth_proxy_url }}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host {{ pulp_auth_proxy_url | urlsplit('hostname') }}; + # The important part: add basic auth header + proxy_set_header Authorization "Basic {{ (pulp_auth_proxy_username ~ ':' ~ pulp_auth_proxy_password) | b64encode }}"; + proxy_pass_header Authorization; + # See https://stackoverflow.com/questions/25329941/nginx-caching-proxy-fails-with-ssl23-get-server-hellosslv3-alert-handshake-fail/25330027#25330027 + proxy_ssl_server_name on; + proxy_ssl_protocols TLSv1.2; + } +} diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index 946759718..f31629357 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -43,7 +43,7 @@ resolv_is_managed: false # Host and port of a package repository mirror. # Build against the development Pulp service repositories. # Use Ark's package repositories to install packages. -stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}" +stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}" stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}" stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}" diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 15407f00c..1a047f33c 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -287,16 +287,21 @@ base_centos_repo_overrides_post_yum_rocky_list: "{{ stackhpc_rocky_9_repos + sta base_centos_repo_overrides_post_yum_list: "{{ base_centos_repo_overrides_post_yum_centos_list if kolla_base_distro == 'centos' else base_centos_repo_overrides_post_yum_rocky_list }}" stackhpc_yum_repos: "{{ stackhpc_centos_stream_repos if kolla_base_distro == 'centos' else stackhpc_rocky_9_repos }}" +# Apt sources.list entry prefix. +# If using an authenticating Pulp proxy we need to trust the repository because +# the certificate provided by the upstream repo will not match the proxy's IP. +stackhpc_ubuntu_repo_prefix: "deb {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}[trusted=yes] {% endif %}" + # List of base repositories for Ubuntu Focal. stackhpc_ubuntu_focal_base_repos: - - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal main universe" - - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe" - - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe" - - "deb {{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe" # List of UCA repositories for Ubuntu Focal. stackhpc_ubuntu_focal_uca_repos: - - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main" # List of repositories for Ubuntu Focal. stackhpc_ubuntu_focal_repos: "{{ stackhpc_ubuntu_focal_base_repos + stackhpc_ubuntu_focal_uca_repos }}" @@ -317,26 +322,16 @@ kolla_build_blocks: sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \ - {% if stackhpc_repo_mirror_username is truthy %} - -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \ - -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \ - {% endif %} -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \ {% endif %} {% endfor %} {% else %} RUN \ rm /etc/apt/sources.list && \ - rm -f /etc/apt/auth.conf && \ - {% if stackhpc_repo_mirror_url | urlsplit('scheme') == 'https' %} - {# We lack the ca-certificates package at this stage, so don't verify the CA #} + {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %} + {# We lack the ca-certificates package at this stage, so don't verify the CA initially #} echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \ {% endif %} - {% if stackhpc_repo_mirror_username is truthy %} - echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \ - echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \ - echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \ - {% endif %} {% for repo in stackhpc_ubuntu_focal_base_repos %} echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ {% endif %} @@ -354,10 +349,6 @@ kolla_build_blocks: sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \ - {% if stackhpc_repo_mirror_username is truthy %} - -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \ - -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \ - {% endif %} -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \ {% endfor %} {% endif %} @@ -368,13 +359,7 @@ kolla_build_blocks: {% endif %} RUN \ rm /etc/apt/sources.list && \ - rm -f /etc/apt/auth.conf && \ rm -f /etc/apt/apt.conf.d/90no-verify-peer && \ - {% if stackhpc_repo_mirror_username is truthy %} - echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \ - echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \ - echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \ - {% endif %} {% for repo in stackhpc_ubuntu_focal_repos %} echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ {% endif %} diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 21233ab34..513dd56d8 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -8,6 +8,17 @@ stackhpc_repo_mirror_username: # Password of a package repository mirror. stackhpc_repo_mirror_password: +# Whether to use an authenticating reverse proxy to access the package +# repository mirror. This may be used when building container images, to avoid +# injecting package repository mirror credentials into the built images. See +# ansible/roles/pulp_auth_proxy/README.md for details. +stackhpc_repo_mirror_auth_proxy_enabled: false + +# URL of an authenticating reverse proxy used to access the package repository +# mirror. Used during container image builds when +# stackhpc_repo_mirror_auth_proxy_enabled is true. +stackhpc_repo_mirror_auth_proxy_url: "http://localhost" + # Distribution name. Either 'development' or 'production'. stackhpc_repo_distribution: "development" diff --git a/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml new file mode 100644 index 000000000..e9d54f989 --- /dev/null +++ b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a custom playbook (``pulp-auth-proxy.yml``) for deploying an + authenticating proxy for Pulp. This can be used when building container + images to avoid leaking credentials for package repositories into the built + images or their metadata.