From 653c886c75ac0bb97bc71f043544ab01a9e0160d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 14 Sep 2023 11:16:51 +0100 Subject: [PATCH 01/11] Move Kolla image tags to a separate file This will make it easier to automate various tasks. --- etc/kayobe/kolla-image-tags.yml | 23 +++++++++++++++++++++++ etc/kayobe/kolla/globals.yml | 25 ++++++------------------- 2 files changed, 29 insertions(+), 19 deletions(-) create mode 100644 etc/kayobe/kolla-image-tags.yml diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml new file mode 100644 index 000000000..d6416b169 --- /dev/null +++ b/etc/kayobe/kolla-image-tags.yml @@ -0,0 +1,23 @@ +--- +# Dict of Kolla image tags to deploy for each service. +# Each key is the tag variable prefix name, and the value is another dict, +# where the key is the OS distro and the value is the tag to deploy. +kolla_image_tags: + openstack: + rocky: zed-rocky-9-20230921T153510 + ubuntu: zed-ubuntu-jammy-20230921T153510 + bifrost: + rocky: zed-rocky-9-20230927T142529 + ubuntu: zed-ubuntu-jammy-20230927T154443 + ovn: + rocky: zed-rocky-9-20230925T132313 + ubuntu: zed-ubuntu-jammy-20230821T155947 + cloudkitty: + rocky: zed-rocky-9-20231114T124701 + ubuntu: zed-ubuntu-jammy-20231114T124701 + neutron: + rocky: zed-rocky-9-20231115T094053 + ubuntu: zed-ubuntu-jammy-20231115T094053 + opensearch: + rocky: zed-rocky-9-20231214T095452 + ubuntu: zed-ubuntu-jammy-20231214T095452 diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index d2e20b974..911755ce1 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -10,25 +10,12 @@ enable_docker_repo: "{% raw %}{{ 'overcloud' not in group_names }}{% endraw %}" # This is necessary for os migrations where mixed clouds might be deployed kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}" -kayobe_image_tags: - openstack: - rocky: zed-rocky-9-20230921T153510 - ubuntu: zed-ubuntu-jammy-20230921T153510 - bifrost: - rocky: zed-rocky-9-20230927T142529 - ubuntu: zed-ubuntu-jammy-20230927T154443 - ovn: - rocky: zed-rocky-9-20230925T132313 - ubuntu: zed-ubuntu-jammy-20230821T155947 - cloudkitty: - rocky: zed-rocky-9-20231114T124701 - ubuntu: zed-ubuntu-jammy-20231114T124701 - neutron: - rocky: zed-rocky-9-20231115T094053 - ubuntu: zed-ubuntu-jammy-20231115T094053 - opensearch: - rocky: zed-rocky-9-20231214T095452 - ubuntu: zed-ubuntu-jammy-20231214T095452 +# Dict of Kolla image tags to deploy for each service. +# Each key is the tag variable prefix name, and the value is another dict, +# where the key is the OS distro and the value is the tag to deploy. +# NOTE: This is defined in etc/kayobe/kolla-image-tags.yml. +kolla_image_tags: +{{ kolla_image_tags | to_nice_yaml | indent(width=4, first=true) }} openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}" bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}" From 0cfa5489eafd5e56192461a2b51ab299dba412ce Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 3 Oct 2023 11:07:32 +0100 Subject: [PATCH 02/11] Add base distro version to Kolla image tag definitions --- etc/kayobe/kolla-image-tags.yml | 24 ++++++++++++------------ etc/kayobe/pulp.yml | 5 +++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index d6416b169..c7b669fb5 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -4,20 +4,20 @@ # where the key is the OS distro and the value is the tag to deploy. kolla_image_tags: openstack: - rocky: zed-rocky-9-20230921T153510 - ubuntu: zed-ubuntu-jammy-20230921T153510 + rocky-9: zed-rocky-9-20230921T153510 + ubuntu-jammy: zed-ubuntu-jammy-20230921T153510 bifrost: - rocky: zed-rocky-9-20230927T142529 - ubuntu: zed-ubuntu-jammy-20230927T154443 + rocky-9: zed-rocky-9-20230927T142529 + ubuntu-jammy: zed-ubuntu-jammy-20230927T154443 ovn: - rocky: zed-rocky-9-20230925T132313 - ubuntu: zed-ubuntu-jammy-20230821T155947 + rocky-9: zed-rocky-9-20230925T132313 + ubuntu-jammy: zed-ubuntu-jammy-20230821T155947 cloudkitty: - rocky: zed-rocky-9-20231114T124701 - ubuntu: zed-ubuntu-jammy-20231114T124701 + rocky-9: zed-rocky-9-20231114T124701 + ubuntu-jammy: zed-ubuntu-jammy-20231114T124701 neutron: - rocky: zed-rocky-9-20231115T094053 - ubuntu: zed-ubuntu-jammy-20231115T094053 + rocky-9: zed-rocky-9-20231115T094053 + ubuntu-jammy: zed-ubuntu-jammy-20231115T094053 opensearch: - rocky: zed-rocky-9-20231214T095452 - ubuntu: zed-ubuntu-jammy-20231214T095452 + rocky-9: zed-rocky-9-20231214T095452 + ubuntu-jammy: zed-ubuntu-jammy-20231214T095452 diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index d305f1d91..9207f5239 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -522,8 +522,9 @@ stackhpc_pulp_images_kolla: # List of images for each base distribution which should not/cannot be built. stackhpc_kolla_unbuildable_images: - ubuntu: [] - rocky: [] + ubuntu-jammy: [] + rocky-9: + - iscsid # Whitespace-separated list of regular expressions matching Kolla image names. # Usage is similar to kolla-build CLI arguments. From fafc68ae0ced87124cbf3290fa90456a3b5a27e4 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 3 Oct 2023 11:08:05 +0100 Subject: [PATCH 03/11] Add kolla-images.py --- tools/kolla-images.py | 365 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100755 tools/kolla-images.py diff --git a/tools/kolla-images.py b/tools/kolla-images.py new file mode 100755 index 000000000..f30b989b5 --- /dev/null +++ b/tools/kolla-images.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 + +""" +Script to manage Kolla container image tags. + +Background: +In Kolla Ansible each container is deployed using a specific image. +Typically the image is named the same as the container, with underscores +replaced by dashes, however there are some exceptions. Sometimes multiple +containers use the same image. + +The image tag deployed by each container is defined by a Kolla Ansible variable +named _tag. There are also intermediate tag variables to make it +easier to set the tag for all containers in a service, e.g. nova_tag is the +default for nova_api_tag, nova_compute, etc. There is a global default tag +defined by openstack_tag. This setup forms a hierarchy of tag variables. + +This script captures this logic, as well as exceptions to these rules. +""" + +import argparse +import inspect +import json +import os +import pathlib +import re +import subprocess +import sys +from typing import Dict, List, Optional + +import yaml + + +# Dict of Kolla image tags to deploy for each service. +# Each key is the tag variable prefix name, and the value is another dict, +# where the key is the OS distro and the value is the tag to deploy. +# This is the content of etc/kayobe/kolla-image-tags.yml. +KollaImageTags = Dict[str, Dict[str, str]] + +# Maps a Kolla image to a list of containers that use the image. +IMAGE_TO_CONTAINERS_EXCEPTIONS: Dict[str, List[str]] = { + "haproxy": [ + "glance_tls_proxy", + "neutron_tls_proxy", + ], + "mariadb-server": [ + "mariadb", + "mariabackup", + ], + "neutron-eswitchd": [ + "neutron_mlnx_agent", + ], + "neutron-metadata-agent": [ + "neutron_metadata_agent", + "neutron_ovn_metadata_agent", + ], + "nova-conductor": [ + "nova_super_conductor", + "nova_conductor", + ], + "prometheus-v2-server": [ + "prometheus_server", + ], +} + +# Maps a container to the parent tag variable in the hierarchy. +CONTAINER_TO_PREFIX_VAR_EXCEPTIONS: Dict[str, str] = { + "cron": "common", + "fluentd": "common", + "glance_tls_proxy": "haproxy", + "hacluster_corosync": "openstack", + "hacluster_pacemaker": "openstack", + "hacluster_pacemaker_remote": "openstack", + "heat_api_cfn": "heat", + "ironic_neutron_agent": "neutron", + "kolla_toolbox": "common", + "mariabackup": "mariadb", + "neutron_eswitchd": "neutron_mlnx_agent", + "neutron_tls_proxy": "haproxy", + "nova_compute_ironic": "nova", + "redis_sentinel": "openstack", + "swift_object_expirer": "swift", + "tgtd": "iscsi", +} + +# List of supported base distributions and versions. +SUPPORTED_BASE_DISTROS = [ + "rocky-9", + "ubuntu-jammy", +] + + +def parse_args() -> argparse.Namespace: + """Parse command line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument("--base-distros", default=",".join(SUPPORTED_BASE_DISTROS), choices=SUPPORTED_BASE_DISTROS) + subparsers = parser.add_subparsers(dest="command", required=True) + + subparser = subparsers.add_parser("check-hierarchy", help="Check tag variable hierarchy against kolla-ansible") + subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch") + + subparser = subparsers.add_parser("check-tags", help="Check specified tags for each image exist in the Ark registry") + subparser.add_argument("--registry", required=True, help="Hostname of container image registry") + subparser.add_argument("--namespace", required=True, help="Namespace in container image registry") + + subparsers.add_parser("list-containers", help="List supported containers based on pulp.yml") + + subparsers.add_parser("list-images", help="List supported images based on pulp.yml") + + subparsers.add_parser("list-tags", help="List tags for each image based on kolla-image-tags.yml") + + subparsers.add_parser("list-tag-vars", help="List Kolla Ansible tag variables") + + return parser.parse_args() + + +def get_abs_path(relative_path: str) -> str: + """Return the absolute path of a file in SKC.""" + script_path = pathlib.Path(inspect.getfile(inspect.currentframe())) + return script_path.parent.parent / relative_path + + +def read_images(images_file: str) -> List[str]: + """Read image list from pulp.yml config file.""" + with open(get_abs_path(images_file), "r") as f: + variables = yaml.safe_load(f) + return variables["stackhpc_pulp_images_kolla"] + + +def read_unbuildable_images(images_file: str) -> Dict[str, List[str]]: + """Read unbuildable image list from pulp.yml config file.""" + with open(get_abs_path(images_file), "r") as f: + variables = yaml.safe_load(f) + return variables["stackhpc_kolla_unbuildable_images"] + + +def read_kolla_image_tags(tags_file: str) -> KollaImageTags: + """Read kolla image tags kolla-image-tags.yml config file.""" + with open(get_abs_path(tags_file), "r") as f: + variables = yaml.safe_load(f) + return variables["kolla_image_tags"] + + +def get_containers(image): + """Return a list of containers that use the specified image.""" + default_container = image.replace('-', '_') + return IMAGE_TO_CONTAINERS_EXCEPTIONS.get(image, [default_container]) + + +def get_parent_tag_name(kolla_image_tags: KollaImageTags, base_distro: Optional[str], container: str) -> str: + """Return the parent tag variable for a container in the tag variable hierarchy.""" + + if container in CONTAINER_TO_PREFIX_VAR_EXCEPTIONS: + parent = CONTAINER_TO_PREFIX_VAR_EXCEPTIONS[container] + if parent in kolla_image_tags: + return parent + + def tag_key(tag): + """Return a sort key to order the tags.""" + if tag == "openstack": + # This is the default tag. + return 0 + elif tag != container and container.startswith(tag) and (base_distro is None or base_distro in kolla_image_tags[tag]): + # Prefix match - sort by the longest match. + return -len(tag) + else: + # No match. + return 1 + + return sorted(kolla_image_tags.keys(), key=tag_key)[0] + + +def get_parent_tag(kolla_image_tags: KollaImageTags, base_distro: str, container: str) -> str: + """Return the tag used by the parent in the hierarchy.""" + parent_tag_name = get_parent_tag_name(kolla_image_tags, base_distro, container) + return kolla_image_tags[parent_tag_name][base_distro] + + +def get_tag(kolla_image_tags: KollaImageTags, base_distro: str, container: str) -> str: + """Return the tag for a container.""" + container_tag = kolla_image_tags.get(container, {}).get(base_distro) + if container_tag: + return container_tag + + return get_parent_tag(kolla_image_tags, base_distro, container) + + +def get_tags(base_distros: List[str], kolla_image_tags: KollaImageTags) -> Dict[str, List[str]]: + """Return a list of tags used for each image.""" + images = read_images("etc/kayobe/pulp.yml") + unbuildable_images = read_unbuildable_images("etc/kayobe/pulp.yml") + image_tags: Dict[str, List[str]] = {} + for base_distro in base_distros: + for image in images: + if image not in unbuildable_images[base_distro]: + for container in get_containers(image): + tag = get_tag(kolla_image_tags, base_distro, container) + tags = image_tags.setdefault(image, []) + if tag not in tags: + tags.append(tag) + return image_tags + + +def get_openstack_release() -> str: + """Return the OpenStack release.""" + with open(get_abs_path(".gitreview"), "r") as f: + gitreview = f.readlines() + for line in gitreview: + if "=" not in line: + continue + key, value = line.split("=") + if key.strip().strip() == "defaultbranch": + value = value.strip().rstrip() + prefix = "stable/" + assert value.startswith(prefix) + return value[len(prefix):] + raise Exception("Failed to determine OpenStack release") + + +def validate(kolla_image_tags: KollaImageTags): + """Validate the kolla_image_tags variable.""" + tag_var_re = re.compile(r"^[a-z0-9_]+$") + openstack_release = get_openstack_release() + tag_res = { + base_distro: re.compile(f"^{openstack_release}-{base_distro}-[\d]{{8}}T[\d]{{6}}$") + for base_distro in SUPPORTED_BASE_DISTROS + } + errors = [] + if "openstack" not in kolla_image_tags: + errors.append("Missing default openstack tag") + for tag_var, base_distros in kolla_image_tags.items(): + if not tag_var_re.match(tag_var): + errors.append(f"Key {tag_var} does not match expected pattern. It should match {tag_var_re.pattern}") + for base_distro, tag in base_distros.items(): + if base_distro not in SUPPORTED_BASE_DISTROS: + errors.append(f"{tag_var}: base distro {base_distro} not supported. Options: {SUPPORTED_BASE_DISTROS}") + continue + if not tag_res[base_distro].match(tag): + errors.append(f"{tag_var}: {base_distro}: tag {tag} does not match expected pattern. It should match {tag_res[base_distro].pattern}") + if errors: + print("Errors in kolla_image_tags variable:") + for error in errors: + print(error) + sys.exit(1) + + +def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, registry: str, namespace: str): + """Check whether expected tags are present in container image registry.""" + image_tags = get_tags(base_distros, kolla_image_tags) + + missing = {} + for image, tags in image_tags.items(): + for _ in range(3): + try: + output = subprocess.check_output(f"skopeo list-tags docker://{registry}/{namespace}/{image}", shell=True) + except Exception as e: + exc = e + else: + break + else: + raise exc + ark_tags = json.loads(output)["Tags"] + missing_tags = set(tags) - set(ark_tags) + if missing_tags: + missing[image] = list(missing_tags) + + if missing: + print(f"ERROR: Some expected tags not found in {namespace} namespace") + print(yaml.dump(missing, indent=2)) + sys.exit(1) + + +def check_hierarchy(kolla_ansible_path: str): + """Check the tag variable hierarchy against Kolla Ansible variables.""" + cmd = """git grep -h '^[a-z0-9_]*_tag:' ansible/roles/*/defaults/main.yml""" + hierarchy_str = subprocess.check_output(cmd, shell=True, cwd=os.path.realpath(kolla_ansible_path)) + hierarchy = yaml.safe_load(hierarchy_str) + # This one is not a container: + hierarchy.pop("octavia_amp_image_tag") + tag_var_re = re.compile(r"^([a-z0-9_]+)_tag$") + parent_re = re.compile(r"{{[\s]*([a-z0-9_]+)_tag[\s]*}}") + hierarchy = { + tag_var_re.match(tag_var).group(1): parent_re.match(parent).group(1) + for tag_var, parent in hierarchy.items() + } + kolla_image_tags: KollaImageTags = {image: {} for image in hierarchy} + kolla_image_tags["openstack"] = {} + errors = [] + for tag_var, expected in hierarchy.items(): + parent = get_parent_tag_name(kolla_image_tags, None, tag_var) + if parent != expected: + errors.append((tag_var, parent, expected)) + if errors: + print("Errors:") + for tag_var, parent, expected in errors: + print(f"{tag_var} -> {parent} != {expected}") + if errors: + sys.exit(1) + + +def list_containers(base_distros: List[str]): + """List supported containers.""" + images = read_images("etc/kayobe/pulp.yml") + unbuildable_images = read_unbuildable_images("etc/kayobe/pulp.yml") + containers = set() + for base_distro in base_distros: + for image in images: + if image not in unbuildable_images[base_distro]: + containers |= set(get_containers(image)) + print(yaml.dump(sorted(containers))) + + +def list_images(base_distros: List[str]): + """List supported images.""" + images = read_images("etc/kayobe/pulp.yml") + print(yaml.dump(images)) + + +def list_tags(base_distros: List[str], kolla_image_tags: KollaImageTags): + """List tags used by each image.""" + image_tags = get_tags(base_distros, kolla_image_tags) + + print(yaml.dump(image_tags)) + + +def list_tag_vars(kolla_image_tags: KollaImageTags): + """List tag variables.""" + tag_vars = [] + for tag_var in kolla_image_tags: + if tag_var == "openstack": + default = "" + else: + parent_tag_name = get_parent_tag_name(kolla_image_tags, None, tag_var) + default = f" | default({parent_tag_name}_tag)" + tag_vars.append(f"{tag_var}_tag: \"{{{{ kolla_image_tags['{tag_var}'][kolla_base_distro_and_version]{default} }}}}\"") + + for tag_var in tag_vars: + print(tag_var) + + +def main(): + args = parse_args() + kolla_image_tags = read_kolla_image_tags("etc/kayobe/kolla-image-tags.yml") + base_distros = args.base_distros.split(",") + + validate(kolla_image_tags) + + if args.command == "check-hierarchy": + check_hierarchy(args.kolla_ansible_path) + elif args.command == "check-tags": + check_tags(base_distros, kolla_image_tags, args.registry, args.namespace) + elif args.command == "list-containers": + list_containers(base_distros) + elif args.command == "list-images": + list_images(base_distros) + elif args.command == "list-tags": + list_tags(base_distros, kolla_image_tags) + elif args.command == "list-tag-vars": + list_tag_vars(kolla_image_tags) + else: + sys.exit(1) + + +if __name__ == "__main__": + main() From 8d8f6e0a8f330a8d0245b493412f6bf084125c8e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 23 Oct 2023 16:46:34 +0100 Subject: [PATCH 04/11] Remove iscsid from Pulp image list We no longer require it since Ironic dropped the iscsi deploy driver. --- etc/kayobe/pulp.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 9207f5239..736423f28 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -523,8 +523,7 @@ stackhpc_pulp_images_kolla: # List of images for each base distribution which should not/cannot be built. stackhpc_kolla_unbuildable_images: ubuntu-jammy: [] - rocky-9: - - iscsid + rocky-9: [] # Whitespace-separated list of regular expressions matching Kolla image names. # Usage is similar to kolla-build CLI arguments. From eee54814512d0ae187cd12924b2c0d641f4e331e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 24 Oct 2023 16:40:05 +0100 Subject: [PATCH 05/11] CI: Trigger container image promotion automatically after merge No more forgotten image promotions! --- .github/workflows/stackhpc-promote.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stackhpc-promote.yml b/.github/workflows/stackhpc-promote.yml index e5c254381..bbd9279b5 100644 --- a/.github/workflows/stackhpc-promote.yml +++ b/.github/workflows/stackhpc-promote.yml @@ -1,5 +1,5 @@ --- -name: Promote package repositories +name: Promote Pulp repositories on: push: branches: @@ -7,7 +7,7 @@ on: - stackhpc/zed jobs: promote: - name: Trigger package repository promotion + name: Trigger Pulp promotion workflows if: github.repository == 'stackhpc/stackhpc-kayobe-config' runs-on: ubuntu-latest permissions: {} @@ -27,3 +27,19 @@ jobs: - name: Display link to package repository promotion workflows run: | echo "::notice Package repository promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-promote.yml" + + # NOTE(mgoddard): Trigger another CI workflow in the + # stackhpc-release-train repository. + - name: Trigger container image promotion + run: | + gh workflow run \ + container-promote.yml \ + --repo stackhpc/stackhpc-release-train \ + --ref main \ + -f kayobe_config_branch=${{ github.ref_name }} + env: + GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }} + + - name: Display link to container image promotion workflows + run: | + echo "::notice Container image promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-promote.yml" From d1262ba67eb402f2b1668df3811ed943a6efdae8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 24 Oct 2023 16:42:13 +0100 Subject: [PATCH 06/11] Automatically define Kolla Ansible tag variables in globals.yml Use the new kolla-images.py script to automatically define tag variables in globals.yml based on the tags that are defined in kolla_image_tags. --- etc/kayobe/kolla/globals.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 911755ce1..b156f050a 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -10,6 +10,9 @@ enable_docker_repo: "{% raw %}{{ 'overcloud' not in group_names }}{% endraw %}" # This is necessary for os migrations where mixed clouds might be deployed kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}" +# Convenience variable for base distro and version string. +kolla_base_distro_and_version: "{% raw %}{{ kolla_base_distro }}-{{ kolla_base_distro_version }}{% endraw %}" + # Dict of Kolla image tags to deploy for each service. # Each key is the tag variable prefix name, and the value is another dict, # where the key is the OS distro and the value is the tag to deploy. @@ -17,12 +20,11 @@ kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw % kolla_image_tags: {{ kolla_image_tags | to_nice_yaml | indent(width=4, first=true) }} -openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}" -bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}" -ovn_tag: "{% raw %}{{ kayobe_image_tags['ovn'][kolla_base_distro] }}{% endraw %}" -cloudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}" -neutron_tag: "{% raw %}{{ kayobe_image_tags['neutron'][kolla_base_distro] }}{% endraw %}" -opensearch_tag: "{% raw %}{{ kayobe_image_tags['opensearch'][kolla_base_distro] }}{% endraw %}" +# Variables defining which tag to use for each container's image. +{{ lookup('pipe', 'python3 ' ~ kayobe_config_path ~ '/../../tools/kolla-images.py list-tag-vars') }} + +############################################################################# +# RabbitMQ om_enable_rabbitmq_high_availability: true From ae21932916b41d1d4fc2c64d4fd7d08a1f6bcd18 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 25 Oct 2023 14:31:31 +0100 Subject: [PATCH 07/11] CI: Add a job to check whether all expected images are in Test Pulp --- .github/workflows/stackhpc-check-tags.yml | 48 +++++++++++++++++++ .github/workflows/stackhpc-pull-request.yml | 10 ++++ etc/kayobe/ansible/check-tags.yml | 39 +++++++++++++++ etc/kayobe/ansible/requirements.yml | 2 +- .../environments/ci-aio/stackhpc-ci.yml | 6 +++ 5 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/stackhpc-check-tags.yml create mode 100644 etc/kayobe/ansible/check-tags.yml diff --git a/.github/workflows/stackhpc-check-tags.yml b/.github/workflows/stackhpc-check-tags.yml new file mode 100644 index 000000000..1fdb01307 --- /dev/null +++ b/.github/workflows/stackhpc-check-tags.yml @@ -0,0 +1,48 @@ +--- +# This workflow queries the Test Pulp server to check that all image tags +# specified in kolla_image_tags are present. + +name: Check container image tags +on: + workflow_call: + inputs: + kayobe_image: + description: Kayobe container image + type: string + required: true + secrets: + KAYOBE_VAULT_PASSWORD: + required: true + +env: + ANSIBLE_FORCE_COLOR: True +jobs: + check-tags: + name: Check container image tags + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: [self-hosted, stackhpc-kayobe-config-aio] + permissions: {} + env: + KAYOBE_ENVIRONMENT: ci-aio + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + KAYOBE_IMAGE: ${{ inputs.kayobe_image }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + # The same tag may be reused (e.g. pr-123), so ensure we have the latest image. + - name: Pull latest Kayobe image + run: | + sudo docker image pull $KAYOBE_IMAGE + + - name: Check container image tags + run: | + sudo -E docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \ + '$KAYOBE_CONFIG_PATH/ansible/check-tags.yml' + #env: + #KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index c8aad3652..cb236281b 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -76,6 +76,16 @@ jobs: if: ${{ needs.check-changes.outputs.aio == 'true' }} if: github.repository == 'stackhpc/stackhpc-kayobe-config' + check-tags: + name: Check container image tags + needs: + - build-kayobe-image + uses: ./.github/workflows/stackhpc-check-tags.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + secrets: inherit + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + all-in-one-ubuntu-jammy-ovs: name: aio (Ubuntu Jammy OVS) needs: diff --git a/etc/kayobe/ansible/check-tags.yml b/etc/kayobe/ansible/check-tags.yml new file mode 100644 index 000000000..3f0d22a68 --- /dev/null +++ b/etc/kayobe/ansible/check-tags.yml @@ -0,0 +1,39 @@ +--- +# This playbook queries the Pulp server to check that all image tags specified +# in kolla_image_tags are present. + +- name: Check whether tags exist in Pulp container registry + hosts: localhost + tasks: + - name: Query images and tags + command: + cmd: >- + {{ kayobe_config_path }}/../../tools/kolla-images.py list-tags + register: kolla_images_result + changed_when: false + + - name: Set a fact about images and tags + set_fact: + kolla_images: "{{ kolla_images_result.stdout | from_yaml }}" + + - name: Set a fact about the Pulp URL + set_fact: + pulp_url: "{{ stackhpc_repo_mirror_url }}" + + # Use state=read and allow_missing=false to check for missing tags in test pulp. + - import_role: + name: stackhpc.pulp.pulp_container_content + vars: + pulp_container_content: >- + {%- set contents = [] -%} + {%- for image, tags in kolla_images.items() -%} + {%- set repository = kolla_docker_namespace ~ "/" ~ image -%} + {%- set content = { + "allow_missing": False, + "repository": repository, + "state": "read", + "tags": tags, + } -%} + {%- set _ = contents.append(content) -%} + {%- endfor -%} + {{ contents }} diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index fd685f1dc..0bae52204 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -7,7 +7,7 @@ collections: - name: pulp.squeezer version: 0.0.13 - name: stackhpc.pulp - version: 0.5.2 + version: 0.5.4 - name: stackhpc.hashicorp version: 2.4.0 - name: stackhpc.kayobe_workflows diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml index ec5a82578..f9f425f2b 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -62,3 +62,9 @@ stackhpc_docker_registry_password: !vault | 38333133393730633666613965653364316162353337313330346164303631313731646461363461 3963323635373866630a633533376339363734626664333765313665623662613764363038383735 38646138376438643533376161376634653439386230353365316239613430363338 + +# Override Pulp credentials to allow querying container image tags in the +# check-tags.yml custom playbook. +pulp_url: "{{ stackhpc_repo_mirror_url }}" +pulp_username: "{{ stackhpc_docker_registry_username }}" +pulp_password: "{{ stackhpc_docker_registry_password }}" From 82a194a9f6c8daad500712fea988b4a9a88ce272 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 29 Jan 2024 14:35:05 +0000 Subject: [PATCH 08/11] kolla-images.py: Remove unnecessary strips, support unmaintained branches --- tools/kolla-images.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/kolla-images.py b/tools/kolla-images.py index f30b989b5..f2c69e27e 100755 --- a/tools/kolla-images.py +++ b/tools/kolla-images.py @@ -209,11 +209,11 @@ def get_openstack_release() -> str: if "=" not in line: continue key, value = line.split("=") - if key.strip().strip() == "defaultbranch": - value = value.strip().rstrip() - prefix = "stable/" - assert value.startswith(prefix) - return value[len(prefix):] + if key.strip() == "defaultbranch": + value = value.strip() + for prefix in ("stable/", "unmaintained/"): + if value.startswith(prefix): + return value[len(prefix):] raise Exception("Failed to determine OpenStack release") From 4e908585e8549de324b02d9d81ff472f4559a790 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 29 Jan 2024 14:35:26 +0000 Subject: [PATCH 09/11] kolla-images.py: Check for existence of skopeo --- tools/kolla-images.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/kolla-images.py b/tools/kolla-images.py index f2c69e27e..836f86090 100755 --- a/tools/kolla-images.py +++ b/tools/kolla-images.py @@ -246,6 +246,11 @@ def validate(kolla_image_tags: KollaImageTags): def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, registry: str, namespace: str): """Check whether expected tags are present in container image registry.""" + try: + subprocess.check_output("type skopeo", shell=True) + except subprocess.CalledProcessError: + print("Failed to find skopeo. Please install it.") + sys.exit(1) image_tags = get_tags(base_distros, kolla_image_tags) missing = {} From 03371903c04cec806849f0f1c4ab3d8d0a4a045c Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 29 Jan 2024 15:35:21 +0000 Subject: [PATCH 10/11] zed: Update bifrost image for Ubuntu Jammy The previous tag was not in Ark, but this one is present. --- etc/kayobe/kolla-image-tags.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index c7b669fb5..024b247a6 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -8,7 +8,7 @@ kolla_image_tags: ubuntu-jammy: zed-ubuntu-jammy-20230921T153510 bifrost: rocky-9: zed-rocky-9-20230927T142529 - ubuntu-jammy: zed-ubuntu-jammy-20230927T154443 + ubuntu-jammy: zed-ubuntu-jammy-20231101T132522 ovn: rocky-9: zed-rocky-9-20230925T132313 ubuntu-jammy: zed-ubuntu-jammy-20230821T155947 From 0bdc266d94052caa7c60bbdebe98eb4b7790cb2e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 29 Jan 2024 15:38:23 +0000 Subject: [PATCH 11/11] kolla-images.py: Fix container to image tag lookup corner case The failing case was the haproxy image used by the neutron_tls_proxy container, if a tag is defined for the 'neutron' prefix but not 'haproxy'. --- tools/kolla-images.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/kolla-images.py b/tools/kolla-images.py index 836f86090..06e67cb1c 100755 --- a/tools/kolla-images.py +++ b/tools/kolla-images.py @@ -151,16 +151,18 @@ def get_parent_tag_name(kolla_image_tags: KollaImageTags, base_distro: Optional[ """Return the parent tag variable for a container in the tag variable hierarchy.""" if container in CONTAINER_TO_PREFIX_VAR_EXCEPTIONS: - parent = CONTAINER_TO_PREFIX_VAR_EXCEPTIONS[container] - if parent in kolla_image_tags: - return parent + prefix_var = CONTAINER_TO_PREFIX_VAR_EXCEPTIONS[container] + if prefix_var in kolla_image_tags: + return prefix_var + else: + prefix_var = container def tag_key(tag): """Return a sort key to order the tags.""" if tag == "openstack": # This is the default tag. return 0 - elif tag != container and container.startswith(tag) and (base_distro is None or base_distro in kolla_image_tags[tag]): + elif tag != prefix_var and prefix_var.startswith(tag) and (base_distro is None or base_distro in kolla_image_tags[tag]): # Prefix match - sort by the longest match. return -len(tag) else: