From 48a64aa2b7c2321289422b79f6352000db8f7545 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 5 Apr 2024 15:28:26 +0200 Subject: [PATCH] test: allow failures on Fedora 41 Add a list of distro versions that are allowed to fail in CI. When a distro matches one in the list, the 'allow_failure' property of the job is set to 'true'. All distro-specific jobs are allowed to fail: - Manifest generation - Builds - OSTree manifest generation - OSTree builds In addition, the distro is not counted in the coverage check since some image configurations for that distribution may never be built. --- test/scripts/check-build-coverage | 3 +++ test/scripts/configure-generators | 23 ++++++++++++++++++----- test/scripts/generate-build-config | 6 ++++-- test/scripts/generate-ostree-build-config | 6 ++++-- test/scripts/imgtestlib.py | 3 +++ 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/test/scripts/check-build-coverage b/test/scripts/check-build-coverage index ce62edc1ab..50113197d7 100755 --- a/test/scripts/check-build-coverage +++ b/test/scripts/check-build-coverage @@ -49,6 +49,9 @@ def check_build_coverage(cachedir): image = config["image-type"] if image in SKIPS: continue + if distro in testlib.DISTROS_CAN_FAIL: + # if a distro's builds are allowed to fail, don't count them when checking coverage + continue all_combos.add((distro, arch, image)) missing = all_combos - tests diff --git a/test/scripts/configure-generators b/test/scripts/configure-generators index 77c6c243ee..1c9260971c 100755 --- a/test/scripts/configure-generators +++ b/test/scripts/configure-generators @@ -41,6 +41,7 @@ generate-build-config-{distro}-{arch}: key: testcache paths: - {cache} + allow_failure: {can_fail} """ TRIGGER_TEMPLATE = """ @@ -53,6 +54,7 @@ image-build-trigger-{distro}-{arch}: strategy: depend needs: - generate-build-config-{distro}-{arch} + allow_failure: {can_fail} """ OSTREE_GEN_TEMPLATE = """ @@ -76,6 +78,7 @@ generate-ostree-build-config-{distro}-{arch}: key: testcache paths: - {cache} + allow_failure: {can_fail} """ OSTREE_TRIGGER_TEMPLATE = """ @@ -88,6 +91,7 @@ image-build-ostree-trigger-{distro}-{arch}: strategy: depend needs: - generate-ostree-build-config-{distro}-{arch} + allow_failure: {can_fail} """ @@ -101,6 +105,7 @@ generate-manifests-{distro}-{arch}: script: - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches {arch} --distros {distro} --workers 10 + allow_failure: {can_fail} """ @@ -120,11 +125,15 @@ def main(): if combo in combos: continue + can_fail = str(img["distro"] in testlib.DISTROS_CAN_FAIL) combos.add(combo) - gen_stage.append(GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache)) - trigger_stage.append(TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache)) - ostree_gen_stage.append(OSTREE_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache)) - ostree_trigger_stage.append(OSTREE_TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache)) + gen_stage.append(GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache, can_fail=can_fail)) + trigger_stage.append(TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"], + cache=cache, can_fail=can_fail)) + ostree_gen_stage.append(OSTREE_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], + cache=cache, can_fail=can_fail)) + ostree_trigger_stage.append(OSTREE_TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"], + cache=cache, can_fail=can_fail)) man_only_images = testlib.list_images(arches=MANIFEST_ONLY_ARCHES) man_gen_stage = [] @@ -134,7 +143,11 @@ def main(): continue combos.add(combo) - man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"])) + man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format( + distro=img["distro"], + arch=img["arch"], + can_fail=can_fail, + )) with open(config_path, "w", encoding="utf-8") as config_file: config_file.write(BASE_CONFIG) diff --git a/test/scripts/generate-build-config b/test/scripts/generate-build-config index 9d8a81f65c..5e01ecd4f4 100755 --- a/test/scripts/generate-build-config +++ b/test/scripts/generate-build-config @@ -24,6 +24,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: variables: RUNNER: aws/fedora-39-{arch} INTERNAL_NETWORK: "{internal}" + allow_failure: {can_fail} """ @@ -80,8 +81,9 @@ def generate_configs(build_requests, pipeline_file): pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type, config_name=config_name, config=config_path, container_push=container_push_cmd, - internal="true" if "rhel" in distro else "false", - image_path=image_path)) + internal=str("rhel" in distro), + image_path=image_path, + can_fail=str(distro in testlib.DISTROS_CAN_FAIL))) print("✅ DONE!") diff --git a/test/scripts/generate-ostree-build-config b/test/scripts/generate-ostree-build-config index 891f9b581d..1d69c8ca1e 100755 --- a/test/scripts/generate-ostree-build-config +++ b/test/scripts/generate-ostree-build-config @@ -24,6 +24,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: needs: - pipeline: "$PARENT_PIPELINE_ID" job: generate-ostree-build-config-{distro}-{arch} + allow_failure: {can_fail} """ @@ -296,8 +297,9 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type, config_name=config_name, config=build_config_path, start_container=container_cmd, - internal="true" if "rhel" in distro else "false", - image_path=image_path)) + internal=str("rhel" in distro), + image_path=image_path, + can_fail=str(distro in testlib.DISTROS_CAN_FAIL))) print("✅ DONE!") diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib.py index 0df01847b2..c88f3a4985 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib.py @@ -17,6 +17,9 @@ SCHUTZFILE = "Schutzfile" OS_RELEASE_FILE = "/etc/os-release" +# allow failures on devel distro versions like rawhide +DISTROS_CAN_FAIL = ["fedora-41"] + # ostree containers are pushed to the CI registry to be reused by dependants OSTREE_CONTAINERS = [ "iot-container",