test: allow failures on Fedora 41

Add a list of distro versions that are allowed to fail in CI.  When a
distro matches one in the list, the 'allow_failure' property of the job
is set to 'true'.

All distro-specific jobs are allowed to fail:
- Manifest generation
- Builds
- OSTree manifest generation
- OSTree builds

In addition, the distro is not counted in the coverage check since some
image configurations for that distribution may never be built.

test/scripts/check-build-coverage

@@ -49,6 +49,9 @@ def check_build_coverage(cachedir):
         image = config["image-type"]
         if image in SKIPS:
             continue
+        if distro in testlib.DISTROS_CAN_FAIL:
+            # if a distro's builds are allowed to fail, don't count them when checking coverage
+            continue
         all_combos.add((distro, arch, image))
 
     missing = all_combos - tests

test/scripts/configure-generators

@@ -41,6 +41,7 @@ generate-build-config-{distro}-{arch}:
     key: testcache
     paths:
       - {cache}
+  allow_failure: {can_fail}
 """
 
 TRIGGER_TEMPLATE = """
@@ -53,6 +54,7 @@ image-build-trigger-{distro}-{arch}:
     strategy: depend
   needs:
     - generate-build-config-{distro}-{arch}
+  allow_failure: {can_fail}
 """
 
 OSTREE_GEN_TEMPLATE = """
@@ -76,6 +78,7 @@ generate-ostree-build-config-{distro}-{arch}:
     key: testcache
     paths:
       - {cache}
+  allow_failure: {can_fail}
 """
 
 OSTREE_TRIGGER_TEMPLATE = """
@@ -88,6 +91,7 @@ image-build-ostree-trigger-{distro}-{arch}:
     strategy: depend
   needs:
     - generate-ostree-build-config-{distro}-{arch}
+  allow_failure: {can_fail}
 """
 
 
@@ -101,6 +105,7 @@ generate-manifests-{distro}-{arch}:
   script:
     - sudo ./test/scripts/install-dependencies
     - go run ./cmd/gen-manifests --arches {arch} --distros {distro} --workers 10
+  allow_failure: {can_fail}
 """
 
 
@@ -120,11 +125,15 @@ def main():
         if combo in combos:
             continue
 
+        can_fail = str(img["distro"] in testlib.DISTROS_CAN_FAIL)
         combos.add(combo)
-        gen_stage.append(GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache))
-        trigger_stage.append(TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache))
-        ostree_gen_stage.append(OSTREE_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache))
-        ostree_trigger_stage.append(OSTREE_TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache))
+        gen_stage.append(GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], cache=cache, can_fail=can_fail))
+        trigger_stage.append(TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"],
+                                                     cache=cache, can_fail=can_fail))
+        ostree_gen_stage.append(OSTREE_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"],
+                                                           cache=cache, can_fail=can_fail))
+        ostree_trigger_stage.append(OSTREE_TRIGGER_TEMPLATE.format(distro=img["distro"], arch=img["arch"],
+                                                                   cache=cache, can_fail=can_fail))
 
     man_only_images = testlib.list_images(arches=MANIFEST_ONLY_ARCHES)
     man_gen_stage = []
@@ -134,7 +143,11 @@ def main():
             continue
 
         combos.add(combo)
-        man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"]))
+        man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(
+            distro=img["distro"],
+            arch=img["arch"],
+            can_fail=can_fail,
+        ))
 
     with open(config_path, "w", encoding="utf-8") as config_file:
         config_file.write(BASE_CONFIG)

test/scripts/generate-build-config

@@ -24,6 +24,7 @@ build/{distro}/{arch}/{image_type}/{config_name}:
   variables:
     RUNNER: aws/fedora-39-{arch}
     INTERNAL_NETWORK: "{internal}"
+  allow_failure: {can_fail}
 """
 
 
@@ -80,8 +81,9 @@ def generate_configs(build_requests, pipeline_file):
         pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type,
                                                 config_name=config_name, config=config_path,
                                                 container_push=container_push_cmd,
-                                                internal="true" if "rhel" in distro else "false",
-                                                image_path=image_path))
+                                                internal=str("rhel" in distro),
+                                                image_path=image_path,
+                                                can_fail=str(distro in testlib.DISTROS_CAN_FAIL)))
     print("✅ DONE!")
 
 

test/scripts/generate-ostree-build-config

@@ -24,6 +24,7 @@ build/{distro}/{arch}/{image_type}/{config_name}:
   needs:
     - pipeline: "$PARENT_PIPELINE_ID"
       job: generate-ostree-build-config-{distro}-{arch}
+  allow_failure: {can_fail}
 """
 
 
@@ -296,8 +297,9 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d
         pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type,
                                                 config_name=config_name, config=build_config_path,
                                                 start_container=container_cmd,
-                                                internal="true" if "rhel" in distro else "false",
-                                                image_path=image_path))
+                                                internal=str("rhel" in distro),
+                                                image_path=image_path,
+                                                can_fail=str(distro in testlib.DISTROS_CAN_FAIL)))
     print("✅ DONE!")
 
 

test/scripts/imgtestlib.py

@@ -17,6 +17,9 @@
 SCHUTZFILE = "Schutzfile"
 OS_RELEASE_FILE = "/etc/os-release"
 
+# allow failures on devel distro versions like rawhide
+DISTROS_CAN_FAIL = ["fedora-41"]
+
 # ostree containers are pushed to the CI registry to be reused by dependants
 OSTREE_CONTAINERS = [
     "iot-container",