test: run stuff in parallel

.github/workflows/tests.yml

@@ -130,7 +130,9 @@ jobs:
         # podman needs (parts of) the environment but will break when
         # XDG_RUNTIME_DIR is set.
         # TODO: figure out what exactly podman needs
-        sudo -E XDG_RUNTIME_DIR= pytest-3 --basetemp=/mnt/var/tmp/bib-tests
+        # TODO2: figure out how much wecan run in parallel before running
+        # out of diskspace
+        sudo -E XDG_RUNTIME_DIR= pytest-3 --basetemp=/mnt/var/tmp/bib-tests -n 8 --dist worksteal --maxschedchunk=1
     - name: Diskspace (after)
       if: ${{ always() }}
       run: |

test/conftest.py

@@ -20,3 +20,10 @@ def pytest_make_parametrize_id(config, val):  # pylint: disable=W0613
     if isinstance(val, TestCase):
         return f"{val}"
     return None
+
+
+@pytest.fixture(name="shared_tmpdir", scope='session')
+def shared_tmpdir_fixture(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp().parent / "shared"
+    tmp_path.mkdir(exist_ok=True)
+    yield tmp_path

test/containerbuild.py

@@ -1,4 +1,5 @@
 import os
+import pathlib
 import platform
 import random
 import string
@@ -7,6 +8,7 @@
 from contextlib import contextmanager
 
 import pytest
+from filelock import FileLock
 
 
 @contextmanager
@@ -33,67 +35,74 @@ def make_container(container_path, arch=None):
 
 
 @pytest.fixture(name="build_container", scope="session")
-def build_container_fixture():
+def build_container_fixture(shared_tmpdir):
     """Build a container from the Containerfile and returns the name"""
     if tag_from_env := os.getenv("BIB_TEST_BUILD_CONTAINER_TAG"):
         return tag_from_env
 
     container_tag = "bootc-image-builder-test"
-    subprocess.check_call([
-        "podman", "build",
-        "-f", "Containerfile",
-        "-t", container_tag,
-    ])
+    with FileLock(shared_tmpdir / pathlib.Path(container_tag + ".lock")):
+        if not os.path.exists(shared_tmpdir / container_tag):
+            subprocess.check_call([
+                "podman", "build",
+                "-f", "Containerfile",
+                "-t", container_tag,
+            ])
+            (shared_tmpdir / container_tag).write_text("done")
     return container_tag
 
 
 @pytest.fixture(name="build_fake_container", scope="session")
-def build_fake_container_fixture(tmpdir_factory, build_container):
+def build_fake_container_fixture(shared_tmpdir, build_container):
     """Build a container with a fake osbuild and returns the name"""
-    tmp_path = tmpdir_factory.mktemp("build-fake-container")
-
-    # see https://github.com/osbuild/osbuild/blob/main/osbuild/testutil/__init__.py#L91
-    tracing_podman_path = tmp_path / "tracing-podman"
-    tracing_podman_path.write_text(textwrap.dedent("""\
-    #!/bin/sh -e
-
-    TRACE_PATH=/output/"$(basename $0)".log
-    for arg in "$@"; do
-        echo "$arg" >> "$TRACE_PATH"
-    done
-    # extra separator to differenciate between calls
-    echo >> "$TRACE_PATH"
-    exec "$0".real "$@"
-    """), encoding="utf8")
-
-    fake_osbuild_path = tmp_path / "fake-osbuild"
-    fake_osbuild_path.write_text(textwrap.dedent("""\
-    #!/bin/bash -e
-
-    # injest generated manifest from the images library, if we do not
-    # do this images may fail with "broken" pipe errors
-    cat - >/dev/null
-
-    mkdir -p /output/qcow2
-    echo "fake-disk.qcow2" > /output/qcow2/disk.qcow2
-
-    """), encoding="utf8")
-
-    cntf_path = tmp_path / "Containerfile"
-
-    cntf_path.write_text(textwrap.dedent(f"""\n
-    FROM {build_container}
-    COPY fake-osbuild /usr/bin/osbuild
-    RUN chmod 755 /usr/bin/osbuild
-    COPY --from={build_container} /usr/bin/podman /usr/bin/podman.real
-    COPY tracing-podman /usr/bin/podman
-    RUN chmod 755 /usr/bin/podman
-    """), encoding="utf8")
+    tmp_path = shared_tmpdir / "build-fake-container"
 
     container_tag = "bootc-image-builder-test-faked-osbuild"
-    subprocess.check_call([
-        "podman", "build",
-        "-t", container_tag,
-        tmp_path,
-    ])
-    return container_tag
+    with FileLock(tmp_path + ".lock"):
+        if tmp_path.exists():
+            return container_tag
+        tmp_path.mkdir()
+        # see https://github.com/osbuild/osbuild/blob/main/osbuild/testutil/__init__.py#L91
+        tracing_podman_path = tmp_path / "tracing-podman"
+        tracing_podman_path.write_text(textwrap.dedent("""\
+        #!/bin/sh -e
+
+        TRACE_PATH=/output/"$(basename $0)".log
+        for arg in "$@"; do
+            echo "$arg" >> "$TRACE_PATH"
+        done
+        # extra separator to differenciate between calls
+        echo >> "$TRACE_PATH"
+        exec "$0".real "$@"
+        """), encoding="utf8")
+
+        fake_osbuild_path = tmp_path / "fake-osbuild"
+        fake_osbuild_path.write_text(textwrap.dedent("""\
+        #!/bin/bash -e
+
+        # injest generated manifest from the images library, if we do not
+        # do this images may fail with "broken" pipe errors
+        cat - >/dev/null
+
+        mkdir -p /output/qcow2
+        echo "fake-disk.qcow2" > /output/qcow2/disk.qcow2
+
+        """), encoding="utf8")
+
+        cntf_path = tmp_path / "Containerfile"
+
+        cntf_path.write_text(textwrap.dedent(f"""\n
+        FROM {build_container}
+        COPY fake-osbuild /usr/bin/osbuild
+        RUN chmod 755 /usr/bin/osbuild
+        COPY --from={build_container} /usr/bin/podman /usr/bin/podman.real
+        COPY tracing-podman /usr/bin/podman
+        RUN chmod 755 /usr/bin/podman
+        """), encoding="utf8")
+
+        subprocess.check_call([
+            "podman", "build",
+            "-t", container_tag,
+            tmp_path,
+        ])
+        return container_tag

test/requirements.txt

@@ -4,3 +4,5 @@ paramiko==2.12.0
 boto3==1.33.13
 qmp==1.1.0
 pylint==3.2.5
+pytest-xdist==3.6.1
+filelock==3.16.1