From bd6bc07b2c97eb7a780d867e86eaed6f2ef5d348 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Thu, 11 Jul 2024 16:12:03 +0300 Subject: [PATCH] Add rockcraft specs for harbor-db:v2.10.2 Signed-off-by: Nashwan Azhari --- v2.10.2/harbor-db/README.md | 3 + v2.10.2/harbor-db/rockcraft.yaml | 177 +++++++++++++++++++++++++++ v2.10.2/harbor-db/tests/test_rock.py | 70 +++++++++++ v2.10.2/harbor-db/tox.ini | 47 +++++++ 4 files changed, 297 insertions(+) create mode 100644 v2.10.2/harbor-db/README.md create mode 100644 v2.10.2/harbor-db/rockcraft.yaml create mode 100644 v2.10.2/harbor-db/tests/test_rock.py create mode 100644 v2.10.2/harbor-db/tox.ini diff --git a/v2.10.2/harbor-db/README.md b/v2.10.2/harbor-db/README.md new file mode 100644 index 0000000..3cb8a5d --- /dev/null +++ b/v2.10.2/harbor-db/README.md @@ -0,0 +1,3 @@ +# ROCK specs for harbor-db. + +Aims to be compatible with `docker.io/goharbor/harbor-db`. diff --git a/v2.10.2/harbor-db/rockcraft.yaml b/v2.10.2/harbor-db/rockcraft.yaml new file mode 100644 index 0000000..6965afc --- /dev/null +++ b/v2.10.2/harbor-db/rockcraft.yaml @@ -0,0 +1,177 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +# Rockcraft definition for Harbor db image, which is basically a +# re-packaged PostgreSQL with some added configs/setup files. +# docker.io/goharbor/harbor-db:v2.10.2 + +name: harbor-db +summary: Rock containing Harbor DB PostgreSQL component. +description: | + Packages the PostgreSQL DB of Harbor. +license: Apache-2.0 + +version: "2.10.2" + +# NOTE(aznashwan): the base for the DB image is VMware's Photon, +# but rockcraft only currently supports bare/ubuntu-based bases. +base: ubuntu@22.04 +build-base: ubuntu@22.04 +platforms: + amd64: + arm64: + + +# NOTE(aznashwan): The PhotonOS PostgreSQL package releases are +# simply built from the upstream sources from postgresql.org: +# https://github.com/vmware/photon/blob/5.0/SPECS/postgresql/postgresql13.spec#L23 +# To avoid needlessly building the packages ourselves, we simply +# add the upstream posgresql.org deb repos to the build host: +package-repositories: + - type: apt + components: [main] + suites: [jammy-pgdg] + key-id: B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 + url: http://apt.postgresql.org/pub/repos/apt + priority: always + + +environment: + # https://github.com/goharbor/harbor/blob/v2.10.2/make/photon/db/Dockerfile.base#L3 + PGDATA: /var/lib/postgresql/data + + +services: + + # HACK(aznashwan): the upstream image's entrypoint calls `initdb.sh`, + # which in turn calls postgres' `initdb` with a hardcoded en_US.UTF-8 locale: + # https://github.com/goharbor/harbor/blob/v2.10.2/make/photon/db/initdb.sh#L34 + # Because we can't run any of the commands from the `locales` package within + # the primed filesystem's context (they'll just change locale settings on + # the build host), and locale-related files are apparently excluded + # from the final image (1), we have to set this up as its own service: + # (1): https://github.com/canonical/craft-parts/blob/1.33.0/craft_parts/packages/deb.py#L140 + locale_setup: + startup: enabled + override: replace + command: bash -c "apt-get update && apt-get install -y language-pack-en-base" + on-success: ignore + + harbor_db: + startup: enabled + override: replace + requires: + - locale_setup + + # NOTE(aznashwan) set docker-entrypoint.sh for compatibility with upstream image. + # It takes as arguments the older and current ProstgreSQL versions to start: + # https://github.com/goharbor/harbor/blob/v2.10.2/make/photon/db/Dockerfile#L15 + command: /docker-entrypoint.sh 13 14 + + user: postgres + group: postgres + + # TODO(aznashwan): original Docker image includes Healthcheck should/can we also? + # https://github.com/goharbor/harbor/blob/v2.10.2/make/photon/db/Dockerfile#L16 + + +parts: + create-postgres-user: + plugin: nil + overlay-script: | + groupadd -R $CRAFT_OVERLAY -r postgres --gid=999 + useradd -R $CRAFT_OVERLAY -m -r -g postgres --uid=999 postgres + + # Sourced from: + # https://github.com/goharbor/harbor/blob/v2.10.2/make/photon/db/Dockerfile.base + # https://github.com/goharbor/harbor/blob/v2.10.2/make/photon/db/Dockerfile + setup-harbor-db: + after: [create-postgres-user] + plugin: nil + + source-type: git + source: https://github.com/goharbor/harbor + source-tag: v2.10.2 + source-depth: 1 + + build-environment: + - PGDATA: /var/lib/postgresql/data + + stage-packages: + # NOTE(aznashwan): upstream image installs both Postgres 13 and 14: + # https://github.com/goharbor/harbor/blob/v2.10.2/make/photon/db/Dockerfile.base#L9-L10 + - postgresql-13 + - postgresql-14 + - findutils + - bc + - util-linux + - net-tools + # HACK(aznashwan): the upstream image's entrypoint calls `initdb.sh`, + # which in turn calls postgres' `initdb` with a hardcoded en_US.UTF-8 locale: + # https://github.com/goharbor/harbor/blob/v2.10.2/make/photon/db/initdb.sh#L34 + # Because we can't run any of the commands from the `locales` package within + # the primed filesystem's context (they'll just change locale settings on + # the build host), we simply stage this umbrella package for English: + # + # Doesn't seem to work due to `locales` being explicitly excluded: + # https://github.com/canonical/craft-parts/blob/1.33.0/craft_parts/packages/deb.py#L140 + # - locales + # - language-pack-en-base + + override-build: | + set -eux + + # Setup Postgres files: + mkdir -p "$CRAFT_PART_INSTALL/run/postgresql" + chown -R 999:999 "$CRAFT_PART_INSTALL/run/postgresql" + chmod 2777 "$CRAFT_PART_INSTALL/run/postgresql" + + mkdir -p "$CRAFT_PART_INSTALL/$PGDATA" + chown -R 999:999 "$CRAFT_PART_INSTALL/$PGDATA" + chmod 777 "$CRAFT_PART_INSTALL/$PGDATA" + + sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" \ + "$CRAFT_PART_INSTALL/usr/share/postgresql/14/postgresql.conf.sample" + sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" \ + "$CRAFT_PART_INSTALL/usr/share/postgresql/14/postgresql.conf.sample" + + # Copy over auxiliary files: + cd "$CRAFT_PART_SRC" + OUTDIR="$CRAFT_PART_INSTALL" + + cp ./make/photon/db/docker-entrypoint.sh "$OUTDIR/docker-entrypoint.sh" + chown 999:999 "$OUTDIR/docker-entrypoint.sh" + chmod u+x "$OUTDIR/docker-entrypoint.sh" + + cp ./make/photon/db/docker-healthcheck.sh "$OUTDIR/docker-healthcheck.sh" + chown 999:999 "$OUTDIR/docker-healthcheck.sh" + chmod u+x "$OUTDIR/docker-healthcheck.sh" + + cp ./make/photon/db/initdb.sh "$OUTDIR/initdb.sh" + chown 999:999 "$OUTDIR/initdb.sh" + # NOTE(aznashwan): initdb.sh is NOT chmod'd for some reason... + + cp ./make/photon/db/upgrade.sh "$OUTDIR/upgrade.sh" + # NOTE(aznashwan): upgrade.sh is NOT chown'd or chmod'd for some reason... + + mkdir -p "$OUTDIR/docker-entrypoint-initdb.d" + chown -R 999:999 "$OUTDIR/docker-entrypoint-initdb.d" + + cp ./make/photon/db/initial-registry.sql \ + "$OUTDIR/docker-entrypoint-initdb.d/initial-registry.sql" + cp ./make/photon/db/upgrade.sh "$OUTDIR/" + + # HACK(aznashwan): the upstream harbor-db image sets up + # update-alternatives for initdb and a handful of other + # commands bundled with PostgreSQL, which will use the + # versions from postgresql-14, so we simply symlink them: + # https://github.com/vmware/photon/blob/5.0/SPECS/postgresql/postgresql14.spec#L374-L387 + for full_path in "$CRAFT_PART_INSTALL/usr/lib/postgresql/14/bin/"*; do + exe=$(basename $full_path) + if [ ! -f "$CRAFT_PART_INSTALL/usr/bin/$exe" ]; then + ln -s \ + "/usr/lib/postgresql/14/bin/$exe" \ + "$CRAFT_PART_INSTALL/usr/bin/$exe" + fi + done + diff --git a/v2.10.2/harbor-db/tests/test_rock.py b/v2.10.2/harbor-db/tests/test_rock.py new file mode 100644 index 0000000..29e29eb --- /dev/null +++ b/v2.10.2/harbor-db/tests/test_rock.py @@ -0,0 +1,70 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging +import random +import pytest +import string +import subprocess +import sys + +from charmed_kubeflow_chisme.rock import CheckRock + +logger: logging.Logger = logging.getLogger(__name__) + +logger.addHandler(logging.FileHandler(f"{__name__}.log")) +logger.addHandler(logging.StreamHandler(sys.stdout)) + + +@pytest.fixture() +def rock_test_env(tmpdir): + """Yields a temporary directory and random docker container name, then cleans them up after.""" + container_name = "".join( + [str(i) for i in random.choices(string.ascii_lowercase, k=8)] + ) + yield tmpdir, container_name + + try: + subprocess.run(["docker", "rm", container_name]) + except Exception: + pass + # tmpdir fixture we use here should clean up the other files for us + + + def _check_file_present_in_image(image: str, path_to_check: str): + """Checks whether a file with the given path is present within an image.""" + subprocess.run( + [ + "docker", + "run", + image, + "exec", + "ls", + "-la", + path_to_check, + ], + check=True, + ) + + +@pytest.mark.abort_on_fail +def test_rock(rock_test_env): + """Test rock.""" + _, container_name = rock_test_env + check_rock = CheckRock("rockcraft.yaml") + rock_image = check_rock.get_name() + rock_version = check_rock.get_version() + LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}" + + image_files_to_check = [ + "/var/lib/postgresql/data", + "/run/postgresq", + "/docker-entrypoint.sh", + "/initdb.sh", + "/upgrade.sh", + "/docker-healthcheck.sh", + "/docker-entrypoint-initdb.d/initial-registry.sql", + ] + + for file in image_files_to_check: + _check_file_present_in_image(LOCAL_ROCK_IMAGE, file) diff --git a/v2.10.2/harbor-db/tox.ini b/v2.10.2/harbor-db/tox.ini new file mode 100644 index 0000000..6a3616a --- /dev/null +++ b/v2.10.2/harbor-db/tox.ini @@ -0,0 +1,47 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +[tox] +skipsdist = True +skip_missing_interpreters = True +envlist = pack, export-to-docker, sanity + +[testenv] +setenv = + PYTHONPATH={toxinidir} + PYTHONBREAKPOINT=ipdb.set_trace + +[testenv:pack] +passenv = * +allowlist_externals = + rockcraft +commands = + rockcraft pack -v + +[testenv:export-to-docker] +passenv = * +allowlist_externals = + bash + skopeo + yq +commands = + # export already packed rock to docker + bash -c 'NAME="$(yq -r .name rockcraft.yaml)" && \ + VERSION="$(yq -r .version rockcraft.yaml)" && \ + ARCH="$(yq -r ".platforms | keys | .[0]" rockcraft.yaml)" && \ + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}.rock" && \ + DOCKER_IMAGE=$NAME:$VERSION && \\ + echo "Exporting $ROCK to docker as $DOCKER_IMAGE" && \ + rockcraft.skopeo --insecure-policy copy \ + oci-archive:$ROCK docker-daemon:$DOCKER_IMAGE' + +[testenv:sanity] +passenv = * +deps = + pytest + charmed-kubeflow-chisme +allowlist_externals = + echo +commands = + # run rock tests + pytest -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests