From 47cca038a7c1abf0f2ab404a776614ef8058a8e0 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Wed, 30 May 2018 17:18:10 +0200 Subject: [PATCH 1/3] Build garage_deploy.deb on Ubuntu Xenial Same environment as aktualizr.deb, so it saves us one build environment! --- .travis.yml | 5 +- Dockerfile.deb-stable | 60 -------------------- Dockerfile.garage-deploy.deb-stable | 20 ------- Dockerfile.test-install-garage-deploy.xenial | 13 +++++ Jenkinsfile | 28 +-------- README.adoc | 2 +- scripts/build_garage_deploy.sh | 14 ----- scripts/build_ubuntu.sh | 2 +- scripts/test.sh | 2 +- scripts/test_garage_deploy_deb.sh | 2 +- 10 files changed, 22 insertions(+), 126 deletions(-) delete mode 100644 Dockerfile.deb-stable delete mode 100644 Dockerfile.garage-deploy.deb-stable create mode 100644 Dockerfile.test-install-garage-deploy.xenial delete mode 100755 scripts/build_garage_deploy.sh diff --git a/.travis.yml b/.travis.yml index 43fb84f018..9cd32f3a5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ env: DARGS="-eTEST_TESTSUITE_ONLY=crypto" - DOCKERFILE=Dockerfile.deb-testing CACHE=latest-deb-unstable SCRIPT=src/scripts/test.sh DARGS="-eTEST_WITH_LOAD_TESTS=1 -eTEST_WITH_TESTSUITE=0 -eTEST_WITH_STATICTESTS=1" -- DOCKERFILE=Dockerfile.deb-stable CACHE=latest-deb-stable SCRIPT=src/scripts/build_garage_deploy.sh - DOCKERFILE=Dockerfile.noostree CACHE=latest SCRIPT=src/scripts/build_ubuntu.sh services: - docker @@ -23,7 +22,7 @@ script: - ci_env=`bash <(curl -s https://codecov.io/env)` - docker run -v /persistent:/persistent $ci_env $DARGS --name "$CACHE" -it advancedtelematic/aktualizr ${SCRIPT} -- if [ $DOCKERFILE = Dockerfile.deb-stable ]; then ./scripts/test_garage_deploy_deb.sh; fi +- if [ $DOCKERFILE = Dockerfile.noostree ]; then ./scripts/test_garage_deploy_deb.sh; fi - if [ $DOCKERFILE = Dockerfile.noostree ]; then ./scripts/test_aktualizr_deb_ubuntu.sh; fi deploy: - provider: releases @@ -33,7 +32,7 @@ deploy: tags: true all_branches: true repo: advancedtelematic/aktualizr - condition: $DOCKERFILE = Dockerfile.deb-stable + condition: $DOCKERFILE = Dockerfile.noostree api_key: secure: "Z9IEs+GbPW+pxxfYofmeT4Jwjz4OpXJ6WZbv8nyN0MOZV146QExKhZA64mJ7nuXRepS7M5wFgVd0QVlmE7lLa8oNVTKujn+DmQ6701HeP9bITwv6wcDyhNOULLRBwRD2YN5lR29vHGWsjMUm8R13Wtgr/XyOG4L8fcg021B0BMtvepO9HJD4kHSNqB8pJXWVSMd5+d77BRz8Yr72oP98iBMAm94XNd5Gd2RB77YBRKlR7XEV2DxW346C9xI48crMRAKAyp/35vTCTMT04In4FpSexSue0q5dGqfQZf2I5fpGtSbwwJIb3ct/T0CUcW8mDQL01LtG3Hm2qNXKn8aCse6MrVoktUWBTA+tn7DMGq6zY4XXkyC7OisBYg9eel8HacSPrCXDB9C80aJht9a7AZIHehLC4yUomhYmnQDotvHoc3JrJXMSq4HE31WwIBn0xmOpr4ts0nnAprl3bRCQcv10J4MU4zK+BEYET+sLs1stqeION/AYTr0OxWVfbUhRdVepAjDh/mNLYO7raDb/PTZb4upunjS5a1RK+U+aW8ct85RPamZ+8+ZTo8Ofj5VGQiVJmmmrlVkDyBzXwJL3WonnZwuU6QDaaYCFUuzyVqmEuYZNc3i6jNUs0Yptw2YjKww6ayV5msyGaA0QZ3vdGWRk9K7JQQhE7CII2N4yzvI=" diff --git a/Dockerfile.deb-stable b/Dockerfile.deb-stable deleted file mode 100644 index 1d57a11472..0000000000 --- a/Dockerfile.deb-stable +++ /dev/null @@ -1,60 +0,0 @@ -FROM debian:stable-20170620 -LABEL Description="Aktualizr testing dockerfile for Debian Stable with p11" - -ENV DEBIAN_FRONTEND noninteractive - -RUN echo "deb http://ftp.de.debian.org/debian stable main" > /etc/apt/sources.list -RUN echo "deb http://ftp.de.debian.org/debian stable-updates main" >> /etc/apt/sources.list -RUN echo "deb http://security.debian.org stable/updates main" >> /etc/apt/sources.list - -# It is important to run these in the same RUN command, because otherwise -# Docker layer caching breaks us -RUN apt-get update && apt-get -y install \ - apt-transport-https \ - asn1c \ - autoconf \ - automake \ - bison \ - cmake \ - curl \ - dpkg-dev \ - e2fslibs-dev \ - g++ \ - gcc \ - git \ - lcov \ - libarchive-dev \ - libboost-dev \ - libboost-log-dev \ - libboost-program-options-dev \ - libboost-random-dev \ - libboost-regex-dev \ - libboost-system-dev \ - libboost-test-dev \ - libboost-thread-dev \ - libcurl4-openssl-dev \ - libdpkg-dev \ - libengine-pkcs11-openssl \ - libexpat1-dev \ - libglib2.0-dev \ - libgpgme11-dev \ - libgtest-dev \ - liblzma-dev \ - libostree-dev \ - libp11-dev \ - libsodium-dev \ - libsqlite3-dev \ - libssl1.0-dev \ - libsystemd-dev \ - libtool \ - make \ - opensc \ - ostree \ - pkg-config \ - psmisc \ - softhsm2 \ - wget - -RUN useradd testuser -WORKDIR aktualizr -ADD . src diff --git a/Dockerfile.garage-deploy.deb-stable b/Dockerfile.garage-deploy.deb-stable deleted file mode 100644 index 52d9b5d32a..0000000000 --- a/Dockerfile.garage-deploy.deb-stable +++ /dev/null @@ -1,20 +0,0 @@ -FROM debian:stable -LABEL Description="Garage-deploy deb package testing dockerfile" - -ENV DEBIAN_FRONTEND noninteractive - -RUN echo "deb http://ftp.de.debian.org/debian stable main" > /etc/apt/sources.list -RUN echo "deb http://ftp.de.debian.org/debian stable-updates main" >> /etc/apt/sources.list -RUN echo "deb http://security.debian.org stable/updates main" >> /etc/apt/sources.list - -RUN apt-get update && apt-get install -y \ - libarchive13 \ - libcurl3 \ - libengine-pkcs11-openssl \ - libglib2.0-0 \ - libostree-1-1 \ - libsodium18 \ - openjdk-8-jre \ - opensc - -ENTRYPOINT dpkg -i /persistent/garage_deploy.deb && garage-deploy --version && garage-sign --help diff --git a/Dockerfile.test-install-garage-deploy.xenial b/Dockerfile.test-install-garage-deploy.xenial new file mode 100644 index 0000000000..9a43ef60f6 --- /dev/null +++ b/Dockerfile.test-install-garage-deploy.xenial @@ -0,0 +1,13 @@ +FROM ubuntu:xenial +LABEL Description="Garage-deploy deb package testing dockerfile" + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get install -y \ + libarchive13 \ + libcurl3 \ + libglib2.0-0 \ + libstdc++6 \ + openjdk-8-jre + +ENTRYPOINT dpkg -i /persistent/garage_deploy.deb && garage-deploy --version && garage-sign --help diff --git a/Jenkinsfile b/Jenkinsfile index e2cd50418f..2c0ce9fa2c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,29 +99,6 @@ pipeline { sh 'scripts/test.sh' } } - // build garage_deploy.deb - stage('garage_deploy') { - agent any - environment { - TEST_INSTALL_DESTDIR = "${env.WORKSPACE}/build-debstable/pkg" - } - steps { - // build package inside docker - sh ''' - IMG_TAG=deb-$(cat /proc/sys/kernel/random/uuid) - mkdir -p ${TEST_INSTALL_DESTDIR} - docker build -t ${IMG_TAG} -f Dockerfile.deb-stable . - docker run -u $(id -u):$(id -g) -v $PWD:$PWD -v ${TEST_INSTALL_DESTDIR}:/persistent -w $PWD --rm ${IMG_TAG} $PWD/scripts/build_garage_deploy.sh - ''' - // test package installation in another docker - sh 'scripts/test_garage_deploy_deb.sh ${TEST_INSTALL_DESTDIR}' - } - post { - always { - archiveArtifacts artifacts: "build-debstable/pkg/*garage_deploy.deb", fingerprint: true - } - } - } // run crypto tests with Openssl 1.1 stage('openssl11') { agent { @@ -153,7 +130,7 @@ pipeline { } } } - // build and test aktualizr.deb + // build and test aktualizr.deb and garage_deploy.deb stage('debian_pkg') { agent any environment { @@ -168,11 +145,12 @@ pipeline { docker run -u $(id -u):$(id -g) -v $PWD:$PWD -v ${TEST_INSTALL_DESTDIR}:/persistent -w $PWD --rm ${IMG_TAG} $PWD/scripts/build_ubuntu.sh ''' // test package installation in another docker + sh 'scripts/test_garage_deploy_deb.sh ${TEST_INSTALL_DESTDIR}' sh 'scripts/test_aktualizr_deb_ubuntu.sh ${TEST_INSTALL_DESTDIR}' } post { always { - archiveArtifacts artifacts: "build-ubuntu/pkg/*aktualizr.deb", fingerprint: true + archiveArtifacts artifacts: "build-ubuntu/pkg/*.deb", fingerprint: true } } } diff --git a/README.adoc b/README.adoc index c3d30792af..241a9a3e62 100644 --- a/README.adoc +++ b/README.adoc @@ -170,7 +170,7 @@ docker build -t advancedtelematic/aktualizr . While the default Dockerfile image is usually the best one to use, you can select another one with the `-f ` flag, e.g.: ---- -docker build -t advancedtelematic/aktualizr -f Dockerfile.deb-stable . +docker build -t advancedtelematic/aktualizr -f Dockerfile.noostree . ---- Once your docker image is built, Aktualizr can be built and tested with: diff --git a/scripts/build_garage_deploy.sh b/scripts/build_garage_deploy.sh deleted file mode 100755 index dcee019c5c..0000000000 --- a/scripts/build_garage_deploy.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -exuo pipefail - -# configure test.sh -export GITREPO_ROOT=${1:-$(readlink -f "$(dirname "$0")/..")} -export TEST_INSTALL_DESTDIR=${TEST_INSTALL_DESTDIR:-/persistent} -export TEST_BUILD_DIR=build-debstable -export TEST_CMAKE_BUILD_TYPE=Release -export TEST_WITH_INSTALL_DEB_PACKAGES=1 -export TEST_WITH_TESTSUITE=0 - -# build and copy aktualizr.deb to $TEST_INSTALL_DESTDIR -"${GITREPO_ROOT}/scripts/test.sh" diff --git a/scripts/build_ubuntu.sh b/scripts/build_ubuntu.sh index cffe5ee526..c1d5f771ce 100755 --- a/scripts/build_ubuntu.sh +++ b/scripts/build_ubuntu.sh @@ -11,7 +11,7 @@ export TEST_WITH_INSTALL_DEB_PACKAGES=1 export TEST_WITH_OSTREE=0 export TEST_WITH_TESTSUITE=0 -# build and copy aktualizr.deb to $TEST_INSTALL_DESTDIR +# build and copy aktualizr.deb and garage_deploy.deb to $TEST_INSTALL_DESTDIR "${GITREPO_ROOT}/scripts/test.sh" # copy provisioning data and scripts diff --git a/scripts/test.sh b/scripts/test.sh index e997cc8ee3..1754e8fe78 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -126,7 +126,7 @@ if [[ $TEST_WITH_INSTALL_DEB_PACKAGES = 1 ]]; then make package -j8 || add_failure "make package" # install garage-deploy - cp ./*garage_deploy.deb "${TEST_INSTALL_DESTDIR}" + cp ./*garage_deploy.deb "${TEST_INSTALL_DESTDIR}/garage_deploy.deb" # install aktualizr.deb cp ./*aktualizr.deb "${TEST_INSTALL_DESTDIR}/aktualizr.deb" diff --git a/scripts/test_garage_deploy_deb.sh b/scripts/test_garage_deploy_deb.sh index b1af0ef0bf..309c8bb958 100755 --- a/scripts/test_garage_deploy_deb.sh +++ b/scripts/test_garage_deploy_deb.sh @@ -6,6 +6,6 @@ PKG_SRCDIR="${1:-/persistent}" IMG_TAG=garage-deploy-$(cat /proc/sys/kernel/random/uuid) echo "Building docker for testing garage-deploy deb package inside it." -docker build -t "${IMG_TAG}" -f Dockerfile.garage-deploy.deb-stable . +docker build -t "${IMG_TAG}" -f Dockerfile.test-install-garage-deploy.xenial . echo "Running docker container with garage-deploy debian package inside." docker run --rm -v "${PKG_SRCDIR}":/persistent -t "${IMG_TAG}" From b8a5edd0239b2953752ddd3398b44952e5cfaa18 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Thu, 31 May 2018 10:12:30 +0200 Subject: [PATCH 2/3] Simplify the deb package testing docker dance --- Dockerfile.test-install-garage-deploy.xenial | 13 ------------- Dockerfile.test-install.xenial | 7 ++++--- scripts/build_ubuntu.sh | 1 - scripts/test_aktualizr_deb_ubuntu.sh | 2 +- scripts/test_garage_deploy_deb.sh | 4 ++-- ...date.sh => test_install_aktualizr_and_update.sh} | 0 scripts/test_install_garage_deploy.sh | 7 +++++++ 7 files changed, 14 insertions(+), 20 deletions(-) delete mode 100644 Dockerfile.test-install-garage-deploy.xenial rename scripts/{test_aktualizr_deb_and_update.sh => test_install_aktualizr_and_update.sh} (100%) create mode 100755 scripts/test_install_garage_deploy.sh diff --git a/Dockerfile.test-install-garage-deploy.xenial b/Dockerfile.test-install-garage-deploy.xenial deleted file mode 100644 index 9a43ef60f6..0000000000 --- a/Dockerfile.test-install-garage-deploy.xenial +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:xenial -LABEL Description="Garage-deploy deb package testing dockerfile" - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && apt-get install -y \ - libarchive13 \ - libcurl3 \ - libglib2.0-0 \ - libstdc++6 \ - openjdk-8-jre - -ENTRYPOINT dpkg -i /persistent/garage_deploy.deb && garage-deploy --version && garage-sign --help diff --git a/Dockerfile.test-install.xenial b/Dockerfile.test-install.xenial index a64bc182e9..84e1551d69 100644 --- a/Dockerfile.test-install.xenial +++ b/Dockerfile.test-install.xenial @@ -1,5 +1,5 @@ FROM ubuntu:xenial -LABEL Description="Aktualizr deb package testing dockerfile" +LABEL Description="Deb package testing dockerfile" ENV DEBIAN_FRONTEND noninteractive @@ -8,12 +8,13 @@ RUN apt-get update && apt-get install -y \ libarchive13 \ libc6 \ libcurl3 \ - libgcc1 \ + libglib2.0-0 \ libsodium18 \ libsqlite3-0 \ libssl1.0.0 \ libstdc++6 \ lshw \ + openjdk-8-jre \ python3 -ENTRYPOINT /persistent/test_aktualizr_deb_and_update.sh +ADD . / diff --git a/scripts/build_ubuntu.sh b/scripts/build_ubuntu.sh index c1d5f771ce..2632a2ab75 100755 --- a/scripts/build_ubuntu.sh +++ b/scripts/build_ubuntu.sh @@ -18,7 +18,6 @@ export TEST_WITH_TESTSUITE=0 cp -rf "${GITREPO_ROOT}/tests/test_data/prov_selfupdate" "${TEST_INSTALL_DESTDIR}" cp -rf "${GITREPO_ROOT}/tests/config/selfupdate.toml" "${TEST_INSTALL_DESTDIR}" cp -rf "${GITREPO_ROOT}/scripts/selfupdate_server.py" "${TEST_INSTALL_DESTDIR}" -cp -f "${GITREPO_ROOT}/scripts/test_aktualizr_deb_and_update.sh" "${TEST_INSTALL_DESTDIR}/test_aktualizr_deb_and_update.sh" cp -rf "${GITREPO_ROOT}/tests/test_data/fake_root" "${TEST_INSTALL_DESTDIR}" git -C "${GITREPO_ROOT}" fetch --unshallow || true diff --git a/scripts/test_aktualizr_deb_ubuntu.sh b/scripts/test_aktualizr_deb_ubuntu.sh index b4310fb81a..dc77e56d86 100755 --- a/scripts/test_aktualizr_deb_ubuntu.sh +++ b/scripts/test_aktualizr_deb_ubuntu.sh @@ -7,4 +7,4 @@ IMG_TAG=aktualizr-deb-$(cat /proc/sys/kernel/random/uuid) echo "Building docker for testing aktualizr deb package inside it." docker build -t "${IMG_TAG}" -f Dockerfile.test-install.xenial . echo "Running docker container with aktualizr debian package inside." -docker run --rm -v "${PKG_SRCDIR}":/persistent -t "${IMG_TAG}" +docker run --rm -v "${PKG_SRCDIR}":/persistent -t "${IMG_TAG}" /scripts/test_install_aktualizr_and_update.sh diff --git a/scripts/test_garage_deploy_deb.sh b/scripts/test_garage_deploy_deb.sh index 309c8bb958..928f554f3a 100755 --- a/scripts/test_garage_deploy_deb.sh +++ b/scripts/test_garage_deploy_deb.sh @@ -6,6 +6,6 @@ PKG_SRCDIR="${1:-/persistent}" IMG_TAG=garage-deploy-$(cat /proc/sys/kernel/random/uuid) echo "Building docker for testing garage-deploy deb package inside it." -docker build -t "${IMG_TAG}" -f Dockerfile.test-install-garage-deploy.xenial . +docker build -t "${IMG_TAG}" -f Dockerfile.test-install.xenial . echo "Running docker container with garage-deploy debian package inside." -docker run --rm -v "${PKG_SRCDIR}":/persistent -t "${IMG_TAG}" +docker run --rm -v "${PKG_SRCDIR}":/persistent -t "${IMG_TAG}" /scripts/test_install_garage_deploy.sh diff --git a/scripts/test_aktualizr_deb_and_update.sh b/scripts/test_install_aktualizr_and_update.sh similarity index 100% rename from scripts/test_aktualizr_deb_and_update.sh rename to scripts/test_install_aktualizr_and_update.sh diff --git a/scripts/test_install_garage_deploy.sh b/scripts/test_install_garage_deploy.sh new file mode 100755 index 0000000000..2893d0ee53 --- /dev/null +++ b/scripts/test_install_garage_deploy.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +set -exuo pipefail + +dpkg -i /persistent/garage_deploy.deb +garage-deploy --version +garage-sign --help From 34d0251ca5820dbce82b2932fcbdc601d7a9a7c0 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Thu, 31 May 2018 11:14:53 +0200 Subject: [PATCH 3/3] Move some includes out of `#ifdef BUILD_P11` --- src/libaktualizr/crypto/crypto_test.cc | 2 +- src/libaktualizr/crypto/keymanager_test.cc | 1 - src/libaktualizr/uptane/uptane_test.cc | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libaktualizr/crypto/crypto_test.cc b/src/libaktualizr/crypto/crypto_test.cc index f5c1c2456e..93c60d58e5 100644 --- a/src/libaktualizr/crypto/crypto_test.cc +++ b/src/libaktualizr/crypto/crypto_test.cc @@ -10,11 +10,11 @@ #include #include "crypto/crypto.h" +#include "crypto/p11engine.h" #include "storage/fsstorage.h" #include "utilities/utils.h" #ifdef BUILD_P11 -#include "crypto/p11engine.h" #ifndef TEST_PKCS11_MODULE_PATH #define TEST_PKCS11_MODULE_PATH "/usr/local/softhsm/libsofthsm2.so" #endif diff --git a/src/libaktualizr/crypto/keymanager_test.cc b/src/libaktualizr/crypto/keymanager_test.cc index fb1acfc787..230900e88f 100644 --- a/src/libaktualizr/crypto/keymanager_test.cc +++ b/src/libaktualizr/crypto/keymanager_test.cc @@ -9,7 +9,6 @@ #include "utilities/utils.h" #ifdef BUILD_P11 -#include "crypto/p11engine.h" #ifndef TEST_PKCS11_MODULE_PATH #define TEST_PKCS11_MODULE_PATH "/usr/local/softhsm/libsofthsm2.so" #endif diff --git a/src/libaktualizr/uptane/uptane_test.cc b/src/libaktualizr/uptane/uptane_test.cc index 38d6d7bcab..a2afed2afe 100644 --- a/src/libaktualizr/uptane/uptane_test.cc +++ b/src/libaktualizr/uptane/uptane_test.cc @@ -9,6 +9,7 @@ #include +#include "crypto/p11engine.h" #include "httpfake.h" #include "logging/logging.h" #include "primary/initializer.h" @@ -20,7 +21,6 @@ #include "utilities/utils.h" #ifdef BUILD_P11 -#include "crypto/p11engine.h" #ifndef TEST_PKCS11_MODULE_PATH #define TEST_PKCS11_MODULE_PATH "/usr/local/softhsm/libsofthsm2.so" #endif