From 801e5a0091a9d1232e74a62883b99353f655c1f4 Mon Sep 17 00:00:00 2001 From: Bindea Cristian Date: Fri, 6 Sep 2024 11:27:52 +0300 Subject: [PATCH] ci: ship Docker Image with all the necessary tools for build - this was done to ensure that all workflows already contain the required tools, without the need to download them at runtime Signed-off-by: Bindea Cristian --- .github/workflows/appimage-armhf.yml | 2 +- ci/armhf/.dockerignore | 4 ++++ ci/armhf/armhf_build_config.sh | 8 +++----- ci/armhf/armhf_build_process.sh | 14 +++++++++++++- ci/armhf/create_docker_image.sh | 6 +++--- ci/armhf/docker/Dockerfile | 6 +++++- ci/x86_64/docker/Dockerfile | 3 ++- ci/x86_64/x86-64_appimage_process.sh | 11 ++++++++++- 8 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 ci/armhf/.dockerignore diff --git a/.github/workflows/appimage-armhf.yml b/.github/workflows/appimage-armhf.yml index 06159b35d0..edd6137dcd 100644 --- a/.github/workflows/appimage-armhf.yml +++ b/.github/workflows/appimage-armhf.yml @@ -34,7 +34,7 @@ jobs: /bin/bash -c 'cd $HOME && \ sudo chown -R runner:runner scopy && \ cd $HOME/scopy && \ - ./ci/armhf/armhf_build_process.sh install_packages download_cmake download_crosscompiler move_sysroot build_scopy build_iio-emu create_appdir create_appimage move_appimage + ./ci/armhf/armhf_build_process.sh install_packages move_tools move_sysroot build_scopy build_iio-emu create_appdir create_appimage move_appimage ' - name: Set short git commit SHA shell: bash diff --git a/ci/armhf/.dockerignore b/ci/armhf/.dockerignore new file mode 100644 index 0000000000..5fd00763d8 --- /dev/null +++ b/ci/armhf/.dockerignore @@ -0,0 +1,4 @@ +* +!sysroot.tar.gz +!armhf_build_process.sh +!armhf_build_config.sh \ No newline at end of file diff --git a/ci/armhf/armhf_build_config.sh b/ci/armhf/armhf_build_config.sh index cd120367ff..44a5f7f690 100644 --- a/ci/armhf/armhf_build_config.sh +++ b/ci/armhf/armhf_build_config.sh @@ -1,6 +1,6 @@ #!/bin/bash -LIBIIO_VERSION=v0.25 +LIBIIO_VERSION=libiio-v0 LIBAD9361_BRANCH=main GLOG_BRANCH=v0.4.0 LIBM2K_BRANCH=main @@ -18,12 +18,10 @@ export APPIMAGE=1 PYTHON_VERSION=python3.9 # default python version used in CI scripts, can be changed to match locally installed python -STAGING_AREA=$SRC_DIR/CI/armhf/staging - -STAGING_AREA=$SRC_DIR/ci/armhf/staging +SRC_SCRIPT=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +STAGING_AREA=$SRC_SCRIPT/staging SYSROOT=$STAGING_AREA/sysroot SYSROOT_TAR=$STAGING_AREA/sysroot.tar.gz -SYSROOT_DOCKER=$SRC_DIR/ci/armhf/docker/sysroot.tar.gz TOOLCHAIN=$STAGING_AREA/cross-pi-gcc TOOLCHAIN_BIN=$TOOLCHAIN/bin TOOLCHAIN_HOST="arm-linux-gnueabihf" diff --git a/ci/armhf/armhf_build_process.sh b/ci/armhf/armhf_build_process.sh index 631ae58fbf..7447d70b08 100755 --- a/ci/armhf/armhf_build_process.sh +++ b/ci/armhf/armhf_build_process.sh @@ -4,7 +4,9 @@ set -ex git config --global --add safe.directory $HOME/scopy SRC_DIR=$(git rev-parse --show-toplevel 2>/dev/null ) || \ SRC_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && cd ../../ && pwd ) -source $SRC_DIR/ci/armhf/armhf_build_config.sh +SRC_SCRIPT=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SRC_SCRIPT/armhf_build_config.sh echo -- USING CMAKE COMMAND: echo $CMAKE @@ -347,6 +349,16 @@ move_sysroot(){ fi } +# move the staging folder that contains the tools needed for the build to the known location +move_tools(){ + [ -d /home/runner/staging ] && mv /home/runner/staging $STAGING_AREA || echo "Staging folder not found or already moved" + if [ ! -d $STAGING_AREA ]; then + echo "Missing tools folder, downloading now" + download_cmake + download_crosscompiler + fi +} + # move and rename the AppImage artifact move_appimage(){ mv $APP_IMAGE $SRC_DIR/Scopy-armhf.AppImage diff --git a/ci/armhf/create_docker_image.sh b/ci/armhf/create_docker_image.sh index 9cc79c3569..4619f94572 100755 --- a/ci/armhf/create_docker_image.sh +++ b/ci/armhf/create_docker_image.sh @@ -28,13 +28,13 @@ create_sysroot(){ tar_and_move_sysroot(){ pushd $STAGING_AREA sudo tar -czvf "${SYSROOT_TAR##*/}" sysroot - sudo mv $SYSROOT_TAR $SYSROOT_DOCKER + sudo mv $SYSROOT_TAR $SRC_DIR/ci/armhf popd } create_image(){ - pushd ${SRC_DIR}/ci/armhf/docker - sudo docker build --load --tag cristianbindea/scopy2-armhf-appimage:testing . + pushd ${SRC_DIR}/ci/armhf + sudo docker build --load --tag cristianbindea/scopy2-armhf-appimage:testing -f docker/Dockerfile . # sudo DOCKER_BUILDKIT=0 docker build --tag cristianbindea/scopy2-armhf-appimage:testing . # build the image using old backend popd } diff --git a/ci/armhf/docker/Dockerfile b/ci/armhf/docker/Dockerfile index 6d88c76fbe..b79a1e1dda 100644 --- a/ci/armhf/docker/Dockerfile +++ b/ci/armhf/docker/Dockerfile @@ -56,4 +56,8 @@ COPY --from=sysroot_builder /home/${USER}/scopy/ci/armhf/staging/sysroot/share / COPY --from=sysroot_builder /home/${USER}/scopy/ci/armhf/staging/sysroot/include /home/${USER}/sysroot/include WORKDIR /home/${USER}/sysroot RUN ln -s usr/bin bin && ln -s usr/lib lib && ln -s usr/sbin sbin -WORKDIR /home/${USER} \ No newline at end of file +WORKDIR /home/${USER} +COPY armhf_build_process.sh /home/${USER}/armhf_build_process.sh +COPY armhf_build_config.sh /home/${USER}/armhf_build_config.sh +RUN /home/${USER}/armhf_build_process.sh install_packages download_cmake download_crosscompiler && \ + rm /home/${USER}/armhf_build_process.sh /home/${USER}/armhf_build_config.sh diff --git a/ci/x86_64/docker/Dockerfile b/ci/x86_64/docker/Dockerfile index b2c2d644b5..fcb6bfcce7 100644 --- a/ci/x86_64/docker/Dockerfile +++ b/ci/x86_64/docker/Dockerfile @@ -24,7 +24,8 @@ RUN echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-$USER && \ USER $USER WORKDIR /home/${USER}/scripts COPY x86-64_appimage_process.sh . -RUN ./x86-64_appimage_process.sh configure_system +RUN ./x86-64_appimage_process.sh download_tools configure_system +RUN mv /home/${USER}/scripts/staging /home/${USER}/staging ENV CI_SCRIPT=ON WORKDIR /home/${USER} diff --git a/ci/x86_64/x86-64_appimage_process.sh b/ci/x86_64/x86-64_appimage_process.sh index 2c156c11d2..f0ee6078e1 100755 --- a/ci/x86_64/x86-64_appimage_process.sh +++ b/ci/x86_64/x86-64_appimage_process.sh @@ -424,6 +424,15 @@ generate_ci_envs(){ $SRC_DIR/ci/general/gen_ci_envs.sh > $SRC_DIR/ci/x86_64/gh-actions.envs } +# move the staging folder that contains the tools needed for the build to the known location +move_tools(){ + [ -d /home/runner/staging ] && mv /home/runner/staging $STAGING_AREA || echo "Staging folder not found or already moved" + if [ ! -d $STAGING_AREA ]; then + echo "Missing tools folder, downloading now" + download_tools + fi +} + move_appimage(){ mv $APP_IMAGE $SRC_DIR } @@ -450,7 +459,7 @@ build_deps(){ } run_workflow(){ - download_tools + [ "$CI_SCRIPT" == "ON" ] && move_tools || download_tools build_iio-emu build_scopy create_appdir