From 692df67cb9ad3c4b23eea11d6fb6a798664cc7c3 Mon Sep 17 00:00:00 2001 From: Kevin Hunter Kesling Date: Thu, 25 Apr 2024 12:07:03 -0400 Subject: [PATCH] Jenkins WIP --- compute_endpoint/packaging/JenkinsFile | 85 +++++++++++++++++++ compute_endpoint/packaging/Makefile | 69 ++++++++++----- .../packaging/create-prereqs-tarball.sh | 6 +- compute_endpoint/packaging/debian/rules | 12 ++- 4 files changed, 143 insertions(+), 29 deletions(-) create mode 100644 compute_endpoint/packaging/JenkinsFile diff --git a/compute_endpoint/packaging/JenkinsFile b/compute_endpoint/packaging/JenkinsFile new file mode 100644 index 000000000..32d9330c0 --- /dev/null +++ b/compute_endpoint/packaging/JenkinsFile @@ -0,0 +1,85 @@ +#! groovy + +@Library('gcs-build-scripts') _ + +pipeline { + agent none + options { + buildDiscarder( + logRotator( + numToKeepStr: '5', + artifactNumToKeepStr: '1', + artifactDaysToKeepStr: '7' + ) + ) + timeout(time: 3, unit: 'HOURS') + disableConcurrentBuilds() + } + stages { + stage ("Prep source") { + agent {label "package_creator"} + steps { + checkout scm + script { + def py_full_version = sh(script: "/opt/globus-python/bin/python3 -c 'import sys; print(\"{}.{}.{}\".format(*sys.version_info))'", returnStdout: true).trim() + def (py_epoch_version, py_major_version, py_minor_version) = py_full_version.tokenize(".") + def pkg_version = "not yet set" + + def venv_py = "${env.WORKSPACE}/compute_endpoint/packaging/venv-${py_epoch_version}${py_major_version}/bin/python" + sh "git clean -fdx" + sh "git checkout kevin_package_test" + + dir("compute_endpoint/packaging/") { + env.DEB_SOURCE_STASH_NAME = "${UUID.randomUUID()}"; +// env.RPM_SOURCE_STASH_NAME = "${UUID.randomUUID()}"; + sh(script: "make show_vars") + sh(script: "make setup_dist_for_deb setup_dist_for_rpm") + sh(script: "make show_vars") + echo "PY_MAJOR_VERSION = ${env.PY_MAJOR_VERSION}" + } + dir("compute_endpoint/") { + pkg_version = sh(script: "${venv_py} setup.py --version", returnStdout: true).trim().replace("-", "~") + echo "pkg_version = ${pkg_version}" + } + env.PKG_TARBALL = "globus_compute_endpoint-${pkg_version.replace("~", "")}.tar.gz" + env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${py_epoch_version}${py_major_version}-${pkg_version}.tar.gz" + dir("compute_endpoint/packaging/dist/") { + sh "ls -l ./" + sh "cp ${env.PKG_TARBALL} ${env.PREREQS_TARBALL} debbuild/globus-compute-agent/" + sh "ls -lR debbuild/globus-compute-agent/" + sh "mv debbuild/globus-compute-agent/ ${env.DEB_SOURCE_STASH_NAME}" +// sh "mv dist/rpmbuild/globus-compute-agent/ ${env.DEB_SOURCE_STASH_NAME}" + stash(name: env.DEB_SOURCE_STASH_NAME, includes: "${env.DEB_SOURCE_STASH_NAME}/**/*") +// stash(name: env.RPM_SOURCE_STASH_NAME, includes: "${env.RPM_SOURCE_STASH_NAME}/**/*") + } + } + } + } + stage ("Build packages") { + steps { + script { + lock(resource: 'globus-compute-agent-build') { + parallel "debian": { + def extra_tarball_map = [ + prereqs: env.PREREQS_TARBALL + ] + echo "extra_tarball_map = ${extra_tarball_map}" + env.DEB_ARTIFACTS_STASH = buildDebian( + env.DEB_SOURCE_STASH_NAME, + env.PKG_TARBALL, + require_gcs5_repo: true, + epic: null, + exclude: null, + extra_tarball_map: extra_tarball_map) +// }, "rpm": { +// env.RPM_ARTIFACTS_STASH = buildMock( +// env.RPM_SOURCE_STASH_NAME, +// env.PKG_TARBALL, +// true, +// null) + }, "failFast": true + } + } + } + } } +} diff --git a/compute_endpoint/packaging/Makefile b/compute_endpoint/packaging/Makefile index ff842d4ab..a665c2691 100644 --- a/compute_endpoint/packaging/Makefile +++ b/compute_endpoint/packaging/Makefile @@ -27,7 +27,8 @@ PIP_NAME_U := $(shell echo $(PIP_NAME_D) | tr '-' '_') PKG_VERSION := $(shell cd ../; "$(VENV_PY)" setup.py --version | tr '-' '~') PKG_WHEEL = $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl -PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.xz +PKG_TARBALL = $(PIP_NAME_U)-$(PKG_VERSION).tar.gz +PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.gz OS_CODENAME := $(shell test -f /etc/os-release && . /etc/os-release; echo $${VERSION_CODENAME:-focal}) @@ -63,6 +64,7 @@ show_vars: ##-For debugging, show the Makefile variables; will install a venv @echo "PKG_NAME : $(PKG_NAME)" @echo "PKG_VERSION : $(PKG_VERSION)" @echo "PREREQS_TARBALL_NAME : $(PREREQS_TARBALL_NAME)" + @echo "PKG_TARBALL : $(PKG_TARBALL)" @echo "PKG_WHEEL : $(PKG_WHEEL)" @echo @echo " Override python path with PYTHON3 variable:" @@ -107,7 +109,7 @@ $(PKG_WHEEL): $(VENV_PY) { /bin/echo -e "\nBUILD COMPUTE FROM A RELEASE TAG (current branch: $$(git branch --show-current))"; exit 1; } \ fi \ && rm -rf tests/ \ - && "$(VENV_PY)" -m build --wheel -o ../../ \ + && "$(VENV_PY)" -m build -o ../../ \ ) wheel: $(PKG_WHEEL) ##-Make the wheel (note that this does *not* include dependencies) @@ -117,21 +119,35 @@ $(PREREQS_TARBALL_NAME): $(VENV_PY) $(PKG_WHEEL) prereq_tarball: $(PREREQS_TARBALL_NAME) ##-Make a tarball of wheel dependencies -dist: $(PREREQS_TARBALL_NAME) ##-Make the dist/ directory with prereqs and wheel ready for packaging step - rm -rf dist/ \ - && mkdir dist/ \ - && cp $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) dist/ +_dist: + rm -rf dist/ && mkdir dist/ + cp "$(PKG_WHEEL)" dist/ + + # For CI, which does things differently than this Makefile: + # - append PKG_WHEEL to *prereqs* tarball + # - append package_shim.sh to *package* tarball + ( _prereq_dir="$$(tar -tf "$(PREREQS_TARBALL_NAME)" | head -1)" \ + && _pkg_dir="$$(tar -tf "$(PKG_TARBALL)" | head -1)" \ + && cd dist/ \ + && tar -xf "../$(PREREQS_TARBALL_NAME)" \ + && cp "$(PKG_WHEEL)" "$${_prereq_dir}" \ + && tar -czf "$(PREREQS_TARBALL_NAME)" "$${_prereq_dir}" \ + && rm -r "$${_prereq_dir}" \ + && tar -xf "../$(PKG_TARBALL)" \ + && cp "../package_shim.sh" "$${_pkg_dir}" \ + && tar -czf "$(PKG_TARBALL)" "$${_pkg_dir}" \ + && rm -r "$${_pkg_dir}" \ + ) -deb_build_needs: ##-Check that necessary executables are available before starting the DEB build. - @[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; } - @dpkg-checkbuilddeps +dist: $(PREREQS_TARBALL_NAME) _dist ##-Make the dist/ directory with prereqs and wheel ready for packaging step + rm "$(PKG_WHEEL)" "$(PKG_TARBALL)" "$(PREREQS_TARBALL_NAME)" -deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpoint (.deb) +setup_dist_for_deb: dist ##-Place Debian-build required files in dist/ ( cd dist/ \ && rm -rf debbuild/ \ - && mkdir -p debbuild/$(PKG_NAME)/wheels/ \ - && tar -C debbuild/$(PKG_NAME)/wheels/ -xf "$(PREREQS_TARBALL_NAME)" \ - && cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/wheels/ \ + && mkdir -p debbuild/$(PKG_NAME)/prereqs/ \ + && tar -C debbuild/$(PKG_NAME)/prereqs/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \ + && cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/prereqs/ \ && cp -R ../debian debbuild/$(PKG_NAME)/ \ && cp ../package_shim.sh debbuild/$(PKG_NAME)/ \ && cd debbuild/$(PKG_NAME)/ \ @@ -148,21 +164,23 @@ deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpo -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \ -e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \ -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \ - && dpkg-buildpackage -uc -us \ ) + +deb_build_needs: ##-Check that necessary executables are available before starting the DEB build. + @[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; } + @dpkg-checkbuilddeps + +deb: deb_build_needs dist setup_dist_for_deb ##-Build a Debian package of the Globus Compute Endpoint (.deb) + (cd dist/debbuild/$(PKG_NAME)/; dpkg-buildpackage -uc -us) @echo "\nDEB package successfully built:" @ls -lh dist/debbuild/*deb -rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build. - @[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; } - -rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoint (.rpm) +setup_dist_for_rpm: dist ##-Place RPM-build required files in dist/ (separated for CI building purposes) ( cd dist/ \ - && pwd && ls \ && rm -rf rpmbuild/ \ - && mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/wheels \ - && tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels -xf "$(PREREQS_TARBALL_NAME)" \ - && cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/wheels/ \ + && mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/prereqs/ \ + && tar -C rpmbuild/SOURCES/$(PKG_NAME)/prereqs/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \ + && cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/prereqs/ \ && cp ../package_shim.sh rpmbuild/SOURCES/$(PKG_NAME)/ \ && sed \ -e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \ @@ -170,7 +188,12 @@ rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoin -e "s/@PACKAGE_WHEEL@/$(PKG_WHEEL)/g" \ -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \ < ../fedora/$(PKG_NAME).spec.in > ./$(PKG_NAME).spec \ - && HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec \ ) + +rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build. + @[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; } + +rpm: rpm_build_needs dist setup_dist_for_rpm ##-Build an RPM package of the Globus Compute Endpoint (.rpm) + (cd dist/; HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec) @echo -e "\nRPM package successfully built:" @ls -lh dist/rpmbuild/RPMS/**/*rpm diff --git a/compute_endpoint/packaging/create-prereqs-tarball.sh b/compute_endpoint/packaging/create-prereqs-tarball.sh index 4fccbd594..402f20ad2 100755 --- a/compute_endpoint/packaging/create-prereqs-tarball.sh +++ b/compute_endpoint/packaging/create-prereqs-tarball.sh @@ -90,12 +90,12 @@ done > dependent-prereqs.txt modified_time="$(TZ=UTC0 date --rfc-3339=seconds)" -TZ=UTC0 tar -C "$download_dir/" --format=posix \ +TZ=UTC0 tar --format=posix \ --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime,delete=mtime \ --mtime="$modified_time" \ --numeric-owner \ --owner=0 \ --group=0 \ --mode="go-rwx,u+rw" \ - -cf - . \ - | xz -9 + -cf - "$download_dir/" \ + | gzip -9 diff --git a/compute_endpoint/packaging/debian/rules b/compute_endpoint/packaging/debian/rules index 5391a1679..fcce70efb 100755 --- a/compute_endpoint/packaging/debian/rules +++ b/compute_endpoint/packaging/debian/rules @@ -30,10 +30,13 @@ _unitdir=/lib/systemd/system .PHONY: override_dh_auto_configure override_dh_auto_configure: $(PYTHON3) -mvenv $${PWD}/$(TMP_VIRTUAL_ENV) - . "$(TMP_VIRTUAL_ENV)/bin/activate"; \ + . "$${PWD}/$(TMP_VIRTUAL_ENV)/bin/activate"; \ set -x; \ - python3 -mpip install --no-index --no-cache-dir -I --compile -U wheels/pip-*.whl; \ - python3 -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/wheels/ "@PIP_NAME@" + pwd; \ + ls -lR; \ + pwd; \ + python -mpip install --no-index --no-cache-dir -I --compile -U prereqs/pip-*.whl; \ + python -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/prereqs/ "@PIP_NAME@" .PHONY: override_dh_auto_clean override_dh_auto_clean: @@ -78,5 +81,8 @@ override_dh_python3: override_dh_builddeb: dh_builddeb -- -Zxz +.PHONY: override_dh_auto_build +override_dh_auto_build: # empty == we do it ourselves in auto_install (above) + %: dh $@ --with python3 --with systemd