From c58d015084af005ffa654b20dfa7c1700142e0bf Mon Sep 17 00:00:00 2001 From: Kevin Hunter Kesling Date: Thu, 25 Apr 2024 12:07:03 -0400 Subject: [PATCH] Jenkins WIP --- compute_endpoint/packaging/JenkinsFile | 89 +++++++++++++++++ compute_endpoint/packaging/Makefile | 98 ++++++++++++++----- .../packaging/create-prereqs-tarball.sh | 6 +- compute_endpoint/packaging/debian/control | 2 +- compute_endpoint/packaging/debian/rules | 12 ++- .../fedora/globus-compute-agent.spec.in | 17 ++-- 6 files changed, 180 insertions(+), 44 deletions(-) create mode 100644 compute_endpoint/packaging/JenkinsFile diff --git a/compute_endpoint/packaging/JenkinsFile b/compute_endpoint/packaging/JenkinsFile new file mode 100644 index 000000000..853a992c5 --- /dev/null +++ b/compute_endpoint/packaging/JenkinsFile @@ -0,0 +1,89 @@ +#! groovy + +@Library('gcs-build-scripts') _ + +pipeline { + agent none + options { + buildDiscarder( + logRotator( + numToKeepStr: '5', + artifactNumToKeepStr: '1', + artifactDaysToKeepStr: '7' + ) + ) + timeout(time: 3, unit: 'HOURS') + disableConcurrentBuilds() + } + stages { + stage ("Prep source") { + agent {label "package_creator"} + steps { + checkout scm + script { + env.DEB_SOURCE_STASH_NAME = "${UUID.randomUUID()}"; + env.RPM_SOURCE_STASH_NAME = "${UUID.randomUUID()}"; + def py_full_version = sh(script: "/opt/globus-python/bin/python3 -c 'import sys; print(\"{}.{}.{}\".format(*sys.version_info))'", returnStdout: true).trim() + def (py_epoch_version, py_major_version, py_minor_version) = py_full_version.tokenize(".") + def pkg_version = "not yet set" + + def venv_py = "${env.WORKSPACE}/compute_endpoint/packaging/venv-${py_epoch_version}${py_major_version}/bin/python" + sh "git clean -fdx" + sh "git checkout kevin_package_test" + + dir("compute_endpoint/packaging/") { + sh(script: "make show_vars") +// sh(script: "make setup_dist_for_deb setup_dist_for_rpm") + sh(script: "make setup_dist_for_deb") +// sh(script: "make setup_dist_for_rpm") + } + dir("compute_endpoint/") { + pkg_version = sh(script: "${venv_py} setup.py --version", returnStdout: true).trim().replace("-", "~") + echo "pkg_version = ${pkg_version}" + } + env.PKG_TARBALL = "globus_compute_endpoint-${pkg_version.replace("~", "")}.tar.gz" + env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${py_epoch_version}${py_major_version}-${pkg_version}.tar.gz" + dir("compute_endpoint/packaging/") { + sh "ls -lR ./" +// sh "cp ${env.PKG_TARBALL} ${env.PREREQS_TARBALL} debbuild/globus-compute-agent/" +// sh "ls -lR debbuild/globus-compute-agent/" + sh "mv dist/ ${env.DEB_SOURCE_STASH_NAME}" + stash(name: env.DEB_SOURCE_STASH_NAME, includes: "${env.DEB_SOURCE_STASH_NAME}/**/*") +// sh "mv dist/ ${env.RPM_SOURCE_STASH_NAME}" +// stash(name: env.RPM_SOURCE_STASH_NAME, includes: "${env.RPM_SOURCE_STASH_NAME}/**/*") + } + } + } + } + stage ("Build packages") { + steps { + script { + lock(resource: 'globus-compute-agent-build') { + parallel "debian": { +// parallel "rpm": { + def extra_tarball_map = [ + prereqs: env.PREREQS_TARBALL + ] + echo "extra_tarball_map = ${extra_tarball_map}" + env.DEB_ARTIFACTS_STASH = buildDebian( + env.DEB_SOURCE_STASH_NAME, + env.PKG_TARBALL, + require_gcs5_repo: true, + epic: null, + exclude: null, + exclude_arch: ["arm64"], + extra_tarball_map: extra_tarball_map) +// }, "rpm": { + +// env.RPM_ARTIFACTS_STASH = buildMock( +// env.RPM_SOURCE_STASH_NAME, +// env.PKG_TARBALL, +// true, +// null) + }, "failFast": true + } + } + } + } + } +} diff --git a/compute_endpoint/packaging/Makefile b/compute_endpoint/packaging/Makefile index ff842d4ab..0c71d070a 100644 --- a/compute_endpoint/packaging/Makefile +++ b/compute_endpoint/packaging/Makefile @@ -26,8 +26,11 @@ PIP_NAME_D := $(shell cd ../; "$(VENV_PY)" setup.py --name) PIP_NAME_U := $(shell echo $(PIP_NAME_D) | tr '-' '_') PKG_VERSION := $(shell cd ../; "$(VENV_PY)" setup.py --version | tr '-' '~') -PKG_WHEEL = $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl -PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.xz +PKG_WHEEL := $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl +PKG_SOURCE_DIR := $(PIP_NAME_U)-$(PKG_VERSION) +PKG_TARBALL := $(PKG_SOURCE_DIR).tar.gz +PREREQS_DIR := $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION) +PREREQS_TARBALL_NAME = $(PREREQS_DIR).tar.gz OS_CODENAME := $(shell test -f /etc/os-release && . /etc/os-release; echo $${VERSION_CODENAME:-focal}) @@ -62,7 +65,10 @@ show_vars: ##-For debugging, show the Makefile variables; will install a venv @echo "PIP_NAME_U : $(PIP_NAME_U)" @echo "PKG_NAME : $(PKG_NAME)" @echo "PKG_VERSION : $(PKG_VERSION)" + @echo "PREREQS_DIR : $(PREREQS_DIR)" @echo "PREREQS_TARBALL_NAME : $(PREREQS_TARBALL_NAME)" + @echo "PKG_SOURCE_DIR : $(PKG_SOURCE_DIR)" + @echo "PKG_TARBALL : $(PKG_TARBALL)" @echo "PKG_WHEEL : $(PKG_WHEEL)" @echo @echo " Override python path with PYTHON3 variable:" @@ -107,7 +113,7 @@ $(PKG_WHEEL): $(VENV_PY) { /bin/echo -e "\nBUILD COMPUTE FROM A RELEASE TAG (current branch: $$(git branch --show-current))"; exit 1; } \ fi \ && rm -rf tests/ \ - && "$(VENV_PY)" -m build --wheel -o ../../ \ + && "$(VENV_PY)" -m build -o ../../ \ ) wheel: $(PKG_WHEEL) ##-Make the wheel (note that this does *not* include dependencies) @@ -117,24 +123,32 @@ $(PREREQS_TARBALL_NAME): $(VENV_PY) $(PKG_WHEEL) prereq_tarball: $(PREREQS_TARBALL_NAME) ##-Make a tarball of wheel dependencies -dist: $(PREREQS_TARBALL_NAME) ##-Make the dist/ directory with prereqs and wheel ready for packaging step - rm -rf dist/ \ - && mkdir dist/ \ - && cp $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) dist/ +_dist: + rm -rf dist/ && mkdir dist/ + cp "$(PKG_WHEEL)" dist/ + + # For CI, which does things differently than this Makefile: + # - append PKG_WHEEL to *prereqs* tarball + # - append package_shim.sh to *package* tarball + ( _prereq_dir="$$(tar -tf "$(PREREQS_TARBALL_NAME)" | head -1)" \ + && _pkg_dir="$$(tar -tf "$(PKG_TARBALL)" | head -1)" \ + && cd dist/ \ + && tar -xf "../$(PREREQS_TARBALL_NAME)" \ + && cp "$(PKG_WHEEL)" "$${_prereq_dir}" \ + && tar -czf "$(PREREQS_TARBALL_NAME)" "$${_prereq_dir}" \ + && rm -r "$${_prereq_dir}" \ + && tar -xf "../$(PKG_TARBALL)" \ + && cp "../package_shim.sh" "$${_pkg_dir}" \ + && tar -czf "$(PKG_TARBALL)" "$${_pkg_dir}" \ + && rm -r "$${_pkg_dir}" \ + ) -deb_build_needs: ##-Check that necessary executables are available before starting the DEB build. - @[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; } - @dpkg-checkbuilddeps +dist: $(PREREQS_TARBALL_NAME) _dist ##-Make the dist/ directory with prereqs and wheel ready for packaging step + rm "$(PKG_WHEEL)" "$(PKG_TARBALL)" "$(PREREQS_TARBALL_NAME)" -deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpoint (.deb) +_setup_dist_for_deb: ( cd dist/ \ - && rm -rf debbuild/ \ - && mkdir -p debbuild/$(PKG_NAME)/wheels/ \ - && tar -C debbuild/$(PKG_NAME)/wheels/ -xf "$(PREREQS_TARBALL_NAME)" \ - && cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/wheels/ \ - && cp -R ../debian debbuild/$(PKG_NAME)/ \ - && cp ../package_shim.sh debbuild/$(PKG_NAME)/ \ - && cd debbuild/$(PKG_NAME)/ \ + && cp -R ../debian ./ \ && mv debian/changelog.in.in debian/changelog \ && sed -i debian/changelog \ -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \ @@ -148,29 +162,59 @@ deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpo -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \ -e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \ -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \ + ) + +setup_dist_for_deb: dist _setup_dist_for_deb ##-Place Debian-build required files in dist/ + +deb_build_needs: ##-Check that necessary executables are available before starting the DEB build. + @[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; } + @dpkg-checkbuilddeps + +_deb: + ( cd dist/ \ + && rm -rf debbuild/ \ + && mkdir -p debbuild/$(PKG_NAME)/prereqs/ \ + && cp -R debian debbuild/$(PKG_NAME)/ \ + && tar -C debbuild/$(PKG_NAME)/prereqs/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \ + && cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/prereqs/ \ + && cp ../package_shim.sh debbuild/$(PKG_NAME)/ \ + && cd debbuild/$(PKG_NAME)/ \ && dpkg-buildpackage -uc -us \ ) @echo "\nDEB package successfully built:" @ls -lh dist/debbuild/*deb -rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build. - @[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; } +# Reminder: deb rule is not invoked by CI; this is for one-off needs +deb: deb_build_needs setup_dist_for_deb _deb ##-Build a Debian package of the Globus Compute Endpoint (.deb) -rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoint (.rpm) +_setup_dist_for_rpm: ( cd dist/ \ - && pwd && ls \ - && rm -rf rpmbuild/ \ - && mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/wheels \ - && tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels -xf "$(PREREQS_TARBALL_NAME)" \ - && cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/wheels/ \ - && cp ../package_shim.sh rpmbuild/SOURCES/$(PKG_NAME)/ \ && sed \ -e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \ -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \ -e "s/@PACKAGE_WHEEL@/$(PKG_WHEEL)/g" \ + -e "s/@PACKAGE_TARBALL_NAME@/$(PKG_TARBALL)/g" \ + -e "s/@PREREQS_TARBALL_NAME@/$(PREREQS_TARBALL_NAME)/g" \ -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \ + -e "s/@PACKAGE_SOURCE_DIR@/$(PKG_SOURCE_DIR)/" \ + -e "s/@PREREQS_DIR@/$(PREREQS_DIR)/" \ < ../fedora/$(PKG_NAME).spec.in > ./$(PKG_NAME).spec \ + ) + +setup_dist_for_rpm: dist _setup_dist_for_rpm ##-Place RPM-build required files in dist/ (separated for CI building purposes) + +rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build. + @[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; } + +# Reminder: rpm rule is not invoked by CI; this is for one-off needs +_rpm: + ( cd dist/ \ + && rm -rf rpmbuild/ \ + && mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}/ \ + && cp *.tar.gz rpmbuild/SOURCES/ \ && HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec \ ) @echo -e "\nRPM package successfully built:" @ls -lh dist/rpmbuild/RPMS/**/*rpm + +rpm: rpm_build_needs setup_dist_for_rpm _rpm ##-Build an RPM package of the Globus Compute Endpoint (.rpm) diff --git a/compute_endpoint/packaging/create-prereqs-tarball.sh b/compute_endpoint/packaging/create-prereqs-tarball.sh index 4fccbd594..402f20ad2 100755 --- a/compute_endpoint/packaging/create-prereqs-tarball.sh +++ b/compute_endpoint/packaging/create-prereqs-tarball.sh @@ -90,12 +90,12 @@ done > dependent-prereqs.txt modified_time="$(TZ=UTC0 date --rfc-3339=seconds)" -TZ=UTC0 tar -C "$download_dir/" --format=posix \ +TZ=UTC0 tar --format=posix \ --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime,delete=mtime \ --mtime="$modified_time" \ --numeric-owner \ --owner=0 \ --group=0 \ --mode="go-rwx,u+rw" \ - -cf - . \ - | xz -9 + -cf - "$download_dir/" \ + | gzip -9 diff --git a/compute_endpoint/packaging/debian/control b/compute_endpoint/packaging/debian/control index 642c7fa1d..68c0a8920 100644 --- a/compute_endpoint/packaging/debian/control +++ b/compute_endpoint/packaging/debian/control @@ -2,7 +2,7 @@ Source: @PACKAGE_NAME@ Priority: optional Section: contrib/science Maintainer: Globus Support -Build-Depends: debhelper (>= 13.6), globus-python (>= 3.9), dh-exec, dh-python +Build-Depends: debhelper (>= 12), globus-python (>= 3.9), dh-exec, dh-python Standards-Version: 4.6.0 Homepage: https://globus-compute.readthedocs.io/ diff --git a/compute_endpoint/packaging/debian/rules b/compute_endpoint/packaging/debian/rules index 5391a1679..fcce70efb 100755 --- a/compute_endpoint/packaging/debian/rules +++ b/compute_endpoint/packaging/debian/rules @@ -30,10 +30,13 @@ _unitdir=/lib/systemd/system .PHONY: override_dh_auto_configure override_dh_auto_configure: $(PYTHON3) -mvenv $${PWD}/$(TMP_VIRTUAL_ENV) - . "$(TMP_VIRTUAL_ENV)/bin/activate"; \ + . "$${PWD}/$(TMP_VIRTUAL_ENV)/bin/activate"; \ set -x; \ - python3 -mpip install --no-index --no-cache-dir -I --compile -U wheels/pip-*.whl; \ - python3 -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/wheels/ "@PIP_NAME@" + pwd; \ + ls -lR; \ + pwd; \ + python -mpip install --no-index --no-cache-dir -I --compile -U prereqs/pip-*.whl; \ + python -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/prereqs/ "@PIP_NAME@" .PHONY: override_dh_auto_clean override_dh_auto_clean: @@ -78,5 +81,8 @@ override_dh_python3: override_dh_builddeb: dh_builddeb -- -Zxz +.PHONY: override_dh_auto_build +override_dh_auto_build: # empty == we do it ourselves in auto_install (above) + %: dh $@ --with python3 --with systemd diff --git a/compute_endpoint/packaging/fedora/globus-compute-agent.spec.in b/compute_endpoint/packaging/fedora/globus-compute-agent.spec.in index 0d9837c77..2c078c4ea 100644 --- a/compute_endpoint/packaging/fedora/globus-compute-agent.spec.in +++ b/compute_endpoint/packaging/fedora/globus-compute-agent.spec.in @@ -5,7 +5,6 @@ Name: @PACKAGE_NAME@ %global pythonversion py39 %global VIRTUAL_ENV /opt/%{name}/venv-%{pythonversion} %global src_dir %{_sourcedir}/%{name} -%global wheels %{src_dir}/wheels %global package_executable @PIP_NAME@ %global globus_python3_version 3.9 %global __python /opt/globus-python/bin/python3 @@ -33,6 +32,8 @@ Requires(pre): /usr/sbin/useradd, /usr/bin/getent Requires(postun): /usr/sbin/userdel Group: System Environment/Daemons +Source: @PACKAGE_TARBALL_NAME@ +Source1: @PREREQS_TARBALL_NAME@ License: Apache 2.0 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} @@ -48,24 +49,20 @@ want to install the `@PIP_NAME@` directly from PyPI. %prep %global DEST_VIRTUAL_ENV "$PWD/TMP_RPM_BUILD_ROOT%{VIRTUAL_ENV}" -[ -f %{wheels}/@PACKAGE_WHEEL@ ] || { - C="" R="" - [ -t 2 ] && { C="\033[91;1;40m"; R="\033[m"; } - >&2 echo -e "${C}Main source wheel not found. Reminder: the Makefile sets that up. (Hint: make rpm)${R}"; - exit 1 -} +%setup -n "@PACKAGE_SOURCE_DIR@" +%setup -n "@PREREQS_DIR@" -T -D -b 1 rm -rf "%{DEST_VIRTUAL_ENV}" %__python -mvenv "%{DEST_VIRTUAL_ENV}" %build . "%{DEST_VIRTUAL_ENV}/bin/activate" -python3 -mpip install --no-index --no-cache-dir -I --compile -U %{wheels}/pip-*.whl +python3 -mpip install --no-index --no-cache-dir -I --compile -U pip-*.whl +python3 -mpip install --pre --compile --no-index --no-cache-dir -I --find-links=./ "@PIP_NAME@" deactivate %install . "%{DEST_VIRTUAL_ENV}/bin/activate" -python3 -mpip install --pre --compile --no-index --no-cache-dir -I --find-links=file://%{wheels} "@PIP_NAME@" tar -C "${PWD}/TMP_RPM_BUILD_ROOT" -cf - . | tar -C "${RPM_BUILD_ROOT}" -xf - @@ -84,7 +81,7 @@ sed -i "${RPM_BUILD_ROOT}%{VIRTUAL_ENV}/bin/activate" \ -e "s|^VIRTUAL_ENV=.*|VIRTUAL_ENV=%{VIRTUAL_ENV}|" install -d -m 755 ${RPM_BUILD_ROOT}%{_sbindir} -install -m 755 %{src_dir}/package_shim.sh "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}" +install -m 755 "%{_builddir}/@PACKAGE_SOURCE_DIR@"/package_shim.sh "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}" sed -i "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}" -e "s|@VIRTUAL_ENV@|%{VIRTUAL_ENV}|" %pre