Skip to content

Commit

Permalink
Jenkins WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
khk-globus committed May 2, 2024
1 parent 6483f86 commit c02a83a
Show file tree
Hide file tree
Showing 10 changed files with 206 additions and 55 deletions.
103 changes: 103 additions & 0 deletions compute_endpoint/packaging/JenkinsFile
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#! groovy

@Library('gcs-build-scripts') _

pipeline {
agent none
options {
buildDiscarder(
logRotator(
numToKeepStr: '5',
artifactNumToKeepStr: '1',
artifactDaysToKeepStr: '7'
)
)
timeout(time: 3, unit: 'HOURS')
disableConcurrentBuilds()
}
stages {
stage ("Prep source") {
agent {label "package_creator"}
steps {
checkout scm
script {
env.SOURCE_STASH_NAME = "${UUID.randomUUID()}";
sh "git clean -fdx"
sh "git checkout kevin_package_test"

dir("compute_endpoint/packaging/") {
sh(script: "make show_vars")
sh(script: "make setup_dist_for_deb setup_dist_for_rpm")
sh "mv dist/ ${env.SOURCE_STASH_NAME}"
stash(name: env.SOURCE_STASH_NAME, includes: "${env.SOURCE_STASH_NAME}/**/*")
}

def venv_py = "${env.WORKSPACE}/compute_endpoint/packaging/venv/bin/python"
def py_full_version = sh(script: "'${venv_py}' -c 'import sys; print(\"{}.{}\".format(*sys.version_info))'", returnStdout: true).trim()
def (py_epoch_version, py_major_version) = py_full_version.tokenize(".")
def pkg_version = "not yet set"

dir("compute_endpoint/") {
env.PACKAGE_VERSION = sh(script: "${venv_py} setup.py --version", returnStdout: true).trim().replace("-", "~")
echo "pkg_version = ${env.PACKAGE_VERSION}"
}
env.PKG_TARBALL = "globus_compute_endpoint-${env.PACKAGE_VERSION.replace("~", "")}.tar.gz"
env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${py_epoch_version}${py_major_version}-${env.PACKAGE_VERSION}.tar.gz"
}
}
}
stage ("Build packages") {
steps {
script {
lock(resource: 'globus-compute-agent-build') {
parallel "debian": {
def extra_tarball_map = [
prereqs: env.PREREQS_TARBALL
]
env.DEB_ARTIFACTS_STASH = buildDebian(
env.SOURCE_STASH_NAME,
env.PKG_TARBALL,
require_gcs5_repo: true,
extra_tarball_map: extra_tarball_map
)
}, "rpm": {
env.RPM_ARTIFACTS_STASH = buildMock(
env.SOURCE_STASH_NAME,
env.PKG_TARBALL,
true
)
}, "failFast": false
}
}
}
}
stage ("Publish packages") {
agent { label "master" }
steps {
script {
def stashname = "${UUID.randomUUID()}"

sh "ls -lR"
dir("artifacts") {
if (env.DEB_ARTIFACTS_STASH) {
unstash(name: env.DEB_ARTIFACTS_STASH)
}
if (env.RPM_ARTIFACTS_STASH) {
unstash(name: env.RPM_ARTIFACTS_STASH)
}
dir("dist/") {
unstash(name: env.SOURCE_STASH_NAME)
sh "ls -lR"
// sh(script: "make distclean")
}

// stash(name: stashname, includes: "**/*")
deleteDir()
}
// publishResults(stashname, "globus-compute-agent", env.PACKAGE_VERSION, false)
deleteDir()
}
}
}
}
}
111 changes: 78 additions & 33 deletions compute_endpoint/packaging/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ VENV_PY := $(shell pwd)/$(VIRTUALENV)/bin/python
# This if conditional before the PKG_ variables, so as to ensure we have setuptools;
# not so ideal, but at the moment, I don't know how to do better. Hmm.
ifeq ($(wildcard $(VENV_PY)),)
_DUMMY := $(shell "$(PYTHON3)" -mvenv "$(VIRTUALENV)")
_DUMMY := $(shell "$(PYTHON3)" -mvenv "$(VIRTUALENV)"; ln -sf "$(VIRTUALENV)" venv)
_DUMMY := $(shell "$(VENV_PY)" -mpip install -U pip -U setuptools)
endif

Expand All @@ -26,8 +26,11 @@ PIP_NAME_D := $(shell cd ../; "$(VENV_PY)" setup.py --name)
PIP_NAME_U := $(shell echo $(PIP_NAME_D) | tr '-' '_')

PKG_VERSION := $(shell cd ../; "$(VENV_PY)" setup.py --version | tr '-' '~')
PKG_WHEEL = $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl
PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.xz
PKG_WHEEL := $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl
PKG_SOURCE_DIR := $(PIP_NAME_U)-$(PKG_VERSION)
PKG_TARBALL := $(PKG_SOURCE_DIR).tar.gz
PREREQS_DIR := $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION)
PREREQS_TARBALL_NAME = $(PREREQS_DIR).tar.gz

OS_CODENAME := $(shell test -f /etc/os-release && . /etc/os-release; echo $${VERSION_CODENAME:-focal})

Expand Down Expand Up @@ -62,15 +65,18 @@ show_vars: ##-For debugging, show the Makefile variables; will install a venv
@echo "PIP_NAME_U : $(PIP_NAME_U)"
@echo "PKG_NAME : $(PKG_NAME)"
@echo "PKG_VERSION : $(PKG_VERSION)"
@echo "PREREQS_DIR : $(PREREQS_DIR)"
@echo "PREREQS_TARBALL_NAME : $(PREREQS_TARBALL_NAME)"
@echo "PKG_SOURCE_DIR : $(PKG_SOURCE_DIR)"
@echo "PKG_TARBALL : $(PKG_TARBALL)"
@echo "PKG_WHEEL : $(PKG_WHEEL)"
@echo
@echo " Override python path with PYTHON3 variable:"
@echo " $(MAKE) PYTHON3=/path/to/python target(s)"

.PHONY: clean
clean: ##-Remove the venv, build/ dist/, prereqs tarball, and the package wheel
rm -rf -- "$(VIRTUALENV)" build/ dist/ "$(PREREQS_TARBALL_NAME)" "$(PKG_WHEEL)"
rm -rf -- "$(VIRTUALENV)" venv build/ dist/ "$(PREREQS_TARBALL_NAME)" "$(PKG_WHEEL)"

.PHONY: distclean
distclean: clean ##-Run `clean` target, then additionally remove venv-*, *tar.xz, *whl
Expand All @@ -95,19 +101,20 @@ $(VENV_PY): _build_needs
exit 1; \
fi
"$(PYTHON3)" -mvenv "$(VIRTUALENV)"
. "$(VIRTUALENV)/bin/activate"
ln -sf "$(VIRTUALENV)" venv
. venv/bin/activate
@"$(VENV_PY)" -m pip install -U pip -U setuptools -U build

$(PKG_WHEEL): $(VENV_PY)
( rm -rf build/ \
&& git clone ../../ build/ \
&& cd build/compute_endpoint/ \
&& echo -n " Git Tag: " \
&& if ! git describe --tags --exact-match; then \
{ /bin/echo -e "\nBUILD COMPUTE FROM A RELEASE TAG (current branch: $$(git branch --show-current))"; exit 1; } \
&& if git describe --tags --exact-match 2> /dev/null; \
then { echo " Git Tag: $$(git describe --tags --exact-match)"; }; \
else { echo " Git Branch: $$(git rev-parse HEAD) ($$(git describe --tags))"; }; \
fi \
&& rm -rf tests/ \
&& "$(VENV_PY)" -m build --wheel -o ../../ \
&& "$(VENV_PY)" -m build -o ../../ \
)

wheel: $(PKG_WHEEL) ##-Make the wheel (note that this does *not* include dependencies)
Expand All @@ -117,24 +124,32 @@ $(PREREQS_TARBALL_NAME): $(VENV_PY) $(PKG_WHEEL)

prereq_tarball: $(PREREQS_TARBALL_NAME) ##-Make a tarball of wheel dependencies

dist: $(PREREQS_TARBALL_NAME) ##-Make the dist/ directory with prereqs and wheel ready for packaging step
rm -rf dist/ \
&& mkdir dist/ \
&& cp $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) dist/
_dist:
rm -rf dist/ && mkdir dist/
cp "$(PKG_WHEEL)" dist/

# For CI, which does things differently than this Makefile:
# - append PKG_WHEEL to *prereqs* tarball
# - append package_shim.sh to *package* tarball
( _prereq_dir="$$(tar -tf "$(PREREQS_TARBALL_NAME)" | head -1)" \
&& _pkg_dir="$$(tar -tf "$(PKG_TARBALL)" | head -1)" \
&& cd dist/ \
&& tar -xf "../$(PREREQS_TARBALL_NAME)" \
&& cp "$(PKG_WHEEL)" "$${_prereq_dir}" \
&& tar -czf "$(PREREQS_TARBALL_NAME)" "$${_prereq_dir}" \
&& rm -r "$${_prereq_dir}" \
&& tar -xf "../$(PKG_TARBALL)" \
&& cp "../package_shim.sh" "$${_pkg_dir}" \
&& tar -czf "$(PKG_TARBALL)" "$${_pkg_dir}" \
&& rm -r "$${_pkg_dir}" \
)

deb_build_needs: ##-Check that necessary executables are available before starting the DEB build.
@[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; }
@dpkg-checkbuilddeps
dist: $(PREREQS_TARBALL_NAME) _dist ##-Make the dist/ directory with prereqs and wheel ready for packaging step
rm "$(PKG_WHEEL)" "$(PKG_TARBALL)" "$(PREREQS_TARBALL_NAME)"

deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpoint (.deb)
_setup_dist_for_deb:
( cd dist/ \
&& rm -rf debbuild/ \
&& mkdir -p debbuild/$(PKG_NAME)/wheels/ \
&& tar -C debbuild/$(PKG_NAME)/wheels/ -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/wheels/ \
&& cp -R ../debian debbuild/$(PKG_NAME)/ \
&& cp ../package_shim.sh debbuild/$(PKG_NAME)/ \
&& cd debbuild/$(PKG_NAME)/ \
&& cp -R ../debian ./ \
&& mv debian/changelog.in.in debian/changelog \
&& sed -i debian/changelog \
-e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
Expand All @@ -148,29 +163,59 @@ deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpo
-e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
-e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \
-e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \
)

setup_dist_for_deb: dist _setup_dist_for_deb ##-Place Debian-build required files in dist/

deb_build_needs: ##-Check that necessary executables are available before starting the DEB build.
@[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; }
@dpkg-checkbuilddeps

_deb:
( cd dist/ \
&& rm -rf debbuild/ \
&& mkdir -p debbuild/$(PKG_NAME)/prereqs/ \
&& cp -R debian debbuild/$(PKG_NAME)/ \
&& tar -C debbuild/$(PKG_NAME)/prereqs/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/prereqs/ \
&& cp ../package_shim.sh debbuild/$(PKG_NAME)/ \
&& cd debbuild/$(PKG_NAME)/ \
&& dpkg-buildpackage -uc -us \
)
@echo "\nDEB package successfully built:"
@ls -lh dist/debbuild/*deb

rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build.
@[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; }
# Reminder: deb rule is not invoked by CI; this is for one-off needs
deb: deb_build_needs setup_dist_for_deb _deb ##-Build a Debian package of the Globus Compute Endpoint (.deb)

rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoint (.rpm)
_setup_dist_for_rpm:
( cd dist/ \
&& pwd && ls \
&& rm -rf rpmbuild/ \
&& mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/wheels \
&& tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/wheels/ \
&& cp ../package_shim.sh rpmbuild/SOURCES/$(PKG_NAME)/ \
&& sed \
-e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \
-e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
-e "s/@PACKAGE_WHEEL@/$(PKG_WHEEL)/g" \
-e "s/@PACKAGE_TARBALL_NAME@/$(PKG_TARBALL)/g" \
-e "s/@PREREQS_TARBALL_NAME@/$(PREREQS_TARBALL_NAME)/g" \
-e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \
-e "s/@PACKAGE_SOURCE_DIR@/$(PKG_SOURCE_DIR)/" \
-e "s/@PREREQS_DIR@/$(PREREQS_DIR)/" \
< ../fedora/$(PKG_NAME).spec.in > ./$(PKG_NAME).spec \
)

setup_dist_for_rpm: dist _setup_dist_for_rpm ##-Place RPM-build required files in dist/ (separated for CI building purposes)

rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build.
@[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; }

# Reminder: rpm rule is not invoked by CI; this is for one-off needs
_rpm:
( cd dist/ \
&& rm -rf rpmbuild/ \
&& mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}/ \
&& cp *.tar.gz rpmbuild/SOURCES/ \
&& HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec \
)
@echo -e "\nRPM package successfully built:"
@ls -lh dist/rpmbuild/RPMS/**/*rpm

rpm: rpm_build_needs setup_dist_for_rpm _rpm ##-Build an RPM package of the Globus Compute Endpoint (.rpm)
6 changes: 3 additions & 3 deletions compute_endpoint/packaging/create-prereqs-tarball.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,12 @@ done > dependent-prereqs.txt

modified_time="$(TZ=UTC0 date --rfc-3339=seconds)"

TZ=UTC0 tar -C "$download_dir/" --format=posix \
TZ=UTC0 tar --format=posix \
--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime,delete=mtime \
--mtime="$modified_time" \
--numeric-owner \
--owner=0 \
--group=0 \
--mode="go-rwx,u+rw" \
-cf - . \
| xz -9
-cf - "$download_dir/" \
| gzip -9
4 changes: 2 additions & 2 deletions compute_endpoint/packaging/debian/changelog.in.in
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
@PACKAGE_NAME@ (@PACKAGE_VERSION@.@distro@) @distro@; urgency=low
@PACKAGE_NAME@ (@PACKAGE_VERSION@-1.@distro@) @distro@; urgency=low

* Initial release as a Debian package

* Includes support for multi-user Globus Compute Endpoints; see
https://globus-compute.readthedocs.io/ for more information.

-- Maintainer: Globus Support <[email protected]> Mon, 08 Apr 2024 11:01:30 -0500
-- Globus Toolkit <[email protected]> Mon, 08 Apr 2024 11:01:30 -0500
6 changes: 3 additions & 3 deletions compute_endpoint/packaging/debian/control
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
Source: @PACKAGE_NAME@
Priority: optional
Section: contrib/science
Maintainer: Globus Support <[email protected]>
Build-Depends: debhelper (>= 13.6), globus-python (>= 3.9), dh-exec, dh-python
Standards-Version: 4.6.0
Maintainer: Globus Toolkit <[email protected]>
Build-Depends: debhelper (>= 12), globus-python (>= 3.9), dh-exec, dh-python
Standards-Version: 3.9.6
Homepage: https://globus-compute.readthedocs.io/

Package: @PACKAGE_NAME@
Expand Down
12 changes: 9 additions & 3 deletions compute_endpoint/packaging/debian/rules
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,13 @@ _unitdir=/lib/systemd/system
.PHONY: override_dh_auto_configure
override_dh_auto_configure:
$(PYTHON3) -mvenv $${PWD}/$(TMP_VIRTUAL_ENV)
. "$(TMP_VIRTUAL_ENV)/bin/activate"; \
. "$${PWD}/$(TMP_VIRTUAL_ENV)/bin/activate"; \
set -x; \
python3 -mpip install --no-index --no-cache-dir -I --compile -U wheels/pip-*.whl; \
python3 -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/wheels/ "@PIP_NAME@"
pwd; \
ls -lR; \
pwd; \
python -mpip install --no-index --no-cache-dir -I --compile -U prereqs/pip-*.whl; \
python -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/prereqs/ "@PIP_NAME@"

.PHONY: override_dh_auto_clean
override_dh_auto_clean:
Expand Down Expand Up @@ -78,5 +81,8 @@ override_dh_python3:
override_dh_builddeb:
dh_builddeb -- -Zxz

.PHONY: override_dh_auto_build
override_dh_auto_build: # empty == we do it ourselves in auto_install (above)

%:
dh $@ --with python3 --with systemd
Loading

0 comments on commit c02a83a

Please sign in to comment.