From ae506ec85589e68f13331a4c0dac28d9b5dd2cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 20 Dec 2024 10:24:18 +0100 Subject: [PATCH 01/28] use latest bootstrap script --- bootstrap-prefix.sh | 1579 ++++++++++++++++++++++++++++--------------- 1 file changed, 1029 insertions(+), 550 deletions(-) diff --git a/bootstrap-prefix.sh b/bootstrap-prefix.sh index 3633641a..47fd388a 100755 --- a/bootstrap-prefix.sh +++ b/bootstrap-prefix.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash -# Copyright 2006-2023 Gentoo Authors +#shellcheck disable=SC1091,SC2015,SC2016,SC2030,SC2031,SC2038,SC2185,SC2120 +# Copyright 2006-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 trap 'exit 1' TERM INT QUIT ABRT # RAP (libc) mode is triggered on Linux kernel and glibc. is-rap() { [[ ${PREFIX_DISABLE_RAP} != "yes" && ${CHOST} = *linux-gnu* ]]; } -rapx() { is-rap && echo $1 || echo $2; } +rapx() { is-rap && echo "$1" || echo "$2"; } ## Functions Start Here @@ -15,32 +16,38 @@ estatus() { # disturbing -- if it works, it makes it easy to see where we are in # the bootstrap from the terminal status line (usually the window # name) - printf '\033]2;'"$*"'\007' + printf '\033]2;%s\007' "$*" } -eerror() { estatus $*; echo "!!! $*" 1>&2; } +eerror() { estatus "$*"; echo "!!! $*" 1>&2; } einfo() { echo "* $*"; } v() { echo "$@"; "$@"; } econf() { estatus "stage1: configuring ${PWD##*/}" - v ${CONFIG_SHELL} ./configure \ - --host=${CHOST} \ + v ${CONFIG_SHELL:+"${CONFIG_SHELL}"} ./configure \ + --host="${CHOST}" \ --prefix="${ROOT}"/tmp/usr \ --mandir="${ROOT}"/tmp/usr/share/man \ --infodir="${ROOT}"/tmp/usr/share/info \ --datadir="${ROOT}"/tmp/usr/share \ --sysconfdir="${ROOT}"/tmp/etc \ --localstatedir="${ROOT}"/tmp/var/lib \ - --build=${CHOST} \ + --build="${CHOST}" \ "$@" || return 1 } emake() { - [[ $* == *install* ]] \ - && estatus "stage1: installing ${PWD##*/}" \ - || estatus "stage1: building ${PWD##*/}" - v ${MAKE} ${MAKEOPTS} "$@" || return 1 + if [[ $* == *install* ]] ; then + estatus "stage1: installing ${PWD##*/}" + else + estatus "stage1: building ${PWD##*/}" + fi + read -r -a makeopts <<< "${MAKEOPTS}" + if ! v "${MAKE}" "${makeopts[@]}" "$@" ; then + estatus "stage1: retry with -j1 for clearer error message in ${PWD##*/}" + v "${MAKE}" "${makeopts[@]}" "$@" -j1 || return 1 + fi } efetch() { @@ -56,7 +63,7 @@ efetch() { if [[ ${OFFLINE_MODE} ]] ; then echo "I need ${1##*/} from $1 in $DISTDIR, can you give it to me?" - read + read -r [[ -e ${DISTDIR}/${1##*/} ]] && return 0 # Give fetch a try fi @@ -64,15 +71,15 @@ efetch() { if [[ -z ${FETCH_COMMAND} ]] ; then # Try to find a download manager, we only deal with wget, # curl, FreeBSD's fetch and ftp. - if [[ x$(type -t wget) == "xfile" ]] ; then - FETCH_COMMAND="wget" + if [[ $(type -t wget) == "file" ]] ; then + FETCH_COMMAND="wget -t 3 -T 3" # 3x3s wait [[ $(wget -h) == *"--no-check-certificate"* ]] \ && FETCH_COMMAND+=" --no-check-certificate" - elif [[ x$(type -t curl) == "xfile" ]] ; then + elif [[ $(type -t curl) == "file" ]] ; then FETCH_COMMAND="curl -f -L -O" - elif [[ x$(type -t fetch) == "xfile" ]] ; then + elif [[ $(type -t fetch) == "file" ]] ; then FETCH_COMMAND="fetch" - elif [[ x$(type -t ftp) == "xfile" ]] ; then + elif [[ $(type -t ftp) == "file" ]] ; then FETCH_COMMAND="ftp" else eerror "no suitable download manager found!" @@ -84,7 +91,7 @@ efetch() { einfo "Fetching ${1##*/}" estatus "stage1: fetching ${1##*/}" - pushd "${DISTDIR}" > /dev/null + pushd "${DISTDIR}" > /dev/null || exit 1 # Try for mirrors first, fall back to distfiles, then try given location local locs=( ) @@ -97,21 +104,25 @@ efetch() { done locs=( "${locs[@]}" "$1" ) + read -r -a fetchcmd <<< "${FETCH_COMMAND}" for loc in "${locs[@]}" ; do - v ${FETCH_COMMAND} "${loc}" < /dev/null + v "${fetchcmd[@]}" "${loc}" < /dev/null [[ -f ${1##*/} ]] && break done if [[ ! -f ${1##*/} ]] ; then eerror "downloading ${1} failed!" return 1 fi - popd > /dev/null + popd > /dev/null || exit 1 fi return 0 } configure_cflags() { export CPPFLAGS="-I${ROOT}/tmp/usr/include" + # keep it fairly reasonable (no -march or whatever) + export OVERRIDE_CFLAGS="-O2 -pipe" + export OVERRIDE_CXXFLAGS="-O2 -pipe" case ${CHOST} in *-darwin*) @@ -151,35 +162,23 @@ configure_cflags() { configure_toolchain() { linker="sys-devel/binutils" - local gcc_deps="dev-libs/gmp dev-libs/mpfr dev-libs/mpc" + local gcc_deps="dev-libs/gmp dev-libs/mpfr dev-libs/mpc dev-libs/libffi" compiler="${gcc_deps} sys-devel/gcc-config sys-devel/gcc" compiler_stage1="${gcc_deps} sys-devel/gcc-config" compiler_type="gcc" - case ${CHOST} in - *-darwin*) - # handled below - ;; - *-freebsd* | *-openbsd*) - # comes with clang, handled below - ;; - *) - # The host may not have a functioning c++ toolchain, so use a - # stage1 compiler that can build with C only. - # But gcc-4.7 fails to build with gcc-5.4, so we check for - # >gcc-4.7, as anything newer provides c++ anyway (#619542). - # gcc-4.7 is the last version not to require a c++ compiler to - # build - eval $( (gcc -E - | grep compiler_stage1) <<-EOP - #if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 7)) - compiler_stage1+=" sys-devel/gcc" - #elif defined(__GNUC__) && __GNUC__ >= 4 - compiler_stage1+=" "${MAKE_CONF_DIR}/0100_bootstrap_prefix_make.conf" fi if is-rap ; then if [[ ! -f ${ROOT}/etc/passwd ]]; then - if grep -q $(id -un) /etc/passwd; then + if grep -q "^$(id -un):" /etc/passwd; then ln -sf {,"${ROOT}"}/etc/passwd else getent passwd > "${ROOT}"/etc/passwd # add user if it's not in /etc/passwd, bug #766417 - getent passwd $(id -un) >> "${ROOT}"/etc/passwd + getent passwd "$(id -un)" >> "${ROOT}"/etc/passwd fi fi if [[ ! -f ${ROOT}/etc/group ]]; then - if grep -q $(id -gn) /etc/group; then + if grep -q "^$(id -gn):" /etc/group; then ln -sf {,"${ROOT}"}/etc/group else getent group > "${ROOT}"/etc/group # add group if it's not in /etc/group, bug #766417 - getent group $(id -gn) >> "${ROOT}"/etc/group + getent group "$(id -gn)" >> "${ROOT}"/etc/group fi fi [[ -f ${ROOT}/etc/resolv.conf ]] || ln -s {,"${ROOT}"}/etc/resolv.conf @@ -360,10 +351,11 @@ bootstrap_setup() { } bootstrap_profile() { - local profile="" + local profile + local profile_linux # 2.6.32.1 -> 2*256^3 + 6*256^2 + 32 * 256 + 1 = 33955841 - kver() { uname -r|cut -d\- -f1|awk -F. '{for (i=1; i<=NF; i++){s+=lshift($i,(4-i)*8)};print s}'; } + kver() { uname -r|cut -d- -f1|awk -F. '{for (i=1; i<=NF; i++){s+=lshift($i,(4-i)*8)};print s}'; } # >=glibc-2.20 requires >=linux-2.6.32. profile-kernel() { if [[ $(kver) -ge 50462720 ]] ; then # 3.2 @@ -378,9 +370,9 @@ bootstrap_profile() { } if is-rap ; then - local profile_linux=default/linux/ARCH/17.0/prefix/$(profile-kernel) + profile_linux="default/linux/ARCH/17.0/prefix/$(profile-kernel)" else - local profile_linux=prefix/linux/ARCH + profile_linux="prefix/linux/ARCH" fi case ${CHOST} in @@ -388,18 +380,24 @@ bootstrap_profile() { rev=${CHOST##*darwin} profile="prefix/darwin/macos/10.$((rev - 4))/ppc" ;; - i*86-apple-darwin1[578]) + i*86-apple-darwin9) rev=${CHOST##*darwin} profile="prefix/darwin/macos/10.$((rev - 4))/x86" ;; + i*86-apple-darwin1[578]) + eerror "REMOVED ARCH: this 32-bit MacOS architecture was removed," + eerror "bootstrapping is impossible" + exit 1 + ;; x86_64-apple-darwin1[5789]) rev=${CHOST##*darwin} profile="prefix/darwin/macos/10.$((rev - 4))/x64" ;; *64-apple-darwin2[0123456789]) - # Big Sur is 11.0 - # Monterey is 12.0 - # Ventura is 13.0 + # Big Sur is 11.0 darwin20 + # Monterey is 12.0 darwin21 + # Ventura is 13.0 darwin22 + # Sanoma is 14.0 darwin23 rev=${CHOST##*darwin} case ${CHOST%%-*} in x86_64) arch=x64 ;; @@ -411,7 +409,7 @@ bootstrap_profile() { i*86-pc-linux-gnu) profile=${profile_linux/ARCH/x86} ;; - riscv64-pc-linux-gnu) + riscv64-*-linux-gnu) profile=${profile_linux/ARCH/riscv} profile=${profile/17.0/20.0/rv64gc/lp64d} ;; @@ -435,8 +433,9 @@ bootstrap_profile() { aarch64-unknown-linux-gnu) profile=${profile_linux/ARCH/arm64} ;; - armv7l-pc-linux-gnu) + armv7*-unknown-linux-gnueabi*) profile=${profile_linux/ARCH/arm} + profile=${profile/17.0/17.0/armv7a} ;; x86_64-pc-solaris2.11) profile="prefix/sunos/solaris/5.11/x64" @@ -458,23 +457,11 @@ bootstrap_profile() { ;; esac - if [[ ${CHOST} == *-darwin* ]] ; then - # setup MacOSX.sdk symlink for GCC, this should probably be - # managed using an eselect module in the future - rm -f "${ROOT}"/MacOSX.sdk - local SDKPATH=$(xcrun --show-sdk-path --sdk macosx) - if [[ ! -e ${SDKPATH} ]] ; then - SDKPATH=$(xcodebuild -showsdks | sort -nr \ - | grep -o "macosx.*" | head -n1) - SDKPATH=$(xcode-select -print-path)/SDKs/MacOSX${SDKPATH#macosx}.sdk - fi - ( cd "${ROOT}" && ln -s "${SDKPATH}" MacOSX.sdk ) - einfo "using system sources from ${SDKPATH}" - fi - if [[ ${DARWIN_USE_GCC} == 1 ]] ; then # amend profile, to use gcc one profile="${profile}/gcc" + elif [[ ${CHOST} == *-darwin* ]] ; then + [[ "${BOOTSTRAP_STAGE}" != stage2 ]] && profile+="/clang" fi [[ -n ${PROFILE_BASE}${PROFILE_VARIANT} ]] && @@ -492,10 +479,6 @@ bootstrap_profile() { # Disable bootstrapping libcxx* with libunwind sys-libs/libcxxabi -libunwind sys-libs/libcxx -libunwind - # Most binary Linux distributions seem to fancy toolchains that - # do not do c++ support (need to install a separate package). - sys-libs/ncurses -cxx - sys-devel/binutils -cxx EOF # On Darwin we might need this to bootstrap the compiler, since @@ -505,19 +488,6 @@ bootstrap_profile() { # For Darwin bootstraps sys-devel/native-cctools EOF - - # Strange enough, -cxx causes wrong libtool config on Cygwin, - # but we require a C++ compiler there anyway - so just use it. - cat >> "${ROOT}"/etc/portage/make.profile/package.use <<-EOF - # gmp has cxx flag enabled by default. When dealing with a host - # compiler without cxx support this causes configure failure. - # In addition, The stage2 g++ is only for compiling stage3 compiler, - # because the host libstdc++.so runtime may be not compatible and - # stage2 libstdc++.so might conflict with that of stage3. The - # trade-off is just not to use cxx. - dev-libs/gmp -cxx - sys-devel/binutils -gold - EOF } do_tree() { @@ -546,29 +516,35 @@ do_tree() { # latest tree cannot be fetched from mirrors, always have to # respect the source to get the latest if [[ -n ${LATEST_TREE_YES} ]] ; then - echo "$1" - ( export GENTOO_MIRRORS= DISTFILES_G_O= DISTFILES_PFX= ; + ( export GENTOO_MIRRORS='' DISTFILES_G_O='' DISTFILES_PFX='' ; efetch "$1/$2" ) || return 1 else - efetch "$1/$2" || return 1 + # use only Prefix mirror + ( export GENTOO_MIRRORS='' DISTFILES_G_O='' ; + efetch "$1/$2" ) || return 1 fi einfo "Unpacking, this may take a while" estatus "stage1: unpacking Portage tree" - bzip2 -dc ${DISTDIR}/$2 | tar -xf - -C ${PORTDIR} --strip-components=1 + bzip2 -dc "${DISTDIR}/$2" \ + | tar -xf - -C "${PORTDIR}" --strip-components=1 [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 - touch ${PORTDIR}/.unpacked + touch "${PORTDIR}"/.unpacked fi } bootstrap_tree() { + # retain this comment and the line below to + # keep this snapshot around in the snapshots + # MKSNAPSHOT-ANCHOR -- directory of rsync slaves + local PV="20240930" + # RAP uses the latest gentoo main repo snapshot to bootstrap. is-rap && LATEST_TREE_YES=1 - local PV="20230601" - if [[ -n ${LATEST_TREE_YES} ]]; then - do_tree "${SNAPSHOT_URL}" portage-latest.tar.bz2 - else - do_tree http://dev.gentoo.org/~grobian/distfiles prefix-overlay-${PV}.tar.bz2 - fi + + [[ -n ${LATEST_TREE_YES} ]] && PV=latest + + do_tree "${SNAPSHOT_URL}" portage-${PV}.tar.bz2 + local ret=$? if [[ -n ${TREE_FROM_SRC} ]]; then estatus "stage1: rsyncing Portage tree" @@ -593,7 +569,7 @@ bootstrap_startscript() { fi einfo "Trying to emerge the shell you use, if necessary by running:" einfo "emerge -u ${theshell}" - if ! emerge -u ${theshell} ; then + if ! emerge -u "${theshell}" ; then eerror "Your shell is not available in portage, hence we cannot" > /dev/stderr eerror "automate starting your prefix, set SHELL and rerun this script" > /dev/stderr return 1 @@ -603,8 +579,10 @@ bootstrap_startscript() { einfo "To start Gentoo Prefix, run the script ${ROOT}/startprefix" # see if PATH is kept/respected - local minPATH="preamble:${BASH%/*}:postlude" - local theirPATH="$(echo 'echo "${PATH}"' | env LS_COLORS= PATH="${minPATH}" $SHELL -l 2>/dev/null | grep "preamble:.*:postlude")" + local minPATH + local theirPATH + minPATH="preamble:${BASH%/*}:postlude" + theirPATH="$(echo 'echo "${PATH}"' | env LS_COLORS= PATH="${minPATH}" "${SHELL}" -l 2>/dev/null | grep "preamble:.*:postlude")" if [[ ${theirPATH} != *"preamble:"*":postlude"* ]] ; then einfo "WARNING: your shell initialisation (.cshrc, .bashrc, .profile)" einfo " seems to overwrite your PATH, this effectively kills" @@ -630,24 +608,24 @@ bootstrap_portage() { # STABLE_PV that is known to work. Intended for power users only. ## It is critical that STABLE_PV is the lastest (non-masked) version that is ## included in the snapshot for bootstrap_tree. - STABLE_PV="3.0.30.1" - [[ ${TESTING_PV} == latest ]] && TESTING_PV="3.0.30.1" + STABLE_PV="3.0.56.1" + [[ ${TESTING_PV} == latest ]] && TESTING_PV="3.0.56.1" PV="${TESTING_PV:-${STABLE_PV}}" A=prefix-portage-${PV}.tar.bz2 einfo "Bootstrapping ${A%.tar.*}" - efetch ${DISTFILES_URL}/${A} || return 1 + efetch "${DISTFILES_URL}/${A}" || return 1 einfo "Unpacking ${A%.tar.*}" export S="${PORTAGE_TMPDIR}"/portage-${PV} ptmp=${S} rm -rf "${S}" >& /dev/null mkdir -p "${S}" >& /dev/null - cd "${S}" + cd "${S}" || return 1 bzip2 -dc "${DISTDIR}/${A}" | tar -xf - [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}/prefix-portage-${PV}" - cd "${S}" + cd "${S}" || return 1 fix_config_sub @@ -673,8 +651,8 @@ bootstrap_portage() { einfo "Compiling ${A%.tar.*}" econf \ --with-offset-prefix="${ROOT}"/tmp \ - --with-portage-user="`id -un`" \ - --with-portage-group="`id -gn`" \ + --with-portage-user="$(id -un)" \ + --with-portage-group="$(id -gn)" \ --with-extra-path="${PATH}" \ || return 1 emake || return 1 @@ -682,8 +660,8 @@ bootstrap_portage() { einfo "Installing ${A%.tar.*}" emake install || return 1 - cd "${ROOT}" - rm -Rf ${ptmp} >& /dev/null + cd "${ROOT}" || return 1 + rm -Rf "${ptmp}" >& /dev/null # Some people will skip the tree() step and hence var/log is not created # As such, portage complains.. @@ -692,11 +670,11 @@ bootstrap_portage() { # in Prefix the sed wrapper is deadly, so kill it rm -f "${ROOT}"/tmp/usr/lib/portage/bin/ebuild-helpers/sed - local tmpportdir=${ROOT}/tmp/${PORTDIR#${ROOT}} + local tmpportdir=${ROOT}/tmp/${PORTDIR#"${ROOT}"} [[ -e "${tmpportdir}" ]] || ln -s "${PORTDIR}" "${tmpportdir}" for d in "${ROOT}"/tmp/usr/lib/python$(python_ver); do - [[ -e ${d}/portage ]] || ln -s "${ROOT}"/tmp/usr/lib/portage/lib/portage ${d}/portage - [[ -e ${d}/_emerge ]] || ln -s "${ROOT}"/tmp/usr/lib/portage/lib/_emerge ${d}/_emerge + [[ -e ${d}/portage ]] || ln -s "${ROOT}"/tmp/usr/lib/portage/lib/portage "${d}"/portage + [[ -e ${d}/_emerge ]] || ln -s "${ROOT}"/tmp/usr/lib/portage/lib/_emerge "${d}"/_emerge done if [[ -s ${PORTDIR}/profiles/repo_name ]]; then @@ -714,6 +692,7 @@ fix_config_sub() { # for various versions of autoconf if [[ ${CHOST} == arm64-apple-darwin* ]] ; then # Apple Silicon doesn't use aarch64, but arm64 + # note: macOS /usr/bin/find knows no -print0 or -exec find . -name "config.sub" | \ xargs sed -i -e 's/ arm\(-\*\)* / arm\1 | arm64\1 /' find . -name "config.sub" | \ @@ -728,41 +707,46 @@ bootstrap_simple() { A=${PN}-${PV}.tar.${3:-gz} einfo "Bootstrapping ${A%.tar.*}" - efetch ${4:-${DISTFILES_G_O}/distfiles}/${A} || return 1 + efetch "${4:-${DISTFILES_G_O}/distfiles}/${A}" || return 1 einfo "Unpacking ${A%.tar.*}" S="${PORTAGE_TMPDIR}/${PN}-${PV}" rm -rf "${S}" - mkdir -p "${S}" - cd "${S}" + mkdir -p "${S}" || return 1 + cd "${S}" || return 1 case $3 in + zstd) decomp=zstd ;; xz) decomp=xz ;; bz2) decomp=bzip2 ;; gz|"") decomp=gzip ;; esac - ${decomp} -dc "${DISTDIR}"/${A} | tar -xf - + ${decomp} -dc "${DISTDIR}/${A}" | tar -xf - [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}"/${PN}-${PV} - cd "${S}" + cd "${S}" || return 1 fix_config_sub # for libressl, only provide static lib, such that wget (above) # links it in and we don't have to bother about RPATH or something if [[ ${PN} == "libressl" ]] ; then - myconf="${myconf} --enable-static --disable-shared" + myconf=( + "${myconf[@]}" + --enable-static + --disable-shared + ) fi einfo "Compiling ${A%.tar.*}" if [[ -x configure ]] ; then - econf ${myconf} || return 1 + econf "${myconf[@]}" || return 1 fi emake || return 1 einfo "Installing ${A%.tar.*}" emake PREFIX="${ROOT}"/tmp/usr install || return 1 - cd "${ROOT}" + cd "${ROOT}" || return 1 rm -Rf "${S}" einfo "${A%.tar.*} successfully bootstrapped" } @@ -774,6 +758,7 @@ bootstrap_gnu() { einfo "Bootstrapping ${A%.tar.*}" + # GNU does not use zstd (yet?) for t in tar.xz tar.bz2 tar.gz tar ; do A=${PN}-${PV}.${t} @@ -789,19 +774,19 @@ bootstrap_gnu() { fi URL=${GNU_URL}/${PN}/${A} - efetch ${URL} || continue + efetch "${URL}" || continue einfo "Unpacking ${A%.tar.*}" S="${PORTAGE_TMPDIR}/${PN}-${PV}" rm -rf "${S}" - mkdir -p "${S}" - cd "${S}" + mkdir -p "${S}" || return 1 + cd "${S}" || return 1 case ${t} in tar.xz) decomp=xz ;; tar.bz2) decomp=bzip2 ;; tar.gz) decomp=gzip ;; tar) - tar -xf "${DISTDIR}"/${A} || continue + tar -xf "${DISTDIR}/${A}" || continue break ;; *) @@ -809,7 +794,7 @@ bootstrap_gnu() { return 1 ;; esac - ${decomp} -dc "${DISTDIR}"/${URL##*/} | tar -xf - + ${decomp} -dc "${DISTDIR}/${URL##*/}" | tar -xf - [[ ${PIPESTATUS[*]} == '0 0' ]] || continue break done @@ -828,13 +813,29 @@ bootstrap_gnu() { # have a suffix. Remove suffix by copy, not move, to not # trigger refetch on repeated invocations of this script. if [[ -f "${DISTDIR}/${tar_patch_file}?${tar_patch_id}" ]]; then - cp ${DISTDIR}/${tar_patch_file}{?${tar_patch_id},} || return 1 + cp "${DISTDIR}/${tar_patch_file}"{"?${tar_patch_id}",} || return 1 fi - patch -p1 < ${DISTDIR}/${tar_patch_file} || return 1 + patch -p1 < "${DISTDIR}/${tar_patch_file}" || return 1 fi - local myconf="" - if [[ ${PN} == "make" && ${PV} == "4.2.1" ]] ; then + # gcc14 fails to build bash if host lacks /usr/include/termcap.h + # fixed upstream in devel branch + if [[ ${PN}-${PV} == "bash-5.2" ]] ; then + local bash_patch_file="tparam.c" + local bash_patch_id="id=5b239ebbd2b1251c03b8e5591fe797a791266799" + local bash_patch_url="https://git.savannah.gnu.org/cgit/bash.git/patch/lib/termcap/${bash_patch_file}?${bash_patch_id}" + efetch "${bash_patch_url}" || return 1 + # If fetched from upstream url instead of mirror, filename will + # have a suffix. Remove suffix by copy, not move, to not + # trigger refetch on repeated invocations of this script. + if [[ -f "${DISTDIR}/${bash_patch_file}?${bash_patch_id}" ]]; then + cp "${DISTDIR}/${bash_patch_file}"{"?${bash_patch_id}",} || return 1 + fi + patch -p1 < "${DISTDIR}/${bash_patch_file}" || return 1 + fi + + local -a myconf + if [[ ${PN}-${PV} == "make-4.2.1" ]] ; then if [[ ${CHOST} == *-linux-gnu* ]] ; then # force this, macros aren't set correctly with newer glibc export CPPFLAGS="${CPPFLAGS} -D__alloca=alloca -D__stat=stat" @@ -865,21 +866,47 @@ bootstrap_gnu() { # so just don't set it at all. # Solaris 11 has a messed up prce installation. We don't need # it anyway, so just disable it - myconf="${myconf} --disable-perl-regexp" + myconf+=( "--disable-perl-regexp" ) + fi + + if [[ ${PN} == "mpfr" || ${PN} == "mpc" || ${PN} == "gcc" ]] ; then + [[ -e "${ROOT}"/tmp/usr/include/gmp.h ]] \ + && myconf+=( "--with-gmp=${ROOT}/tmp/usr" ) + fi + if [[ ${PN} == "mpc" || ${PN} == "gcc" ]] ; then + [[ -e "${ROOT}"/tmp/usr/include/mpfr.h ]] \ + && myconf+=( "--with-mpfr=${ROOT}/tmp/usr" ) + fi + if [[ ${PN} == "gcc" ]] ; then + [[ -e "${ROOT}"/tmp/usr/include/mpc.h ]] \ + && myconf+=( "--with-mpc=${ROOT}/tmp/usr" ) + + myconf+=( + "--enable-languages=c,c++" + "--disable-bootstrap" + "--disable-multilib" + "--disable-nls" + "--disable-libsanitizer" + ) + + if [[ ${CHOST} == *-darwin* ]] ; then + myconf+=( + "--with-native-system-header-dir=${ROOT}/MacOSX.sdk/usr/include" + "--with-ld=${ROOT}/tmp/usr/bin/ldwrapper" + ) + fi + + export CFLAGS="-O1 -pipe" + export CXXFLAGS="-O1 -pipe" fi # pod2man may be too old (not understanding --utf8) but we don't # care about manpages at this stage export ac_cv_path_POD2MAN=no - # Darwin9 in particular doesn't compile when using system readline, - # but we don't need any groovy input handling at all, so just disable it - [[ ${PN} == "bash" ]] && myconf="${myconf} --disable-readline" - # On e.g. musl systems bash will crash with a malloc error if we use # bash' internal malloc, so disable it during it this stage - [[ ${PN} == "bash" ]] && \ - myconf="${myconf} --without-bash-malloc" + [[ ${PN} == "bash" ]] && myconf+=( "--without-bash-malloc" ) # Ensure we don't read system-wide shell initialisation, it may # contain cruft, bug #650284 @@ -896,30 +923,35 @@ bootstrap_gnu() { # stdbuf is giving many problems, and we don't really care about it # at this level, so disable it too if [[ ${PN} == "coreutils" ]] ; then - myconf="${myconf} --disable-acl --without-gmp" - myconf="${myconf} --enable-no-install-program=stdbuf" + myconf+=( + "--disable-acl" + "--without-gmp" + "--enable-no-install-program=stdbuf" + ) fi # Gentoo Bug 400831, fails on Ubuntu with libssl-dev installed if [[ ${PN} == "wget" ]] ; then if [[ -x ${ROOT}/tmp/usr/bin/openssl ]] ; then - myconf="${myconf} --with-ssl=openssl" - myconf="${myconf} --with-libssl-prefix=${ROOT}/tmp/usr" + myconf+=( + "-with-ssl=openssl" + "--with-libssl-prefix=${ROOT}/tmp/usr" + ) export CPPFLAGS="${CPPFLAGS} -I${ROOT}/tmp/usr/include" export LDFLAGS="${LDFLAGS} -L${ROOT}/tmp/usr/lib" else - myconf="${myconf} --without-ssl" + myconf+=( "--without-ssl" ) fi fi # SuSE 11.1 has GNU binutils-2.20, choking on crc32_x86 - [[ ${PN} == "xz" ]] && myconf="${myconf} --disable-assembler" + [[ ${PN} == "xz" ]] && myconf+=( "--disable-assembler" ) if [[ ${PN} == "libffi" ]] ; then # we do not have pkg-config to find lib/libffi-*/include/ffi.h sed -i -e '/includesdir =/s/=.*/= $(includedir)/' include/Makefile.in # force install into libdir - myconf="${myconf} --libdir=${ROOT}/tmp/usr/lib" + myconf+=( "--libdir=${ROOT}/tmp/usr/lib" ) sed -i -e '/toolexeclibdir =/s/=.*/= $(libdir)/' Makefile.in # we have to build the libraries for correct bitwidth case $CHOST in @@ -937,7 +969,7 @@ bootstrap_gnu() { fi einfo "Compiling ${A%.tar.*}" - econf ${myconf} || return 1 + econf "${myconf[@]}" || return 1 if [[ ${PN} == "make" && $(type -t $MAKE) != "file" ]]; then estatus "stage1: building ${A%.tar.*}" v ./build.sh || return 1 @@ -953,7 +985,7 @@ bootstrap_gnu() { emake install || return 1 fi - cd "${ROOT}" + cd "${ROOT}" || return 1 rm -Rf "${S}" einfo "${A%.tar.*} successfully bootstrapped" } @@ -964,11 +996,14 @@ python_ver() { # snapshot for stage3, else packages will break with some python # mismatch error due to Portage using a different version after it # upgraded itself with a newer Python - echo 3.11 # keep this number in line with PV below for stage1,2 + echo 3.11 + export PYTHON_FULL_VERSION="3.11.7-gentoo-prefix-patched" + # keep this number in line with PV below for stage1,2 } bootstrap_python() { - PV=$(python_ver).3-gentoo-prefix-patched + python_ver # to get full version + PV=${PYTHON_FULL_VERSION} A=Python-${PV}.tar.xz einfo "Bootstrapping ${A%.tar.*}" @@ -981,8 +1016,8 @@ bootstrap_python() { einfo "Unpacking ${A%.tar.*}" export S="${PORTAGE_TMPDIR}/python-${PV}" rm -rf "${S}" - mkdir -p "${S}" - cd "${S}" + mkdir -p "${S}" || return 1 + cd "${S}" || return 1 case ${A} in *bz2) bzip2 -dc "${DISTDIR}"/${A} | tar -xf - ;; *xz) xz -dc "${DISTDIR}"/${A} | tar -xf - ;; @@ -990,7 +1025,7 @@ bootstrap_python() { esac [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}"/Python-${PV%%-*} - cd "${S}" + cd "${S}" || return 1 rm -rf Modules/_ctypes/libffi* || return 1 rm -rf Modules/zlib || return 1 @@ -1015,11 +1050,11 @@ bootstrap_python() { # make Python's selectors resort to poll() or select() for the # time being sed -i \ - -e 's/KQUEUE/KQUEUE_DISABLED/' \ + -e 's/kqueue/kqueue_DISABLED/' \ configure # fixup thread id detection (only needed on vanilla Python tar) - #efetch "https://dev.gentoo.org/~sam/distfiles/dev-lang/python/python-3.9.6-darwin9_pthreadid.patch" - #patch -p1 < "${DISTDIR}"/python-3.9.6-darwin9_pthreadid.patch + efetch "https://dev.gentoo.org/~sam/distfiles/dev-lang/python/python-3.9.6-darwin9_pthreadid.patch" + patch -p1 < "${DISTDIR}"/python-3.9.6-darwin9_pthreadid.patch ;; (*-openbsd*) # OpenBSD is not a multilib system @@ -1040,8 +1075,6 @@ bootstrap_python() { fix_config_sub - local myconf="" - case ${CHOST} in (x86_64-*-*|sparcv9-*-*) export CFLAGS="-m64" @@ -1055,7 +1088,9 @@ bootstrap_python() { *-linux*) # Bug 382263: make sure Python will know about the libdir in use for # the current arch - libdir="-L/usr/lib/$(gcc ${CFLAGS} -print-multi-os-directory)" + local -a flgarg + read -r -a flgarg <<< "${CFLAGS}" + libdir="-L/usr/lib/$(gcc "${flgarg[@]}" -print-multi-os-directory)" ;; x86_64-*-solaris*|sparcv9-*-solaris*) # Like above, make Python know where GCC's 64-bits @@ -1102,21 +1137,21 @@ bootstrap_python() { # configure to not find them using HAS_HG (TODO: obsolete?) # - Do not find libffi via pkg-config using PKG_CONFIG HAS_HG=no \ - PKG_CONFIG= \ + PKG_CONFIG='' \ econf \ --with-system-ffi \ --without-ensurepip \ --disable-ipv6 \ --disable-shared \ --libdir="${ROOT}"/tmp/usr/lib \ - ${myconf} || return 1 + || return 1 emake || return 1 einfo "Installing ${A%.tar.*}" emake -k install || echo "??? Python failed to install *sigh* continuing anyway" - cd "${ROOT}"/tmp/usr/bin + cd "${ROOT}"/tmp/usr/bin || return 1 ln -sf python${PV%.*} python - cd "${ROOT}"/tmp/usr/lib + cd "${ROOT}"/tmp/usr/lib || return 1 # messes up python emerges, and shouldn't be necessary for anything # http://forums.gentoo.org/viewtopic-p-6890526.html rm -f libpython${PV%.*}.a @@ -1130,18 +1165,18 @@ bootstrap_cmake_core() { einfo "Bootstrapping ${A%.tar.*}" - efetch https://github.com/Kitware/CMake/releases/download/v${PV}/${A} \ + efetch "https://github.com/Kitware/CMake/releases/download/v${PV}/${A}" \ || return 1 einfo "Unpacking ${A%.tar.*}" export S="${PORTAGE_TMPDIR}/cmake-${PV}" rm -rf "${S}" - mkdir -p "${S}" - cd "${S}" - gzip -dc "${DISTDIR}"/${A} | tar -xf - + mkdir -p "${S}" || return 1 + cd "${S}" || return 1 + gzip -dc "${DISTDIR}/${A}" | tar -xf - [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}"/cmake-${PV} - cd "${S}" + cd "${S}" || return 1 # don't set a POSIX standard, system headers don't like that, #757426 sed -i -e 's/^#if !defined(_WIN32) && !defined(__sun)/& \&\& !defined(__APPLE__)/' \ @@ -1162,18 +1197,19 @@ bootstrap_cmake_core() { # we need sysroot crap to build cmake itself, but it makes trouble # later on, so kill it in the installed version - ver=${A%-*} ; ver=${ver%.*} sed -i -e '/cmake_gnu_set_sysroot_flag/d' \ - "${ROOT}"/tmp/usr/share/${ver}/Modules/Platform/Apple-GNU-*.cmake || die + "${ROOT}"/tmp/usr/share/cmake*/Modules/Platform/Apple-GNU-*.cmake || die # disable isysroot usage with clang as well sed -i -e '/_SYSROOT_FLAG/d' \ - "${ROOT}"/tmp/usr/share/${ver}/Modules/Platform/Apple-Clang.cmake || die + "${ROOT}"/tmp/usr/share/cmake*/Modules/Platform/Apple-Clang.cmake || die einfo "${A%.tar.*} bootstrapped" } bootstrap_cmake() { - bootstrap_cmake_core 3.16.5 || bootstrap_cmake_core 3.0.2 + bootstrap_cmake_core 3.20.6 || \ + bootstrap_cmake_core 3.16.5 || \ + bootstrap_cmake_core 3.0.2 } bootstrap_zlib_core() { @@ -1183,21 +1219,21 @@ bootstrap_zlib_core() { einfo "Bootstrapping ${A%.tar.*}" - efetch ${DISTFILES_G_O}/distfiles/${A} || return 1 + efetch "${DISTFILES_G_O}/distfiles/${A}" || return 1 einfo "Unpacking ${A%.tar.*}" export S="${PORTAGE_TMPDIR}/zlib-${PV}" rm -rf "${S}" - mkdir -p "${S}" - cd "${S}" + mkdir -p "${S}" || return 1 + cd "${S}" || return 1 case ${A} in *.tar.gz) decomp=gzip ;; *) decomp=bzip2 ;; esac - ${decomp} -dc "${DISTDIR}"/${A} | tar -xf - + ${decomp} -dc "${DISTDIR}/${A}" | tar -xf - [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}"/zlib-${PV} - cd "${S}" + cd "${S}" || return 1 if [[ ${CHOST} == x86_64-*-* || ${CHOST} == sparcv9-*-* ]] ; then # 64-bits targets need zlib as library (not just to unpack), @@ -1212,13 +1248,13 @@ bootstrap_zlib_core() { # compiler to 32-bits code generation if requested here export CC="${CC} -m32" fi - local makeopts=( ${MAKEOPTS} ) + local makeopts=() # 1.2.5 suffers from a concurrency problem - [[ ${PV} == 1.2.5 ]] && makeopts=() + [[ ${PV} == 1.2.5 ]] || read -r -a makeopts <<< "${MAKEOPTS}" einfo "Compiling ${A%.tar.*}" - CHOST= ${CONFIG_SHELL} ./configure --prefix="${ROOT}"/tmp/usr || return 1 - MAKEOPTS= + CHOST='' ${CONFIG_SHELL} ./configure --prefix="${ROOT}"/tmp/usr || return 1 + MAKEOPTS='' emake "${makeopts[@]}" || return 1 einfo "Installing ${A%.tar.*}" @@ -1242,16 +1278,64 @@ bootstrap_zlib() { } bootstrap_libffi() { + # 3.0.8: last version to bootstrap on Darwin 9 x86 + bootstrap_gnu libffi 3.4.5 || \ bootstrap_gnu libffi 3.3 || \ - bootstrap_gnu libffi 3.2.1 + bootstrap_gnu libffi 3.2.1 || \ + bootstrap_gnu libffi 3.0.8 +} + +bootstrap_gmp() { + bootstrap_gnu gmp 6.2.1 +} + +bootstrap_mpfr() { + bootstrap_gnu mpfr 4.1.0 +} + +bootstrap_mpc() { + bootstrap_gnu mpc 1.2.1 +} + +bootstrap_ldwrapper() { + A=ldwrapper.c + + einfo "Bootstrapping ${A%.c}" + + efetch "https://rsync.prefix.bitzolder.nl/sys-devel/binutils-config/files/${A}" || return 1 + + export S="${PORTAGE_TMPDIR}/ldwrapper" + rm -rf "${S}" + mkdir -p "${S}" || return 1 + cd "${S}" || return 1 + cp "${DISTDIR}/${A}" . || return 1 + + einfo "Compiling ${A%.c}" + ${CC:-gcc} \ + -o ldwrapper \ + -DCHOST="\"${CHOST}\"" \ + -DEPREFIX="\"${ROOT}\"" \ + ldwrapper.c || return 1 + + einfo "Installing ${A%.c}" + mkdir -p "${ROOT}"/tmp/usr/bin + cp -a ldwrapper "${ROOT}"/tmp/usr/bin/ || return 1 + + einfo "${A%.c} bootstrapped" +} + +bootstrap_gcc5() { + # bootstraps with gcc-4.0.1 (Darwin 8), provides C11 + bootstrap_gnu gcc 5.5.0 } bootstrap_sed() { - bootstrap_gnu sed 4.5 || \ + bootstrap_gnu sed 4.9 || bootstrap_gnu sed 4.5 || \ bootstrap_gnu sed 4.2.2 || bootstrap_gnu sed 4.2.1 } bootstrap_findutils() { + bootstrap_gnu findutils 4.9.0 || bootstrap_gnu findutils 4.7.0 || bootstrap_gnu findutils 4.5.10 || bootstrap_gnu findutils 4.2.33 @@ -1275,6 +1359,7 @@ bootstrap_coreutils() { # 8.16 is the last version released as tar.gz # 8.18 is necessary for macOS High Sierra (darwin17) and converted # to tar.gz for this case + bootstrap_gnu coreutils 9.5 || \ bootstrap_gnu coreutils 8.32 || bootstrap_gnu coreutils 8.30 || \ bootstrap_gnu coreutils 8.16 || bootstrap_gnu coreutils 8.17 } @@ -1288,7 +1373,7 @@ bootstrap_make() { bootstrap_gnu make 4.2.1 || return 1 if [[ ${MAKE} == gmake ]] ; then # make make available as gmake - ( cd ${ROOT}/tmp/usr/bin && ln -s make gmake ) + ( cd "${ROOT}"/tmp/usr/bin && ln -s make gmake ) fi } @@ -1314,9 +1399,9 @@ bootstrap_texinfo() { } bootstrap_bash() { + bootstrap_gnu bash 5.2 || bootstrap_gnu bash 5.1 || - bootstrap_gnu bash 4.3 || - bootstrap_gnu bash 4.2 + bootstrap_gnu bash 5.0 } bootstrap_bison() { @@ -1336,6 +1421,7 @@ bootstrap_gzip() { } bootstrap_xz() { + GNU_URL=http://tukaani.org/xz bootstrap_gnu xz 5.4.5 || \ GNU_URL=http://tukaani.org/xz bootstrap_gnu xz 5.2.4 || \ GNU_URL=http://tukaani.org/xz bootstrap_gnu xz 5.2.3 } @@ -1395,15 +1481,132 @@ bootstrap_stage1() { # See comments in do_tree(). local portroot=${PORTDIR%/*} - mkdir -p "${ROOT}"/tmp/${portroot#${ROOT}/} + mkdir -p "${ROOT}/tmp/${portroot#"${ROOT}"/}" for x in lib sbin bin; do mkdir -p "${ROOT}"/tmp/usr/${x} [[ -e ${ROOT}/tmp/${x} ]] || ( cd "${ROOT}"/tmp && ln -s usr/${x} ) done - configure_toolchain + BOOTSTRAP_STAGE="stage1" configure_toolchain || return 1 export CC CXX + # default: empty = NO + local USEGCC5= + + if [[ ${CHOST} == *-darwin* ]] ; then + # setup MacOSX.sdk symlink for GCC, this should probably be + # managed using an eselect module in the future + # FWIW, just use system (/) if it seems OK, for some reason + # early versions of TAPI-based SDKs did not include some symbols + # like fclose, which ld64 is able to resolve from the dylibs + # although they are unvisible using e.g. nm. + rm -f "${ROOT}"/MacOSX.sdk + local SDKPATH + if [[ -e /usr/lib/libSystem.B.dylib && -d /usr/include ]] ; then + SDKPATH=/ + else + SDKPATH=$(xcrun --show-sdk-path --sdk macosx) + if [[ -e ${SDKPATH} ]] ; then + local fsdk + local osvers + # try and find a matching OS SDK, xcrun seems to return + # the latest installed, so not necessarily the one + # matching the macOS version + [[ -L ${SDKPATH} ]] && fsdk="$(readlink "${SDKPATH}")" + # note readlink -f is not supported on older versions of + # macOS so need to emulate it + if [[ ${fsdk} != /* ]] ; then + # this is not proper, but Apple does not use ../ + # constructs here, as far as we know + fsdk="${SDKPATH%/*}/${fsdk}" + fi + osvers="$(sw_vers -productVersion)" + if [[ ${osvers%%.*} -le 10 ]] ; then + osvers=$(echo "${osvers}" | cut -d'.' -f1-2) + else + osvers=${osvers%%.*} + fi + fsdk=${fsdk%/MacOSX*.sdk} + fsdk=${fsdk}/MacOSX${osvers}.sdk + [[ -e ${fsdk} ]] && SDKPATH=${fsdk} + fi + if [[ ! -e ${SDKPATH} ]] ; then + SDKPATH=$(xcodebuild -showsdks | sort -nr \ + | grep -o "macosx.*" | head -n1) + SDKPATH=$(xcode-select -print-path)/SDKs/MacOSX${SDKPATH#macosx}.sdk + fi + fi + ( cd "${ROOT}" && ln -s "${SDKPATH}" MacOSX.sdk ) + einfo "using system sources from ${SDKPATH}" + + # GCC 14 cannot be compiled by versions of Clang at least on + # Darwin17, so go the safe route and get GCC-5 which is sufficient + # and the last one we can compile without C11. This also compiles + # on Darwin 8 and 9. + # see also configure_toolchain + case ${CHOST} in + *-darwin2[23456789]) : ;; # host toolchain can compile gcc-14 + *-darwin[89]) USEGCC5=yes ;; + *86*-darwin*) USEGCC5=yes ;; + # arm64/M1 isn't supported by old GCC-5! + esac + fi + + if [[ -n ${USEGCC5} ]] ; then + # benefit from 4.2 if it's present + if [[ -e /usr/bin/gcc-4.2 ]] ; then + export CC=gcc-4.2 + export CXX=g++-4.2 + fi + + [[ -e ${ROOT}/tmp/usr/include/gmp.h ]] \ + || (bootstrap_gmp) || return 1 + [[ -e ${ROOT}/tmp/usr/include/mpfr.h ]] \ + || (bootstrap_mpfr) || return 1 + [[ -e ${ROOT}/tmp/usr/include/mpc.h ]] \ + || (bootstrap_mpc) || return 1 + [[ -x ${ROOT}/tmp/usr/bin/ldwrapper ]] \ + || (bootstrap_ldwrapper) || return 1 + # get ldwrapper target in PATH + export BINUTILS_CONFIG_LD="$(type -P ld)" + # force deployment target in GCCs build, GCC-5 doesn't quite get + # the newer macOS versions (20+) and thus confuses ld when it + # passes on the deployment version. Use High Sierra as it has + # everything we need + [[ ${CHOST##*darwin} -gt 10 ]] && export MACOSX_DEPLOYMENT_TARGET=10.13 + [[ -x ${ROOT}/tmp/usr/bin/gcc ]] \ + || (bootstrap_gcc5) || return 1 + + if [[ ${CHOST##*darwin} -gt 10 ]] ; then + # install wrappers in tmp/usr/local/bin which comes before + # /tmp/usr/bin in PATH + mkdir -p "${ROOT}"/tmp/usr/local/bin + rm -f "${ROOT}"/tmp/usr/local/bin/{gcc,${CHOST}-gcc} + cat > "${ROOT}/tmp/usr/local/bin/${CHOST}-gcc" <<-EOS + #!/usr/bin/env sh + export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} + export BINUTILS_CONFIG_LD="$(type -P ld)" + exec "${ROOT}"/tmp/usr/bin/${CHOST}-gcc "\$@" + EOS + chmod 755 "${ROOT}/tmp/usr/local/bin/${CHOST}-gcc" + ln -s ${CHOST}-gcc "${ROOT}"/tmp/usr/local/bin/gcc + + rm -f "${ROOT}"/tmp/usr/local/bin/{g++,${CHOST}-g++} + cat > "${ROOT}"/tmp/usr/local/bin/${CHOST}-g++ <<-EOS + #!/usr/bin/env sh + export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} + export BINUTILS_CONFIG_LD="$(type -P ld)" + exec "${ROOT}"/tmp/usr/bin/${CHOST}-g++ "\$@" + EOS + chmod 755 "${ROOT}"/tmp/usr/local/bin/${CHOST}-g++ + ln -s ${CHOST}-g++ "${ROOT}"/tmp/usr/local/bin/g++ + fi + + # reset after gcc-4.2 usage + export CC=gcc + export CXX=g++ + fi + # Run all bootstrap_* commands in a subshell since the targets # frequently pollute the environment using exports which affect # packages following (e.g. zlib builds 64-bits) @@ -1430,8 +1633,8 @@ bootstrap_stage1() { || [[ $(m4 --version 2>&1) == *GNU*1.4.1?* ]] \ || (bootstrap_m4) || return 1 [[ -x ${ROOT}/tmp/usr/bin/bison ]] \ - || [[ $(bison --version 2>&1) == *GNU" "Bison") "2.[3-7]* ]] \ - || [[ $(bison --version 2>&1) == *GNU" "Bison") "[3-9]* ]] \ + || [[ $(bison --version 2>&1) == *"GNU Bison) "2.[3-7]* ]] \ + || [[ $(bison --version 2>&1) == *"GNU Bison) "[3-9]* ]] \ || (bootstrap_bison) || return 1 if [[ ! -x ${ROOT}/tmp/usr/bin/uniq ]]; then # If the system has a uniq, let's use it to test whether @@ -1458,7 +1661,7 @@ bootstrap_stage1() { || (bootstrap_grep) || return 1 [[ -x ${ROOT}/tmp/usr/bin/gawk ]] \ || [[ $(awk --version < /dev/null 2>&1) == *GNU" Awk "[456789]* ]] \ - || bootstrap_gawk || return 1 + || (bootstrap_gawk) || return 1 # always build our own bash, for we don't know what devilish thing # we're working with now, bug #650284 [[ -x ${ROOT}/tmp/usr/bin/bash ]] \ @@ -1490,24 +1693,39 @@ bootstrap_stage1() { # If the version of our binutils an older one, they may not # provide what the system gcc is configured to use. # We need to direct the system gcc to find the system binutils. + EXEC="$(PATH="${ORIGINAL_PATH}" type -P gcc)" + if [[ -z ${EXEC} ]] ; then + eerror "could not find 'gcc' in your PATH!" + eerror "please install gcc or provide access via PATH or symlink" + return 1 + fi cat >> "${ROOT}"/tmp/usr/local/bin/gcc <<-EOF #! /bin/sh PATH="${ORIGINAL_PATH}" export PATH - exec "$(type -P gcc)" "\$@" + exec "${EXEC}" "\$@" EOF + EXEC="$(PATH="${ORIGINAL_PATH}" type -P g++)" + if [[ -z ${EXEC} ]] ; then + eerror "could not find 'g++' in your PATH!" + eerror "please install g++ or provide access via PATH or symlink" + return 1 + fi cat >> "${ROOT}"/tmp/usr/local/bin/g++ <<-EOF #! /bin/sh PATH="${ORIGINAL_PATH}" export PATH - exec "$(type -P g++)" "\$@" + exec "${EXEC}" "\$@" EOF chmod 755 "${ROOT}"/tmp/usr/local/bin/g{cc,++} fi ;; esac + # Host compiler can output a variety of libdirs. At stage1, # they should be the same as lib. Otherwise libffi may not be - # found by python. - if is-rap ; then + # found by python. Don't do this when we're using a Gentoo host to + # speed up bootstrapping, it should be good, and we shouldn't be + # touching the host either. Bug #927957 + if is-rap && [[ ! -L "${ROOT}"/tmp ]] ; then [[ -d ${ROOT}/tmp/usr/lib ]] || mkdir -p "${ROOT}"/tmp/usr/lib local libdir for libdir in lib64 lib32 libx32; do @@ -1525,12 +1743,12 @@ bootstrap_stage1() { # Portage) and binutils use it # note that this actually breaks the concept of stage1, this will be # compiled for the target prefix - for zlib in ${ROOT}/tmp/usr/lib/libz.* ; do + for zlib in "${ROOT}"/tmp/usr/lib*/libz.* ; do [[ -e ${zlib} ]] && break zlib= done [[ -n ${zlib} ]] || (bootstrap_zlib) || return 1 - for libffi in ${ROOT}/tmp/usr/lib*/libffi.* ; do + for libffi in "${ROOT}"/tmp/usr/lib*/libffi.* ; do [[ -e ${libffi} ]] && break libffi= done @@ -1555,9 +1773,16 @@ bootstrap_stage1() { # setup a profile for stage2 mkdir -p "${ROOT}"/tmp/etc/. || return 1 [[ -e ${ROOT}/tmp/etc/portage/make.profile ]] || \ - ( "${CP}" -dpR "${ROOT}"/etc/portage "${ROOT}"/tmp/etc && \ + ( + "${CP}" -dpR "${ROOT}"/etc/portage "${ROOT}"/tmp/etc && \ rm -f "${ROOT}"/tmp/etc/portage/make.profile && \ - (ROOT="${ROOT}"/tmp PREFIX_DISABLE_RAP=yes bootstrap_profile) ) || return 1 + ( + ROOT="${ROOT}"/tmp \ + PREFIX_DISABLE_RAP="yes" \ + BOOTSTRAP_STAGE="stage2" \ + bootstrap_profile + ) + ) || return 1 # setup portage [[ -e ${ROOT}/tmp/usr/bin/emerge ]] || (bootstrap_portage) || return 1 @@ -1568,10 +1793,16 @@ bootstrap_stage1() { } bootstrap_stage1_log() { - bootstrap_stage1 "${@}" 2>&1 | tee -a ${ROOT}/stage1.log + { + echo "===== stage 1 -- $(date -u +%Y-%m-%dT%H:%M:%SZ) =====" + echo "CHOST: ${CHOST}" + echo "IDENT: ${CHOST_IDENTIFY}" + echo "===========================================" + } >> "${ROOT}"/stage1.log + bootstrap_stage1 "${@}" 2>&1 | tee -a "${ROOT}"/stage1.log local ret=${PIPESTATUS[0]} - [[ ${ret} == 0 ]] && touch ${ROOT}/.stage1-finished - return ${ret} + [[ ${ret} == 0 ]] && touch "${ROOT}"/.stage1-finished + return "${ret}" } do_emerge_pkgs() { @@ -1589,42 +1820,62 @@ do_emerge_pkgs() { else vdb=${vdb}-\* fi - for pvdb in ${EPREFIX}/var/db/pkg/${vdb%-*}-* ; do + for pvdb in "${EPREFIX}/var/db/pkg/${vdb%-*}"-* ; do if [[ -d ${pvdb} ]] ; then evdb=${pvdb##*/} if [[ ${pkg} == "="* ]] ; then # exact match required (* should work here) - [[ ${evdb} == ${vdb##*/} ]] && break + [[ ${evdb} == "${vdb##*/}" ]] && break else vdb=${vdb%-*} evdb=${evdb%-r*} evdb=${evdb%_p*} evdb=${evdb%-*} - [[ ${evdb} == ${vdb#*/} ]] && break + [[ ${evdb} == "${vdb#*/}" ]] && break fi fi pvdb= done [[ -n ${pvdb} ]] && continue + # avoid many deps at this stage which aren't necessary, e.g. + # having a bash without readline is OK, we're not using the + # shell interactive local myuse=( - -acl - -berkdb - -fortran - -gdbm - -git - -libcxx - -nls - -pcre - -python - -qmanifest -qtegrity - -readline - -sanitize - bootstrap - clang - internal-glib + "${DISABLE_USE[@]}" + "-acl" + "-berkdb" + "-fortran" # gcc + "-gdbm" + "-nls" + "-pcre" + "-python" + "-qmanifest" # portage-utils + "-qtegrity" # portage-utils + "-readline" # bash + "-sanitize" + "system-bootstrap" + "clang" + "internal-glib" ) - local override_make_conf_dir="${PORTAGE_OVERRIDE_EPREFIX}${MAKE_CONF_DIR#${ROOT}}" + + local skip_llvm_pkg_setup= + if [[ ${CHOST}:${DARWIN_USE_GCC} == *-darwin*:0 ]] ; then + # Clang-based Darwin + myuse+=( + "-binutils-plugin" + "default-compiler-rt" + "default-libcxx" + "default-lld" + ) + if [[ "${BOOTSTRAP_STAGE}" == stage2 ]] ; then + myuse+=( "bootstrap-prefix" ) + skip_llvm_pkg_setup="yes" + fi + fi + + local override_make_conf_dir="${PORTAGE_OVERRIDE_EPREFIX}" + override_make_conf_dir+="${MAKE_CONF_DIR#"${ROOT}"}" if [[ " ${USE} " == *" prefix-stack "* ]] && [[ ${PORTAGE_OVERRIDE_EPREFIX} == */tmp ]] && @@ -1639,37 +1890,56 @@ do_emerge_pkgs() { echo "USE=\"\${USE} ${myuse[*]}\" # by bootstrap-prefix.sh" \ >> "${override_make_conf_dir}/0101_bootstrap_prefix_stack.conf" fi - myuse=" ${myuse[*]} " + local smyuse=" ${myuse[*]} " local use for use in ${USE} ; do - myuse=" ${myuse/ ${use} /} " - myuse=" ${myuse/ -${use} /} " - myuse=" ${myuse/ ${use#-} /} " - myuse+=" ${use} " + smyuse=" ${smyuse/ ${use} /} " + smyuse=" ${smyuse/ -${use} /} " + smyuse=" ${smyuse/ ${use#-} /} " + smyuse=" ${smyuse} ${use} " done - myuse=( ${myuse} ) + read -r -a myuse <<< "${smyuse}" # Disable the STALE warning because the snapshot frequently gets stale. # - # Need need to spam the user about news until the emerge -e system - # because the tools aren't available to read the news item yet anyway. + # No need to spam the user about news until the final emerge @world + # because the tools aren't available to read the news items yet anyway. # # Avoid circular deps caused by the default profiles (and IUSE # defaults). echo "USE=${myuse[*]} PKG=${pkg}" ( + local -a eopts + read -r -a eopts <<< "${opts}" + eopts=( + "--color" "n" + "-v" + "--oneshot" + "--root-deps" + "${eopts[@]}" + ) estatus "${STAGE}: emerge ${pkg}" unset CFLAGS CXXFLAGS [[ -n ${OVERRIDE_CFLAGS} ]] \ - && export CFLAGS=${OVERRIDE_CFLAGS} + && export CFLAGS="${OVERRIDE_CFLAGS}" [[ -n ${OVERRIDE_CXXFLAGS} ]] \ - && export CXXFLAGS=${OVERRIDE_CXXFLAGS} + && export CXXFLAGS="${OVERRIDE_CXXFLAGS}" + # In the stage3 bootstrap we always prefer to use tools that + # have been built for stage3; to accomplish this we ensure + # that it is the first thing evaluated in PATH. + # Unfortunately, Portage, Python, and Python-exec are often + # pulled into the depgraph at some point before we're fully + # boostrapped. To ensure that we don't try and execute + # ${EPREFIX}/usr/bin/emerge before we're ready, always + # provide the full path to the bootstrap Python interpreter + # and emerge script. PORTAGE_SYNC_STALE=0 \ FEATURES="-news ${FEATURES}" \ USE="${myuse[*]}" \ - emerge --color n -v --oneshot --root-deps ${opts} "${pkg}" - ) - [[ $? -eq 0 ]] || return 1 + LLVM_ECLASS_SKIP_PKG_SETUP="${skip_llvm_pkg_setup}" \ + "${ROOT}"/tmp/bin/python \ + "${ROOT}"/tmp/usr/bin/emerge "${eopts[@]}" "${pkg}" + ) || return 1 done } @@ -1683,7 +1953,7 @@ bootstrap_stage2() { # Find out what toolchain packages we need, and configure LDFLAGS # and friends. - configure_toolchain || return 1 + BOOTSTRAP_STAGE="stage2" configure_toolchain || return 1 configure_cflags || return 1 export CONFIG_SHELL="${ROOT}"/tmp/bin/bash export CC CXX @@ -1697,7 +1967,7 @@ bootstrap_stage2() { # bison's configure checks for perl, but doesn't use it, # except for tests. Since we don't want to pull in perl at this # stage, fake it - export PERL=$(which touch) + PERL="$(which touch)" ; export PERL # GCC sometimes decides that it needs to run makeinfo to update some # info pages from .texi files. Obviously we don't care at this # stage and rather have it continue instead of abort the build @@ -1745,14 +2015,52 @@ bootstrap_stage2() { # Disable RAP directory hacks of binutils and gcc. If libc.so # linker script provides no hint of ld-linux*.so*, ld should - # look into its default library path. Prefix library pathes + # look into its default library path. Prefix library paths # are taken care of by LDFLAGS in configure_cflags(). + # see profiles/features/prefix/standalone/profile.bashrc export BOOTSTRAP_RAP_STAGE2=yes + # elt-patches needs gentoo-functions, but gentoo-functions these + # days needs meson to install, which requires a properly installed + # Python -- at this stage we don't have that + # so fake gentoo-functions with some dummies to make elt-patches + # and others install + if [[ ! -e "${ROOT}"/tmp/lib/gentoo/functions.sh ]] ; then + mkdir -p "${ROOT}"/tmp/lib/gentoo + cat > "${ROOT}"/tmp/lib/gentoo/functions.sh <<-EOF + #!${BASH} + + ewarn() { + echo $* + } + + eerror() { + echo "!!! $*" + } + EOF + fi + + # provide active SDK link on Darwin + if [[ ${CHOST} == *-darwin* ]] ; then + rm -f "${ROOT}"/tmp/MacOSX.sdk + ( cd "${ROOT}"/tmp && ln -s ../MacOSX.sdk MacOSX.sdk ) + if [[ ${DARWIN_USE_GCC} == 0 ]] ; then + # Until proper clang is installed, just redirect calls to it + # to the system's one. Libtool is here because its path is + # passed to the compiler-rt and llvm's ebuilds. + for bin in libtool clang clang++ ; do + { + echo "#!${ROOT}/tmp/bin/sh" + echo "exec ${bin}"' "$@"' + } > "${ROOT}/tmp/usr/bin/${CHOST}-${bin}" + chmod +x "${ROOT}/tmp/usr/bin/${CHOST}-${bin}" + done + fi + fi + # Build a basic compiler and portage dependencies in $ROOT/tmp. pkgs=( sys-devel/gnuconfig - sys-apps/gentoo-functions app-portage/elt-patches sys-libs/ncurses sys-libs/readline @@ -1760,7 +2068,6 @@ bootstrap_stage2() { app-arch/xz-utils sys-apps/sed sys-apps/baselayout - dev-libs/libffi sys-devel/m4 sys-devel/flex sys-apps/diffutils # needed by bison-3 build system @@ -1769,52 +2076,44 @@ bootstrap_stage2() { sys-devel/binutils-config ) - # Old versions of gcc has been masked. We need gcc-4.7 to bootstrap - # on systems without a c++ compiler. - echo '> "${ROOT}"/tmp/etc/portage/package.unmask - - # libffi-3.0_rc0 has broken Solaris ld support, which we still - # use at this stage (host compiler) - [[ ${CHOST} == *-solaris* ]] && echo "=dev-libs/libffi-3.3_rc0" \ - >> "${ROOT}"/tmp/etc/portage/package.mask + # cmake has some external dependencies which require autoconf, etc. + # unless we only build the buildtool, bug #603012 + echo "dev-build/cmake -server" >> "${ROOT}"/tmp/etc/portage/package.use - # provide active SDK link on Darwin - if [[ ${CHOST} == *-darwin* ]] ; then - rm -f "${ROOT}"/tmp/MacOSX.sdk - ( cd "${ROOT}"/tmp && ln -s ../MacOSX.sdk ) + mkdir -p "${ROOT}"/tmp/etc/portage/profile # site-specific overrides + if [[ ${CHOST} == *-solaris* ]] ; then + # avoid complexities with the host toolchain + echo "sys-devel/gcc -pie" >> \ + "${ROOT}"/tmp/etc/portage/profile/package.use.force + echo "sys-devel/gcc -pie" >> "${ROOT}"/tmp/etc/portage/package.use fi - # cmake has some external dependencies which require autoconf, etc. - # unless we only build the buildtool, bug #603012 - echo "dev-util/cmake -server" >> "${ROOT}"/tmp/etc/portage/package.use + # don't use CET, we don't know if the host compiler supports it + echo "sys-devel/binutils -cet" >> \ + "${ROOT}"/tmp/etc/portage/profile/package.use.force emerge_pkgs --nodeps "${pkgs[@]}" || return 1 - # Debian multiarch supported by RAP needs ld to support sysroot. - EXTRA_ECONF=$(rapx --with-sysroot=/) \ - emerge_pkgs --nodeps ${linker} || return 1 - - # During Gentoo prefix bootstrap stage2, GCC is built with - # "--disable-bootstrap". For Darwin, it means that rather than letting - # GCC to eventually build itself using multiple passes, we're forcing - # it to build with the host LLVM/clang toolchain in a single pass. - # It's not officially supported, but practically it worked. However, - # since >=gcc-12.2.0, in order to support the new embedded rpath - # feature on Darwin, two incompatible options, "-nodefaultrpaths" and - # "-nodefaultexport" are introduced. This causes linking failures, - # since these options are only recognized by GCC and are unknown to - # LLVM/clang (hypothetically, using an older GCC possibly causes the - # same problem as well). - # - # Thus, embedded rpath should be disabled during prefix bootstrap stage2 - # and passed into EXTRA_ECONF. - # https://bugs.gentoo.org/895334 - if [[ ${CHOST} == *-darwin* ]] ; - then - local disable_darwin_rpath="--disable-darwin-at-rpath" - else - local disable_darwin_rpath="" - fi + for pkg in ${linker} ; do + # Debian multiarch supported by RAP needs ld to support sysroot. + EXTRA_ECONF=$(rapx --with-sysroot=/) \ + emerge_pkgs --nodeps "${pkg}" || return 1 + done + + # GCC doesn't respect CPPFLAGS because of its own meddling as well + # as toolchain.eclass, so provide a wrapper here to force just + # installed packages to be found + mkdir -p "${ROOT}"/tmp/usr/local/bin + rm -f "${ROOT}"/tmp/usr/local/bin/my{gcc,g++} + cat > "${ROOT}/tmp/usr/local/bin/mygcc" <<-EOS + #!/usr/bin/env sh + exec ${CC} "\$@" ${CPPFLAGS} + EOS + cat > "${ROOT}/tmp/usr/local/bin/myg++" <<-EOS + #!/usr/bin/env sh + exec ${CXX} "\$@" ${CPPFLAGS} + EOS + chmod 755 "${ROOT}/tmp/usr/local/bin/my"{gcc,g++} for pkg in ${compiler_stage1} ; do # =gcc-12.2.0, rpath needs to be disabled in stage2 on - # Darwin, see above. - EXTRA_ECONF="--disable-bootstrap $(rapx --with-linker-hash-style=both) --with-local-prefix=${ROOT} ${disable_darwin_rpath}" \ + EXTRA_ECONF="$(rapx --with-linker-hash-style=both) --with-local-prefix=${ROOT}" \ MYCMAKEARGS="-DCMAKE_USE_SYSTEM_LIBRARY_LIBUV=OFF" \ GCC_MAKE_TARGET=all \ - OVERRIDE_CXXFLAGS="${CPPFLAGS} -O2 -pipe" \ - TPREFIX="${ROOT}" \ + OVERRIDE_CFLAGS="${CPPFLAGS} ${OVERRIDE_CFLAGS}" \ + OVERRIDE_CXXFLAGS="${CPPFLAGS} ${OVERRIDE_CXXFLAGS}" \ + CC=mygcc CXX=myg++ \ PYTHON_COMPAT_OVERRIDE=python$(python_ver) \ - emerge_pkgs --nodeps ${pkg} || return 1 + emerge_pkgs --nodeps "${pkg}" || return 1 if [[ "${pkg}" == *sys-devel/llvm* || ${pkg} == *sys-devel/clang* ]] ; then @@ -1842,12 +2140,22 @@ bootstrap_stage2() { done if [[ ${compiler_type} == clang ]] ; then + if [[ ${CHOST} == *-darwin* ]] ; then + # Stop using host's compilers, but still need 'libtool' in PATH. + rm "${ROOT}/tmp/usr/bin/${CHOST}"-{libtool,clang,clang++} + mkdir -p "${ROOT}"/usr/bin + ln -s "${ROOT}"/tmp/usr/lib/llvm/*/bin/llvm-libtool-darwin \ + "${ROOT}"/usr/bin/libtool + fi + # We use Clang as our toolchain compiler, so we need to make # sure we actually use it mkdir -p -- "${MAKE_CONF_DIR}" { echo echo "# System compiler on $(uname) Prefix is Clang, do not remove this" + echo "AS=\"${CHOST}-clang -c\"" + echo "CPP=${CHOST}-clang-cpp" echo "CC=${CHOST}-clang" echo "CXX=${CHOST}-clang++" echo "OBJC=${CHOST}-clang" @@ -1861,12 +2169,14 @@ bootstrap_stage2() { # multilib.eclass -- can't blame it at this point really) # do it ourselves here to make the bootstrap continue if [[ -x "${ROOT}"/tmp/usr/bin/${CHOST}-clang ]] ; then - ( cd "${ROOT}"/tmp/usr/bin && ln -s clang ${CHOST}-clang && ln -s clang++ ${CHOST}-clang++ ) + ( cd "${ROOT}"/tmp/usr/bin && \ + ln -s clang "${CHOST}-clang" && \ + ln -s clang++ "${CHOST}-clang++" ) fi elif ! is-rap ; then # make sure the EPREFIX gcc shared libraries are there - mkdir -p "${ROOT}"/usr/${CHOST}/lib/gcc - cp "${ROOT}"/tmp/usr/${CHOST}/lib/gcc/* "${ROOT}"/usr/${CHOST}/lib/gcc + mkdir -p "${ROOT}/usr/${CHOST}/lib/gcc" + cp "${ROOT}/tmp/usr/${CHOST}/lib/gcc"/* "${ROOT}/usr/${CHOST}/lib/gcc" fi estatus "stage2 finished" @@ -1874,21 +2184,42 @@ bootstrap_stage2() { } bootstrap_stage2_log() { - bootstrap_stage2 "${@}" 2>&1 | tee -a ${ROOT}/stage2.log + { + echo "===== stage 2 -- $(date -u +%Y-%m-%dT%H:%M:%SZ) =====" + echo "CHOST: ${CHOST}" + echo "IDENT: ${CHOST_IDENTIFY}" + echo "===========================================" + } >> "${ROOT}"/stage2.log + bootstrap_stage2 "${@}" 2>&1 | tee -a "${ROOT}"/stage2.log local ret=${PIPESTATUS[0]} [[ ${ret} == 0 ]] && touch "${ROOT}/.stage2-finished" - return ${ret} + return "${ret}" } bootstrap_stage3() { export PORTAGE_CONFIGROOT="${ROOT}" + # We need the stage2 in PATH for bootstrapping. We rely on + # emerge running on some benign package before running anything + # that would rely on 98stage2 coming before 99host + mkdir -p "${ROOT}"/etc/env.d/ + cat > "${ROOT}"/etc/env.d/98stage2 <<-EOF + PATH="$(unset PATH; + source "${ROOT}"/tmp/etc/profile.env; + echo "$PATH")" + EOF + if ! type -P emerge > /dev/null ; then eerror "emerge not found, did you bootstrap stage1?" return 1 fi - configure_toolchain || return 1 + # At this point, we should have a proper GCC, and don't need to + # rely on the system wrappers. Let's get rid of them, so that + # they stop mucking up builds. + rm -f "${ROOT}"/tmp/usr/local/bin/{,my,${CHOST}-}{gcc,g++} + + BOOTSTRAP_STAGE=stage3 configure_toolchain || return 1 if [[ ${compiler_type} == clang ]] ; then if ! type -P clang > /dev/null ; then @@ -1906,21 +2237,27 @@ bootstrap_stage3() { # tmp, we basically made the system unusable, so remove python-exec # here so we can use the python in tmp for pef in python{,3} python{,3}-config ; do - rm -f "${ROOT}"/tmp/usr/bin/${pef} + rm -f "${ROOT}/tmp/usr/bin/${pef}" [[ ${pef} == *-config ]] && ppf=-config || ppf= - ( cd "${ROOT}"/tmp/usr/bin && ln -s python$(python_ver)${ppf} ${pef} ) + ( cd "${ROOT}"/tmp/usr/bin && \ + ln -s "python$(python_ver)${ppf}" "${pef}" ) done get_libdir() { - local l=$(portageq envvar LIBDIR_$(portageq envvar ABI) 2>/dev/null) + local l + l="$(portageq envvar "LIBDIR_$(portageq envvar ABI)" 2>/dev/null)" [[ -z ${l} ]] && l=lib - echo ${l} + echo "${l}" } + # Remember: binutils-config and gcc were built in ROOT/tmp, so they + # are looking for includes and libraries under ROOT/tmp, *NOT* ROOT, + # therefore we need to export search paths for ROOT (the final + # destination Prefix) here until we've installed the toolchain export CONFIG_SHELL="${ROOT}"/tmp/bin/bash [[ ${compiler_type} == gcc ]] && \ export CPPFLAGS="-isystem ${ROOT}/usr/include" - export LDFLAGS="-L${ROOT}/usr/$(get_libdir)" + LDFLAGS="-L${ROOT}/usr/$(get_libdir)" ; export LDFLAGS [[ ${CHOST} == *-darwin* ]] || \ LDFLAGS+=" -Wl,-rpath=${ROOT}/usr/$(get_libdir)" unset CC CXX @@ -1935,33 +2272,31 @@ bootstrap_stage3() { # (CBUILD, BDEPEND) and with the system being built # (CHOST, RDEPEND). To correctly bootstrap stage3, # PORTAGE_OVERRIDE_EPREFIX as BROOT is needed. - PREROOTPATH="${ROOT}"$(echo /{,tmp/}{usr/,}{,lib/llvm/{12,11,10}/}{s,}bin | sed "s, ,:${ROOT},g") \ EPREFIX="${ROOT}" PORTAGE_TMPDIR="${PORTAGE_TMPDIR}" \ EMERGE_LOG_DIR="${ROOT}"/var/log \ STAGE=stage3 \ do_emerge_pkgs "$@" } - with_stack_emerge_pkgs() { - # keep FEATURES=stacked-prefix until we bump portage in stage1 - FEATURES="${FEATURES} stacked-prefix" \ - USE="${USE} prefix-stack" \ - PORTAGE_OVERRIDE_EPREFIX="${ROOT}/tmp" \ - emerge_pkgs "$@" - } - - without_stack_emerge_pkgs() { + # retained in case we *do* need this, but using this will cause + # packages installed end up in ROOT/tmp, which means we keep using + # stage2 area and config which breaks things like binutils-config' + # path search, so don't use this + #with_stack_emerge_pkgs() { + # # keep FEATURES=stacked-prefix until we bump portage in stage1 + # FEATURES="${FEATURES} stacked-prefix" \ + # USE="${USE} prefix-stack" \ + # PORTAGE_OVERRIDE_EPREFIX="${ROOT}/tmp" \ + # emerge_pkgs "$@" + #} + + # pre_emerge_pkgs relies on stage 2 portage, but installs into the + # final destination Prefix + pre_emerge_pkgs() { PORTAGE_OVERRIDE_EPREFIX="${ROOT}" \ emerge_pkgs "$@" } - # pre_emerge_pkgs relies on stage 2 portage. - pre_emerge_pkgs() { - is-rap \ - && without_stack_emerge_pkgs "$@" \ - || with_stack_emerge_pkgs "$@" - } - # Some packages fail to properly depend on sys-apps/texinfo. # We don't really need that package, so we fake it instead, # explicitly emerging it later on will overwrite the fakes. @@ -1979,6 +2314,20 @@ bootstrap_stage3() { cp -a "${ROOT}"{/tmp,}/usr/share/portage fi + local -a linker_pkgs compiler_pkgs + read -r -a linker_pkgs <<< "${linker}" + read -r -a compiler_pkgs <<< "${compiler}" + + # We need gentoo-functions but it meson is still a no-go, because we + # don't have a Python. Why would such simple package with a silly + # script file need meson is beyond me. So, we have no other way + # than to fake it here for the time being like in stage2. + if [[ ! -e "${ROOT}"/lib/gentoo/functions.sh ]] ; then + mkdir -p "${ROOT}"/lib/gentoo + cp "${ROOT}"/tmp/lib/gentoo/functions.sh \ + "${ROOT}"/lib/gentoo/functions.sh + fi + if is-rap ; then # We need ${ROOT}/usr/bin/perl to merge glibc. if [[ ! -x "${ROOT}"/usr/bin/perl ]]; then @@ -2008,13 +2357,13 @@ bootstrap_stage3() { # Tell dynamic loader the path of libgcc_s.so of stage2 if [[ ! -f "${ROOT}"/etc/ld.so.conf.d/stage2.conf ]]; then mkdir -p "${ROOT}"/etc/ld.so.conf.d - dirname $(gcc -print-libgcc-file-name) \ + dirname "$(gcc -print-libgcc-file-name)" \ > "${ROOT}"/etc/ld.so.conf.d/stage2.conf fi pkgs=( + sys-devel/gnuconfig sys-apps/baselayout - sys-apps/gentoo-functions app-portage/elt-patches sys-kernel/linux-headers sys-libs/glibc @@ -2027,13 +2376,25 @@ bootstrap_stage3() { grep -q 'esac' "${ROOT}"/usr/bin/rsync && \ rm "${ROOT}"/usr/bin/rsync + # sys-apps/baselayout will install a dummy openrc-run wrapper + # for any package that installs an init.d script, like rsync and + # python will need openrc-run to exist, else we'll die with a QA + # error, bug #858596. However it only does this for + # prefix-guest, so NOT For RAP, which results in bug #913856. + if [[ ! -x "${ROOT}"/sbin/openrc-run ]]; then + [[ -e "${ROOT}"/sbin ]] || mkdir -p "${ROOT}"/sbin + echo "We need openrc-run at ${ROOT}/sbin to merge some packages." \ + > "${ROOT}"/sbin/openrc-run + chmod +x "${ROOT}"/sbin/openrc-run + fi + pkgs=( sys-devel/binutils-config sys-libs/zlib - ${linker} + "${linker_pkgs[@]}" ) # use the new dynamic linker in place of rpath from now on. - RAP_DLINKER=$(echo "${ROOT}"/$(get_libdir)/ld*.so.[0-9] | sed s"!${ROOT}/$(get_libdir)/ld-lsb.*!!") + RAP_DLINKER=$(echo "${ROOT}/$(get_libdir)"/ld*.so.[0-9] | sed s"!${ROOT}/$(get_libdir)/ld-lsb.*!!") export CPPFLAGS="--sysroot=${ROOT}" export LDFLAGS="-Wl,--dynamic-linker=${RAP_DLINKER}" # make sure these flags are used even in places that ignore/strip CPPFLAGS/LDFLAGS @@ -2053,7 +2414,7 @@ bootstrap_stage3() { pre_emerge_pkgs --nodeps "${pkgs[@]}" || return 1 else pkgs=( - sys-apps/gentoo-functions + sys-devel/gnuconfig app-portage/elt-patches app-arch/xz-utils sys-apps/sed @@ -2062,14 +2423,14 @@ bootstrap_stage3() { sys-devel/flex sys-devel/binutils-config sys-libs/zlib - ${linker} + "${linker_pkgs[@]}" ) pre_emerge_pkgs --nodeps "${pkgs[@]}" || return 1 fi # remove stage2 ld so that stage3 ld is used by stage2 gcc. is-rap && [[ -f ${ROOT}/tmp/usr/${CHOST}/bin/ld ]] && \ - mv ${ROOT}/tmp/usr/${CHOST}/bin/ld{,.stage2} + mv "${ROOT}/tmp/usr/${CHOST}/bin"/ld{,.stage2} # On some hosts, gcc gets confused now when it uses the new linker, # see for instance bug #575480. While we would like to hide that @@ -2082,16 +2443,41 @@ bootstrap_stage3() { # setup for a scenario where python doesn't live in the target # prefix and no helpers are available ( cd "${ROOT}"/usr/bin && test ! -e python && \ - ln -s "${ROOT}"/tmp/usr/bin/python$(python_ver) ) + ln -s "${ROOT}/tmp/usr/bin/python$(python_ver)" "python$(python_ver)" ) # in addition, avoid collisions - rm -Rf "${ROOT}"/tmp/usr/lib/python$(python_ver)/site-packages/clang + rm -Rf "${ROOT}/tmp/usr/lib/python$(python_ver)/site-packages/clang" # Try to get ourself out of the mud, bug #575324 EXTRA_ECONF="--disable-compiler-version-checks $(rapx '--disable-lto --disable-bootstrap')" \ - GCC_MAKE_TARGET=$(rapx all) \ + GCC_MAKE_TARGET="$(rapx all)" \ MYCMAKEARGS="-DCMAKE_USE_SYSTEM_LIBRARY_LIBUV=OFF" \ - PYTHON_COMPAT_OVERRIDE=python$(python_ver) \ - pre_emerge_pkgs --nodeps ${compiler} || return 1 + PYTHON_COMPAT_OVERRIDE="python$(python_ver)" \ + pre_emerge_pkgs --nodeps "${compiler_pkgs[@]}" || return 1 + + if [[ ${CHOST}:${DARWIN_USE_GCC} == *-darwin*:0 ]] ; then + # At this point our libc++abi.dylib is dynamically linked to + # /usr/lib/libc++abi.dylib. That causes issues with perl later. Force + # rebuild of sys-libs/libcxxabi to break this link. + rm -Rf "${ROOT}/var/db/pkg/sys-libs/libcxxabi"* + PYTHON_COMPAT_OVERRIDE=python$(python_ver) \ + pre_emerge_pkgs --nodeps "sys-libs/libcxxabi" || return 1 + + # Make ${CHOST}-libtool (used by compiler-rt's and llvm's ebuild) to + # point at the correct libtool in stage3. Resolve it in runtime, to + # support llvm version upgrades. + rm -f ${ROOT}/usr/bin/${CHOST}-libtool + { + echo "#!${ROOT}/bin/sh" + echo 'exec llvm-libtool-darwin "$@"' + } > "${ROOT}"/usr/bin/${CHOST}-${bin} + + # Now clang is ready, can use it instead of /usr/bin/gcc + # TODO: perhaps symlink the whole etc/portage instead? + ln -s -f "${ROOT}/etc/portage/make.profile" \ + "${ROOT}/tmp/etc/portage/make.profile" + cp "${ROOT}/etc/portage/make.conf/0100_bootstrap_prefix_clang.conf" \ + "${ROOT}/tmp/etc/portage/make.conf/" + fi # Undo libgcc_s.so path of stage2 # Now we have the compiler right there @@ -2102,15 +2488,42 @@ bootstrap_stage3() { # need special care, it depends on texinfo, #717786 pre_emerge_pkgs --nodeps sys-apps/gawk || return 1 - ( cd "${ROOT}"/usr/bin && test ! -e python && rm -f python$(python_ver) ) + ( cd "${ROOT}"/usr/bin && test ! -e python && rm -f "python$(python_ver)" ) # Use $ROOT tools where possible from now on. if [[ $(readlink "${ROOT}"/bin/sh) == "${ROOT}/tmp/"* ]] ; then rm -f "${ROOT}"/bin/sh ln -s bash "${ROOT}"/bin/sh fi - # Start using apps from new target - export PREROOTPATH="${ROOT}/usr/bin:${ROOT}/bin" + if [[ "${compiler_type}" == clang ]] ; then + if [[ ! -e "${ROOT}"/tmp/etc/env.d/11stage3-llvm ]]; then + ln -s "${ROOT}"/etc/env.d/60llvm-* \ + "${ROOT}"/tmp/etc/env.d/11stage3-llvm + fi + # Prevent usage of AppleClang aka gcc for bad packages which ignore $CC + if [[ ! -e "${ROOT}"/usr/bin/gcc ]]; then + echo "#!${ROOT}/bin/bash" > "${ROOT}"/usr/bin/gcc + echo "false ${CHOST}-clang \"\$@\"" >> "${ROOT}"/usr/bin/gcc + fi + if [[ ! -e "${ROOT}"/usr/bin/g++ ]]; then + echo "#!${ROOT}/bin/bash" > "${ROOT}"/usr/bin/g++ + echo "false ${CHOST}-clang++ \"\$@\"" >> "${ROOT}"/usr/bin/g++ + fi + chmod +x "${ROOT}"/usr/bin/{gcc,g++} + if [[ ${CHOST} == *-darwin* ]]; then + if [[ ! -e "${ROOT}"/usr/bin/ld ]]; then + echo "#!${ROOT}/bin/bash" > "${ROOT}"/usr/bin/ld + echo "false ld64.lld \"\$@\"" >> "${ROOT}"/usr/bin/ld + fi + chmod +x "${ROOT}"/usr/bin/ld + fi + fi + + # Start using apps from the final destination Prefix + cat > "${ROOT}"/tmp/etc/env.d/10stage3 <<-EOF + PATH="${ROOT}/usr/bin:${ROOT}/bin" + EOF + "${ROOT}"/tmp/usr/sbin/env-update # Get a sane bash, overwriting tmp symlinks pre_emerge_pkgs "" "app-shells/bash" || return 1 @@ -2118,14 +2531,14 @@ bootstrap_stage3() { # now we have a shell right there unset CONFIG_SHELL - # Build portage and dependencies. + # Build portage dependencies. pkgs=( sys-apps/coreutils sys-apps/findutils app-arch/gzip app-arch/tar sys-apps/grep - sys-devel/make + dev-build/make sys-apps/file app-admin/eselect ) @@ -2137,19 +2550,14 @@ bootstrap_stage3() { TIME_T_32_BIT_OK=yes \ pre_emerge_pkgs "" "${pkgs[@]}" || return 1 - if [[ ! -x "${ROOT}"/sbin/openrc-run ]]; then - echo "We need openrc-run at ${ROOT}/sbin to merge rsync." \ - > "${ROOT}"/sbin/openrc-run - chmod +x "${ROOT}"/sbin/openrc-run - fi - pkgs=( virtual/os-headers sys-devel/gettext sys-apps/portage + sys-apps/gentoo-functions ) - without_stack_emerge_pkgs "" "${pkgs[@]}" || return 1 + pre_emerge_pkgs "" "${pkgs[@]}" || return 1 # Switch to the proper portage. hash -r @@ -2169,14 +2577,35 @@ bootstrap_stage3() { fi # Avoid installing git or encryption just for fun while completing @system - # e.g. bug #901101 - export USE="-git -crypt -http2" + # e.g. bug #901101, this is a reduced (e.g. as minimal as possible) + # set of DISABLE_USE, to set the stage for solving circular + # dependencies, such as: + export USE="${DISABLE_USE[*]}" # Portage should figure out itself what it needs to do, if anything. - einfo "running emerge -uDNv system" - estatus "stage3: emerge -uDNv system" - unset CFLAGS CXXFLAGS CPPFLAGS - emerge --color n -uDNv system || return 1 + local eflags=( "--deep" "--update" "--changed-use" "@system" ) + einfo "running emerge ${eflags[*]}" + estatus "stage3: emerge ${eflags[*]}" + emerge --color n -v "${eflags[@]}" || return 1 + + # gcc no longer depends on sys-devel/binutils which means it is to + # be depcleaned at this point, quite strange, but to prevent this + # from happening, add to the worldfile #936629#c5 + emerge --color n --noreplace sys-devel/binutils + + # Remove the stage2 hack from above. A future emerge run will + # get env-update to happen. + rm "${ROOT}"/etc/env.d/98stage2 + + # now try and get things in the way they should be according to the + # default USE-flags + unset USE + + # re-emerge anything hopefully not running into circular deps + eflags=( "--deep" "--changed-use" "@world" ) + einfo "running emerge ${eflags[*]}" + estatus "stage3: emerge ${eflags[*]}" + emerge --color n -v "${eflags[@]}" || return 1 # Remove anything that we don't need (compilers most likely) einfo "running emerge --depclean" @@ -2192,10 +2621,16 @@ bootstrap_stage3() { } bootstrap_stage3_log() { - bootstrap_stage3 "${@}" 2>&1 | tee -a ${ROOT}/stage3.log + { + echo "===== stage 3 -- $(date -u +%Y-%m-%dT%H:%M:%SZ) =====" + echo "CHOST: ${CHOST}" + echo "IDENT: ${CHOST_IDENTIFY}" + echo "===========================================" + } >> "${ROOT}"/stage3.log + bootstrap_stage3 "${@}" 2>&1 | tee -a "${ROOT}"/stage3.log local ret=${PIPESTATUS[0]} [[ ${ret} == 0 ]] && touch "${ROOT}/.stage3-finished" - return ${ret} + return "${ret}" } set_helper_vars() { @@ -2209,9 +2644,20 @@ set_helper_vars() { DISTFILES_G_O="http://distfiles.prefix.bitzolder.nl" DISTFILES_PFX="http://distfiles.prefix.bitzolder.nl/prefix" GENTOO_MIRRORS=${GENTOO_MIRRORS:="http://distfiles.gentoo.org"} - SNAPSHOT_HOST=$(rapx ${DISTFILES_G_O} http://rsync.prefix.bitzolder.nl) + SNAPSHOT_HOST=$(rapx http://distfiles.gentoo.org http://rsync.prefix.bitzolder.nl) SNAPSHOT_URL=${SNAPSHOT_URL:-"${SNAPSHOT_HOST}/snapshots"} - GCC_APPLE_URL="http://www.opensource.apple.com/darwinsource/tarballs/other" + + # USE-flags to disable during bootstrap for they produce + # unnecessary, or worse: circular deps #901101, #936629 + # - nghttp2 -> cmake -> curl -> nghttp2 (http2) + DISABLE_USE=( + "-crypt" + "-curl_quic_openssl" # curl + "-git" + "-http2" # curl + "-http3" # curl + "-quic" # curl + ) export MAKE CONFIG_SHELL } @@ -2247,7 +2693,7 @@ bootstrap_interactive() { EOF [[ ${TODO} == 'noninteractive' ]] && ans=yes || - read -p "Do you want me to start off now? [Yn] " ans + read -r -p "Do you want me to start off now? [Yn] " ans case "${ans}" in [Yy][Ee][Ss]|[Yy]|"") : ;; @@ -2300,10 +2746,10 @@ EOF # note that this code is so complex because it handles both # C-shell as well as *sh dvar="echo \"((${flag}=\${${flag}}))\"" - dvar="$(echo "${dvar}" | env -i HOME=$HOME $SHELL -l 2>/dev/null)" + dvar="$(echo "${dvar}" | env -i HOME="${HOME}" "$SHELL" -l 2>/dev/null)" if [[ ${dvar} == *"((${flag}="?*"))" ]] ; then badflags="${badflags} ${flag}" - dvar=${dvar#*((${flag}=} + dvar=${dvar#*"((${flag}="} dvar=${dvar%%))*} echo " uh oh, ${flag}=${dvar} :(" else @@ -2454,7 +2900,7 @@ necessary to add to PATH for me to find a compiler. I start off with PATH=${PATH} and will add anything you give me here. EOF [[ ${TODO} == 'noninteractive' ]] && ans="${usergcc%/gcc}" || - read -p "Where can I find your compiler? [] " ans + read -r -p "Where can I find your compiler? [] " ans case "${ans}" in "") : ;; @@ -2483,7 +2929,7 @@ EOF if type -P xcode-select > /dev/null ; then if [[ -d /usr/include ]] ; then # if we have /usr/include we're on an older system - if [[ ${CHOST} == powerpc* ]]; then + if [[ ${CHOST} == *-darwin[89] ]]; then # ancient Xcode (3.0/3.1) cat << EOF @@ -2525,18 +2971,15 @@ EOF *-solaris*) ncpu=$(/usr/sbin/psrinfo | wc -l) ;; *-linux-gnu*) - ncpu=$(cat /proc/cpuinfo | grep processor | wc -l) ;; + ncpu=$(grep -c processor /proc/cpuinfo) ;; *) ncpu=1 ;; esac # get rid of excess spaces (at least Solaris wc does) ncpu=$((ncpu + 0)) - # Suggest usage of 100% to 60% of the available CPUs in the range - # from 1 to 14. We limit to no more than 8, since we easily flood - # the bus on those heavy-core systems and only slow down in that - # case anyway. - local tcpu=$((ncpu / 2 + 1)) - [[ ${tcpu} -gt 8 ]] && tcpu=8 + # Suggest usage of 50% to 75% of the available CPUs + [[ ${tcpu} -eq 0 ]] && tcpu=1 + local tcpu=$((((ncpu * 3) + 1) / 4)) [[ -n ${USE_CPU_CORES} ]] && tcpu=${USE_CPU_CORES} cat << EOF @@ -2548,7 +2991,7 @@ clue what this means, you should go with my excellent default I've chosen below, really! EOF [[ ${TODO} == 'noninteractive' ]] && ans="" || - read -p "How many parallel make jobs do you want? [${tcpu}] " ans + read -r -p "How many parallel make jobs do you want? [${tcpu}] " ans case "${ans}" in "") MAKEOPTS="-j${tcpu}" @@ -2585,9 +3028,11 @@ EOF local candomultilib=no local t64 t32 case "${CHOST}" in - *86*-darwin9|*86*-darwin1[012345]) + *86*-darwin1[012345]) # PPC/Darwin only works in 32-bits mode, so this is Intel - # only, and only starting from Leopard (10.5, darwin9) + # only, and officially starting from Leopard (10.5, darwin9) + # but this is broken, so stick to 32-bits there, and use it + # from Snow Lepard (10.6). # with Big Sur (11.0, darwin20) we have x64 or arm64 only candomultilib=yes t64=x86_64-${CHOST#*-} @@ -2633,11 +3078,11 @@ EOF case "${CHOST}" in x86_64-*|sparcv9-*) # others can't do multilib, so don't bother # 64-bits native - read -p "How many bits do you want your Prefix to target? [64] " ans + read -r -p "How many bits do you want your Prefix to target? [64] " ans ;; *) # 32-bits native - read -p "How many bits do you want your Prefix to target? [32] " ans + read -r -p "How many bits do you want your Prefix to target? [32] " ans ;; esac case "${ans}" in @@ -2662,7 +3107,8 @@ EOF # Figure out if we are bootstrapping from an existing Gentoo # It can be forced by setting HOST_GENTOO_EROOT manually - local t_GENTOO_EROOT=$(env -u EPREFIX portageq envvar EROOT 2> /dev/null) + local t_GENTOO_EROOT + t_GENTOO_EROOT=$(env -u EPREFIX portageq envvar EROOT 2> /dev/null) if [[ ! -d ${HOST_GENTOO_EROOT} && -d ${t_GENTOO_EROOT} ]]; then cat < you asked me to. -EOF - return 0; - fi if ! [[ -e ${EPREFIX}/.stage1-finished ]] && ! bootstrap_stage1_log ; then # stage 1 fail @@ -2877,6 +3317,9 @@ in #gentoo-prefix on irc.gentoo.org, gentoo-alt@lists.gentoo.org mailing list, or file a bug at bugs.gentoo.org under Gentoo/Alt, Prefix Support. Sorry that I have failed you master. I shall now return to my humble cave. You can find a log of what happened in ${EPREFIX}/stage1.log + + CHOST: ${CHOST} + IDENT: ${CHOST_IDENTIFY} EOF exit 1 fi @@ -2891,7 +3334,8 @@ EOF # This happens at least on 32-bits Darwin, with i386 and i686. # https://bugs.gentoo.org/show_bug.cgi?id=433948 unset CHOST - export CHOST=$(portageq envvar CHOST) + CHOST=$(portageq envvar CHOST) + export CHOST # after stage1 and stage2 we should have a bash of our own, which # is preferable over the host-provided one, because we know it can @@ -2900,7 +3344,7 @@ EOF local https_needed=no if ! [[ -e ${EPREFIX}/.stage2-finished ]] \ - && ! ${BASH} ${BASH_SOURCE[0]} "${EPREFIX}" stage2_log ; then + && ! ${BASH} "${BASH_SOURCE[0]}" "${EPREFIX}" stage2_log ; then # stage 2 fail cat << EOF @@ -2928,6 +3372,9 @@ I have no clue, really. Please find friendly folks in #gentoo-prefix on irc.gentoo.org, gentoo-alt@lists.gentoo.org mailing list, or file a bug at bugs.gentoo.org under Gentoo/Alt, Prefix Support. Remember you might find some clues in ${EPREFIX}/stage2.log + + CHOST: ${CHOST} + IDENT: ${CHOST_IDENTIFY} EOF exit 1 fi @@ -2938,7 +3385,7 @@ EOF hash -r if ! [[ -e ${EPREFIX}/.stage3-finished ]] \ - && ! bash ${BASH_SOURCE[0]} "${EPREFIX}" stage3_log ; then + && ! bash "${BASH_SOURCE[0]}" "${EPREFIX}" stage3_log ; then # stage 3 fail hash -r # previous cat (tmp/usr/bin/cat) may have been removed cat << EOF @@ -2968,51 +3415,24 @@ irc.gentoo.org, gentoo-alt@lists.gentoo.org mailing list, or file a bug at bugs.gentoo.org under Gentoo/Alt, Prefix Support. This is most inconvenient, and it crushed my ego. Sorry, I give up. Should you want to give it a try, there is ${EPREFIX}/stage3.log + + CHOST: ${CHOST} + IDENT: ${CHOST_IDENTIFY} EOF exit 1 fi [[ ${STOP_BOOTSTRAP_AFTER} == stage3 ]] && exit 0 - local cmd="emerge -v -e system" - if [[ -e ${EPREFIX}/var/cache/edb/mtimedb ]] && \ - grep -q resume "${EPREFIX}"/var/cache/edb/mtimedb ; - then - cmd="emerge -v --resume" - fi - einfo "running ${cmd}" - if ${cmd} ; then - # Now, after 'emerge -e system', we can get rid of the temporary tools. - if [[ -d ${EPREFIX}/tmp/var/tmp ]] ; then - rm -Rf "${EPREFIX}"/tmp || return 1 - mkdir -p "${EPREFIX}"/tmp || return 1 - fi - - hash -r # tmp/* stuff is removed in stage3 - else - # emerge -e system fail - cat << EOF - -Oh yeah, I thought I was almost there, and then this! I did - ${cmd} -and it failed at some point :( Details might be found in the build log: -EOF - for log in "${EPREFIX}"/var/tmp/portage/*/*/temp/build.log ; do - [[ -e ${log} ]] || continue - echo " ${log}" - done - [[ -e ${log} ]] || echo " (no build logs found?!?)" - cat << EOF -I have no clue, really. Please find friendly folks in #gentoo-prefix on -irc.gentoo.org, gentoo-alt@lists.gentoo.org mailing list, or file a bug -at bugs.gentoo.org under Gentoo/Alt, Prefix Support. -You know, I got the feeling you just started to like me, but I guess -that's all gone now. I'll bother you no longer. -EOF - exit 1 + # Now, we've got everything in $ROOT, we can get rid of /tmp + if [[ -d ${EPREFIX}/tmp/var/tmp ]] ; then + rm -Rf "${EPREFIX}"/tmp || return 1 + mkdir -p "${EPREFIX}"/tmp || return 1 fi - if ! bash ${BASH_SOURCE[0]} "${EPREFIX}" startscript ; then + hash -r # tmp/* stuff is removed in stage3 + + if ! bash "${BASH_SOURCE[0]}" "${EPREFIX}" startscript ; then # startscript fail? cat << EOF @@ -3020,7 +3440,7 @@ Ok, let's be honest towards each other. If $(type -P bash) ${BASH_SOURCE[0]} "${EPREFIX}" startscript fails, then who cheated on who? Either you use an obscure shell, or your PATH isn't really sane afterall. Despite, I can't really -congratulate you here, you basically made it to the end. +congratulate you here, but you basically made it to the end. Please find friendly folks in #gentoo-prefix on irc.gentoo.org, gentoo-alt@lists.gentoo.org mailing list, or file a bug at bugs.gentoo.org under Gentoo/Alt, Prefix Support. @@ -3048,28 +3468,36 @@ unset TMP TMPDIR TEMP # Try to guess the CHOST if not set. We currently only support guessing # on a very sloppy base. if [[ -z ${CHOST} ]]; then - if [[ x$(type -t uname) == "xfile" ]]; then - case `uname -s` in + if [[ $(type -t uname) == "file" ]]; then + case $(uname -s) in Linux) - plt="gnu" - for f in /lib/ld-musl-*.so.1; do - [[ -e "$f" ]] && plt="musl" - done - sfx="unknown-linux-${plt}" - case `uname -m` in - ppc*) - CHOST="`uname -m | sed -e 's/^ppc/powerpc/'`-${sfx}" - ;; - powerpc*|aarch64*) - CHOST="`uname -m`-${sfx}" - ;; + CHOST=$(uname -m) + CHOST=${CHOST/#ppc/powerpc} + case "${CHOST}" in + x86_64|i*86) + CHOST+=-pc ;; *) - CHOST="`uname -m`-${sfx/unknown/pc}" + CHOST+=-unknown ;; + esac + plt=gnu + for f in /lib/ld-musl-*.so.1; do + [[ -e $f ]] && plt=musl + done + CHOST+=-linux-${plt} + case "${CHOST}" in + arm*) + CHOST+=eabi + for f in /lib/ld-*hf.so.*; do + if [[ -e $f ]]; then + CHOST+=hf + break + fi + done ;; esac ;; Darwin) - rev="`uname -r | cut -d'.' -f 1`" + rev=$(uname -r | cut -d'.' -f 1) if [[ ${rev} -ge 11 && ${rev} -le 19 ]] ; then # Lion and up are 64-bits default (and 64-bits CPUs) CHOST="x86_64-apple-darwin$rev" @@ -3077,40 +3505,46 @@ if [[ -z ${CHOST} ]]; then # uname -p returns arm, -m returns arm64 on this # release while on Darwin 9 -m returns something # like "PowerPC Machine", hence the distinction - CHOST="`uname -m`-apple-darwin$rev" + CHOST="$(uname -m)-apple-darwin$rev" else - CHOST="`uname -p`-apple-darwin$rev" + CHOST="$(uname -p)-apple-darwin$rev" fi ;; SunOS) - case `uname -p` in + case $(isainfo -n) in + amd64) + CHOST="x86_64-pc-solaris$(uname -r | sed 's|5|2|')" + ;; i386) - CHOST="i386-pc-solaris`uname -r | sed 's|5|2|'`" + CHOST="i386-pc-solaris$(uname -r | sed 's|5|2|')" + ;; + sparcv9) + CHOST="sparcv9-sun-solaris$(uname -r | sed 's|5|2|')" ;; sparc) - CHOST="sparc-sun-solaris`uname -r | sed 's|5|2|'`" + CHOST="sparc-sun-solaris$(uname -r | sed 's|5|2|')" ;; esac ;; CYGWIN*) - CHOST="`uname -m`-pc-cygwin" + CHOST="$(uname -m)-pc-cygwin" ;; FreeBSD) - case `uname -m` in + case $(uname -m) in amd64) - CHOST="x86_64-pc-freebsd`uname -r | sed 's|-.*$||'`" + CHOST="x86_64-pc-freebsd$(uname -r | sed 's|-.*$||')" ;; esac ;; OpenBSD) - case `uname -m` in + case $(uname -m) in amd64) - CHOST="x86_64-pc-openbsd`uname -r | sed 's|-.*$||'`" + CHOST="x86_64-pc-openbsd$(uname -r | sed 's|-.*$||')" ;; esac ;; *) - eerror "Nothing known about platform `uname -s`." + eerror "Nothing known about platform $(uname -s)." eerror "Please set CHOST appropriately for your system" eerror "and rerun $0" exit 1 @@ -3119,19 +3553,53 @@ if [[ -z ${CHOST} ]]; then fi fi +CHOST_IDENTIFY=${CHOST} +# massage CHOST on Linux systems +if [[ ${CHOST} == *-linux-* ]] ; then + # two choices here: x86_64_ubuntu16-linux-gnu + # x86_64-pc-linux-ubuntu16 + # I choose the latter because it is compatible with most + # UNIX vendors and it allows to fit RAP into platform + dist=$(lsb_release -si) + rel=$(lsb_release -sr) + if [[ -z ${dist} ]] || [[ -z ${rel} ]] ; then + source /etc/os-release # this may fail if the file isn't there + [[ -z ${dist} ]] && dist=${ID} + [[ -z ${dist} ]] && dist=${NAME} + [[ -z ${rel} ]] && rel=${VERSION_ID} + fi + [[ -z ${dist} ]] && dist=linux + + # Gentoo's versioning isn't really relevant, since it is + # a rolling distro + if [[ ${dist,,} == "gentoo" ]] ; then + rel= + [[ ${CHOST##*-} == "musl" ]] && rel="musl" + fi + + # leave rel unset/empty if we don't know about it + while [[ ${rel} == *.*.* ]] ; do + rel=${rel%.*} + done + + platform=${CHOST#*-}; platform=${platform%%-*} + platform=$(rapx rap "${platform}") + CHOST_IDENTIFY=${CHOST%%-*}-${platform}-linux-${dist,,}${rel} +fi + # Now based on the CHOST set some required variables. Doing it here # allows for user set CHOST still to result in the appropriate variables # being set. case ${CHOST} in *-*-solaris*) if type -P gmake > /dev/null ; then - MAKE=gmake + MAKE="gmake" else - MAKE=make + MAKE="make" fi ;; *) - MAKE=make + MAKE="make" ;; esac @@ -3206,7 +3674,15 @@ case $ROOT in chost.guess) # undocumented feature that sort of is our own config.guess, if # CHOST was unset, it now contains the guessed CHOST - echo "$CHOST" + echo "${CHOST}" + exit 0 + ;; + chost.identify) + # another undocumented feature, produces a pseudo CHOST that + # identifies the system for bootstraps, currently only Linux is + # different from CHOST + + echo "${CHOST_IDENTIFY}" exit 0 ;; /*) ;; @@ -3219,10 +3695,11 @@ esac einfo "Bootstrapping Gentoo prefixed portage installation using" einfo "host: ${CHOST}" +einfo "ident: ${CHOST_IDENTIFY}" einfo "prefix: ${ROOT}" TODO=${2} -if [[ ${TODO} != "noninteractive" && $(type -t bootstrap_${TODO}) != "function" ]]; +if [[ ${TODO} != "noninteractive" && $(type -t "bootstrap_${TODO}") != "function" ]]; then eerror "bootstrap target ${TODO} unknown" exit 1 @@ -3244,18 +3721,20 @@ fi einfo "ready to bootstrap ${TODO}" -# part of bootstrap_interactive should be executed before any bootstrap_${TODO} -# to properly setup environment variables and guarantee that bootstrap_${TODO} -# executes inside a prefix -if [[ ${TODO} != "noninteractive" && $(type -t bootstrap_${TODO} == "function") ]]; then - PARTIAL_BOOTSTRAP=true - TODO='noninteractive' bootstrap_interactive || exit 1 - bootstrap_${TODO} || exit 1 - exit 0 +# When we call individual stages separately (e.g. not from +# bootstrap_interactive) we might need some env to be setup in order to +# function properly. Basically do a non-interactive call for each stage +# that will only set whatever needs to be set. +if [[ ${TODO} != "interactive" && ${TODO} != "noninteractive" ]] ; then + # squelch the output, we've seen it already when running from + # interactive proper + SETUP_ENV_ONLY=true TODO=noninteractive \ + bootstrap_interactive > /dev/null || exit 1 fi -# bootstrap_interactive proceeds with guessed defaults when TODO=noninteractive -bootstrap_${TODO#non} || exit 1 +# call the appropriate function, +# beware noninteractive is just a mode of interactive +bootstrap_"${TODO#non}" || exit 1 # Local Variables: # sh-indentation: 4 From 8cbb4cbbd59363ed2df03d2041ac25d950c4df5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 20 Dec 2024 10:29:26 +0100 Subject: [PATCH 02/28] bump version to 2025.01 --- install_compatibility_layer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh index c8cdc996..f3707c1c 100755 --- a/install_compatibility_layer.sh +++ b/install_compatibility_layer.sh @@ -10,7 +10,7 @@ REPOSITORY="software.eessi.io" RESUME= RETAIN_TMP=0 STORAGE= -VERSION=2023.06 +VERSION=2025.01 VERBOSE= display_help() { From ea89f5e5c4aa513eb3a9a63bb9eaab2b86cb3599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 20 Dec 2024 10:30:14 +0100 Subject: [PATCH 03/28] bump version to 2025.01, stick to GCC 11, use latest gentoo commit, remove openssl 3 mask --- .../roles/compatibility_layer/defaults/main.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index 58cb21ae..b17060b1 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -1,6 +1,6 @@ # Defaults file for the compatibility layer role. --- -eessi_version: "2023.06" +eessi_version: "2025.01" custom_overlays: - name: eessi @@ -16,24 +16,17 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/c gentoo_git_repo: https://github.com/gentoo/gentoo.git # Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, # e.g. by checking: https://github.com/gentoo/gentoo/commits/master -# April 17 (29492845e41ea6a0a4a9769c7e0ce287d106079b) commit is after fix for Lmod -# gentoo_git_commit: 29492845e41ea6a0a4a9769c7e0ce287d106079b -# June 8 (aab8473aa90e0287553b3348a5c5b17872df4b7b) commit that was current when fetching luaposix -gentoo_git_commit: aab8473aa90e0287553b3348a5c5b17872df4b7b +gentoo_git_commit: db21a802e879f713cdb8a80235c5982ca257b63f prefix_required_space: 15 GB prefix_user_defined_trusted_dirs: - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib" prefix_mask_packages: | - # stick to GCC 10.x; using a too recent compiler in the compat layer complicates stuff in the software layer, + # stick to GCC 11.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, # see for example https://github.com/EESSI/software-layer/issues/151 - >=sys-devel/gcc-11 - # mask OpenSSL 3.x, stick to OpenSSL 1.1.x for now to avoid problems with: - # - older versions of Rust (see https://github.com/EESSI/software-layer/issues/257) - # - older versions of cryptograhy in Python (see https://github.com/EESSI/software-layer/issues/258) - >=dev-libs/openssl-3 + >=sys-devel/gcc-12 prefix_unmask_packages: | # unmask older GCC to make it installable - =sys-devel/gcc-9* + =sys-devel/gcc-11* prefix_bootstrap_use_flags: | # only build libnss, don't build the daemon (use the one from the host) sys-auth/nss-pam-ldapd -nslcd From 257242e57fc78ca1cac5fde7f8d84d8c7a5ff6e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 20 Dec 2024 16:51:04 +0100 Subject: [PATCH 04/28] use python 3.12 --- ansible/playbooks/roles/compatibility_layer/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index b17060b1..314eb880 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -35,8 +35,8 @@ prefix_bootstrap_use_flags: | # make sure that gold linker is installed with binutils sys-devel/binutils gold # only install Python 3.11 - */* PYTHON_TARGETS: -* python3_11 - */* PYTHON_SINGLE_TARGET: -* python3_11 + */* PYTHON_TARGETS: -* python3_12 + */* PYTHON_SINGLE_TARGET: -* python3_12 prefix_use_builtin_bootstrap: false prefix_custom_bootstrap_script: local: "{{ playbook_dir }}/../../bootstrap-prefix.sh" From 472ac83764228f26572b6be13223aff5fee6cfa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 20 Dec 2024 16:52:12 +0100 Subject: [PATCH 05/28] log ansible output --- install_compatibility_layer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh index f3707c1c..977292f0 100755 --- a/install_compatibility_layer.sh +++ b/install_compatibility_layer.sh @@ -188,7 +188,7 @@ ${RUNTIME} shell ${CONTAINER} < Date: Tue, 24 Dec 2024 15:46:13 +0100 Subject: [PATCH 06/28] add comment about commit, stick to GCC 14 for now --- .../playbooks/roles/compatibility_layer/defaults/main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index 314eb880..c5d3cc19 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -16,17 +16,19 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/c gentoo_git_repo: https://github.com/gentoo/gentoo.git # Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, # e.g. by checking: https://github.com/gentoo/gentoo/commits/master +# December 20, 2024: use latest commit +# TODO: pick commit that includes new Lmod version gentoo_git_commit: db21a802e879f713cdb8a80235c5982ca257b63f prefix_required_space: 15 GB prefix_user_defined_trusted_dirs: - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib" prefix_mask_packages: | - # stick to GCC 11.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, + # stick to GCC 14.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, # see for example https://github.com/EESSI/software-layer/issues/151 - >=sys-devel/gcc-12 + >=sys-devel/gcc-15 prefix_unmask_packages: | # unmask older GCC to make it installable - =sys-devel/gcc-11* + # =sys-devel/gcc-11* prefix_bootstrap_use_flags: | # only build libnss, don't build the daemon (use the one from the host) sys-auth/nss-pam-ldapd -nslcd From 727bd4729f7a65ecc25311a9edb1d8f4ab7f13ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 2 Jan 2025 13:08:02 +0100 Subject: [PATCH 07/28] don't enable stable prefix for x86_64 --- bootstrap-prefix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-prefix.sh b/bootstrap-prefix.sh index 47fd388a..ddbc4666 100755 --- a/bootstrap-prefix.sh +++ b/bootstrap-prefix.sh @@ -3147,7 +3147,7 @@ I can limit your Prefix to use only packages keyworded for stable amd64 by default. Of course, you can still enable testing ~amd64 for the packages you want, when the need arises. EOF - [[ ${TODO} == 'noninteractive' ]] && ans=yes || + [[ ${TODO} == 'noninteractive' ]] && ans=no || read -r -p " Do you want to use stable Prefix? [Yn] " ans case "${ans}" in [Yy][Ee][Ss]|[Yy]|"") From b84779a317f8875fa9fd7a8ccc6f600754cec449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 2 Jan 2025 14:16:40 +0100 Subject: [PATCH 08/28] remove dev-util/hermes --- ansible/playbooks/roles/compatibility_layer/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index c5d3cc19..aac0a9ff 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -75,6 +75,7 @@ prefix_remove_packages: - dev-lang/rust-bin - dev-python/setuptools-rust - dev-util/cmake + - dev-util/hermes - dev-util/ninja - virtual/rust From 27b9ff8fdbb7c8623955146ede8218c54a592923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 2 Jan 2025 17:03:24 +0100 Subject: [PATCH 09/28] overlay configuration template --- .../roles/compatibility_layer/templates/overlay.conf.j2 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 diff --git a/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 b/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 new file mode 100644 index 00000000..33a7c877 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 @@ -0,0 +1,7 @@ +[{{ item.name }}] +location = {{ gentoo_prefix_path }}/var/db/repos/{{ item.name }} +sync-type = {{ item.source }} +sync-uri = {{ item.url }} +{% if item.branch is defined %} +sync-git-clone-extra-opts = --branch {{ item.branch }} +{% endif %} From 30324be3a136728418614604fd03ae02dcab1b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 2 Jan 2025 17:03:53 +0100 Subject: [PATCH 10/28] add overlay with predefined config files --- .../compatibility_layer/tasks/add_overlay.yml | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index dddbdbf8..93ffbe79 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -5,10 +5,10 @@ cmd: emerge gentoolkit creates: "{{ gentoo_prefix_path }}/usr/bin/equery" -- name: Install eselect-repository - community.general.portage: - package: eselect-repository - state: present +#- name: Install eselect-repository +# community.general.portage: +# package: eselect-repository +# state: present # We need git in order to add Gentoo overlays hosted on git repositories. - name: Install git @@ -21,10 +21,17 @@ register: repositories_installed changed_when: false -- name: Add custom overlay configuration - ansible.builtin.command: - cmd: "eselect repository add {{ item.name }} {{ item.source }} {{ item.url }}" - when: item.name not in repositories_installed.stdout +#- name: Add custom overlay configuration +# ansible.builtin.command: +# cmd: "eselect repository add {{ item.name }} {{ item.source }} {{ item.url }}" +# when: item.name not in repositories_installed.stdout +# loop: "{{ custom_overlays }}" + +- name: Add configuration files for custom overlays + ansible.builtin.template: + src: overlay.conf.j2 + dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/{{ item.name }}.conf" + mode: "0644" loop: "{{ custom_overlays }}" - name: Make configuration file with overlays that can override eclasses From d83626256e8a33d4027e9b4278171c6b04bf2534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 2 Jan 2025 17:07:03 +0100 Subject: [PATCH 11/28] add comment to top of file --- .../roles/compatibility_layer/templates/overlay.conf.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 b/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 index 33a7c877..023ef327 100644 --- a/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 +++ b/ansible/playbooks/roles/compatibility_layer/templates/overlay.conf.j2 @@ -1,3 +1,4 @@ +# {{ ansible_managed }} [{{ item.name }}] location = {{ gentoo_prefix_path }}/var/db/repos/{{ item.name }} sync-type = {{ item.source }} From 6b73f531d9329fbc05d0f44271ed2450bce173f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 2 Jan 2025 17:10:24 +0100 Subject: [PATCH 12/28] add spaces to comments --- .../compatibility_layer/tasks/add_overlay.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 93ffbe79..56562806 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -5,10 +5,10 @@ cmd: emerge gentoolkit creates: "{{ gentoo_prefix_path }}/usr/bin/equery" -#- name: Install eselect-repository -# community.general.portage: -# package: eselect-repository -# state: present +# - name: Install eselect-repository +# community.general.portage: +# package: eselect-repository +# state: present # We need git in order to add Gentoo overlays hosted on git repositories. - name: Install git @@ -21,11 +21,11 @@ register: repositories_installed changed_when: false -#- name: Add custom overlay configuration -# ansible.builtin.command: -# cmd: "eselect repository add {{ item.name }} {{ item.source }} {{ item.url }}" -# when: item.name not in repositories_installed.stdout -# loop: "{{ custom_overlays }}" +# - name: Add custom overlay configuration +# ansible.builtin.command: +# cmd: "eselect repository add {{ item.name }} {{ item.source }} {{ item.url }}" +# when: item.name not in repositories_installed.stdout +# loop: "{{ custom_overlays }}" - name: Add configuration files for custom overlays ansible.builtin.template: From ab80d4387c0f300cadfdf262c46aa8b82d55af72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 3 Jan 2025 09:40:15 +0100 Subject: [PATCH 13/28] hardcode eessi_version to 2025.01 for now --- bot/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 55024619..f4f0cd34 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -81,7 +81,8 @@ host_arch=$(uname -m) eessi_arch=${cpu_target_arch:-${host_arch}} eessi_os=linux job_version=$(cfg_get_value "repository" "repo_version") -eessi_version=${job_version:-2023.06} +#eessi_version=${job_version:-2023.06} +eessi_version=2025.01 job_repo=$(cfg_get_value "repository" "repo_name") eessi_repo=${job_repo:-software.eessi.io} tar_topdir=/cvmfs/${eessi_repo}/versions From c18fede96951fc0a878bd4a2ac5289ad27ff11e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 3 Jan 2025 09:41:25 +0100 Subject: [PATCH 14/28] use 2025.01_set branch of fork --- ansible/playbooks/roles/compatibility_layer/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index aac0a9ff..15332fee 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -5,7 +5,8 @@ eessi_version: "2025.01" custom_overlays: - name: eessi source: git - url: https://github.com/EESSI/gentoo-overlay.git + url: https://github.com/bedroge/gentoo-overlay.git + branch: 2025.01_set eclass-overrides: true cvmfs_repository: software.eessi.io From 6562419cdb2f85556b02e719e40a0c51c35812f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 3 Jan 2025 10:46:39 +0100 Subject: [PATCH 15/28] add reframe test that runs pip check --- test/compat_layer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/compat_layer.py b/test/compat_layer.py index 6ebc249c..7e8d76bc 100644 --- a/test/compat_layer.py +++ b/test/compat_layer.py @@ -231,3 +231,15 @@ def __init__(self): f'user-defined-trusted-dirs={trusted_dir}', self.stdout ) + + +@rfm.simple_test +class PipCheckTest(RunInGentooPrefixTest): + def __init__(self): + super().__init__() + self.descr = 'Verify that "pip check" does not return any errors.' + self.command = 'pip check' + self.sanity_patterns = sn.all([ + sn.assert_eq(self.exit_code, 0), + sn.assert_found('\nNo broken requirements found.\n', self.stdout), + ]) From 2018d360a73fe5aa5f8db9aadbbcebcb7ad23c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 3 Jan 2025 13:16:27 +0100 Subject: [PATCH 16/28] disable eselect repository list step --- .../roles/compatibility_layer/tasks/add_overlay.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 56562806..18a9c6e8 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -16,10 +16,10 @@ package: dev-vcs/git state: present -- name: Check which repositories have been installed - ansible.builtin.command: eselect repository list -i - register: repositories_installed - changed_when: false +#- name: Check which repositories have been installed +# ansible.builtin.command: eselect repository list -i +# register: repositories_installed +# changed_when: false # - name: Add custom overlay configuration # ansible.builtin.command: From 5bebdd8ee584ae464f445aa6966b4ea5b871cd5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 3 Jan 2025 13:18:15 +0100 Subject: [PATCH 17/28] add space --- .../roles/compatibility_layer/tasks/add_overlay.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 18a9c6e8..8ab87efc 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -16,10 +16,10 @@ package: dev-vcs/git state: present -#- name: Check which repositories have been installed -# ansible.builtin.command: eselect repository list -i -# register: repositories_installed -# changed_when: false +# - name: Check which repositories have been installed +# ansible.builtin.command: eselect repository list -i +# register: repositories_installed +# changed_when: false # - name: Add custom overlay configuration # ansible.builtin.command: From 0539b1bb8cd98e77c7185e4cb4e80c31e686ca77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 3 Jan 2025 13:24:50 +0100 Subject: [PATCH 18/28] hardcode version to 2025.01 for now --- bot/check-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/check-build.sh b/bot/check-build.sh index 6948e66b..61ca9bd5 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -89,6 +89,7 @@ eessi_arch=${cpu_target_arch:-${host_arch}} # eessi_os=linux job_version=$(cfg_get_value "repository" "repo_version") eessi_version=${job_version:-2023.09} +eessi_version=2025.01 # job_repo=$(cfg_get_value "repository" "repo_name") # eessi_repo=${job_repo:-pilot.nessi.no} # tar_topdir=/cvmfs/${eessi_repo}/versions From 930abba602bb159e096f220ba297075da2c7c6d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 3 Jan 2025 20:11:02 +0100 Subject: [PATCH 19/28] look for ld.bfd instead of ld.gold --- test/compat_layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/compat_layer.py b/test/compat_layer.py index 7e8d76bc..81675820 100644 --- a/test/compat_layer.py +++ b/test/compat_layer.py @@ -63,7 +63,7 @@ def __init__(self): @rfm.simple_test class ToolsAvailableTest(RunInGentooPrefixTest): - tool = parameter(['archspec', 'emerge', 'equery', 'ld.gold', 'make', 'patch', 'patchelf']) + tool = parameter(['archspec', 'emerge', 'equery', 'ld.bfd', 'make', 'patch', 'patchelf']) # removed ld.gold def __init__(self): # patchelf is only installed since 2021.06 compat layer From 92b619234435fda8a14ec8c916086282c1eff77c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 7 Jan 2025 11:36:49 +0100 Subject: [PATCH 20/28] try to run reframe tests with bot --- bot/check-test.sh | 238 ++++++++++++++++++++++++++++++++++++++++++++++ bot/test.sh | 143 ++++++++++++++++++++++++++++ 2 files changed, 381 insertions(+) create mode 100755 bot/check-test.sh create mode 100755 bot/test.sh diff --git a/bot/check-test.sh b/bot/check-test.sh new file mode 100755 index 00000000..2731e754 --- /dev/null +++ b/bot/check-test.sh @@ -0,0 +1,238 @@ +#!/bin/bash +# +# Dummy script that only creates test result file for the bot, without actually checking anything +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Kenneth Hoste (HPC-UGent) +# +# license: GPLv2 +# +job_dir=${PWD} +job_out="slurm-${SLURM_JOB_ID}.out" +job_test_result_file="_bot_job${SLURM_JOB_ID}.test" + +# Check that job output file is found +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" +if [[ -f ${job_out} ]]; then + SLURM_OUTPUT_FOUND=1 + [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" +else + SLURM_OUTPUT_FOUND=0 + [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" +fi + +# ReFrame prints e.g. +#[----------] start processing checks +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default +#[ FAIL ] (1/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default +#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_d597cff4' +#[ OK ] (2/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default +#P: perf: 8.441 ns/day (r:0, l:None, u:None) +#[ FAIL ] (3/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default +#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_f4194106' +#[----------] all spawned checks have finished +#[ FAILED ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted) + +# We will grep for the last and final line, since this reflects the overall result +# Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails +FAILED=-1 +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + GP_failed='\[\s*FAILED\s*\].*Ran .* test case' + grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") + [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_failed}" +fi + +# Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step +# I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests +ERROR=-1 +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + GP_error='ERROR: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") + [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + +SUCCESS=-1 +# Grep for the success pattern, so we can report the amount of tests run +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + GP_success='\[\s*PASSED\s*\].*Ran .* test case' + grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") + [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_success}" +fi + +if [[ ! -z ${grep_reframe_failed} ]]; then + grep_reframe_result=${grep_reframe_failed} +else + # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report + GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' + # Grab the full ReFrame report, than cut the irrelevant parts + # Note that the character limit for messages in github is around 65k, so cutting is important + grep_reframe_success_full=$( \ + grep -v "^>> searching for " ${job_dir}/${job_out} | \ + # Use -z + grep -Pzo "${GP_success_full}" | \ + # Replace null character with newline, to undo the -z option + sed 's/\x00/\n/g' | \ + # Remove the [ RUN ] lines from reframe, they are not very informative + grep -v -P '\[\s*RUN\s*]' | \ + # Remove the line '[----------] all spawned checks have finished' + grep -v '\[-*\]' | \ + # Remove the line '[==========] Finished on Mon Oct 7 21' + grep -v '\[=*\]' | \ + # Remove blank line(s) from the report + grep -v '^$' | \ + # Remove warnings about the local spawner not supporting memory requests + grep -v 'WARNING\: hooks\.req_memory_per_node does not support the scheduler you configured .local.*$' | \ + # Strip color coding characters + sed 's/\x1B\[[0-9;]*m//g' | \ + # Replace all newline characters with
+ sed ':a;N;$!ba;s/\n//g' | \ + # Replace % with %%. Use \%\% to interpret both %% as (non-special) characters + sed 's/\%/\%\%/g' \ + ) + # TODO (optional): we could impose a character limit here, and truncate if too long + # (though we should do that before inserting the
statements). + # If we do, we should probably re-append the final summary, e.g. + # [ PASSED ] Ran 10/10 test case(s) from 10 check(s) (0 failure(s), 0 skipped, 0 aborted) + # so that that is always displayed + # However, that's not implemented yet - let's see if this ever even becomes an issue + grep_reframe_result=${grep_reframe_success_full} +fi +echo "grep_reframe_result: ${grep_reframe_result}" + +echo "[TEST]" > ${job_test_result_file} +if [[ ${SLURM_OUTPUT_FOUND} -eq 0 ]]; then + summary=":cry: FAILURE" + reason="Job output file not found, cannot check test results." + status="FAILURE" +# Should come before general errors: if SUCCESS==1, it indicates the test suite ran succesfully +# regardless of other things that might have gone wrong +elif [[ ${SUCCESS} -eq 1 ]]; then + summary=":grin: SUCCESS" + reason="" + status="SUCCESS" +# Should come before general errors: if FAILED==1, it indicates the test suite ran +# otherwise the pattern wouldn't have been there +elif [[ ${FAILED} -eq 1 ]]; then + summary=":cry: FAILURE" + reason="EESSI test suite produced failures." + status="FAILURE" +elif [[ ${ERROR} -eq 1 ]]; then + summary=":cry: FAILURE" + reason="EESSI test suite was not run, test step itself failed to execute." + status="FAILURE" +else + summary=":cry: FAILURE" + reason="Failed for unknown reason" + status="FAILURE" +fi + + +echo "[TEST]" > ${job_test_result_file} +echo -n "comment_description = " >> ${job_test_result_file} + +# Use template for writing PR comment with details +# construct and write complete PR comment details: implements third alternative +comment_template="
__SUMMARY_FMT__
__REASON_FMT____REFRAME_FMT____DETAILS_FMT__
" +comment_success_item_fmt=":white_check_mark: __ITEM__" +comment_failure_item_fmt=":x: __ITEM__" + +# Initialize comment_description +comment_description=${comment_template} + +# Now, start replacing template items one by one +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" +comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" +comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}} + + +# Only add if there is a reason (e.g. no reason for successful runs) +if [[ ! -z ${reason} ]]; then + comment_reason_fmt="
_Reason_
__REASONS__
" + reason_details="${comment_reason_fmt/__REASONS__/${reason}}" + comment_description=${comment_description/__REASON_FMT__/${reason_details}} +else + comment_description=${comment_description/__REASON_FMT__/""} +fi + +# Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully) +echo "ReFrame result:" +echo "${grep_reframe_result}" +if [[ ! -z ${grep_reframe_result} ]]; then + comment_reframe_fmt="
_ReFrame Summary_
__REFRAME_SUMMARY__
" + reframe_summary=${comment_reframe_fmt/__REFRAME_SUMMARY__/${grep_reframe_result}} + comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}} +else + comment_description=${comment_description/__REFRAME_FMT__/""} +fi + +# Declare functions +function print_br_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}
" +} + +function success() { + format="${comment_success_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + +function failure() { + format="${comment_failure_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + +function add_detail() { + actual=${1} + expected=${2} + success_msg="${3}" + failure_msg="${4}" + if [[ ${actual} -eq ${expected} ]]; then + success "${success_msg}" + else + failure "${failure_msg}" + fi +} + +# first construct comment_details_list, abbreviated comment_details_list +# then use it to set comment_details +comment_details_list="" + +success_msg="job output file ${job_out}" +failure_msg="no job output file ${job_out}" +comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_error}" +failure_msg="found message matching ${GP_error}" +comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") + +# Add an escape character to every *, for it to be printed correctly in the comment on GitHub +GP_failed="${GP_failed//\*/\\*}" +success_msg="no message matching ""${GP_failed}""" +failure_msg="found message matching ""${GP_failed}""" +comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") + +comment_details_fmt="
_Details_
__DETAILS_LIST__
" +comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} + +# Actually writing the comment description to the result file +echo "${comment_description}" >> ${job_test_result_file} +echo "status = ${status}" >> ${job_test_result_file} + +exit 0 diff --git a/bot/test.sh b/bot/test.sh new file mode 100755 index 00000000..9996121a --- /dev/null +++ b/bot/test.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# +# Script to run tests for the whole EESSI compatibility software layer. +# Intended use is that it is called at the end of a (batch) job running on a compute node. +# +# This script is part of the EESSI compatibility layer, see +# https://github.com/EESSI/compatibility-layer.git +# +# author: Thomas Roeblitz (@trz42) +# author: Caspar van Leeuwen (@casparvl) +# author: Bob Dröge (@bedroge) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# + assumption for the build step (as run through bot/build.sh which is provided +# in this repository too) +# - working directory has been prepared by the bot with a checkout of a +# pull request (OR by some other means) +# - the working directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited +# - the directory may contain any additional files referenced in job.cfg +# + assumptions for the test step +# - temporary storage is still available +# example +# Using /tmp/bot/EESSI/eessi.7l3zm2x7qH as temporary storage... +# - run test/compat_layer.py with ReFrame inside build container using tmp storage from build step +# plus possibly additional settings (repo, etc.) + +# stop as soon as something fails +set -e + +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +HOST_ARCH=$(uname -m) + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "bot/test.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/test.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} + +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} +echo "bot/test.sh: HTTP_PROXY='${HTTP_PROXY}'" + +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} +echo "bot/test.sh: HTTPS_PROXY='${HTTPS_PROXY}'" + +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") +echo "bot/test.sh: LOCAL_TMP='${LOCAL_TMP}'" + +# try to determine tmp directory from build job +EESSI_TMPDIR=$(grep -oP "To resume work add '--resume \K.*(?=')" slurm-${SLURM_JOBID}.out) + +if [[ -z ${EESSI_TMPDIR} ]]; then + echo "bot/test.sh: no information about tmp directory build step; --> giving up" + exit 2 +fi + +# obtain list of modules to be loaded +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") +echo "bot/test.sh: LOAD_MODULES='${LOAD_MODULES}'" + +# load modules if LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/test.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/test.sh: no modules to be loaded" +fi + +cpu_target_arch=$(cfg_get_value "architecture" "software_subdir" | cut -d/ -f1) +host_arch=$(uname -m) +eessi_arch=${cpu_target_arch:-${host_arch}} +eessi_os=linux +job_version=$(cfg_get_value "repository" "repo_version") +eessi_version=${job_version:-2023.06} +job_repo=$(cfg_get_value "repository" "repo_name") +eessi_repo=${job_repo:-software.eessi.io} +tar_topdir=/cvmfs/${eessi_repo}/versions + +if [ "${eessi_arch}" != "${host_arch}" ]; then + echo "Requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!" + exit 1 +fi + +RUNTIME=$(get_container_runtime) +exit_code=$? +[[ ${VERBOSE} == '-vvv' ]] && echo "RUNTIME='${RUNTIME}'" +check_exit_code ${exit_code} "using runtime ${RUNTIME}" "oh no, neither apptainer nor singularity available" + +# Set up paths and mount points for Apptainer +if [[ -z ${APPTAINER_CACHEDIR} ]]; then + export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache + [[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_CACHEDIR='${APPTAINER_CACHEDIR}'" +fi +export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${PWD}:/compatibility-layer" +export APPTAINER_BIND="${APPTAINER_BIND},${EESSI_TMPDIR}/tmp:/tmp" +[[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_BIND='${APPTAINER_BIND}'" +export APPTAINER_HOME="${EESSI_TMPDIR}/home:/home/${USER}" +[[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_HOME='${APPTAINER_HOME}'" + +# also define SINGULARITY_* env vars +if [[ -z ${SINGULARITY_CACHEDIR} ]]; then + export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache + [[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +fi +export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${PWD}:/compatibility-layer" +export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/tmp:/tmp" +[[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" +export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" +[[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'" + +CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian11 + +${RUNTIME} shell ${CONTAINER} < Date: Tue, 7 Jan 2025 17:16:56 +0100 Subject: [PATCH 21/28] script for testing the compat layer with reframe --- test_compatibility_layer.sh | 126 ++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100755 test_compatibility_layer.sh diff --git a/test_compatibility_layer.sh b/test_compatibility_layer.sh new file mode 100755 index 00000000..4b7e9c3a --- /dev/null +++ b/test_compatibility_layer.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +EESSI_REPO_DIR=${EESSI_CVMFS_REPO:-/cvmfs/software.eessi.io} +EESSI_VERSION=${EESSI_VERSION:-2023.06} +EESSI_ARCH=${EESSI_CPU_FAMILY:-$(uname -m)} +EESSI_OS=${EESSI_OS_TYPE:-linux} + +display_help() { + echo "usage: $0 [OPTIONS]" + echo "OPTIONS:" + echo " -a | --arch ARCHITECTURE" + echo " Architecture of compatibility layer to be tested" + echo " [default: \$EESSI_CPU_FAMILY or current host's architecture]" + echo "" + echo " -g | --storage DIRECTORY" + echo " directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo "" + echo " -h | --help" + echo " display this usage information" + echo "" + echo " -o | --os" + echo " Operating system of compatibility to be tested" + echo " [default: \$EESSI_OS_TYPE or linux]" + echo "" + echo " -r | --repository REPO" + echo " CVMFS repository [default: \$EESSI_CVMFS_REPO or software.eessi.io]" + echo " Note that this has to be mounted as /cvmfs/${REPOSITORY}!" + echo "" + echo " -v | --version VERSION" + echo " version of EESSI stack to be tested [default: \$EESSI_VERSION or 2023.06]" + echo "" + echo " --verbose" + echo " increase verbosity of output [default: not set]" + echo +} + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -a|--arch) + EESSI_ARCH="$2" + shift 2 + ;; + -g|--storage) + STORAGE="$2" + shift 2 + ;; + -h|--help) + display_help + exit 0 + ;; + -o|--os) + EESSI_OS="$2" + shift 2 + ;; + -r|--repository) + EESSI_REPO_DIR="/cvmfs/$2" + shift 2 + ;; + -v|--version) + EESSI_VERSION="$2" + shift 2 + ;; + --verbose) + VERBOSE="-vvv" + shift 1 + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + + +COMPAT_LAYER_PREFIX="${EESSI_REPO_DIR}/versions/${EESSI_VERSION}/compat/${EESSI_OS}/${EESSI_ARCH}" +if [ ! -d ${COMPAT_LAYER_PREFIX} ]; then + echo "Directory ${COMPAT_LAYER_PREFIX} does not exist, please provide a correct path, version, and architecture." + exit 1 +fi +if [ ! -f ${COMPAT_LAYER_PREFIX}/startprefix ]; then + echo "Cannot find a startprefix file in ${COMPAT_LAYER_PREFIX}!" + exit 1 +fi + +# We assume that this script is located in a directory containing a full checkout of the git repo, +# and verify this by checking for the existance of the ReFrame test script. +SCRIPT_DIR=$(dirname $(realpath $0)) +if [ ! -f "${SCRIPT_DIR}/test/compat_layer.py" ]; then + echo "ReFrame test script compat_layer.py cannot be found!" + echo "Make sure to run this script from a directory containing a the compatibility-layer git repository." + exit 1 +fi + +if [ -z ${EESSI_TMPDIR} ]; +then + EESSI_TMPDIR=$(mktemp -t -d eessi.XXXXXXXXXX) +fi + +echo "Using ${EESSI_TMPDIR} as temporary directory." + +if ! command -v "reframe" &>/dev/null; then + echo "ReFrame command not found, trying to install it to a temporary directory..." + pip3 install --ignore-installed -t ${EESSI_TMPDIR}/reframe reframe-hpc + export PYTHONPATH=${EESSI_TMPDIR}/reframe + export PATH="${EESSI_TMPDIR}/reframe/bin/:${PATH}" +fi + +echo "Trying to run 'reframe --version' as sanity check..." +if ! reframe --version; then + echo "Cannot run ReFrame, giving up. Please install it manually and add it to your \$PATH." + exit 1 +fi + +echo "Running the tests with: ${EESSI_TMPDIR}/reframe/bin/reframe -r -v -c ${SCRIPT_DIR}/test/compat_layer.py" +export EESSI_REPO_DIR EESSI_VERSION EESSI_ARCH EESSI_OS +export RFM_PREFIX=$PWD/reframe_runs +${EESSI_TMPDIR}/reframe/bin/reframe --nocolor -r -v -c ${SCRIPT_DIR}/test/compat_layer.py + From 072f77dcfe6c0fe980b3f90db8a0ca496c4654cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 7 Jan 2025 17:19:30 +0100 Subject: [PATCH 22/28] use test_compatibility.sh, hardcode version number for now --- bot/test.sh | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index 9996121a..d4747d46 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -90,7 +90,8 @@ host_arch=$(uname -m) eessi_arch=${cpu_target_arch:-${host_arch}} eessi_os=linux job_version=$(cfg_get_value "repository" "repo_version") -eessi_version=${job_version:-2023.06} +#eessi_version=${job_version:-2023.06} +eessi_version=2025.01 job_repo=$(cfg_get_value "repository" "repo_name") eessi_repo=${job_repo:-software.eessi.io} tar_topdir=/cvmfs/${eessi_repo}/versions @@ -110,7 +111,7 @@ if [[ -z ${APPTAINER_CACHEDIR} ]]; then export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache [[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_CACHEDIR='${APPTAINER_CACHEDIR}'" fi -export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${PWD}:/compatibility-layer" +export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${PWD}" export APPTAINER_BIND="${APPTAINER_BIND},${EESSI_TMPDIR}/tmp:/tmp" [[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_BIND='${APPTAINER_BIND}'" export APPTAINER_HOME="${EESSI_TMPDIR}/home:/home/${USER}" @@ -121,7 +122,7 @@ if [[ -z ${SINGULARITY_CACHEDIR} ]]; then export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache [[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" fi -export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${PWD}:/compatibility-layer" +export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${PWD}" export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/tmp:/tmp" [[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" @@ -129,15 +130,6 @@ export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian11 -${RUNTIME} shell ${CONTAINER} < Date: Tue, 7 Jan 2025 17:21:17 +0100 Subject: [PATCH 23/28] use SLURM_JOB_ID instead of SLURM_JOBID --- bot/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/test.sh b/bot/test.sh index d4747d46..12966f5c 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -63,7 +63,7 @@ LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") echo "bot/test.sh: LOCAL_TMP='${LOCAL_TMP}'" # try to determine tmp directory from build job -EESSI_TMPDIR=$(grep -oP "To resume work add '--resume \K.*(?=')" slurm-${SLURM_JOBID}.out) +EESSI_TMPDIR=$(grep -oP "To resume work add '--resume \K.*(?=')" slurm-${SLURM_JOB_ID}.out) if [[ -z ${EESSI_TMPDIR} ]]; then echo "bot/test.sh: no information about tmp directory build step; --> giving up" From 4540dba876f7b45ca806518588afe0472c3c27bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 28 Jan 2025 15:34:57 +0100 Subject: [PATCH 24/28] get eessi_version from work dir --- bot/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index f4f0cd34..a53c8cb5 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -81,8 +81,6 @@ host_arch=$(uname -m) eessi_arch=${cpu_target_arch:-${host_arch}} eessi_os=linux job_version=$(cfg_get_value "repository" "repo_version") -#eessi_version=${job_version:-2023.06} -eessi_version=2025.01 job_repo=$(cfg_get_value "repository" "repo_name") eessi_repo=${job_repo:-software.eessi.io} tar_topdir=/cvmfs/${eessi_repo}/versions @@ -96,7 +94,9 @@ fi # store output in local file such that the temporary directory ${STORAGE}/eessi.XXXXXXXXXX # can be determined script_out="install_stdout.log" -./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -g ${STORAGE} -k 2>&1 | tee -a ${script_out} +./install_compatibility_layer.sh -a ${eessi_arch} -r ${eessi_repo} -g ${STORAGE} -k 2>&1 | tee -a ${script_out} + +eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) # TODO handle errors (no outfile, no tmp directory found) eessi_tmp=$(cat ${script_out} | grep 'To resume work add' | cut -f 2 -d \' | cut -f 2 -d ' ') From 73e7b8cd9b7845fbde53c9e2ddd4ccbd585c1c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 28 Jan 2025 15:37:13 +0100 Subject: [PATCH 25/28] get eessi_version from tmp dir --- bot/test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index 12966f5c..bbfb9612 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -90,11 +90,10 @@ host_arch=$(uname -m) eessi_arch=${cpu_target_arch:-${host_arch}} eessi_os=linux job_version=$(cfg_get_value "repository" "repo_version") -#eessi_version=${job_version:-2023.06} -eessi_version=2025.01 job_repo=$(cfg_get_value "repository" "repo_name") eessi_repo=${job_repo:-software.eessi.io} tar_topdir=/cvmfs/${eessi_repo}/versions +eessi_version=$(ls -1 ${EESSI_TMPDIR}/${tar_topdir}) if [ "${eessi_arch}" != "${host_arch}" ]; then echo "Requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!" From 9f4464d49eb5c92d0f1dd34a784aae842cea11da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 28 Jan 2025 15:39:11 +0100 Subject: [PATCH 26/28] just find any tarball --- bot/check-build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 61ca9bd5..35e94e79 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -88,8 +88,8 @@ host_arch=$(uname -m) eessi_arch=${cpu_target_arch:-${host_arch}} # eessi_os=linux job_version=$(cfg_get_value "repository" "repo_version") -eessi_version=${job_version:-2023.09} -eessi_version=2025.01 +e# essi_version=${job_version:-2023.09} +# eessi_version=2025.01 # job_repo=$(cfg_get_value "repository" "repo_name") # eessi_repo=${job_repo:-pilot.nessi.no} # tar_topdir=/cvmfs/${eessi_repo}/versions @@ -135,7 +135,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi found_tarballs=0 -tarballs=$(ls eessi-${eessi_version}-compat-linux-${eessi_arch}-*.tar.gz 2>&1) +tarballs=$(ls eessi-*-compat-linux-${eessi_arch}-*.tar.gz 2>&1) ec=$? [[ ${VERBOSE} -ne 0 ]] && echo "TARBALLS.ec=${ec}" if [[ ${ec} -eq 0 ]]; then From 41a9e41a42a2e852666328da83bc7d8cfcb8e83a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 28 Jan 2025 15:41:30 +0100 Subject: [PATCH 27/28] remove typo --- bot/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 35e94e79..98fab58e 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -88,7 +88,7 @@ host_arch=$(uname -m) eessi_arch=${cpu_target_arch:-${host_arch}} # eessi_os=linux job_version=$(cfg_get_value "repository" "repo_version") -e# essi_version=${job_version:-2023.09} +# eessi_version=${job_version:-2023.09} # eessi_version=2025.01 # job_repo=$(cfg_get_value "repository" "repo_name") # eessi_repo=${job_repo:-pilot.nessi.no} From cd3f7beea82b8af00747575a2d39245b7f776aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 29 Jan 2025 14:11:28 +0100 Subject: [PATCH 28/28] set eessi_version after eessi_tmp has been set --- bot/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index a53c8cb5..170d6927 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -96,10 +96,9 @@ fi script_out="install_stdout.log" ./install_compatibility_layer.sh -a ${eessi_arch} -r ${eessi_repo} -g ${STORAGE} -k 2>&1 | tee -a ${script_out} -eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) - # TODO handle errors (no outfile, no tmp directory found) eessi_tmp=$(cat ${script_out} | grep 'To resume work add' | cut -f 2 -d \' | cut -f 2 -d ' ') +eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) # create tarball -> should go into a separate script when this is supported by the bot target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz if [ -d ${eessi_tmp}${tar_topdir}/${eessi_version} ]; then