deploy.lib

# vim: ft=sh:

# # A library for deployment via Spack
#
# This library assumes the following environmental variables:
#
# * `DEPLOYMENT_ROOT` for the installation directory
# * `DEPLOYMENT_DATA` containing tarballs of proprietary software
# * `DEPLOYMENT_DATE` to force a date for the installation directory

DEFAULT_DEPLOYMENT_DATE=${DEFAULT_DEPLOYMENT_DATE:-$(date +%Y-%m-%d)}

BASE_GCC_VERSION=${DEFAULT_BASE_GCC_VERSION:-9.3.0}

# A list of stages in the order they will be built
stages="compilers externals libraries applications"

# A list of all stage names ever used, to be used for module creation
historic_stages="compilers tools externals external-libraries serial-libraries parallel-libraries libraries applications"

# Detect installed compilers and allow future spack stages to use them
#
# Note that *all* compilers need to be mentioned in the
# `configure_compilers` function, otherwise they will not be detected!
declare -A detect_compilers=([compilers]=yes
                             [externals]=yes
                             [libraries]=no
                             [applications]=no)

# Export installed software as packages to be re-used later
# Allowed values: yes/no
#
# Rationale:
# * compilers end up in their configuration file
# * tools + serial libraries should be exported
# * for serial, parallel libraries: use in chains (see below) to get
#   dependency DAG
declare -A export_packages=([compilers]=no
                            [externals]=yes
                            [libraries]=no
                            [applications]=no)

# Re-use installed software via the chain mechanism
# Allowed values: yes/no
declare -A include_in_chain=([compilers]=yes
                             [externals]=no
                             [libraries]=yes
                             [applications]=yes)

# Set up the dependency graph
declare -A spec_parentage
last=""
for stage in $stages; do
    if [[ -n "$last" ]]; then
        spec_parentage[$stage]="$last"
    fi
    last="$stage"
done

if [[ "x${TERM:-dumb}" == "xdumb" ]]; then
    UL_START=
    UL_END=
    B_START=
    B_END=
else
    UL_START=$(tput smul)
    UL_END=$(tput rmul)
    B_START=$(tput bold)
    B_END=$(tput sgr0)
fi

log() {
    date="${UL_START}$(date +%H:%M:%S)${UL_END}"
    echo "${B_START}### ${date} $@${B_END}" >&2
}

install_dir_raw() {
    # Create an installation directory name based on the environment
    # variables set.
    what=$1
    date="${DEPLOYMENT_DATE:-${DEFAULT_DEPLOYMENT_DATE}}"
    name="${DEPLOYMENT_ROOT}/deploy/${what}/${date}"
    echo "${name}"
}

install_dir_name() {
    # Create an installation directory name based on the environment
    # variables set. Resovles any symlinks.
    what=$1
    name=$(install_dir_raw ${what})
    if [[ -L "${name}" ]]; then
        echo "$(readlink -f ${name})"
    else
        echo "${name}"
    fi
}

set_latest() {
    # Set the "latest" symlink to the current deployment date
    what=$1
    source="$(install_dir_name ${what})"
    local OLD_DEPLOYMENT_DATE="${DEPLOYMENT_DATE}"
    DEPLOYMENT_DATE="latest"
    target="$(install_dir_raw ${what})"
    DEPLOYMENT_DATE="${OLD_DEPLOYMENT_DATE}"
    if [[ -e "${target}" && ! -L "${target}" ]]; then
        log "link target is not a symlink: ${target}"
        return 1
    elif [[ "${source}" = "${target}" ]]; then
        log "cannot link to itself: ${source}"
        log "returning!"
        return 0
    elif [[ ! -d "${source}" ]]; then
        log "link source is not a directory: ${source}"
        return 1
    fi
    log "creating symlink for ${what}"
    log "...from ${source}"
    log "...to   ${target}"
    rm -f "${target}"
    ln -s "${source}" "${target}"
}

install_dir() {
    # Create an installation directory based on the environment variables
    # set.
    #
    # When DEPLOYMENT_DATE is set to "latest" and the corresponding symlink
    # does not resolve/is not existing, find the last installation
    # directory and symlink it to "latest".
    what=$1
    date="${DEPLOYMENT_DATE:-${DEFAULT_DEPLOYMENT_DATE}}"
    name="$(install_dir_name ${what})"
    if [[ ! -e "${name}" && "${date}" = "latest" ]]; then
        log "creating symlink for ${date}"
        target=$(last_install_dir ${what})
        if [[ ! -d "${target}" ]]; then
            DEPLOYMENT_DATE=${DEFAULT_DEPLOYMENT_DATE}
            target="$(install_dir_name ${what})"
            log "...creating ${target}"
            mkdir -p "${target}"
        fi
        log "...from ${target}"
        log "...to   ${name}"
        ln -sf "${target}" "${name}"
    fi

    if [[ -L "${name}" ]]; then
        echo "$(readlink -f ${name})"
    else
        echo "${name}"
    fi
}

last_install_dir() {
    # Obtain the installation directory of a parental stage, i.e.,
    # compilers when building tools. Based on some assumptions:
    #
    # 1. Attempt to use the globally set `DEPLOYMENT_DATE` or default via
    #    `install_dir`
    # 2. Otherwise, use the latest directory present
    what=$1
    name="$(install_dir_name ${what})"
    parent="$(dirname ${name})"
    if [[ ! -d "${name}" && -d "${parent}" ]]; then
        name=$(find "${parent}" -mindepth 1 -maxdepth 1 -type d|sort|tail -n1)
    fi
    echo "$(readlink -f ${name})"
}

list_module_paths() {
    # Print all module paths up to and including the stage passed as first
    # argument
    local what="$1"
    for stage in ${stages}; do
        local base="$(last_install_dir ${stage})"
        for modpath in ${base}/modules/tcl/*; do
            if [[ -d "${modpath}" ]]; then
                echo "${modpath}"
            fi
        done

        if [[ "${stage}" = "${what}" ]]; then
            return
        fi
    done
}

list_module_archive_paths() {
    # Print all module paths up to and including the stage passed as first
    # argument. Module paths are taken from the archive location.
    local what="$1"
    for stage in ${stages}; do
        local moddir="${DEPLOYMENT_ROOT}/modules/${stage}"
        if [[ -d "${moddir}" ]]; then
            local base="$(find ${moddir} -maxdepth 1 -type d -regex '.*/[0-9]+-[0-9]+'|sort|tail -n 1)"
            for modpath in ${base}/tcl/*; do
                if [[ -d "${modpath}" ]]; then
                    echo "${modpath}"
                fi
            done
        fi

        if [[ "${stage}" = "${what}" ]]; then
            return
        fi
    done
}

update_module_path() {
    # Export MODULEPATH including all stages up to and excluding the one
    # passed as first argument
    local what="$1"
    local include_self="${2:-no}"
    local modpaths=$(list_module_paths ${what})

    for modpath in ${modpaths}; do
        if [[ "${modpath}" = *${what}* && "${include_self}" != "yes" ]]; then
            return
        fi
        grep -oF "${modpath}" <<< ${MODULEPATH} || {
            log "prepending to MODULEPATH: ${modpath}"
            MODULEPATH="${modpath}${MODULEPATH:+:}${MODULEPATH}"
            export MODULEPATH
        }
        if [[ "${modpath}" = *${what}* ]]; then
            return
        fi
    done
}

update_chain_config() {
    local what="$1"
    local add="${2:-}"
    local suffix="${3:-}"

    conf="${HOME}/.spack/upstreams.yaml"
    ckey="^upstreams:"

    chains=""
    for stage in ${stages}; do
        if [[ "${stage}" = "${what}" ]]; then
            break
        fi
        if [[ "${include_in_chain[${stage}]}" = "yes" ]]; then
            chains="${stage}${chains:+ }${chains}"
        else
            chains=""
        fi
    done
    chains="${add}${add:+ }${chains}"

    if [[ -n "${chains}" ]]; then
        grep -e ${ckey} ${conf} &> /dev/null || {
            echo "upstreams:" >> "${conf}"
        }
        log "chaining up ${what}"
        for stage in ${chains}; do
            log "...adding ${stage}"
            cat << EOF >> "${conf}"
  ${stage}${suffix}:
    install_tree: $(last_install_dir ${stage})
    modules:
      tcl: $(last_install_dir ${stage})/modules/tcl
EOF
        done
    fi
}

configure_compilers() {
    # Add modules for compilers themselves. Should be called after modules
    # have been generated!
    local what="$1"
    update_module_path "${what}" "yes"
    while read -r line; do
        sha="${line%% *}"  # Remove everything after and including the first space
        spec="${line#* }"  # Remove everything up to and including the first space
        set +o nounset


        # Skip all but compilers. Compiler variants need to be specified
        # *after* the version specification!
        case "${spec}" in
            gcc@*)
                ;;
            intel@*)
                ;;
            intel-parallel-studio@*)
                ;;
            llvm@*)
                ;;
            nvhpc@*)
                ;;
            pgi@*)
                ;;
            *)
                continue
                ;;
        esac

        # When a compiler is compiled with *anything but the base
        # compiler*, it's module will not be generated in the initial
        # module refresh
        #
        # As the input should be sorted by compilers, the module generation
        # will now find the compilers that were used when bootstrapping.
        log "...processing ${line}"
        cmd=$(spack module tcl loads ${sha}|tail -n 1)
        echo "${cmd}"
        ${cmd}
        set -o nounset
        if [[ ${spec} != *"intel-parallel-studio"* ]]; then
            spack compiler find --scope=user
        fi

        GCC_DIR=$(spack location --install-dir gcc@${BASE_GCC_VERSION})
        log "...using gcc ${BASE_GCC_VERSION} from ${GCC_DIR}"
        if [[ ${spec} = *"intel"* ]]; then
            # update intel modules to use newer gcc in .cfg files
            INTEL_DIR=$(spack location --install-dir ${sha})
            if [[ "${INTEL_DIR}" != "${SPACK_INSTALL_PREFIX}"* ]]; then
                log "...installation does not match install prefix, skipping modification of files"
            fi
            for f in $(find ${INTEL_DIR} -name "icc.cfg" -o -name "icpc.cfg" -o -name "ifort.cfg"); do
                if ! grep -q "${GCC_DIR}" $f; then
                    echo "-gcc-name=${GCC_DIR}/bin/gcc" >> ${f}
                    echo "-Xlinker -rpath=${GCC_DIR}/lib" >> ${f}
                    echo "-Xlinker -rpath=${GCC_DIR}/lib64" >> ${f}
                    log "updated ${f} with newer GCC"
                fi
            done
        elif [[ ${line} = *"pgi"* || ${line} = *"nvhpc"* ]]; then
            #update pgi modules for network installation
            PGI_DIR=$(dirname $(which makelocalrc))
            PGI_TMPDIR=$(mktemp -d -p .)
            makelocalrc ${PGI_DIR} -d "${PGI_TMPDIR}" -gcc ${GCC_DIR}/bin/gcc -gpp ${GCC_DIR}/bin/g++ -g77 ${GCC_DIR}/bin/gfortran -x -net
            #configure pgi network license
            template=$(ls -rt "${PGI_TMPDIR}"/localrc* | tail -n 1)
            echo "set PREOPTIONS=-D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1;" >> "${template}"
            log "NVIDIA localrc template"
            cat "${template}"
            gpu_nodes=$(sinfo -h -N -o %n\ %f|awk '/volta/ {if (!c[$1]) {print $1}; c[$1]=1}')
            jenkins_nodes=$(squeue -h -o %N\ %j -u bbprelman|awk '/ BB5-[0-9]+$/ {print $1}')
            if [[ "${PGI_DIR}" != "${SPACK_INSTALL_PREFIX}"* ]]; then
                log "...installation does not match install prefix, skipping modification of files"
            fi
            for node in bbpv1 bbpv2 bbptadm tds03 tds04 ${gpu_nodes} ${jenkins_nodes}; do
                cp $template $PGI_DIR/localrc.$node || true
            done
            rm -rf "${PGI_TMPDIR}"
        fi
        cmd=${cmd/load/unload}
        echo "${cmd}"
        ${cmd}
    done

    sed  -i 's#.*f\(77\|c\): null#      f\1: /usr/bin/gfortran#' ${HOME}/.spack/compilers.yaml
}

setup_mirror() {
    what="$1"
    log "adding mirrors"
    if [[ "${what}" = "compilers" ]]; then
        if [[ -z "${SPACK_PROPRIETARY_MIRROR_DIR:+x}" ]]; then
            log "...need to have \$SPACK_PROPRIETARY_MIRROR_DIR set!"
            return 1
        fi
        log "...source mirror: ${SPACK_PROPRIETARY_MIRROR_DIR}"
        spack mirror add --scope=user my_proprietary_mirror file://${SPACK_PROPRIETARY_MIRROR_DIR} || log "proprietary mirror already added!"
    fi

    if [[ "${SOURCECACHE:-x}" = "true" ]]; then
        if [[ -z "${SPACK_SOURCE_MIRROR_DIR:+x}" ]]; then
            log "...need to have \$SPACK_SOURCE_MIRROR_DIR set!"
            return 1
        fi
        log "...source mirror: ${SPACK_SOURCE_MIRROR_DIR}"
        spack mirror add --scope=user my_source_mirror file://${SPACK_SOURCE_MIRROR_DIR} || log "source mirror already added!"
    fi

    if [[ "${BUILDCACHE:-x}" = "true" ]]; then
        if [[ -z "${SPACK_BINARY_MIRROR_DIR:+x}" ]]; then
            log "need to have \$SPACK_BINARY_MIRROR_DIR set!"
            return 1
        fi
        log "...binary mirror: ${SPACK_BINARY_MIRROR_DIR}"
        spack mirror add --scope=user my_binary_mirror file://${SPACK_BINARY_MIRROR_DIR} || log "binary mirror already added!"
    fi
}

populate_build_cache() {
    start_date="$1"

    local created cached spec

    if [[ "${BUILDCACHE:-x}" != "true" ]]; then
        return
    fi

    log "gpg keys available"
    spack gpg list
    log "creating build caches"
    log "...fetching existing cache"
    cached=$(spack buildcache list -l|awk '!/^--|^==/ {print $1}')
    log "...determining new installations"
    created=$(spack find -l --start-date "${start_date}"|awk '!/^--|^==/ {print $1}')
    for checksum in ${created}; do
        spec=$(spack find /${checksum}|tail -n 1)
        if [[ "${cached}" = *${checksum}* ]]; then
            log "...already in build cache: ${spec}"
        else
            log "...caching ${spec}"
            spack buildcache create --only package -d "${HOME}" "/${checksum}" || true
        fi
    done

    if [[ ! -d "${HOME}/build_cache" ]]; then
        log "nothing in build cache"
        return
    fi

    log "cleaning up failed build caches"
    garbage=$(find "${HOME}/build_cache" -name "*.tar.gz")
    for tarball in ${garbage}; do
        if [[ ! -f "${tarball%%.tar.gz}.spack" ]]; then
            log "...cleaning $(basename ${tarball})"
            rm -f "${tarball}"
            rm -f "${HOME}/build_cache/$(basename ${tarball%%.tar.gz}).spec.yaml"
        fi
    done

    log "syncing build cache to mirror"
    log "...from ${HOME}/build_cache"
    log "...to   ${SPACK_BINARY_MIRROR_DIR}"
    mkdir -p "${SPACK_BINARY_MIRROR_DIR}"
    rsync -av "${HOME}/build_cache" "${SPACK_BINARY_MIRROR_DIR}"
}

populate_mirror() {
    what=$1
    shift
    spec_list="$*"

    if [[ "${what}" = "compilers" ]]; then
        log "populating mirror with proprietary compilers"
        for compiler in intel intel-parallel-studio pgi nvhpc; do
            mkdir -p ${SPACK_PROPRIETARY_MIRROR_DIR}/${compiler}
            cp ${DEPLOYMENT_DATA}/${compiler}/* ${SPACK_PROPRIETARY_MIRROR_DIR}/${compiler}/
        done
    fi

    if [[ "${SOURCECACHE:-x}" != "true" ]]; then
        return
    fi

    log "populating mirror for ${what}"

    if [[ -z "${spec_list}" ]]; then
        log "...found no new packages"
        return 0
    fi

    log "found the following specs"
    echo "${spec_list}"
    spack -d mirror create -D -d ${SPACK_SOURCE_MIRROR_DIR} ${spec_list}
}

copy_configuration() {
    what="$1"

    log "copying configuration"
    log "...into ${HOME}"
    rm -rf "${HOME}/.spack"
    mkdir -p "${HOME}/.spack"
    cp configs/*.yaml "${HOME}/.spack"

    if [[ ${spec_parentage[${what}]+_} ]]; then
        parent="${spec_parentage[$what]}"
        pdir="$(last_install_dir ${parent})"
        log "...using configuration output of ${parent}"
        cp "${pdir}/data/packages.yaml" "${HOME}/.spack"
        cp "${pdir}/data/compilers.yaml" "${HOME}/.spack"
    fi

    if [[ -d "configs/${what}" ]]; then
        log "...using specialized configuration files: $(ls configs/${what})"
        cp configs/${what}/*.yaml "${HOME}/.spack"
    fi

    log "...copying gpg setup"
    mkdir -p ${DEPLOYMENT_ROOT}/spack/opt/spack
    cp -r gpg "${DEPLOYMENT_ROOT}/spack/opt/spack"
    chmod 700 "${DEPLOYMENT_ROOT}/spack/opt/spack/gpg"
    chmod 600 "${DEPLOYMENT_ROOT}/spack/opt/spack/gpg/"*
}

# Outputs the contents of a module file to set the module path
# corresponding to a software deployment "release".
generate_archive_module() {
    time="${1}"

    modulepaths=""
    for stage in ${historic_stages}; do
        expansion="${DEPLOYMENT_ROOT}/modules/${stage}/${time}/tcl/*"
        paths="$(echo ${expansion})"
        if [ "${paths}" != "${expansion}" ]; then
            modulepaths="${modulepaths}${modulepaths:+ }${paths}"
        fi
    done

    # Did not find any paths, return
    if [ -z "${modulepaths}" ]; then
        return
    fi

    cat <<-EOF
	#%Module1.0

	conflict archive
	conflict unstable

EOF

    for path in ${modulepaths}; do
        if [ -d "${path}" ]; then
            echo "append-path MODULEPATH \"${path}\""
        fi
    done
}

generate_archive_modules() {
    local root="${1}"

    mkdir -p "${root}/archive"

    for year in $(seq 2019 $(date +%Y)); do
        for month in $(seq 1 12); do
            time="${year}-$(printf '%02d' ${month})"

            if [ "${time}" = "$(date +%Y-%m)" ]; then
                module="unstable"
            else
                module="archive/${time}"
                if [ -f "${root}/${module}" ]; then
                    log "...skipping ${module}"
                    continue
                fi
            fi

            contents=$(generate_archive_module ${time})

            if [ -n "${contents}" ]; then
                echo "${contents}" > "${root}/${module}"
            fi

            if [ "${module}" = "unstable" ]; then
                break
            fi
        done
    done
}

copy_user_configuration() {
    what="$1"

    local source="$(install_dir ${what})/data"
    local target="${DEPLOYMENT_ROOT}/config/"

    local root="${DEPLOYMENT_ROOT}/modules/all"

    log "regenerating archive modules"
    generate_archive_modules "${root}"

    log "copying configuration"
    log "...from ${source}"
    log "...into ${target}"

    mkdir -p "${target}"
    cp ${source}/{compilers,.spack/upstreams,packages}.yaml ${target}

    rm -f ${target}/modules.{,c}sh
    echo "export MODULEPATH=\"\${MODULEPATH}\${MODULEPATH:+:}${root}\"" > ${target}/modules.sh
    cat <<EOF > ${target}/modules.csh
if ( \$?MODULEPATH ) then
    setenv MODULEPATH \${MODULEPATH}:${root}
else
    setenv MODULEPATH ${root}
endif
EOF

    ./modules.rb > "${target}/modules.yaml"

    cat << EOF > "${target}/config.yaml"
config:
  install_tree:
    projections:
      all: \${ARCHITECTURE}/\${COMPILERNAME}-\${COMPILERVER}/\${PACKAGE}-\${VERSION}-\${HASH:6}
  build_jobs: 36
EOF
}

copy_modules() {
    what="$1"

    local source="$(install_dir ${what})/modules"
    local target="${DEPLOYMENT_ROOT}/modules/${what}/$(date +%Y-%m)"

    log "copying module files"
    log "...from ${source}"
    log "...into ${target}"
    mkdir -p ${target}
    rsync -av "${source}/" "${target}"
}

bogus_packages() {
    for toplevel in $(ls "${1}"); do
        if [ "${toplevel}" = "modules" ]; then
            continue
        elif [ "${toplevel}" = "data" ]; then
            continue
        fi
        find "${SPACK_INSTALL_PREFIX}/${toplevel}" -mindepth 2 -maxdepth 2 -type d | while read dirname; do
            [ -z "$(ls ${dirname})" ] && [ -f "${dirname}/.spack/spec.yaml" ] && {
                pkgname=$(sed -ne '/spec:/ { n; s/^- //; s/:$//; p }' ${dirname}/.spack/spec.yaml)
                pkgdir=$(find ${dirname}/.spack -name ${pkgname} -type d)

                log "...found empty installation directory for $(basename ${dirname})"
                if [ -f "${pkgdir}/package.py" ]; then
                    [ -z "$(grep Bundle ${pkgdir}/package.py)" ] && {
                        echo ${dirname}
                    }
                else
                    echo ${dirname}
                fi
            }
        done
    done
}

install_specs() {
    what="$1"

    location="$(install_dir ${what})"
    HOME="${location}/data"
    SPACK_INSTALL_PREFIX="${location}"
    export HOME SPACK_INSTALL_PREFIX
    mkdir -p "${HOME}"

    cat > "${HOME}/.profile" <<EOF
. ${DEPLOYMENT_ROOT}/spack/share/spack/setup-env.sh

SPACK_INSTALL_PREFIX="${location}"

export SPACK_INSTALL_PREFIX
EOF

    copy_configuration "${what}"
    update_chain_config "${what}"
    update_module_path "${what}"

    if [[ "${DEPLOYMENT_UPSTREAM:-${DEPLOYMENT_ROOT}}" != "${DEPLOYMENT_ROOT}" ]]; then
        local OLD_DEPLOYMENT_ROOT="${DEPLOYMENT_ROOT}"
        DEPLOYMENT_ROOT="${DEPLOYMENT_UPSTREAM}"
        export DEPLOYMENT_ROOT
        log "setting up upstream chains, modules"
        update_chain_config "${what}" "${what}" "-upstream"
        update_module_path "${what}" "yes"
        DEPLOYMENT_ROOT="${OLD_DEPLOYMENT_ROOT}"
        export DEPLOYMENT_ROOT
    fi

    # This directory may fail intel builds, pre-emptively remove it.
    rm -rf "${HOME}/intel/.pset"

    log "sourcing spack environment"
    . ${DEPLOYMENT_ROOT}/spack/share/spack/setup-env.sh
    env &> "${HOME}/spack.env"
    (cd "${DEPLOYMENT_ROOT}/spack" && git rev-parse HEAD) &> "${HOME}/spack.version"

    log "stage directory:"
    ls -l "${SPACK_ROOT}/var/spack/stage" || true
    log "stage directory in depth:"
    find "${SPACK_ROOT}/var/spack/stage" || true
    log "cleaning up after a bad spack"
    spack clean -a

    log "cleaning failed installations"
    bogus_packages "${SPACK_INSTALL_PREFIX}" | while read bogus_package; do
        log "...removing potentially failed installation of $(basename ${bogus_package})"
        rm -rf "${bogus_package}"
    done

    log "reindexing software"
    spack reindex

    setup_mirror "${what}"

    log "starting deployment"
    install_start=$(date "+%Y-%m-%d %H:%M:%S")
    rm -f "${HOME}/"stack*.xml
    rm -f "${HOME}/specs.txt"
    export PATH="${SPACK_ROOT}/deploy/bin:${PATH}"
    # deploy -e "${HOME}/build_environment" -j 12 "${SPACK_ROOT}/deploy/environments/${what}.yaml" "${HOME}"
    mkdir -p "${HOME}/build_environment"
    cp "${SPACK_ROOT}/deploy/environments/${what}.yaml" "${HOME}/build_environment/spack.yaml"
    log "concretizing environment"
    spack -D "${HOME}/build_environment" concretize -f
    log "installing environment"
    srun -N${BUILD_NODES:-1} --ntasks-per-node 4 -c12 -t24:0:0 -Aproj16 -Ccpu -pprod -ppre_prod -J"spack!" slurmspack "${HOME}/build_environment"

    cat > "${HOME}/stack.xml" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
</testsuites>
EOF

    installed-hashes -e "${HOME}/build_environment" > "${HOME}/specs.txt"

    populate_build_cache "${install_start}"

    mkdir -p "${WORKSPACE:-.}/stacks/${what}"
    rm -rf "${WORKSPACE:-.}/stacks/${what}/"*.xml
    cp "${HOME}"/stack*.xml "${WORKSPACE:-.}/stacks/${what}"

    log "refreshing modules"
    shas="$(cut -d" " -f1 ${HOME}/specs.txt)"
    spack module tcl refresh -y --delete-tree --upstream-modules ${shas}

    . ${DEPLOYMENT_ROOT}/spack/share/spack/setup-env.sh

    if [[ -f "${HOME}/packages.yaml" ]]; then
        cp "${HOME}/packages.yaml" "${HOME}/packages.yaml.old"
    fi

    if [[ "${export_packages[${what}]}" = "yes" ]]; then
        log "augmenting packages.yaml"
        spack export --scope=user --module tcl --explicit --exclude 'allinea|py-.*|neuron|hdf5|intel-parallel-studio|pgi|nvhpc' ${shas} > "${HOME}/packages.yaml"
    else
        log "copying packages.yaml"
        cp "${HOME}/.spack/packages.yaml" "${HOME}"
    fi

    if [[ -f "${HOME}/packages.yaml.old" ]]; then
        log "differences to old packages.yaml"
        diff -u "${HOME}/packages.yaml.old" "${HOME}/packages.yaml" || true
    fi

    if [[ "${detect_compilers[${what}]}" = "yes" ]]; then
        # When adding the compilers, first add everything compiled with the
        # system compiler, then other compilers bootstrapped with new
        # compilers (LLVM, I'm looking at you)
        #
        # Sort by the compiler spec part to achieve this.
        log "adding compilers"
        configure_compilers "${what}" < "${HOME}/specs.txt"
    fi

    cp "${HOME}/.spack/compilers.yaml" "${HOME}" || true

    log "cleaning up after a bad spack"
    spack clean -a
}