Install toolchain when starting a jail

Before this change, the toolchain was installed when a jail is created.
That approach modifies a root filesystem directory which a user might
want to use for other purposes than Poudriere. Also, it doesn't use the
latest available toolchain to build packages but a version available
when creating the jail.

In order to use the latest available toolchain and not disrupt the root
filesystem directory, install the toolchain when starting a jail.

This change also adds support to install the toolchain to build packages
for both Arm Morello and CHERI-RISC-V when using Poudriere on
CHERI-extended hardware or with the user mode.

src/share/poudriere/common.sh

@@ -2939,7 +2939,7 @@ jail_start() {
 	local name=$1
 	local ptname=$2
 	local setname=$3
-	local arch host_arch version
+	local arch host_arch os version
 	local mnt
 	local needfs="${NULLFSREF}"
 	local needkld kldpair kld kldmodname
@@ -2957,6 +2957,7 @@ jail_start() {
 	else
 		_mastermnt tomnt
 	fi
+	_jget os ${name} os || err 1 "Missing os metadata for jail"
 	_jget arch ${name} arch || err 1 "Missing arch metadata for jail"
 	get_host_arch host_arch
 	_jget mnt ${name} mnt || err 1 "Missing mnt metadata for jail"
@@ -3203,6 +3204,11 @@ jail_start() {
 	if [ -n "${RESOLV_CONF}" ]; then
 		cp -v "${RESOLV_CONF}" "${tomnt}/etc/"
 	fi
+
+	if [ "${os}" = "CheriBSD" ]; then
+		download_toolchain_from_repo
+	fi
+
 	msg "Starting jail ${MASTERNAME}"
 	jstart
 	# Safe to release the lock now as jail_runs() will block further bulks.
@@ -3730,45 +3736,46 @@ download_from_repo_check_pkg() {
 	echo "${pkgname}" >> "${output}"
 }
 
-download_hybridset_from_repo() {
-	local host_abi
-	local oldpkgname pkgname
+download_toolchain_from_repo() {
+	local arch os version
 
-	get_host_abi host_abi
+	_jget os ${JAILNAME} os || err 1 "Missing os metadata for jail"
+	[ "${os}" = "CheriBSD" ] || err 1 "Unexpected OS: ${os}"
+	_jget version ${JAILNAME} version || err 1 "Missing os metadata for jail"
+	_jget arch ${JAILNAME} arch || err 1 "Missing os metadata for jail"
 
-	if [ "${host_abi}" = "purecap" ]; then
-		msg "Hybrid ABI package fetch: bootstrapping pkg64."
-		JNETNAME="n" injail env ASSUME_ALWAYS_YES=yes \
-		    pkg64 update -f
-	else
-		msg "Hybrid ABI package fetch: bootstrapping host pkg."
-		env ASSUME_ALWAYS_YES=yes \
-		    ABI_FILE=/usr/bin/uname \
-		    IGNORE_OSVERSION=yes \
-		    PKG_DBDIR="${MASTERMNT}/var/db/pkg64" \
-		    PKG_CACHEDIR="${MASTERMNT}/var/cache/pkg64" \
-		    pkg -r "${MASTERMNT}/toolchain" update -f
+	msg "Installing toolchain for ${os} ${version} ${arch}"
+
+	cp -a "${SCRIPTPREFIX}/toolchain" "${MASTERMNT}/toolchain"
+	mkdir -p "${MASTERMNT}/toolchain/usr/share/keys/pkg"
+	cp -a "${MASTERMNT}/usr/share/keys/pkg/trusted" \
+	    "${MASTERMNT}/toolchain/usr/share/keys/pkg/trusted"
+
+	case "${arch#*.}" in
+	aarch64*)
+		toolchain="llvm-morello"
+		;;
+	riscv64*)
+		toolchain="llvm-cheri"
+		;;
+	*)
+		err 1 "Unexpected architecture: ${arch}"
+		;;
+	esac
+	hybridset_pkgcmd "${MASTERMNT}" "/toolchain" install -q "${toolchain}"
+	if [ $? -ne 0 ]; then
+		err 1 "Failed to install llvm-morello"
 	fi
+	hybridset_pkgcmd "${MASTERMNT}" "/toolchain" clean -aq
+}
+
+download_hybridset_from_repo() {
+	local oldpkgname pkgname
 
 	hybridset_list | while mapfile_read_loop_redir pkgname; do
 		msg "Hybrid ABI package fetch: installing ${pkgname}."
-		if [ "${host_abi}" = "purecap" ]; then
-			JNETNAME="n" injail env ASSUME_ALWAYS_YES=yes \
-			    pkg64 install "${pkgname}"
-		else
-			# Install replacement packages from pkg.CheriBSD.org
-			# that are compiled for the host and will be installed
-			# in /usr/local64.
-			env ASSUME_ALWAYS_YES=yes \
-			    ABI_FILE=/usr/bin/uname \
-			    IGNORE_OSVERSION=yes \
-			    PKG_DBDIR="${MASTERMNT}/var/db/pkg64" \
-			    PKG_CACHEDIR="${MASTERMNT}/var/cache/pkg64" \
-			    pkg \
-			    -r "${MASTERMNT}" \
-			    -R "${MASTERMNT}/etc/pkg64" \
-			    install "${pkgname}"
-		fi
+
+		hybridset_pkgcmd "${MASTERMNT}" "/" install -q "${pkgname}"
 	done
 
 	# Regenerate hints files for installed packages.

src/share/poudriere/include/hybridset.sh

@@ -68,3 +68,31 @@ hybridset_list() {
 
 	(cd "${MASTER_DATADIR_ABS}" && find "hybridset" -type d -depth 2 | cut -d / -f 3 | sort -u)
 }
+
+hybridset_pkgcmd() {
+	[ $# -ge 2 ] || eargs hybridset_pkgcmd rootdir pkgrootdir
+	local rootdir="$1"
+	local pkgrootdir="$2"
+	shift 2
+	local host_abi
+	local pkgcmd
+
+	get_host_abi host_abi
+	if [ "${host_abi}" = "purecap" ]; then
+		pkgcmd="pkg64"
+		abifile="/usr/sbin/pkg64"
+	else
+		pkgcmd="pkg"
+		abifile="/usr/bin/uname"
+	fi
+
+	env ABI_FILE="${abifile}" \
+	    IGNORE_OSVERSION=yes \
+	    PKG_DBDIR="${rootdir}${pkgrootdir}/var/db/pkg64" \
+	    PKG_CACHEDIR="${rootdir}${pkgrootdir}/var/cache/pkg64" \
+	    ASSUME_ALWAYS_YES=yes \
+	    ${pkgcmd} \
+	    -R "${rootdir}/etc/pkg64" \
+	    -r "${rootdir}${pkgrootdir}" \
+	    "${@}"
+}

src/share/poudriere/jail.sh

@@ -867,21 +867,6 @@ install_from_tar() {
 	build_native_xtools
 }
 
-install_toolchain() {
-	if [ "${OS}" = "CheriBSD" ]; then
-		msg "Installing toolchain for ${OS} ${VERSION} ${ARCH}"
-		cp -a "${SCRIPTPREFIX}/toolchain" "${JAILMNT}/toolchain"
-		mkdir -p "${JAILMNT}/toolchain/usr/share/keys/pkg"
-		cp -a "${JAILMNT}/usr/share/keys/pkg/trusted" \
-		    "${JAILMNT}/toolchain/usr/share/keys/pkg/trusted"
-		pkg64 -r "${JAILMNT}/toolchain" install -qy llvm-base
-		if [ $? -ne 0 ]; then
-			err 1 "Failed to install llvm-base"
-		fi
-		sudo pkg64 -r "${JAILMNT}/toolchain" clean -aqy
-	fi
-}
-
 create_jail() {
 	local IFS
 
@@ -1028,10 +1013,6 @@ create_jail() {
 
 	setup_rc_conf
 
-	if [ "${METHOD}" != "null" ]; then
-		install_toolchain
-	fi
-
 	if [ -r "${SRC_BASE:?}/sys/conf/newvers.sh" ]; then
 		RELEASE=$(update_version "${version_extra}")
 	else