Skip to content

Commit

Permalink
Merge pull request #421 from perftool-incubator/dev-kmr
Browse files Browse the repository at this point in the history
update crucble-install.sh to natively create ${CRUCIBLE_HOME}/config/registries.json
  • Loading branch information
k-rister authored Nov 8, 2024
2 parents f5cec70 + afae46b commit 07e312f
Show file tree
Hide file tree
Showing 4 changed files with 280 additions and 52 deletions.
74 changes: 74 additions & 0 deletions bin/_registries.lib
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/usr/bin/env bash
# -*- mode: sh; indent-tabs-mode: nil; sh-basic-offset: 4 -*-
# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=bash

# create a registries.json
function create_registries_json() {
local CONTROLLER_IMAGE
CONTROLLER_IMAGE=$1
local ENGINES_REPO
ENGINES_REPO=$2
local ENGINES_REPO_AUTH_TOKEN
ENGINES_REPO_AUTH_TOKEN=$3
local ENGINES_REPO_TLS_VERIFY
ENGINES_REPO_TLS_VERIFY=$4

if [ -z "${REGISTRIES_CFG}" ]; then
exit_error "ERROR: \%REGISTRIES_CFG must be defined when calling $0"
fi

# create an empty JSON file that jq will be able to add to
echo "{}" > ${REGISTRIES_CFG}

# populate the new JSON file with the information from
# /etc/sysconfig/crucible
CONTROLLER_URL=$(echo ${CONTROLLER_IMAGE} | awk -F: '{ print $1 }')
CONTROLLER_TAG=$(echo ${CONTROLLER_IMAGE} | awk -F: '{ print $2 }')
jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-controller" \
--arg CONTROLLER_URL "${CONTROLLER_URL}" \
--arg CONTROLLER_TAG "${CONTROLLER_TAG}" \
'. += { "controller": { "url": $CONTROLLER_URL, "tag": $CONTROLLER_TAG } }'

jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-engines-public" \
--arg ENGINES_PUBLIC_URL "${ENGINES_REPO}" \
'. += { "engines": { "public": { "url": $ENGINES_PUBLIC_URL } } }'

if [ -n "${ENGINES_REPO_AUTH_TOKEN}" ]; then
jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-engines-public-push-token" \
--arg ENGINES_PUBLIC_PUSH_TOKEN "${ENGINES_REPO_AUTH_TOKEN}" \
'.engines.public += { "push-token": $ENGINES_PUBLIC_PUSH_TOKEN }'
fi

if [ -n "${ENGINES_REPO_TLS_VERIFY}" ]; then
jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-engines-public-tls-verify" \
--argjson ENGINES_PUBLIC_TLS_VERIFY "${ENGINES_REPO_TLS_VERIFY}" \
'.engines.public += { "tls-verify": $ENGINES_PUBLIC_TLS_VERIFY }'
fi
}

# add quay specific information to an existing registries.json
function registries_json_add_quay() {
local ENGINES_QUAY_EXPIRATION_LENGTH
ENGINES_QUAY_EXPIRATION_LENGTH=$1
local ENGINES_QUAY_EXPIRATION_REFRESH_TOKEN
ENGINES_QUAY_EXPIRATION_REFRESH_TOKEN=$2
local ENGINES_QUAY_EXPIRATION_REFRESH_API_URL
ENGINES_QUAY_EXPIRATION_REFRESH_API_URL=$3

if [ -z "${REGISTRIES_CFG}" ]; then
exit_error "ERROR: \%REGISTRIES_CFG must be defined when calling $0"
fi

if [ "${ENGINES_QUAY_EXPIRATION_LENGTH}" != "SKIP_QUAY" ]; then
jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-engines-quay-expiration-length" \
--arg ENGINES_PUBLIC_QUAY_EXPIRATION_LENGTH "${ENGINES_QUAY_EXPIRATION_LENGTH}" \
'.engines.public += { "quay": { "expiration-length": $ENGINES_PUBLIC_QUAY_EXPIRATION_LENGTH } }'

if [ -n "${ENGINES_QUAY_EXPIRATION_REFRESH_TOKEN}" -a -n "${ENGINES_QUAY_EXPIRATION_REFRESH_API_URL}" ]; then
jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-engines-quay-expiration-refresh-token" \
--arg ENGINES_PUBLIC_QUAY_EXPIRATION_REFRESH_TOKEN "${ENGINES_QUAY_EXPIRATION_REFRESH_TOKEN}" \
--arg ENGINES_PUBLIC_QUAY_EXPIRATION_REFRESH_API_URL "${ENGINES_QUAY_EXPIRATION_REFRESH_API_URL}" \
'.engines.public.quay += { "refresh-expiration": { "token-file": $ENGINES_PUBLIC_QUAY_EXPIRATION_REFRESH_TOKEN, "api-url": $ENGINES_PUBLIC_QUAY_EXPIRATION_REFRESH_API_URL } }'
fi
fi
}
35 changes: 9 additions & 26 deletions bin/base
Original file line number Diff line number Diff line change
Expand Up @@ -985,33 +985,14 @@ if [ ! -e ${REGISTRIES_CFG} ]; then

echo "Creating ${REGISTRIES_CFG}"

# create an empty JSON file that jq will be able to add to
echo "{}" > ${REGISTRIES_CFG}

# populate the new JSON file with the information from
# /etc/sysconfig/crucible
CONTROLLER_URL=$(echo ${CRUCIBLE_CONTROLLER_IMAGE} | awk -F: '{ print $1 }')
CONTROLLER_TAG=$(echo ${CRUCIBLE_CONTROLLER_IMAGE} | awk -F: '{ print $2 }')
jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-controller" \
--arg CONTROLLER_URL "${CONTROLLER_URL}" \
--arg CONTROLLER_TAG "${CONTROLLER_TAG}" \
'. += { "controller": { "url": $CONTROLLER_URL, "tag": $CONTROLLER_TAG } }'

jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-engines-public" \
--arg ENGINES_PUBLIC_URL "${CRUCIBLE_ENGINE_REPO}" \
'. += { "engines": { "public": { "url": $ENGINES_PUBLIC_URL } } }'

if [ -n "${CRUCIBLE_ENGINE_REPO_AUTH_TOKEN}" ]; then
jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-engines-public-push-token" \
--arg ENGINES_PUBLIC_PUSH_TOKEN "${CRUCIBLE_ENGINE_REPO_AUTH_TOKEN}" \
'.engines.public += { "push-token": $ENGINES_PUBLIC_PUSH_TOKEN }'
fi
source ${CRUCIBLE_HOME}/bin/_registries.lib

if [ -n "${CRUCIBLE_ENGINE_REPO_TLS_VERIFY}" ]; then
jq_update ${REGISTRIES_CFG} "${REGISTRIES_CFG}:create-engines-public-tls-verify" \
--argjson ENGINES_PUBLIC_TLS_VERIFY "${CRUCIBLE_ENGINE_REPO_TLS_VERIFY}" \
'.engines.public += { "tls-verify": $ENGINES_PUBLIC_TLS_VERIFY }'
fi
create_registries_json \
${CRUCIBLE_CONTROLLER_IMAGE} \
${CRUCIBLE_ENGINE_REPO} \
${CRUCIBLE_ENGINE_REPO_AUTH_TOKEN} \
${CRUCIBLE_ENGINE_REPO_TLS_VERIFY} \
"SKIP_QUAY"

echo "Contents of ${REGISTRIES_CFG}:"
cat ${REGISTRIES_CFG}
Expand Down Expand Up @@ -1059,6 +1040,8 @@ if [ -e ${REGISTRIES_CFG} ]; then
validate_json_schema ${REGISTRIES_CFG} ${REGISTRIES_CFG_SCHEMA}
RC=$?
if [ ${RC} -ne 0 ]; then
echo "${REGISTRIES_CFG}:"
cat ${REGISTRIES_CFG}
exit_error "${REGISTRIES_CFG} does not validate against ${REGISTRIES_CFG_SCHEMA}" ${RC}
fi
fi
Expand Down
122 changes: 109 additions & 13 deletions crucible-install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ GIT_INSTALL_LOG="/tmp/crucible-git-install.log"
CRUCIBLE_CONTROLLER_REGISTRY="quay.io/crucible/controller:latest"
DEFAULT_GIT_REPO="https://github.com/perftool-incubator/crucible"
DEFAULT_GIT_BRANCH="master"
DEFAULT_QUAY_EXPIRATION_LENGTH="13w"
GIT_REPO=""
GIT_BRANCH=""
GIT_TAG=""
Expand All @@ -35,6 +36,8 @@ EC_PUSHD_FAIL=15
EC_PULL_FAIL=16
EC_RELEASE_DEFAULT_REPO_ONLY=18
EC_RELEASE_CONFLICTS_WITH_BRANCH=19
EC_INVALID_QUAY_EXPIRATION_LENGTH=20
EC_OAUTH_FILE_NOT_FOUND=21

# remove a previous installation log
if [ -e ${GIT_INSTALL_LOG} ]; then
Expand Down Expand Up @@ -153,6 +156,15 @@ function usage {
optional:
--quay-engine-expiration-refresh-token <authentication file>>
Quay OAuth authentication token file for refreshing engine image expiration timestamps.
--quay-engine-expiration-refresh-api-url <api url>
Quay API URL that is used to operate on the engine registry.
--quay-engine-expiration-length <length>
How long should a Quay repo allow the engine images to live before they expire.
--engine-auth-file <authentication file>
Authentication file for pushing images to the remote registry.
Expand Down Expand Up @@ -189,11 +201,12 @@ _USAGE_

# list available tags from the remote repository
function list_releases {
# only default repo is supported for the release mechanism
# only default repo is supported for the release mechanism
git ls-remote --tags \
--sort='version:refname' \
https://github.com/perftool-incubator/crucible.git \
| awk -F/ '{print$NF}'
--sort='version:refname' \
https://github.com/perftool-incubator/crucible.git \
| awk -F/ '{print$NF}' \
| grep -E '20[0-9]{2}\.[1234]'
}

# cleanup previous installation
Expand Down Expand Up @@ -358,7 +371,8 @@ function update_repos_config() {

longopts="name:,email:,help,list-releases,verbose"
longopts+=",client-server-registry:,client-server-auth-file:,client-server-tls-verify:"
longopts+=",engine-registry:,engine-auth-file:,engine-tls-verify:"
longopts+=",engine-registry:,engine-auth-file:,engine-tls-verify:,quay-engine-expiration-length:"
longopts+=",quay-engine-expiration-refresh-token:,quay-engine-expiration-refresh-api-url:"
longopts+=",controller-registry:,git-repo:,git-branch:,release:"
opts=$(getopt -q -o "" --longoptions "$longopts" -n "$0" -- "$@");
if [ $? -ne 0 ]; then
Expand All @@ -368,6 +382,21 @@ fi
eval set -- "$opts";
while true; do
case "$1" in
--quay-engine-expiration-refresh-token)
shift;
CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_TOKEN="$1"
shift;
;;
--quay-engine-expiration-refresh-api-url)
shift;
CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_API_URL="$1"
shift;
;;
--quay-engine-expiration-length)
shift;
CRUCIBLE_ENGINE_QUAY_EXPIRATION_LENGTH="$1"
shift;
;;
--client-server-tls-verify|--engine-tls-verify)
shift;
CRUCIBLE_ENGINE_TLS_VERIFY="$1"
Expand Down Expand Up @@ -441,7 +470,7 @@ done

# --release conflicts with --git-repo or --git-branch
if [ -n "${GIT_TAG}" ]; then
if [ -n "${GIT_REPO}" ]; then
if [ -n "${GIT_REPO}" -a "${GIT_REPO}" != "${DEFAULT_GIT_REPO}" ]; then
exit_error "Only default repo is supported for installing a release." $EC_RELEASE_DEFAULT_REPO_ONLY
fi
if [ -n "${GIT_BRANCH}" ]; then
Expand Down Expand Up @@ -470,6 +499,18 @@ for dep in $DEPENDENCIES; do
has_dependency $dep
done

if [ -n "${CRUCIBLE_ENGINE_QUAY_EXPIRATION_LENGTH}" ]; then
if ! echo "${CRUCIBLE_ENGINE_QUAY_EXPIRATION_LENGTH}" | grep -q "[1-9][0-9]*[wm]"; then
exit_error "Invalid syntax for engine Quay expiration length. Expecting either '<integer>w' (for weeks) or '<integer>m' (for months)" ${EC_INVALID_QUAY_EXPIRATION_LENGTH}
fi
fi

if [ -n "${CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_TOKEN}" ]; then
if [ ! -f "${CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_TOKEN}" ]; then
exit_error "Crucible Quay engine refresh token file not found. See --quay-engine-expiration-refresh-token for details." $EC_OAUTH_FILE_NOT_FOUND
fi
fi

if [ ! -z ${CRUCIBLE_ENGINE_AUTH_FILE+x} ]; then
if [ ! -f $CRUCIBLE_ENGINE_AUTH_FILE ]; then
exit_error "Crucible authentication file not found. See --engine-auth-file for details." $EC_AUTH_FILE_NOT_FOUND
Expand Down Expand Up @@ -515,23 +556,73 @@ $INSTALL_PATH/bin/subprojects-install $GIT_TAG >>"$GIT_INSTALL_LOG" 2>&1 ||

SYSCONFIG_CRUCIBLE_ENGINE_REGISTRY="${CRUCIBLE_ENGINE_REGISTRY}"
SYSCONFIG_CRUCIBLE_ENGINE_AUTH=""
SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY="\"true\""
if [ ! -z ${CRUCIBLE_ENGINE_AUTH_FILE+x} ]; then
SYSCONFIG_CRUCIBLE_ENGINE_AUTH="\"${CRUCIBLE_ENGINE_AUTH_FILE}\""
SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY=""
SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY="true"
SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_LENGTH="${DEFAULT_QUAY_EXPIRATION_LENGTH}"
SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_TOKEN=""
SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_API_URL=""

if [ -n "${CRUCIBLE_ENGINE_AUTH_FILE}" ]; then
SYSCONFIG_CRUCIBLE_ENGINE_AUTH="${CRUCIBLE_ENGINE_AUTH_FILE}"
fi
if [ ! -z ${CRUCIBLE_ENGINE_TLS_VERIFY+x} ]; then
SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY="\"${CRUCIBLE_ENGINE_TLS_VERIFY}\""
if [ -n "${CRUCIBLE_ENGINE_TLS_VERIFY}" ]; then
SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY="${CRUCIBLE_ENGINE_TLS_VERIFY}"
fi
if [ -n "${CRUCIBLE_ENGINE_QUAY_EXPIRATION_LENGTH}" ]; then
SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_LENGTH="${CRUCIBLE_ENGINE_QUAY_EXPIRATION_LENGTH}"
fi
if [ -n "${CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_TOKEN}" ]; then
SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_TOKEN="${CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_TOKEN}"
fi
if [ -n "${CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_API_URL}" ]; then
SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_API_URL="${CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_API_URL}"
fi

# native crucible install script already created this, only append
cat << _SYSCFG_ >> $SYSCONFIG
REGISTRIES_CFG=${INSTALL_PATH}/config/registries.json
REGISTRIES_CFG_SCHEMA=${INSTALL_PATH}/schema/registries.json

if [ -e ${INSTALL_PATH}/bin/_registries.lib ]; then
source ${INSTALL_PATH}/bin/_registries.lib

cat << _SYSCFG_ >> $SYSCONFIG
CRUCIBLE_USE_CONTAINERS=1
CRUCIBLE_USE_LOGGER=1
_SYSCFG_

create_registries_json \
${CRUCIBLE_CONTROLLER_REGISTRY} \
${SYSCONFIG_CRUCIBLE_ENGINE_REGISTRY} \
${SYSCONFIG_CRUCIBLE_ENGINE_AUTH} \
${SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY}

registries_json_add_quay \
${SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_LENGTH} \
${SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_TOKEN} \
${SYSCONFIG_CRUCIBLE_ENGINE_QUAY_EXPIRATION_REFRESH_API_URL}

# when the 'base' file is sourced with this particular parameter
# set it will force the registries.json to be validated
CRUCIBLE_CFG_JSON_VALIDATION="yes"
SESSION_ID="installer"
CRUCIBLE_HOME=${INSTALL_PATH} source ${INSTALL_PATH}/bin/base
else
SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY="\"${SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY}\""
if [ -n "${SYSCONFIG_CRUCIBLE_ENGINE_AUTH_FILE}" ]; then
SYSCONFIG_CRUCIBLE_ENGINE_AUTH="\"${SYSCONFIG_CRUCIBLE_ENGINE_AUTH}\""
fi
if [ -n "${CRUCIBLE_ENGINE_TLS_VERIFY}" ]; then
SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY="\"${SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY}\""
fi

cat << _SYSCFG_ >> $SYSCONFIG
CRUCIBLE_USE_CONTAINERS=1
CRUCIBLE_USE_LOGGER=1
CRUCIBLE_CONTROLLER_IMAGE=${CRUCIBLE_CONTROLLER_REGISTRY}
CRUCIBLE_ENGINE_REPO=${SYSCONFIG_CRUCIBLE_ENGINE_REGISTRY}
CRUCIBLE_ENGINE_REPO_AUTH_TOKEN=${SYSCONFIG_CRUCIBLE_ENGINE_AUTH}
CRUCIBLE_ENGINE_REPO_TLS_VERIFY=${SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY}
_SYSCFG_
fi

if [ ${VERBOSE} == 1 ]; then
echo
Expand All @@ -543,6 +634,11 @@ if [ ${VERBOSE} == 1 ]; then
echo
${INSTALL_PATH}/bin/crucible repo config show
fi
if [ -e ${REGISTRIES_CFG} ]; then
echo
echo "Contents of registries config file ${REGISTRIES_CFG}:"
cat ${REGISTRIES_CFG}
fi
fi

echo
Expand Down
Loading

0 comments on commit 07e312f

Please sign in to comment.