diff --git a/.github/workflows/crucible-release.yaml b/.github/workflows/crucible-release.yaml index e6955f5f..19a4df8b 100644 --- a/.github/workflows/crucible-release.yaml +++ b/.github/workflows/crucible-release.yaml @@ -119,7 +119,7 @@ jobs: # get 3rd column (repo URL) without the starting '/' (remove first char), # so repo urls /a /b /c are extracted to a b c # and add to a bash array ( a b c ) - projects=( $(grep -v ^# default_subprojects | awk {'print $3'} | cut -c2- ) ) + projects=( $(jq -r '.official[] | select(.name != "crucible") | .name) ) # builtin implict join array "a","b","c" (double quotes for a valid json) printf -v list '"%s",' "${projects[@]}" # convert to a comma separated list ["a","b","c"] diff --git a/README.md b/README.md index 884101aa..e98c9184 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Introduction -Crucible is a performance tool harness that integrates [multiple performance tooling projects](config/default_subprojects). The project's intention is to provide a well rounded, portable, and highly functional end-to-end performance testing harness. Crucible can be used for test automation to execute, measure, store, visualize, and analyze the performance of various systems-under-test (SUT). +Crucible is a performance tool harness that integrates [multiple performance tooling projects](config/repos.json). The project's intention is to provide a well rounded, portable, and highly functional end-to-end performance testing harness. Crucible can be used for test automation to execute, measure, store, visualize, and analyze the performance of various systems-under-test (SUT). ## Features diff --git a/bin/_crucible_completions b/bin/_crucible_completions index c6cc0f4e..6646b13a 100644 --- a/bin/_crucible_completions +++ b/bin/_crucible_completions @@ -29,7 +29,7 @@ _crucible_completions() { COMPREPLY=($(compgen -W "result metric" -- "${COMP_WORDS[2]}")) ;; repo) - COMPREPLY=($(compgen -W "info details" -- "${COMP_WORDS[2]}")) + COMPREPLY=($(compgen -W "info details config" -- "${COMP_WORDS[2]}")) ;; update) COMPREPLY=($(compgen -W "all crucible $(cd $CRUCIBLE_HOME/subprojects/; find . -type l | sed 'sX./XX')" -- "${COMP_WORDS[2]}")) @@ -97,6 +97,9 @@ _crucible_completions() { info|details) COMPREPLY=($(compgen -W "all crucible $(cd $CRUCIBLE_HOME/subprojects/; find . -type l | sed 'sX./XX')" -- "${COMP_WORDS[3]}")) ;; + config) + COMPREPLY=($(compgen -W "help show" -- "${COMP_WORDS[3]}")) + ;; esac elif [ "$(expr $num_words % 2)" -eq 0 -a ${COMP_WORDS[1]} == "rm" ]; then case "${COMP_WORDS[2]}" in diff --git a/bin/_git-fetch b/bin/_git-fetch index c1311c3f..609b5445 100755 --- a/bin/_git-fetch +++ b/bin/_git-fetch @@ -17,6 +17,9 @@ if [ ! -e ${CRUCIBLE_HOME} ]; then fi source ${CRUCIBLE_HOME}/bin/base +source ${CRUCIBLE_HOME}/bin/jqlib + +REPO_FILE=${CRUCIBLE_HOME}/config/repos.json PROCESS_LOCK_FILE="${UPDATE_STATUS_FILE}.process-lock" touch "${PROCESS_LOCK_FILE}" @@ -45,34 +48,47 @@ touch "${PROCESS_LOCK_FILE}" { UPDATE_COUNT=0 - if pushd ${CRUCIBLE_HOME}/subprojects > /dev/null; then - for tmp_repo in $(find . -maxdepth 2 -mindepth 2 -type l | sed 'sX./XX'); do - echo "REPO: ${tmp_repo}" - if pushd ${tmp_repo} > /dev/null; then + for project in $(jq_query ${REPO_FILE} '.official[], .unofficial[] | .name'); do + echo "Processing project: ${project}" + + PROJECT_NAME=$(jq_query ${REPO_FILE} --arg project ${project} '.official[], .unofficial[] | select(.name == $project) | .name') + PROJECT_TYPE=$(jq_query ${REPO_FILE} --arg project ${project} '.official[], .unofficial[] | select(.name == $project) | .type') + PROJECT_CHECKOUT_MODE=$(jq_query ${REPO_FILE} --arg project ${project} '.official[], .unofficial[] | select(.name == $project) | .checkout.mode') + + if [ "${PROJECT_CHECKOUT_MODE}" == "follow" ]; then + if [ "${PROJECT_NAME}" == "crucible" ]; then + repo_dir=${CRUCIBLE_HOME} + else + repo_dir=${CRUCIBLE_HOME}/subprojects/$(get_project_type_dir ${PROJECT_TYPE})/${PROJECT_NAME} + fi + + if pushd ${repo_dir} > /dev/null; then if [ -d "./.git" ]; then - git remote --verbose update --prune + git_fetch_cmd="git fetch --all --prune --prune-tags --tags --verbose" + echo "${git_fetch_cmd}" + ${git_fetch_cmd} git_status=$(git_get_status) git_tracking=$(git_get_tracking "${git_status}") if [ -n "${git_tracking}" -a "${git_tracking}" != "(detached)" ]; then git_delta_behind=$(git_get_delta_behind "${git_status}") + echo "Found ${git_delta_behind} update(s)" if [ "${git_delta_behind}" != "0" ]; then (( UPDATE_COUNT += git_delta_behind )) fi fi else - echo "ERROR: not a git repo" + echo "ERROR: '${repo_dir}' is not a Git repo" fi popd > /dev/null + else + echo "ERROR: Faild to pushd to '${repo_dir}'" fi - echo - done - popd > /dev/null - else - echo "ERROR: Could not pushd to ${CRUCIBLE_HOME}/subprojects" - exit 1 - fi + else + echo "Skipping analysis since checkout.mode is 'locked'" + fi + done CURRENT_TIMESTAMP=$(date +%s) { diff --git a/bin/_help b/bin/_help index 123dd59c..2b432dc2 100755 --- a/bin/_help +++ b/bin/_help @@ -11,7 +11,7 @@ function help() { echo "help | Show this help message" echo "run | Run a benchmark" echo "log | Manage with the crucible log" - echo "repo | Get info on the crucible and subproject git repos" + echo "repo | Get info on or configure the crucible and subproject git repos" echo "update | Update all or part of the crucible software" echo "console | Run user supplied programs inside a crucible wrapper for logging purposes" echo "wrapper | Run a crucible subproject command directly, within a crucible container" diff --git a/bin/_main b/bin/_main index 3e35d77e..5f68c3c8 100755 --- a/bin/_main +++ b/bin/_main @@ -402,6 +402,7 @@ elif [ "${1}" == "run" ]; then ${CRUCIBLE_HOME}/bin/repo info > ${base_run_dir}/config/crucible.repo.info ${CRUCIBLE_HOME}/bin/repo details > ${base_run_dir}/config/crucible.repo.details + ${CRUCIBLE_HOME}/bin/repo config show > ${base_run_dir}/config/crucible.repo.config.show rs_run_cmd="${CRUCIBLE_HOME}/subprojects/core/rickshaw/rickshaw-run\ ${params_args}\ diff --git a/bin/_update-git b/bin/_update-git index bc393f22..12f73906 100755 --- a/bin/_update-git +++ b/bin/_update-git @@ -12,55 +12,106 @@ exec 2>&1 source /etc/sysconfig/crucible if [ -z "${CRUCIBLE_HOME}" ]; then - echo "CRUCIBLE_HOME not defined, exiting." + echo "ERROR: CRUCIBLE_HOME not defined!" exit 1 fi if [ ! -e "${CRUCIBLE_HOME}" ]; then - echo "Could not find ${CRUCIBLE_HOME}, exiting." + echo "ERROR: Could not find ${CRUCIBLE_HOME}!" exit 1 fi source "${CRUCIBLE_HOME}/bin/base" +source "${CRUCIBLE_HOME}/bin/jqlib" + +REPO_FILE="${CRUCIBLE_HOME}/config/repos.json" + +PROJECT=${1} +if [ -z "${PROJECT}" ]; then + echo "ERROR: You must supply a project name!" + exit 1 +else + PROJECT_NAME=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | .name') + if [ "${PROJECT_NAME}" != "${PROJECT}" ]; then + echo "ERROR: You must supply a valid project name!" + exit 1 + fi + + PROJECT_REPOSITORY=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | .repository') + PROJECT_PRIMARY_BRANCH=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | ."primary-branch"') + PROJECT_CHECKOUT_MODE=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | .checkout.mode') + PROJECT_CHECKOUT_TARGET=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | .checkout.target') +fi + +current_dir=$(pwd) +echo "Attempting to update project repo ${PROJECT_NAME} at '${current_dir}' -> '$(readlink -e ${current_dir})'" if [ -d "./.git" ]; then + # validate that the current repository matches the expected repository + ORIGIN_REPOSITORY=$(git remote get-url origin) + if [ "${ORIGIN_REPOSITORY}" != "${PROJECT_REPOSITORY}" ]; then + echo "ERROR: Invalid state! The origin repository does not match the project repository!" + exit 1 + fi + + current_ref=$(git rev-parse --verify HEAD) + expected_ref=$(git rev-parse --verify ${PROJECT_CHECKOUT_TARGET}) + + if [ "${current_ref}" != "${expected_ref}" ]; then + echo "ERROR: Invalid state! The current checked out ref (${current_ref}) does not match the checkout target's ref (${PROJECT_CHECKOUT_TARGET} -> ${expected_ref})!" + exit 1 + fi + # fetch updates from remote(s) - if ! git remote --verbose update --prune > /dev/null 2>&1; then - echo "ERROR: could not git-update" + git_fetch_cmd="git fetch --all --prune --prune-tags --tags --verbose" + echo "${git_fetch_cmd}" + if ! ${git_fetch_cmd} 2>&1; then + echo "ERROR: could not git-fetch" exit 1 fi - git_status=$(git_get_status) - git_local_branch=$(git_get_local_branch "${git_status}") - git_tracking=$(git_get_tracking "${git_status}") + # store any local changes + stash_output=$(git stash) + if [ $? != 0 ]; then + echo "ERROR: could not git-stash" + exit 1 + fi - if [ -n "${git_tracking}" -a "${git_local_branch}" != "(detached)" ]; then - # store any local changes - stash_output=$(git stash) - if [ $? != 0 ]; then - echo "ERROR: could not git-stash" - exit 1 - fi + # return to the primary branch so that we can then return to the + # checkout target in case it's reference has changed (ie. an + # updated tag) + echo "Switching to primary branch to quiesce the repository state" + git checkout ${PROJECT_PRIMARY_BRANCH} - # merge any changes from the remote branch - if ! git pull --verbose --ff-only 2> /dev/null; then - echo "ERROR: could not git-pull" - exit 1 - fi + # return to the checkout target if it is not the primary branch + if [ "${PROJECT_PRIMARY_BRANCH}" != "${PROJECT_CHECKOUT_TARGET}" ]; then + echo "Returning to the checkout target" + git -c advice.detachedHead=false checkout ${PROJECT_CHECKOUT_TARGET} + fi - if ! echo "${stash_output}" | grep -q "No local changes to save"; then - # reapply local changes - if ! git stash pop > /dev/null; then - echo -e "${stash_output}" - echo "ERROR: could not git-stash pop" + if [ "${PROJECT_CHECKOUT_MODE}" == "locked" ]; then + # return to the previous ref if is supposed to be locked there + echo "Returning to the previously checked out ref since I am in locked mode" + git checkout ${current_ref} + elif [ "${PROJECT_CHECKOUT_MODE}" == "follow" ]; then + if [ "${PROJECT_CHECKOUT_TARGET}" == "HEAD" ]; then + echo "Detected CI environment, skipping merge of changes from the remote branch (because they do not exist)" + else + # merge any changes from the remote branch + echo "Merging any available upstream changes because I am in follow mode" + if ! git pull --verbose --ff-only 2> /dev/null; then + echo "ERROR: could not git-pull" exit 1 fi fi - else - if [ "${git_local_branch}" == "(detached)" ]; then - echo "WARNING: This git repo is in a detached state, not modifying the active contents" - else - echo "ERROR: This git repo is in an unknown state" + fi + + if ! echo "${stash_output}" | grep -q "No local changes to save"; then + # reapply local changes + echo "Reapplying stashed local changes" + if ! git stash pop > /dev/null; then + echo -e "${stash_output}" + echo "ERROR: could not git-stash pop" exit 1 fi fi diff --git a/bin/base b/bin/base index 9dbe7b6b..834d286f 100644 --- a/bin/base +++ b/bin/base @@ -944,3 +944,17 @@ function git_get_remote_name() { function git_get_remote_branch() { echo "${1}" | awk -F'/' '{ print $2 }' } + +# map a project type to it's actual on disk directory name +function get_project_type_dir() { + local TYPE + TYPE=$1 + + case "${TYPE}" in + benchmark|tool|doc) + echo "${TYPE}s" + ;; + *) + echo "${TYPE}" + esac +} diff --git a/bin/jqlib b/bin/jqlib new file mode 100644 index 00000000..0fb91dab --- /dev/null +++ b/bin/jqlib @@ -0,0 +1,27 @@ +#!/bin/bash +# -*- mode: sh; indent-tabs-mode: nil; sh-basic-offset: 4 -*- +# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=bash + +function jq_update() { + local FILE TMP_FILE REASON + FILE=${1}; shift + REASON=${1}; shift + + TMP_FILE=${FILE}.tmp + + jq --indent 4 "$@" ${FILE} > ${TMP_FILE} + RC=$? + + if [ ${RC} -ne 0 ]; then + exit_error "Failed to update ${REASON} in ${FILE}" + fi + + mv ${TMP_FILE} ${FILE} +} + +function jq_query() { + local FILE + FILE=${1}; shift + + jq -r "$@" ${FILE} +} diff --git a/bin/repo b/bin/repo index 1a7ac4fb..88e37dc1 100755 --- a/bin/repo +++ b/bin/repo @@ -4,7 +4,9 @@ # repo management infrastructure -if [ ! -e $CRUCIBLE_HOME ]; then +source /etc/sysconfig/crucible + +if [ ! -e "${CRUCIBLE_HOME}" ]; then echo "Could not find $CRUCIBLE_HOME, exiting." exit 1 fi @@ -19,6 +21,7 @@ function help() { echo "" echo "info | Give high level info about the active repositories (default)" echo "details | Give details for the active repositories" + echo "config | Repository configuration control and information" echo "" } @@ -199,21 +202,75 @@ function handle_scope() { fi } -if [ -z "${1}" ]; then - scope="all" -else - scope=${1} - shift -fi +function config_show() { + fmt_line="%-20s %-15s %-13s %-60s %-20s %-20s %-20s" + printf "${fmt_line}\n" "Project Family" "Project Name" "Project Type" "Git Remote URL" "Primary Branch" "Checkout Mode" "Checkout Target" + + for family in official unofficial; do + projects=$(jq_query ${CRUCIBLE_HOME}/config/repos.json --arg family ${family} '.[$family][] | .name') + for project in ${projects}; do + project_type=$(jq_query ${CRUCIBLE_HOME}/config/repos.json --arg family ${family} --arg project_name ${project} '.[$family][] | select(.name == $project_name) | .type') + project_repository=$(jq_query ${CRUCIBLE_HOME}/config/repos.json --arg family ${family} --arg project_name ${project} '.[$family][] | select(.name == $project_name) | .repository') + project_primary_branch=$(jq_query ${CRUCIBLE_HOME}/config/repos.json --arg family ${family} --arg project_name ${project} '.[$family][] | select(.name == $project_name) | ."primary-branch"') + project_checkout_mode=$(jq_query ${CRUCIBLE_HOME}/config/repos.json --arg family ${family} --arg project_name ${project} '.[$family][] | select(.name == $project_name) | .checkout.mode') + project_checkout_target=$(jq_query ${CRUCIBLE_HOME}/config/repos.json --arg family ${family} --arg project_name ${project} '.[$family][] | select(.name == $project_name) | .checkout.target') + + printf "${fmt_line}\n" "${family}" "${project}" "${project_type}" "${project_repository}" "${project_primary_branch}" "${project_checkout_mode}" "${project_checkout_target}" + done + done +} + +function config_help() { + echo "Config Usage:" + echo " repo config [ ]" + echo + echo "The following command are supported" + echo + echo "show | Dump the configuration information in tabular form (default)" + echo +} case "${command}" in - info) - fmt_line="%-15s %-13s %-16s %-58s %-17s %-15s %-13s %-14s %-13s" - printf "${fmt_line}\n" "Project Name" "Project Type" "Git Local Branch" "Git Remote URL" "Git Remote Branch" "Git Remote Name" "Local Changes" "Current Commit" "Updates" + info|details) + if [ -z "${1}" ]; then + scope="all" + else + scope=${1} + shift + fi - handle_scope info ${scope} + case "${command}" in + info) + fmt_line="%-15s %-13s %-16s %-60s %-17s %-15s %-13s %-14s %-13s" + printf "${fmt_line}\n" "Project Name" "Project Type" "Git Local Branch" "Git Remote URL" "Git Remote Branch" "Git Remote Name" "Local Changes" "Current Commit" "Updates" + + handle_scope info ${scope} + ;; + details) + handle_scope details ${scope} + ;; + esac ;; - details) - handle_scope details ${scope} + config) + source "${CRUCIBLE_HOME}/bin/jqlib" + + config_subcommand="show" + if [ -n "${1}" ]; then + config_subcommand=${1} + shift + fi + + case "${config_subcommand}" in + show) + config_show + ;; + help) + config_help + ;; + *) + config_help + exit 1 + ;; + esac ;; esac diff --git a/bin/subprojects-install b/bin/subprojects-install index 064615fa..f74b55b4 100755 --- a/bin/subprojects-install +++ b/bin/subprojects-install @@ -23,6 +23,7 @@ fi # Find the git repo base directory crucible_repo_bin_dir=$(dirname `readlink -e $0`) crucible_repo_dir=$(echo $crucible_repo_bin_dir | sed -e 'sX/binXX') +source ${crucible_repo_dir}/bin/jqlib # The three sections below use sudo to install files and links as root. # This script could be run by root (in which the sudo does not matter), @@ -57,19 +58,17 @@ else exit_error "file $crucible_repo_bin_dir/crucible was not found, exiting" fi -# Load the subprojects from ./config/default_subprojects, then from $HOME/.crucible/subprojects to get local overrides +# Load the subprojects +REPOS_FILE=${crucible_repo_dir}/config/repos.json declare -A subprojects -for conf_file in "$crucible_repo_dir/config/default_subprojects" "$HOME/.crucible/subprojects"; do - if [ -e "$conf_file" ]; then - while read line; do - if echo $line | grep -q "^\#"; then - continue - fi - name=`echo $line | awk '{print $1}'` - subprojects["$name"]="$line" - done < "$conf_file" - fi -done +if [ -e "${REPOS_FILE}" ]; then + for repo in $(jq_query ${REPOS_FILE} '.official[], .unofficial[] | select(.name != "crucible") | .name'); do + subprojects["${repo}"]=1 + done +else + exit_error "Could not locate the repos.json at ${REPOS_FILE}" +fi + no_clone="" pushd "$crucible_repo_dir" >/dev/null || exit_error "Could not change directory to $crucible_repo_dir" @@ -79,7 +78,7 @@ for link in $(find subprojects -type l); do name_link=$(echo "${link}" | awk -F'/' '{ print $3 }') active=0 if [ -n "${subprojects[$name_link]}" ]; then - sp_type=`echo ${subprojects[$name_link]} | awk '{print $2}'` + sp_type=$(jq_query ${REPOS_FILE} --arg name "${name_link}" '.official[], .unofficial[] | select(.name == $name) | .type') case "$sp_type" in "benchmark") if [ "${type_dir}" == "benchmarks" ]; then @@ -109,29 +108,32 @@ for link in $(find subprojects -type l); do fi done for subproject in "${!subprojects[@]}"; do - sp_type=`echo ${subprojects[$subproject]} | awk '{print $2}'` - sp_git_user_host_org_proj=`echo ${subprojects[$subproject]} | awk '{print $3}'` - # sp_git_user_host_org_proj has two possible formats: + sp_type=$(jq_query ${REPOS_FILE} --arg name "${subproject}" '.official[], .unofficial[] | select(.name == $name) | .type') + sp_repository=$(jq_query ${REPOS_FILE} --arg name "${subproject}" '.official[], .unofficial[] | select(.name == $name) | .repository') + # sp_repository has two possible formats: # https://github.com/perftool-incubator/crucible.git # git@github.com:perftool-incubator/crucible.git - sp_git_proj=`echo $sp_git_user_host_org_proj | awk -F/ '{print $NF}'` - sp_git_user_host_org=`echo $sp_git_user_host_org_proj | sed -e s/$sp_git_proj$// -e s,/$,,` - # If the repo location was omitted, get this from existing crucible repo - if [ -z "$sp_git_user_host_org" ]; then - pushd $crucible_repo_dir >/dev/null - cru_git_user_host_org_proj=`git remote -v | grep origin | grep fetch | head -1 | awk '{print $2}'` - cru_git_proj=`echo $cru_git_user_host_org_proj | awk -F/ '{print $NF}'` - sp_git_user_host_org=`echo $cru_git_user_host_org_proj | sed -e s/$cru_git_proj$// -e s,/$,,` - popd >/dev/null - fi - sp_git_host_org=`echo $sp_git_user_host_org | sed -e 'sXhttps://XX' -e 'sXhttp://XX' -e 's/.*@//'` - sp_git_user=`echo $sp_git_user_host_org | sed -e s,$sp_git_host_org,, -e 'sX://X:X'` - sp_git_host_org=`echo $sp_git_host_org | sed -e 's,/,:,g'` - if [ -z "${GIT_TAG}" ]; then - sp_branch=`echo ${subprojects[$subproject]} | awk '{print $4}'` - else - sp_branch=$GIT_TAG + + sp_git_proj=$(echo ${sp_repository} | awk -F/ '{print $NF}') + sp_git_user_host_org=$(echo ${sp_repository} | sed -e s/${sp_git_proj}$// -e s,/$,,) + + sp_git_host_org=$(echo ${sp_git_user_host_org} | sed -e 'sXhttps://XX' -e 'sXhttp://XX' -e 's/.*@//') + sp_git_user=$(echo ${sp_git_user_host_org} | sed -e s,${sp_git_host_org},, -e 'sX://X:X') + sp_git_host_org=$(echo ${sp_git_host_org} | sed -e 's,/,:,g') + + sp_branch=$(jq_query ${REPOS_FILE} --arg name "${subproject}" '.official[], .unofficial[] | select(.name == $name) | .checkout.target') + if [ -n "${GIT_TAG}" ]; then + if [ "${sp_branch}" != "${GIT_TAG}" ]; then + sp_branch=${GIT_TAG} + jq_update ${REPOS_FILE} ${subproject}:checkout.target --arg subproject "${subproject}" --arg checkout_target "${sp_branch}" '(.official[], .unofficial[] | select(.name == $subproject) | .checkout.target) |= $checkout_target' + fi + + current_checkout_mode=$(jq_query ${REPOS_FILE} --arg subproject "${subproject}" '.official[], .unofficial[] | select(.name == $subproject) | .checkout.mode') + if [ "${current_checkout_mode}" != "locked" ]; then + jq_update ${REPOS_FILE} ${subproject}:checkout.mode --arg subproject "${subproject}" --arg checkout_mode "locked" '(.official[], .unofficial[] | select(.name == $subproject) | .checkout.mode) |= $checkout_mode' + fi fi + case "$sp_type" in "benchmark") sp_dir_prefix="benchmarks/" @@ -146,24 +148,25 @@ for subproject in "${!subprojects[@]}"; do sp_dir_prefix="docs/" ;; esac - clone_user_host_org_dir="repos/$sp_git_user$sp_git_host_org" - mkdir -p "$clone_user_host_org_dir" - clone_user_host_org_proj_dir="$clone_user_host_org_dir/$sp_git_proj" - subproject_dir="subprojects/$sp_dir_prefix$subproject" - if [ ! -e "$clone_user_host_org_proj_dir" ]; then - cmd="git clone $sp_git_user_host_org/$sp_git_proj $clone_user_host_org_proj_dir" + + clone_user_host_org_dir="repos/${sp_git_user}${sp_git_host_org}" + mkdir -p "${clone_user_host_org_dir}" + clone_user_host_org_proj_dir="${clone_user_host_org_dir}/${sp_git_proj}" + subproject_dir="subprojects/${sp_dir_prefix}${subproject}" + + if [ ! -e "${clone_user_host_org_proj_dir}" ]; then + echo "Cloning ${subproject}" + + cmd="git clone ${sp_git_user_host_org}/${sp_git_proj} ${clone_user_host_org_proj_dir}" echo "${cmd}" if ! ${cmd}; then echo "failed to clone ${sp_git_user_host_org}/${sp_git_proj}" - cmd="git clone ${default_git_host_org}/${sp_git_proj} $clone_user_host_org_proj_dir" - echo "${cmd}" - if ! ${cmd}; then - echo "failed to clone ${default_git_host_org}/${sp_git_proj}" - exit_error "Could not clone repo ${sp_git_proj}" - fi + exit_error "Could not clone repo ${sp_git_proj}" fi + + # make sure the correct branch/tag/commit is checked out if pushd "${clone_user_host_org_proj_dir}" >/dev/null; then - cmd="git checkout ${sp_branch}" + cmd="git -c advice.detachedHead=false checkout ${sp_branch}" echo "${cmd}" if ! ${cmd}; then exit_error "Could not checkout ${sp_branch} for ${sp_git_user_host_org}/${sp_git_proj}" @@ -172,19 +175,26 @@ for subproject in "${!subprojects[@]}"; do else exit_error "Could not chdir to ${clone_user_host_org_proj_dir}" fi - if ! /bin/rm -f "${subproject_dir}"; then - exit_error "Could not rm ${subproject_dir}" + + if [ -e "${subproject_dir}" ]; then + # delete an existing $subproject_dir so we can create a new one + if ! /bin/rm -f "${subproject_dir}"; then + exit_error "Could not rm ${subproject_dir}" + fi fi + + # create the new $subproject_dir if ! /bin/ln -sf "../../${clone_user_host_org_proj_dir}" "${subproject_dir}"; then exit_error "Symlinking ${subproject_dir} to ../../${clone_user_host_org_proj_dir} failed" fi else - no_clone="$no_clone $subproject" + no_clone+=" ${subproject}" fi done -if [ ! -z "$no_clone" ]; then + +if [ ! -z "${no_clone}" ]; then echo "Not cloning these subprojects as they already exist:" - echo "$no_clone" + echo "${no_clone}" fi -popd >/dev/null +popd >/dev/null diff --git a/bin/update b/bin/update index 1707021f..0dfcd6b0 100755 --- a/bin/update +++ b/bin/update @@ -16,7 +16,10 @@ if [ ! -e $CRUCIBLE_HOME ]; then exit 1 fi -. $CRUCIBLE_HOME/bin/base +source ${CRUCIBLE_HOME}/bin/base +source ${CRUCIBLE_HOME}/bin/jqlib + +REPO_FILE=${CRUCIBLE_HOME}/config/repos.json help_column_width=25 @@ -30,12 +33,9 @@ function help() { printf "%-${help_column_width}s | Update the main crucible repository\n" "crucible" printf "%-${help_column_width}s | Update the controller container image\n" "controller-image" echo - if pushd $CRUCIBLE_HOME/subprojects > /dev/null; then - for repo in $(find . -type l | sed 'sX./XX'); do - printf "%-${help_column_width}s | Update the %s repository\n" "${repo}" "${repo}" - done - popd > /dev/null - fi + for project in $(jq_query ${REPO_FILE} '.official[], .unofficial[] | select(.name != "crucible") | .name'); do + printf "%-${help_column_width}s | Update the %s repository\n" "${project}" "${project}" + done echo } @@ -82,7 +82,7 @@ case "${repo}" in echo "rm ${TMP_SCRIPT}" >> ${TMP_SCRIPT} echo "exec ${CRUCIBLE_HOME}/bin/update __${repo}" >> ${TMP_SCRIPT} - exec ${TMP_SCRIPT} + exec ${TMP_SCRIPT} crucible else echo "ERROR: failed to pushd to ${CRUCIBLE_HOME}" exit 1 @@ -116,48 +116,61 @@ case "${repo}" in fi ;; *) - repo_dir=$(find ${CRUCIBLE_HOME}/subprojects -maxdepth 2 -mindepth 2 -name ${repo} -type l) - if [ -n "${repo_dir}" ]; then + PROJECT_NAME=$(jq_query ${REPO_FILE} --arg project ${repo} '.official[], .unofficial[] | select(.name == $project) | .name') + if [ "${PROJECT_NAME}" != "${repo}" ]; then + echo "ERROR: You must supply a valid project name!" + exit 1 + fi + PROJECT_TYPE=$(jq_query ${REPO_FILE} --arg project ${repo} '.official[], .unofficial[] | select(.name == $project) | .type') + + repo_dir=${CRUCIBLE_HOME}/subprojects/$(get_project_type_dir ${PROJECT_TYPE})/${PROJECT_NAME} + if [ -n "${repo_dir}" -a -L "${repo_dir}" ]; then echo - echo "Updating ${repo}:" + echo "Updating ${PROJECT_NAME}:" echo if pushd "${repo_dir}" > /dev/null; then - if $CRUCIBLE_HOME/bin/_update-git; then + if $CRUCIBLE_HOME/bin/_update-git ${PROJECT_NAME}; then exit 0 else exit 1 fi else - echo "Could not pushd to ${repo_dir}" + echo "ERROR: Could not pushd to ${repo_dir}!" exit 1 fi else - echo "Invalid repo '${repo}'." + echo "ERROR: Invalid project directory '${repo_dir}'!" exit 1 fi ;; esac if [ "${repo}" == "____all" ]; then - if pushd ${CRUCIBLE_HOME}/subprojects > /dev/null; then - for tmp_repo in $(find . -maxdepth 2 -mindepth 2 -type l | sed 'sX./XX'); do + for project in $(jq_query ${REPO_FILE} '.official[], .unofficial[] | select(.name != "crucible") | .name'); do + PROJECT_NAME=$(jq_query ${REPO_FILE} --arg project ${project} '.official[], .unofficial[] | select(.name == $project) | .name') + PROJECT_TYPE=$(jq_query ${REPO_FILE} --arg project ${project} '.official[], .unofficial[] | select(.name == $project) | .type') + + repo_dir=${CRUCIBLE_HOME}/subprojects/$(get_project_type_dir ${PROJECT_TYPE})/${PROJECT_NAME} + if [ -n "${repo_dir}" -a -L "${repo_dir}" ]; then echo - echo "Updating ${tmp_repo}:" + echo "Updating ${PROJECT_NAME}:" echo - if pushd ${tmp_repo} > /dev/null; then - if ! $CRUCIBLE_HOME/bin/_update-git; then + if pushd ${repo_dir} > /dev/null; then + if ! $CRUCIBLE_HOME/bin/_update-git ${PROJECT_NAME}; then RC=1 fi popd > /dev/null + else + echo "ERROR: Could not pushd to ${repo_dir}!" + RC=1 fi - done - popd > /dev/null - else - echo "Could not pushd to ${CRUCIBLE_HOME}/subprojects" - exit 1 - fi + else + echo "ERROR: Invalid project directory '${repo_dir}'!" + RC=1 + fi + done fi case "${repo}" in @@ -177,5 +190,7 @@ esac echo ${CRUCIBLE_HOME}/bin/repo info +echo +${CRUCIBLE_HOME}/bin/repo config show exit ${RC} diff --git a/config/default_subprojects b/config/default_subprojects deleted file mode 100644 index 57b96ec3..00000000 --- a/config/default_subprojects +++ /dev/null @@ -1,31 +0,0 @@ -# All subprojects will be pulled from the same project/user as the Crucible environment. -# By default this is "https://github.com/perftool-incubator" + "/" -#project-name project-type git-repo-url branch -rickshaw core /rickshaw master -multiplex core /multiplex master -roadblock core /roadblock master -workshop core /workshop master -CommonDataModel core /CommonDataModel master -toolbox core /toolbox main -packrat core /packrat master -crucible-ci core /crucible-ci main -fio benchmark /bench-fio master -uperf benchmark /bench-uperf master -oslat benchmark /bench-oslat master -trafficgen benchmark /bench-trafficgen main -cyclictest benchmark /bench-cyclictest main -hwlatdetect benchmark /bench-hwlatdetect main -tracer benchmark /bench-tracer main -sysstat tool /tool-sysstat master -procstat tool /tool-procstat master -ftrace tool /tool-ftrace master -kernel tool /tool-kernel master -ovs tool /tool-ovs master -rt-trace-bpf tool /tool-rt-trace-bpf main -forkstat tool /tool-forkstat main -flexran benchmark /bench-flexran main -iperf benchmark /bench-iperf main -examples doc /crucible-examples main -testing doc /testing-repo master -ilab benchmark /bench-ilab main -nvidia tool /tool-nvidia main diff --git a/spec/repos.json/repos.json b/config/repos.json similarity index 93% rename from spec/repos.json/repos.json rename to config/repos.json index 40f65484..428cd268 100644 --- a/spec/repos.json/repos.json +++ b/config/repos.json @@ -3,7 +3,7 @@ { "name": "crucible", "type": "primary", - "repository": "https://github.com/perftool-incubator/crucible.git", + "repository": "https://github.com/perftool-incubator/crucible", "primary-branch": "master", "checkout": { "mode": "follow", @@ -13,7 +13,7 @@ { "name": "rickshaw", "type": "core", - "repository": "https://github.com/perftool-incubator/rickshaw.git", + "repository": "https://github.com/perftool-incubator/rickshaw", "primary-branch": "master", "checkout": { "mode": "follow", @@ -23,7 +23,7 @@ { "name": "multiplex", "type": "core", - "repository": "https://github.com/perftool-incubator/multiplex.git", + "repository": "https://github.com/perftool-incubator/multiplex", "primary-branch": "master", "checkout": { "mode": "follow", @@ -33,7 +33,7 @@ { "name": "roadblock", "type": "core", - "repository": "https://github.com/perftool-incubator/roadblock.git", + "repository": "https://github.com/perftool-incubator/roadblock", "primary-branch": "master", "checkout": { "mode": "follow", @@ -43,7 +43,7 @@ { "name": "workshop", "type": "core", - "repository": "https://github.com/perftool-incubator/workshop.git", + "repository": "https://github.com/perftool-incubator/workshop", "primary-branch": "master", "checkout": { "mode": "follow", @@ -53,7 +53,7 @@ { "name": "CommonDataModel", "type": "core", - "repository": "https://github.com/perftool-incubator/CommonDataModel.git", + "repository": "https://github.com/perftool-incubator/CommonDataModel", "primary-branch": "master", "checkout": { "mode": "follow", @@ -63,7 +63,7 @@ { "name": "toolbox", "type": "core", - "repository": "https://github.com/perftool-incubator/toolbox.git", + "repository": "https://github.com/perftool-incubator/toolbox", "primary-branch": "main", "checkout": { "mode": "follow", @@ -73,7 +73,7 @@ { "name": "packrat", "type": "core", - "repository": "https://github.com/perftool-incubator/packrat.git", + "repository": "https://github.com/perftool-incubator/packrat", "primary-branch": "master", "checkout": { "mode": "follow", @@ -83,7 +83,7 @@ { "name": "crucible-ci", "type": "core", - "repository": "https://github.com/perftool-incubator/crucible-ci.git", + "repository": "https://github.com/perftool-incubator/crucible-ci", "primary-branch": "main", "checkout": { "mode": "follow", @@ -93,7 +93,7 @@ { "name": "fio", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-fio.git", + "repository": "https://github.com/perftool-incubator/bench-fio", "primary-branch": "master", "checkout": { "mode": "follow", @@ -103,7 +103,7 @@ { "name": "uperf", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-uperf.git", + "repository": "https://github.com/perftool-incubator/bench-uperf", "primary-branch": "master", "checkout": { "mode": "follow", @@ -113,7 +113,7 @@ { "name": "oslat", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-oslat.git", + "repository": "https://github.com/perftool-incubator/bench-oslat", "primary-branch": "master", "checkout": { "mode": "follow", @@ -123,7 +123,7 @@ { "name": "trafficgen", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-trafficgen.git", + "repository": "https://github.com/perftool-incubator/bench-trafficgen", "primary-branch": "main", "checkout": { "mode": "follow", @@ -133,7 +133,7 @@ { "name": "cyclictest", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-cyclictest.git", + "repository": "https://github.com/perftool-incubator/bench-cyclictest", "primary-branch": "main", "checkout": { "mode": "follow", @@ -143,7 +143,7 @@ { "name": "hwlatdetect", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-hwlatdetect.git", + "repository": "https://github.com/perftool-incubator/bench-hwlatdetect", "primary-branch": "main", "checkout": { "mode": "follow", @@ -153,7 +153,7 @@ { "name": "tracer", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-tracer.git", + "repository": "https://github.com/perftool-incubator/bench-tracer", "primary-branch": "main", "checkout": { "mode": "follow", @@ -163,7 +163,7 @@ { "name": "flexran", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-flexran.git", + "repository": "https://github.com/perftool-incubator/bench-flexran", "primary-branch": "main", "checkout": { "mode": "follow", @@ -173,7 +173,7 @@ { "name": "iperf", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-iperf.git", + "repository": "https://github.com/perftool-incubator/bench-iperf", "primary-branch": "main", "checkout": { "mode": "follow", @@ -183,7 +183,7 @@ { "name": "ilab", "type": "benchmark", - "repository": "https://github.com/perftool-incubator/bench-ilab.git", + "repository": "https://github.com/perftool-incubator/bench-ilab", "primary-branch": "main", "checkout": { "mode": "follow", @@ -193,7 +193,7 @@ { "name": "sysstat", "type": "tool", - "repository": "https://github.com/perftool-incubator/tool-sysstat.git", + "repository": "https://github.com/perftool-incubator/tool-sysstat", "primary-branch": "master", "checkout": { "mode": "follow", @@ -203,7 +203,7 @@ { "name": "procstat", "type": "tool", - "repository": "https://github.com/perftool-incubator/tool-procstat.git", + "repository": "https://github.com/perftool-incubator/tool-procstat", "primary-branch": "master", "checkout": { "mode": "follow", @@ -213,7 +213,7 @@ { "name": "ftrace", "type": "tool", - "repository": "https://github.com/perftool-incubator/tool-ftrace.git", + "repository": "https://github.com/perftool-incubator/tool-ftrace", "primary-branch": "master", "checkout": { "mode": "follow", @@ -223,7 +223,7 @@ { "name": "kernel", "type": "tool", - "repository": "https://github.com/perftool-incubator/tool-kernel.git", + "repository": "https://github.com/perftool-incubator/tool-kernel", "primary-branch": "master", "checkout": { "mode": "follow", @@ -233,7 +233,7 @@ { "name": "ovs", "type": "tool", - "repository": "https://github.com/perftool-incubator/tool-ovs.git", + "repository": "https://github.com/perftool-incubator/tool-ovs", "primary-branch": "master", "checkout": { "mode": "follow", @@ -243,7 +243,7 @@ { "name": "rt-trace-bpf", "type": "tool", - "repository": "https://github.com/perftool-incubator/tool-rt-trace-bpf.git", + "repository": "https://github.com/perftool-incubator/tool-rt-trace-bpf", "primary-branch": "main", "checkout": { "mode": "follow", @@ -253,7 +253,7 @@ { "name": "forkstat", "type": "tool", - "repository": "https://github.com/perftool-incubator/tool-forkstat.git", + "repository": "https://github.com/perftool-incubator/tool-forkstat", "primary-branch": "main", "checkout": { "mode": "follow", @@ -263,7 +263,7 @@ { "name": "nvidia", "type": "tool", - "repository": "https://github.com/perftool-incubator/tool-nvidia.git", + "repository": "https://github.com/perftool-incubator/tool-nvidia", "primary-branch": "main", "checkout": { "mode": "follow", @@ -273,7 +273,7 @@ { "name": "examples", "type": "doc", - "repository": "https://github.com/perftool-incubator/crucible-examples.git", + "repository": "https://github.com/perftool-incubator/crucible-examples", "primary-branch": "main", "checkout": { "mode": "follow", @@ -283,7 +283,7 @@ { "name": "testing", "type": "doc", - "repository": "https://github.com/perftool-incubator/testing-repo.git", + "repository": "https://github.com/perftool-incubator/testing-repo", "primary-branch": "master", "checkout": { "mode": "follow", @@ -291,6 +291,5 @@ } } ], - "unofficial": [ - ] + "unofficial": [] } diff --git a/crucible-install.sh b/crucible-install.sh index bf57c7ca..05e2d50e 100755 --- a/crucible-install.sh +++ b/crucible-install.sh @@ -206,12 +206,14 @@ function clean_old_install { old_install_path="/opt/crucible-moved-on-${timestamp}" echo "An existing installation of crucible exists and will be moved to $old_install_path" /bin/mv "$INSTALL_PATH" "$old_install_path" + echo fi if [ -e ${SYSCONFIG} ]; then old_sysconfig="${SYSCONFIG}-moved-on-${timestamp}" echo "An existing crucible sysconfig file exists and will be moved to ${old_sysconfig}" /bin/mv "${SYSCONFIG}" "${old_sysconfig}" + echo fi # reset the update tracker if there is any existing state @@ -291,6 +293,69 @@ function select_release { GIT_TAG="$release" } +function update_repos_config() { + local REPO BRANCH CRUCIBLE_PATH MODE + local current_repo_location primary_branch + local update_mode update_target + REPO=${1} + CRUCIBLE_PATH=${2} + BRANCH=${3} + + REPO_FILE=${CRUCIBLE_PATH}/config/repos.json + + current_repo_location=$(jq_query ${REPO_FILE} '.official[] | select(.name == "crucible") | .repository') + if [ "${REPO}" != "${current_repo_location}" ]; then + echo "Setting crucible repository to '${REPO}' in '${REPO_FILE}'" >> ${GIT_INSTALL_LOG} + jq_update ${REPO_FILE} repository --arg repository "${REPO}" '(.official[] | select(.name == "crucible") | .repository) |= $repository' + fi + + if [ -z "${BRANCH}" -a -z "$(git branch --show-current)" ]; then + # since we are not on a branch right now (immediately after a + # clone) and no branch was requested, let's assume this is in + # the CI environment and we have to do some special handling + BRANCH="HEAD" + echo "Detected CI environment, setting BRANCH=HEAD" + fi + + primary_branch=$(jq_query ${REPO_FILE} '.official[] | select(.name == "crucible") | ."primary-branch"') + checkout_target=$(jq_query ${REPO_FILE} '.official[] | select(.name == "crucible") | .checkout.target') + checkout_mode=$(jq_query ${REPO_FILE} '.official[] | select(.name == "crucible") | .checkout.mode') + if [ -n "${BRANCH}" ]; then + update_mode=0 + update_target=0 + if [ "${BRANCH}" == "${primary_branch}" ]; then + if [ "${BRANCH}" != "${checkout_target}" ]; then + update_target=1 + fi + else + if [ "${BRANCH}" != "${checkout_target}" ]; then + update_target=1 + + if [ "${BRANCH}" == "HEAD" ]; then + echo "Detected CI environment, setting crucible primary-branch to '${BRANCH}' in '${REPO_FILE}'" >> ${GIT_INSTALL_LOG} + jq_update ${REPO_FILE} primary-branch --arg primary_branch ${BRANCH} '(.official[] | select(.name == "crucible") | ."primary-branch") |= $primary_branch' + fi + fi + fi + if echo "${BRANCH}" | grep -q "^\(20[0-9][0-9]\.[1234]\|version-test\)$"; then + # this is a version install so lock it down + update_mode=1 + MODE="locked" + fi + if [ ${update_mode} -eq 1 ]; then + echo "Setting crucible checkout.mode to '${MODE}' in '${REPO_FILE}'" >> ${GIT_INSTALL_LOG} + jq_update ${REPO_FILE} checkout.mode --arg checkout_mode ${MODE} '(.official[] | select(.name == "crucible") | .checkout.mode) |= $checkout_mode' + fi + if [ ${update_target} -eq 1 ]; then + echo "Setting crucible checkout.target to '${BRANCH}' in '${REPO_FILE}'" >> ${GIT_INSTALL_LOG} + jq_update ${REPO_FILE} checkout.target --arg checkout_target ${BRANCH} '(.official[] | select(.name == "crucible") | .checkout.target) |= $checkout_target' + fi + else + git checkout ${checkout_target} >> ${GIT_INSTALL_LOG} 2>&1 || + exit_error "Failed to git checkout ${checkout_target}, checking ${GIT_INSTALL_LOG} for details" ${EC_FAIL_CHECKOUT} + fi +} + longopts="name:,email:,help,list-releases,verbose" longopts+=",client-server-registry:,client-server-auth-file:,client-server-tls-verify:" longopts+=",engine-registry:,engine-auth-file:,engine-tls-verify:" @@ -422,7 +487,9 @@ clean_old_install echo "Installing crucible in $INSTALL_PATH" echo "Using Git repo: ${GIT_REPO}" echo "Using Git branch: ${GIT_BRANCH}" -git clone $GIT_REPO $INSTALL_PATH > $GIT_INSTALL_LOG 2>&1 || +git_clone_cmd="git -c advice.detachedHead=false clone ${GIT_REPO} ${INSTALL_PATH}" +echo ${git_clone_cmd} > ${GIT_INSTALL_LOG} +${git_clone_cmd} >> $GIT_INSTALL_LOG 2>&1 || exit_error "Failed to git clone $GIT_REPO, check $GIT_INSTALL_LOG for details" $EC_FAIL_CLONE if [ -n "${GIT_BRANCH}" ]; then if pushd ${INSTALL_PATH} > /dev/null; then @@ -435,6 +502,14 @@ if [ -n "${GIT_BRANCH}" ]; then else echo "No specific git branch requested, using default" fi + +# make sure the branch being installed has jqlib support before +# attempting to use it +if [ -e ${INSTALL_PATH}/bin/jqlib ]; then + source ${INSTALL_PATH}/bin/jqlib + update_repos_config ${GIT_REPO} ${INSTALL_PATH} ${GIT_BRANCH} +fi + $INSTALL_PATH/bin/subprojects-install $GIT_TAG >>"$GIT_INSTALL_LOG" 2>&1 || exit_error "Failed to execute crucible-project install, check $GIT_INSTALL_LOG for details" $EC_FAIL_INSTALL @@ -459,14 +534,23 @@ CRUCIBLE_ENGINE_REPO_TLS_VERIFY=${SYSCONFIG_CRUCIBLE_ENGINE_TLS_VERIFY} _SYSCFG_ if [ ${VERBOSE} == 1 ]; then + echo + echo "Contents of git install log: '${GIT_INSTALL_LOG}'" cat ${GIT_INSTALL_LOG} echo + ${INSTALL_PATH}/bin/repo info + if [ -e ${INSTALL_PATH}/bin/jqlib ]; then + echo + ${INSTALL_PATH}/bin/repo config show + fi fi +echo echo "Pulling Crucible controller container image:" podman --log-level error pull ${CRUCIBLE_CONTROLLER_REGISTRY} || exit_error "Failed to pull controller image ${CRUCIBLE_CONTROLLER_REGISTRY}" ${EC_PULL_FAIL} +echo echo "Installation is complete. Run \"crucible help\" to see what's possible" echo "You can also source /etc/profile.d/crucible_completions.sh (or re-login) to use tab completion for crucible" echo diff --git a/spec/repos.json/schema.json b/schema/repos.json similarity index 100% rename from spec/repos.json/schema.json rename to schema/repos.json diff --git a/tests/test-installer b/tests/test-installer index e7b9d0c9..b081f52a 100755 --- a/tests/test-installer +++ b/tests/test-installer @@ -1,41 +1,86 @@ +#!/usr/bin/env bash +# -*- mode: sh; indent-tabs-mode: nil; sh-basic-offset: 4 -*- +# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=bash + +exec 2>&1 + set +e set -x +test_number=0 + +CRUCIBLE_DIR=$(pwd) + +function start_test() { + set +x + echo + echo "#################################################################################" + echo "Test Number: $(( test_number += 1 ))" + echo "#################################################################################" + set -x +} + +function stop_test() { + set +x + echo "*********************************************************************************" + echo + set -x +} + +start_test # usage stdout=$(./crucible-install.sh --help) [[ "$stdout" = *"Usage"* ]] || exit 1 +stop_test +start_test # non-root ec=$(grep EC_FAIL_USER= crucible-install.sh | cut -d '=' -f2) -./crucible-install.sh +./crucible-install.sh \ + --git-repo ${CRUCIBLE_DIR} \ + --verbose test "$?" = "$ec" || exit 1 +stop_test +start_test # no args ec=$(grep EC_FAIL_REGISTRY_UNSET= crucible-install.sh | cut -d '=' -f2) -sudo ./crucible-install.sh +sudo ./crucible-install.sh \ + --git-repo ${CRUCIBLE_DIR} \ + --verbose test "$?" = "$ec" || exit 1 +stop_test -for arg_mode in client-server engine; do echo "testing - arg_mode=${arg_mode}" +for arg_mode in client-server engine; do + start_test + echo "testing arg_mode=${arg_mode}" # auth file not found ec=$(grep EC_AUTH_FILE_NOT_FOUND= crucible-install.sh | cut -d '=' -f2) sudo ./crucible-install.sh \ + --git-repo ${CRUCIBLE_DIR} \ --${arg_mode}-registry myregistry.io/crucible \ - --${arg_mode}-auth-file /tmp/auth-file.json + --${arg_mode}-auth-file /tmp/auth-file.json \ + --verbose test "$?" = "$ec" || exit 1 + stop_test done for arg_mode in client-server engine; do + start_test echo "testing arg_mode=${arg_mode}" # invalid tls verify value ec=$(grep EC_TLS_VERIFY_ERROR= crucible-install.sh | cut -d '=' -f2) sudo ./crucible-install.sh \ + --git-repo ${CRUCIBLE_DIR} \ --${arg_mode}-registry myregistry.io/crucible \ - --${arg_mode}-tls-verify foo + --${arg_mode}-tls-verify foo \ + --verbose test "$?" = "$ec" || exit 1 + stop_test done for arg_mode in client-server engine; do + start_test echo "testing arg_mode=${arg_mode}" # default args touch /tmp/auth-file.json @@ -43,15 +88,19 @@ for arg_mode in client-server engine; do # TODO(rfolco): temporary workaround until we make it distro generic sudo mkdir -p $(dirname $cfgfile) sudo ./crucible-install.sh \ + --git-repo ${CRUCIBLE_DIR} \ --${arg_mode}-registry myregistry.io/crucible \ - --${arg_mode}-auth-file /tmp/auth-file.json + --${arg_mode}-auth-file /tmp/auth-file.json \ + --verbose test "$?" = "0" || exit 1 grep "myregistry.io" $cfgfile || exit 1 grep "auth-file.json" $cfgfile || exit 1 command -v crucible || exit 1 + stop_test done for arg_mode in client-server engine; do + start_test echo "testing arg_mode=${arg_mode}" # identity touch /tmp/auth-file.json @@ -62,15 +111,19 @@ for arg_mode in client-server engine; do # TODO(rfolco): temporary workaround until we make it distro generic sudo mkdir -p $(dirname $cfgfile) sudo ./crucible-install.sh \ + --git-repo ${CRUCIBLE_DIR} \ --${arg_mode}-registry myregistry.io/crucible \ --${arg_mode}-auth-file /tmp/auth-file.json \ --name "Nobody" \ - --email "nobody@somewhere.com" + --email "nobody@somewhere.com" \ + --verbose sudo grep "CRUCIBLE_NAME=\"Nobody\"" $idfile || exit 1 sudo grep "CRUCIBLE_EMAIL=\"nobody@somewhere.com\"" $idfile || exit 1 + stop_test done for arg_mode in client-server engine; do + start_test echo "testing arg_mode=${arg_mode}" # override existing installation touch /tmp/auth-file.json @@ -78,52 +131,78 @@ for arg_mode in client-server engine; do # TODO(rfolco): temporary workaround until we make it distro generic sudo mkdir -p $(dirname $cfgfile) sudo ./crucible-install.sh \ + --git-repo ${CRUCIBLE_DIR} \ --${arg_mode}-registry myregistry.io/crucible \ - --${arg_mode}-auth-file /tmp/auth-file.json + --${arg_mode}-auth-file /tmp/auth-file.json \ + --verbose test "$?" = "0" || exit 1 ls /opt/crucible-moved* || exit 1 + stop_test done for arg_mode in client-server engine; do + start_test echo "testing arg_mode=${arg_mode}" # override existing installation, no auth-file arg cfgfile=$(grep SYSCONFIG= crucible-install.sh | cut -d '=' -f2 | sed 's/"//g') # TODO(rfolco): temporary workaround until we make it distro generic sudo mkdir -p $(dirname $cfgfile) sudo ./crucible-install.sh \ - --${arg_mode}-registry myregistry.io/crucible + --git-repo ${CRUCIBLE_DIR} \ + --${arg_mode}-registry myregistry.io/crucible \ + --verbose test "$?" = "0" || exit 1 ls /opt/crucible-moved* || exit 1 + stop_test done +start_test # RELEASE: --release and --list-releases ## prep step: check tags in remote repo tags=$(git ls-remote --tags \ --sort='version:refname' \ https://github.com/perftool-incubator/crucible.git \ | awk -F/ '{print$NF}') +stop_test +start_test ## show releases, test rc and stdout stdout=$(sudo ./crucible-install.sh \ + --git-repo ${CRUCIBLE_DIR} \ --list-releases) test "$?" = "0" || exit 1 [[ "$stdout" = *"$tags"* ]] || exit 1 +stop_test +start_test ## negative test: --release and --git-repo ec=$(grep EC_RELEASE_DEFAULT_REPO_ONLY= crucible-install.sh | cut -d '=' -f2) sudo ./crucible-install.sh \ - --release DoesNotMatter --git-repo DoesNotMatter + --release DoesNotMatter --git-repo DoesNotMatter \ + --verbose test "$?" = "$ec" || exit 1 +stop_test +start_test ## negative test: --release and --git-branch ec=$(grep EC_RELEASE_CONFLICTS_WITH_BRANCH= crucible-install.sh | cut -d '=' -f2) sudo ./crucible-install.sh \ - --release DoesNotMatter --git-branch DoesNotMatter + --release DoesNotMatter --git-branch DoesNotMatter \ + --verbose test "$?" = "$ec" || exit 1 +stop_test +start_test ## negative test: non-existent tag +# for this test we cannot use --git-repo (because it's not allowed in +# conjunction with --release) so we have to "hack" DEFAULT_GIT_REPO to +# point where we need it to +sed -i -e "s|\(DEFAULT_GIT_REPO\)=.*|\1=\"${CRUCIBLE_DIR}\"|" crucible-install.sh +grep "^DEFAULT_GIT_REPO" crucible-install.sh ec=$(grep EC_FAIL_CHECKOUT= crucible-install.sh | cut -d '=' -f2) sudo ./crucible-install.sh \ --release NonExitentTag \ - --client-server-registry myregistry.io/crucible + --client-server-registry myregistry.io/crucible \ + --verbose test "$?" = "$ec" || exit 1 +stop_test