Merge pull request #398 from perftool-incubator/dev-kmr

activate repos.json

.github/workflows/crucible-release.yaml

@@ -119,7 +119,7 @@ jobs:
             # get 3rd column (repo URL) without the starting '/' (remove first char),
             # so repo urls /a /b /c are extracted to a b c
             # and add to a bash array ( a b c )
-            projects=( $(grep -v ^# default_subprojects | awk {'print $3'} | cut -c2- ) )
+            projects=( $(jq -r '.official[] | select(.name != "crucible") | .name) )
             # builtin implict join array "a","b","c" (double quotes for a valid json)
             printf -v list '"%s",' "${projects[@]}"
             # convert to a comma separated list ["a","b","c"]

README.md

@@ -4,7 +4,7 @@
 
 ## Introduction
 
-Crucible is a performance tool harness that integrates [multiple performance tooling projects](config/default_subprojects). The project's intention is to provide a well rounded, portable, and highly functional end-to-end performance testing harness. Crucible can be used for test automation to execute, measure, store, visualize, and analyze the performance of various systems-under-test (SUT).
+Crucible is a performance tool harness that integrates [multiple performance tooling projects](config/repos.json). The project's intention is to provide a well rounded, portable, and highly functional end-to-end performance testing harness. Crucible can be used for test automation to execute, measure, store, visualize, and analyze the performance of various systems-under-test (SUT).
 
 ## Features
 

bin/_crucible_completions

@@ -29,7 +29,7 @@ _crucible_completions() {
                 COMPREPLY=($(compgen -W "result metric" -- "${COMP_WORDS[2]}"))
                 ;;
             repo)
-                COMPREPLY=($(compgen -W "info details" -- "${COMP_WORDS[2]}"))
+                COMPREPLY=($(compgen -W "info details config" -- "${COMP_WORDS[2]}"))
                 ;;
             update)
                 COMPREPLY=($(compgen -W "all crucible $(cd $CRUCIBLE_HOME/subprojects/; find . -type l | sed 'sX./XX')" -- "${COMP_WORDS[2]}"))
@@ -97,6 +97,9 @@ _crucible_completions() {
             info|details)
                 COMPREPLY=($(compgen -W "all crucible $(cd $CRUCIBLE_HOME/subprojects/; find . -type l | sed 'sX./XX')" -- "${COMP_WORDS[3]}"))
                 ;;
+            config)
+                COMPREPLY=($(compgen -W "help show" -- "${COMP_WORDS[3]}"))
+                ;;
         esac
     elif [ "$(expr $num_words % 2)" -eq 0 -a ${COMP_WORDS[1]} == "rm" ]; then
         case "${COMP_WORDS[2]}" in

bin/_git-fetch

@@ -17,6 +17,9 @@ if [ ! -e ${CRUCIBLE_HOME} ]; then
 fi
 
 source ${CRUCIBLE_HOME}/bin/base
+source ${CRUCIBLE_HOME}/bin/jqlib
+
+REPO_FILE=${CRUCIBLE_HOME}/config/repos.json
 
 PROCESS_LOCK_FILE="${UPDATE_STATUS_FILE}.process-lock"
 touch "${PROCESS_LOCK_FILE}"
@@ -45,34 +48,47 @@ touch "${PROCESS_LOCK_FILE}"
     {
         UPDATE_COUNT=0
 
-        if pushd ${CRUCIBLE_HOME}/subprojects > /dev/null; then
-            for tmp_repo in $(find . -maxdepth 2 -mindepth 2 -type l | sed 'sX./XX'); do
-                echo "REPO: ${tmp_repo}"
-                if pushd ${tmp_repo} > /dev/null; then
+        for project in $(jq_query ${REPO_FILE} '.official[], .unofficial[] | .name'); do
+            echo "Processing project: ${project}"
+
+            PROJECT_NAME=$(jq_query ${REPO_FILE} --arg project ${project} '.official[], .unofficial[] | select(.name == $project) | .name')
+            PROJECT_TYPE=$(jq_query ${REPO_FILE} --arg project ${project} '.official[], .unofficial[] | select(.name == $project) | .type')
+            PROJECT_CHECKOUT_MODE=$(jq_query ${REPO_FILE} --arg project ${project} '.official[], .unofficial[] | select(.name == $project) | .checkout.mode')
+
+            if [ "${PROJECT_CHECKOUT_MODE}" == "follow" ]; then
+                if [ "${PROJECT_NAME}" == "crucible" ]; then
+                    repo_dir=${CRUCIBLE_HOME}
+                else
+                    repo_dir=${CRUCIBLE_HOME}/subprojects/$(get_project_type_dir ${PROJECT_TYPE})/${PROJECT_NAME}
+                fi
+
+                if pushd ${repo_dir} > /dev/null; then
                     if [ -d "./.git" ]; then
-                        git remote --verbose update --prune
+                        git_fetch_cmd="git fetch --all --prune --prune-tags --tags --verbose"
+                        echo "${git_fetch_cmd}"
+                        ${git_fetch_cmd}
 
                         git_status=$(git_get_status)
                         git_tracking=$(git_get_tracking "${git_status}")
 
                         if [ -n "${git_tracking}" -a "${git_tracking}" != "(detached)" ]; then
                             git_delta_behind=$(git_get_delta_behind "${git_status}")
+                            echo "Found ${git_delta_behind} update(s)"
                             if [ "${git_delta_behind}" != "0" ]; then
                                 (( UPDATE_COUNT += git_delta_behind ))
                             fi
                         fi
                     else
-                        echo "ERROR: not a git repo"
+                        echo "ERROR: '${repo_dir}' is not a Git repo"
                     fi
                     popd > /dev/null
+                else
+                    echo "ERROR: Faild to pushd to '${repo_dir}'"
                 fi
-                echo
-            done
-            popd > /dev/null
-        else
-            echo "ERROR: Could not pushd to ${CRUCIBLE_HOME}/subprojects"
-            exit 1
-        fi
+            else
+                echo "Skipping analysis since checkout.mode is 'locked'"
+            fi
+        done
 
         CURRENT_TIMESTAMP=$(date +%s)
         {

bin/_help

@@ -11,7 +11,7 @@ function help() {
     echo "help                         |  Show this help message"
     echo "run                          |  Run a benchmark"
     echo "log                          |  Manage with the crucible log"
-    echo "repo                         |  Get info on the crucible and subproject git repos"
+    echo "repo                         |  Get info on or configure the crucible and subproject git repos"
     echo "update                       |  Update all or part of the crucible software"
     echo "console                      |  Run user supplied programs inside a crucible wrapper for logging purposes"
     echo "wrapper                      |  Run a crucible subproject command directly, within a crucible container"

bin/_main

@@ -402,6 +402,7 @@ elif [ "${1}" == "run" ]; then
 
     ${CRUCIBLE_HOME}/bin/repo info > ${base_run_dir}/config/crucible.repo.info
     ${CRUCIBLE_HOME}/bin/repo details > ${base_run_dir}/config/crucible.repo.details
+    ${CRUCIBLE_HOME}/bin/repo config show > ${base_run_dir}/config/crucible.repo.config.show
 
     rs_run_cmd="${CRUCIBLE_HOME}/subprojects/core/rickshaw/rickshaw-run\
       ${params_args}\

bin/_update-git

@@ -12,55 +12,106 @@ exec 2>&1
 source /etc/sysconfig/crucible
 
 if [ -z "${CRUCIBLE_HOME}" ]; then
-    echo "CRUCIBLE_HOME not defined, exiting."
+    echo "ERROR: CRUCIBLE_HOME not defined!"
     exit 1
 fi
 
 if [ ! -e "${CRUCIBLE_HOME}" ]; then
-    echo "Could not find ${CRUCIBLE_HOME}, exiting."
+    echo "ERROR: Could not find ${CRUCIBLE_HOME}!"
     exit 1
 fi
 
 source "${CRUCIBLE_HOME}/bin/base"
+source "${CRUCIBLE_HOME}/bin/jqlib"
+
+REPO_FILE="${CRUCIBLE_HOME}/config/repos.json"
+
+PROJECT=${1}
+if [ -z "${PROJECT}" ]; then
+    echo "ERROR: You must supply a project name!"
+    exit 1
+else
+    PROJECT_NAME=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | .name')
+    if [ "${PROJECT_NAME}" != "${PROJECT}" ]; then
+	echo "ERROR: You must supply a valid project name!"
+	exit 1
+    fi
+
+    PROJECT_REPOSITORY=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | .repository')
+    PROJECT_PRIMARY_BRANCH=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | ."primary-branch"')
+    PROJECT_CHECKOUT_MODE=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | .checkout.mode')
+    PROJECT_CHECKOUT_TARGET=$(jq_query ${REPO_FILE} --arg project ${PROJECT} '.official[], .unofficial[] | select(.name == $project) | .checkout.target')
+fi
+
+current_dir=$(pwd)
+echo "Attempting to update project repo ${PROJECT_NAME} at '${current_dir}' -> '$(readlink -e ${current_dir})'"
 
 if [ -d "./.git" ]; then
+    # validate that the current repository matches the expected repository
+    ORIGIN_REPOSITORY=$(git remote get-url origin)
+    if [ "${ORIGIN_REPOSITORY}" != "${PROJECT_REPOSITORY}" ]; then
+	echo "ERROR: Invalid state!  The origin repository does not match the project repository!"
+	exit 1
+    fi
+
+    current_ref=$(git rev-parse --verify HEAD)
+    expected_ref=$(git rev-parse --verify ${PROJECT_CHECKOUT_TARGET})
+
+    if [ "${current_ref}" != "${expected_ref}" ]; then
+	echo "ERROR: Invalid state!  The current checked out ref (${current_ref}) does not match the checkout target's ref (${PROJECT_CHECKOUT_TARGET} -> ${expected_ref})!"
+	exit 1
+    fi
+
     # fetch updates from remote(s)
-    if ! git remote --verbose update --prune > /dev/null 2>&1; then
-	echo "ERROR: could not git-update"
+    git_fetch_cmd="git fetch --all --prune --prune-tags --tags --verbose"
+    echo "${git_fetch_cmd}"
+    if ! ${git_fetch_cmd} 2>&1; then
+	echo "ERROR: could not git-fetch"
 	exit 1
     fi
 
-    git_status=$(git_get_status)
-    git_local_branch=$(git_get_local_branch "${git_status}")
-    git_tracking=$(git_get_tracking "${git_status}")
+    # store any local changes
+    stash_output=$(git stash)
+    if [ $? != 0 ]; then
+	echo "ERROR: could not git-stash"
+	exit 1
+    fi
 
-    if [ -n "${git_tracking}" -a "${git_local_branch}" != "(detached)" ]; then
-	# store any local changes
-	stash_output=$(git stash)
-	if [ $? != 0 ]; then
-	    echo "ERROR: could not git-stash"
-	    exit 1
-	fi
+    # return to the primary branch so that we can then return to the
+    # checkout target in case it's reference has changed (ie. an
+    # updated tag)
+    echo "Switching to primary branch to quiesce the repository state"
+    git checkout ${PROJECT_PRIMARY_BRANCH}
 
-	# merge any changes from the remote branch
-	if ! git pull --verbose --ff-only 2> /dev/null; then
-	    echo "ERROR: could not git-pull"
-	    exit 1
-	fi
+    # return to the checkout target if it is not the primary branch
+    if [ "${PROJECT_PRIMARY_BRANCH}" != "${PROJECT_CHECKOUT_TARGET}" ]; then
+	echo "Returning to the checkout target"
+	git -c advice.detachedHead=false checkout ${PROJECT_CHECKOUT_TARGET}
+    fi
 
-	if ! echo "${stash_output}" | grep -q "No local changes to save"; then
-            # reapply local changes
-            if ! git stash pop > /dev/null; then
-		echo -e "${stash_output}"
-		echo "ERROR: could not git-stash pop"
+    if [ "${PROJECT_CHECKOUT_MODE}" == "locked" ]; then
+	# return to the previous ref if is supposed to be locked there
+	echo "Returning to the previously checked out ref since I am in locked mode"
+	git checkout ${current_ref}
+    elif [ "${PROJECT_CHECKOUT_MODE}" == "follow" ]; then
+	if [ "${PROJECT_CHECKOUT_TARGET}" == "HEAD" ]; then
+	    echo "Detected CI environment, skipping merge of changes from the remote branch (because they do not exist)"
+	else
+	    # merge any changes from the remote branch
+	    echo "Merging any available upstream changes because I am in follow mode"
+	    if ! git pull --verbose --ff-only 2> /dev/null; then
+		echo "ERROR: could not git-pull"
 		exit 1
 	    fi
 	fi
-    else
-	if [ "${git_local_branch}" == "(detached)" ]; then
-	    echo "WARNING: This git repo is in a detached state, not modifying the active contents"
-	else
-	    echo "ERROR: This git repo is in an unknown state"
+    fi
+
+    if ! echo "${stash_output}" | grep -q "No local changes to save"; then
+        # reapply local changes
+	echo "Reapplying stashed local changes"
+        if ! git stash pop > /dev/null; then
+	    echo -e "${stash_output}"
+	    echo "ERROR: could not git-stash pop"
 	    exit 1
 	fi
     fi

bin/base

@@ -944,3 +944,17 @@ function git_get_remote_name() {
 function git_get_remote_branch() {
     echo "${1}" | awk -F'/' '{ print $2 }'
 }
+
+# map a project type to it's actual on disk directory name
+function get_project_type_dir() {
+    local TYPE
+    TYPE=$1
+
+    case "${TYPE}" in
+        benchmark|tool|doc)
+            echo "${TYPE}s"
+            ;;
+        *)
+            echo "${TYPE}"
+    esac
+}

bin/jqlib

@@ -0,0 +1,27 @@
+#!/bin/bash
+# -*- mode: sh; indent-tabs-mode: nil; sh-basic-offset: 4 -*-
+# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=bash
+
+function jq_update() {
+    local FILE TMP_FILE REASON
+    FILE=${1}; shift
+    REASON=${1}; shift
+
+    TMP_FILE=${FILE}.tmp
+
+    jq --indent 4 "$@" ${FILE} > ${TMP_FILE}
+    RC=$?
+
+    if [ ${RC} -ne 0 ]; then
+        exit_error "Failed to update ${REASON} in ${FILE}"
+    fi
+
+    mv ${TMP_FILE} ${FILE}
+}
+
+function jq_query() {
+    local FILE
+    FILE=${1}; shift
+
+    jq -r "$@" ${FILE}
+}