From a80ae89022425149450745ec3034f39d37ed1351 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 4 Apr 2024 12:02:04 -0600 Subject: [PATCH 1/4] Per #429, add SonarQube workflow for METplotpy --- .github/jobs/configure_sonarqube.sh | 65 ++++++++++++++++ .github/pull_request_template.md | 3 + .github/workflows/sonarqube.yml | 76 +++++++++++++++++++ internal/scripts/sonarqube/run_sonarqube.sh | 42 ++++++---- .../sonarqube/sonar-project.properties | 22 ++---- 5 files changed, 181 insertions(+), 27 deletions(-) create mode 100755 .github/jobs/configure_sonarqube.sh create mode 100644 .github/workflows/sonarqube.yml diff --git a/.github/jobs/configure_sonarqube.sh b/.github/jobs/configure_sonarqube.sh new file mode 100755 index 00000000..929b03fa --- /dev/null +++ b/.github/jobs/configure_sonarqube.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Constants +SONAR_PROPERTIES_DIR=internal/scripts/sonarqube +SONAR_PROPERTIES=sonar-project.properties + +# Check that this is being run from the top-level METdataio directory +if [ ! -e $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES ]; then + echo "ERROR: ${0} -> must be run from the top-level METdataio directory" + exit 1 +fi + +# Check required environment variables +if [ -z ${SOURCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$SOURCE_BRANCH not defined!" + exit 1 +fi +if [ -z ${WD_REFERENCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$WD_REFERENCE_BRANCH not defined!" + exit 1 +fi +if [ -z ${SONAR_HOST_URL+x} ]; then + echo "ERROR: ${0} -> \$SONAR_HOST_URL not defined!" + exit 1 +fi +if [ -z ${SONAR_TOKEN+x} ]; then + echo "ERROR: ${0} -> \$SONAR_TOKEN not defined!" + exit 1 +fi + +# Define the version string +SONAR_PROJECT_VERSION=$(cat docs/version | cut -d'=' -f2 | tr -d '" ') + +# +# Define the $SONAR_REFERENCE_BRANCH as the +# - Target of any requests +# - Manual setting for workflow dispatch +# - Source branch for any pushes (e.g. develop) +# +if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + export SONAR_REFERENCE_BRANCH=$GITHUB_BASE_REF +elif [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then + export SONAR_REFERENCE_BRANCH=$WD_REFERENCE_BRANCH +else + export SONAR_REFERENCE_BRANCH=$SOURCE_BRANCH +fi + +# Configure the sonar-project.properties +[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES +sed -e "s|SONAR_PROJECT_KEY|METdataio-GHA|" \ + -e "s|SONAR_PROJECT_NAME|METdataio GHA|" \ + -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_BRANCH_NAME|$SOURCE_BRANCH|" \ + $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES + +# Define new code when the source and reference branches differ +if [ "$SOURCE_BRANCH" != "$SONAR_REFERENCE_BRANCH" ]; then + echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES +fi + +echo "Contents of the $SONAR_PROPERTIES file:" +cat $SONAR_PROPERTIES + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index dda54835..342abeb9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,6 +11,9 @@ - [ ] Will this PR result in changes to the test suite? **[Yes or No]**
If **yes**, describe the new output and/or changes to the existing output:
+- [ ] Do these changes introduce new SonarQube findings? **[Yes or No]**
+If **yes**, please describe: + - [ ] Please complete this pull request review by **[Fill in date]**.
## Pull Request Checklist ## diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml new file mode 100644 index 00000000..d48e1237 --- /dev/null +++ b/.github/workflows/sonarqube.yml @@ -0,0 +1,76 @@ +name: SonarQube Scan + +# Run SonarQube for Pull Requests and changes to the develop and main_vX.Y branches + +on: + + # Trigger analysis for pushes to develop and main_vX.Y branches + push: + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '**/README.md' + - '**/LICENSE.md' + + # Trigger analysis for pull requests to develop and main_vX.Y branches + pull_request: + types: [opened, synchronize, reopened] + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '**/README.md' + - '**/LICENSE.md' + + workflow_dispatch: + inputs: + reference_branch: + description: 'Reference Branch' + default: develop + type: string + +jobs: + sonarqube: + name: SonarQube Scan + runs-on: ubuntu-latest + + steps: + + - uses: actions/checkout@v4 + with: + # Disable shallow clones for better analysis + fetch-depth: 0 + + - name: Get branch name + id: get_branch_name + run: echo branch_name=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT + + - name: Configure SonarQube + run: .github/jobs/configure_sonarqube.sh + env: + SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }} + WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Scan + uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Quality Gate check + id: sonarqube-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@master + # Force to fail step after specific time. + timeout-minutes: 5 + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/internal/scripts/sonarqube/run_sonarqube.sh b/internal/scripts/sonarqube/run_sonarqube.sh index c98636d3..7cebb003 100755 --- a/internal/scripts/sonarqube/run_sonarqube.sh +++ b/internal/scripts/sonarqube/run_sonarqube.sh @@ -1,10 +1,10 @@ #!/bin/bash # -# Run SonarQube Source Code Analyzer on a specified revision of MET +# Run SonarQube Source Code Analyzer for METplotpy #======================================================================= # # This run_sonarqube.sh script will check out the specified version -# of MET and run the SonarQube Source Code Analyzer on it. First, +# of METplotpy and run the SonarQube Source Code Analyzer on it. First, # go to the directory where you would like the SCA output written and # then run: # @@ -12,9 +12,9 @@ # METplotpy/sonarqube/run_sonarqube.sh name # # Usage: run_sonarqube.sh name -# Test the specified branched version of MET: +# Test the specified branched version of METplotpy: # run_sonarqube.sh {branch name} -# Test the specified tagged version of MET: +# Test the specified tagged version of METplotpy: # run_sonarqube.sh {tag name} # #======================================================================= @@ -24,15 +24,25 @@ GIT_REPO_NAME=METplotpy GIT_REPO="https://github.com/dtcenter/${GIT_REPO_NAME}" function usage { - echo - echo "USAGE: $(basename $0) name" - echo " where \"name\" specifies a branch, tag, or hash." - echo + echo + echo "USAGE: $(basename $0) name" + echo " where \"name\" specifies a branch, tag, or hash." + echo } # Check for arguments if [[ $# -lt 1 ]]; then usage; exit; fi +# Check that SONAR_TOKEN and SONAR_HOST_URL are defined +if [ -z ${SONAR_TOKEN} ]; then + echo "ERROR: SONAR_TOKEN must be set" + exit 1 +fi +if [ -z ${SONAR_HOST_URL} ]; then + echo "ERROR: SONAR_HOST_URL must be set" + exit 1 +fi + # Check that SONARQUBE_WRAPPER_BIN is defined if [ -z ${SONARQUBE_WRAPPER_BIN} ]; then which build-wrapper-linux-x86-64 2> /dev/null @@ -87,7 +97,6 @@ function run_command() { return ${STATUS} } - # Store the full path to the scripts directory SCRIPT_DIR=`dirname $0` if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi @@ -102,14 +111,21 @@ run_command "git clone ${GIT_REPO} ${REPO_DIR}" run_command "cd ${REPO_DIR}" run_command "git checkout ${1}" +# Define the version string +SONAR_PROJECT_VERSION=$(cat docs/version | cut -d'=' -f2 | tr -d '" ') + SONAR_PROPERTIES=sonar-project.properties -# Copy sonar-project.properties for Python code +# Configure the sonar-project.properties [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -cp -p $SCRIPT_DIR/sonar-project.properties $SONAR_PROPERTIES +sed -e "s|SONAR_PROJECT_KEY|METplotpy_NB|" \ + -e "s|SONAR_PROJECT_NAME|METplotpy Nightly Build|" \ + -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_BRANCH_NAME|${1}|" \ + $SCRIPT_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES # Run SonarQube scan for Python code run_command "${SONARQUBE_SCANNER_BIN}/sonar-scanner" -# Run SonarQube report generator to make a PDF file -#TODAY=`date +%Y%m%d` diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index dfb8fee5..19309173 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -1,17 +1,11 @@ -sonar.projectKey=org.sonarqube:METplotpy_NB -sonar.projectName=METplotpy Nightly Build -sonar.projectVersion=1.0 - +# Project and source code settings +sonar.projectKey=SONAR_PROJECT_KEY +sonar.projectName=SONAR_PROJECT_NAME +sonar.projectVersion=SONAR_PROJECT_VERSION +sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=metplotpy,test - -# The build-wrapper output dir - -# Encoding of the source files sonar.sourceEncoding=UTF-8 -#----- Default SonarQube server -#sonar.host.url=http://localhost:9000 -sonar.host.url=http://mandan:9000 - -sonar.login=met -sonar.password=met@sonar.ucar +# SonarQube server +sonar.host.url=SONAR_HOST_URL +sonar.token=SONAR_TOKEN From 3d682d9e1f793f35219cb2ded2effbc7705a3135 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 4 Apr 2024 12:25:52 -0600 Subject: [PATCH 2/4] Per #429, update nightly build email list. --- internal/scripts/sonarqube/run_nightly.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/scripts/sonarqube/run_nightly.sh b/internal/scripts/sonarqube/run_nightly.sh index 95098ff2..1fee1763 100755 --- a/internal/scripts/sonarqube/run_nightly.sh +++ b/internal/scripts/sonarqube/run_nightly.sh @@ -12,7 +12,7 @@ # METplotpy/sonarqube/run_nightly.sh name # # Usage: run_nightly.sh name -# where "name" speci1fies a branch, tag, or hash +# where "name" specifies a branch, tag, or hash # # For example, scan the develop branch: # run_nightly.sh develop @@ -20,8 +20,7 @@ #======================================================================= # Constants -EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu mccabe@ucar.edu jpresto@ucar.edu minnawin@ucar.edu" -TEST_EMAIL_LIST="hsoh@ucar.edu" +EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu jpresto@ucar.edu minnawin@ucar.edu" KEEP_DAYS=5 GIT_REPO_NAME=METplotpy @@ -67,12 +66,9 @@ LOGFILE=${RUN_DIR}/run_sonarqube_${TODAY}.log # Run scan and check for bad return status ${SCRIPT_DIR}/run_sonarqube.sh ${1} >& ${LOGFILE} if [[ $? -ne 0 ]]; then - [ ! -z $2 ] && EMAIL_LIST=$TEST_EMAIL_LIST echo "$0: The nightly SonarQube scanning for $GIT_REPO_NAME FAILED in `basename ${RUN_DIR}`." >> ${LOGFILE} cat ${LOGFILE} | mail -s "$GIT_REPO_NAME SonarQube scanning failed for ${1} in `basename ${RUN_DIR}` (autogen msg)" ${EMAIL_LIST} exit 1 fi -# Convert SonarQube report from pdf to html - exit 0 From bca784c6f4b3352ad3047a2dd9fe7229fc7c1575 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 4 Apr 2024 15:08:16 -0600 Subject: [PATCH 3/4] Per #429, fix cut/paste error configure_sonarqube.sh --- .github/jobs/configure_sonarqube.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/jobs/configure_sonarqube.sh b/.github/jobs/configure_sonarqube.sh index 929b03fa..55138b28 100755 --- a/.github/jobs/configure_sonarqube.sh +++ b/.github/jobs/configure_sonarqube.sh @@ -4,9 +4,9 @@ SONAR_PROPERTIES_DIR=internal/scripts/sonarqube SONAR_PROPERTIES=sonar-project.properties -# Check that this is being run from the top-level METdataio directory +# Check that this is being run from the top-level METplotpy directory if [ ! -e $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES ]; then - echo "ERROR: ${0} -> must be run from the top-level METdataio directory" + echo "ERROR: ${0} -> must be run from the top-level METplotpy directory" exit 1 fi @@ -47,8 +47,8 @@ fi # Configure the sonar-project.properties [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -sed -e "s|SONAR_PROJECT_KEY|METdataio-GHA|" \ - -e "s|SONAR_PROJECT_NAME|METdataio GHA|" \ +sed -e "s|SONAR_PROJECT_KEY|METplotpy-GHA|" \ + -e "s|SONAR_PROJECT_NAME|METplotpy GHA|" \ -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ From 45a49af81f89cc3c5146ac7c3127721b1149c40a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 5 Apr 2024 11:26:38 -0600 Subject: [PATCH 4/4] Per #429, exclude test code from code coverage statistics. --- internal/scripts/sonarqube/sonar-project.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 19309173..f272b9e4 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -4,6 +4,7 @@ sonar.projectName=SONAR_PROJECT_NAME sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=metplotpy,test +sonar.coverage.exclusions=test/** sonar.sourceEncoding=UTF-8 # SonarQube server