Merge branch 'main' into dependabot/pip/scripts/jinja2-3.1.3

.github/workflows/code-scanning-pack-gen.yml

@@ -99,8 +99,8 @@ jobs:
         run: |
           PATH=$PATH:$CODEQL_HOME/codeql
 
-          codeql query compile --threads 0 cpp
-          codeql query compile --threads 0 c
+          codeql query compile --precompile --threads 0 cpp
+          codeql query compile --precompile --threads 0 c
 
           cd ..
           zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas

.github/workflows/finalize-release.yml

@@ -9,8 +9,12 @@ on:
     inputs:
       ref:
         description: |
-          The release branch to finalize.
+          The ref of release to finalize (e.g., 'rc/MAJOR.MINOR.PATCH').
         required: true
+      tool-ref:
+        description: |
+          The ref to the tooling to use for the finalize (e.g., 'rc/MAJOR.MINOR.PATCH').
+        required: false 
 
 jobs:
   finalize-release:
@@ -20,34 +24,56 @@ jobs:
       - name: Determine ref
         env:
           REF_FROM_INPUT: ${{ inputs.ref }}
+          TOOL_REF_FROM_INPUT: ${{ inputs.tool-ref }}
           REF_FROM_PR: ${{ github.event.pull_request.merge_commit_sha  }}
           BASE_REF_FROM_PR: ${{ github.event.pull_request.base.ref }}
         run: |
           if [[ $GITHUB_EVENT_NAME == "workflow_dispatch" ]]; then
             echo "REF=$REF_FROM_INPUT" >> "$GITHUB_ENV"
+            echo "TOOL_REF=$TOOL_REF_FROM_INPUT" >> "$GITHUB_ENV"
             echo "BASE_REF=$REF_FROM_INPUT" >> "$GITHUB_ENV"
           else
             echo "REF=$REF_FROM_PR" >> "$GITHUB_ENV"
+            echo "TOOL_REF=$REF_FROM_PR" >> "$GITHUB_ENV"
             echo "BASE_REF=$BASE_REF_FROM_PR" >> "$GITHUB_ENV"
           fi
 
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{ env.REF }}
+          fetch-depth: 0
+          path: release 
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.TOOL_REF }}
+          path: tooling 
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: pip install -r scripts/release/requirements.txt
+        working-directory: tooling
 
       - name: Configure git
         run: |
           git config user.name "$GITHUB_ACTOR"
           git config user.email "[email protected]"
+        working-directory: release 
 
       - name: Update release tag
         run: |
           version=${BASE_REF#rc/}
           echo "Creating release tag v$version"
 
-          git tag -a v$version -m "Release v$version"
-          git push -f origin v$version
+          git tag -f -a v$version -m "Release v$version"
+          git push --force origin v$version
+        working-directory: release 
 
       - name: Finalize release
         env:
@@ -57,30 +83,51 @@ jobs:
           echo "Finalizing release v$version"
 
           gh release edit "v$version" --draft=false --tag=v$version
+        working-directory: release 
 
       - name: Determine if release was a hotfix release
         run: |
           version=${BASE_REF#rc/}
-          echo "HOTFIX_RELEASE=$(python scripts/release/is-hotfix.py $version)" >> "$GITHUB_ENV"
+          # We are running the script in the tooling directory with the release directory as the working directory
+          echo "HOTFIX_RELEASE=$(python ../tooling/scripts/release/is-hotfix-release.py $version)" >> "$GITHUB_ENV"
+        working-directory: release 
+
+      - name: Determine next release version
+        if: env.HOTFIX_RELEASE == 'false'
+        run: |
+          version=${BASE_REF#rc/}
+          next_version=$(python scripts/release/next-version.py --component minor --pre-release dev -- $version)
+          echo "NEXT_VERSION=$next_version" >> "$GITHUB_ENV"
+        working-directory: tooling 
+
+      - name: Generate token
+        if: env.HOTFIX_RELEASE == 'false'
+        id: generate-token
+        uses: actions/create-github-app-token@eaddb9eb7e4226c68cf4b39f167c83e5bd132b3e
+        with:
+          app-id: ${{ vars.AUTOMATION_APP_ID }}
+          private-key: ${{ secrets.AUTOMATION_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "codeql-coding-standards"
 
       - name: Bump main version
         if: env.HOTFIX_RELEASE == 'false'
         env:
-          GH_TOKEN: ${{ github.token }}
+          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
         run: |
-          version=${BASE_REF#rc/}
-          next_version="$version-dev"
-          echo "Bumping main version to $next_version"
+          echo "Bumping main version to $NEXT_VERSION"
 
           git switch main
           git pull --ff-only origin main
 
-          git switch -c release-automation/bump-version
+          git switch -c "release-automation/bump-version-to-$NEXT_VERSION"
 
-          ./scripts/release/bump-version.sh "$next_version"
+          # We are running the script in the tooling directory with the release directory as the working directory
+          ../tooling/scripts/release/bump-version.sh "$NEXT_VERSION"
 
           git add -u .
-          git commit -m "Bump version to $next_version"
-          git push --set-upstream origin release-automation/bump-version
+          git commit -m "Bump version to $NEXT_VERSION"
+          git push --set-upstream origin "release-automation/bump-version-to-$NEXT_VERSION"
 
-          gh pr create --repo $GITHUB_REPOSITORY --base main --head release-automation/bump-version --body "Bump the version of main to the dev label of the just released version $next_version" --title "Bump version to $next_version"
+          gh pr create --repo $GITHUB_REPOSITORY --base main --head "release-automation/bump-version-to-$NEXT_VERSION" --body "Bump the version of main to $NEXT_VERSION" --title "Bump version to $NEXT_VERSION"
+        working-directory: release 

.github/workflows/update-release.yml

@@ -31,6 +31,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0 # We need the full history to compute the changelog
+          ref: ${{ inputs.head-sha }}
 
       - name: Install Python
         uses: actions/setup-python@v4
@@ -59,7 +60,7 @@ jobs:
             --layout scripts/release/release-layout.yml \
             --repo "$GITHUB_REPOSITORY" \
             --github-token "$GITHUB_REPOSITORY:$GITHUB_TOKEN" "github/codeql-coding-standards-release-engineering:$RELEASE_ENGINEERING_TOKEN" \
-            --skip-checkrun "release-status"
+            --skip-checkrun "release-status" "Update Release"
 
       - name: Update release notes
         env:

.vscode/tasks.json

@@ -140,6 +140,28 @@
             },
             "problemMatcher": []
         },
+        {
+            "label": "🧪 Standards Automation: Build Case Test DB from test file",
+            "type": "shell",
+            "windows": {
+                "command": ".${pathSeparator}scripts${pathSeparator}.venv${pathSeparator}Scripts${pathSeparator}python.exe scripts${pathSeparator}build_test_database.py ${file}"
+            },
+            "linux": {
+                "command": ".${pathSeparator}scripts${pathSeparator}.venv${pathSeparator}bin${pathSeparator}python3 scripts${pathSeparator}build_test_database.py ${file}"
+            },
+            "osx": {
+                "command": ".${pathSeparator}scripts${pathSeparator}.venv${pathSeparator}bin${pathSeparator}python3 scripts${pathSeparator}build_test_database.py ${file}"
+            },
+            "presentation": {
+                "reveal": "always",
+                "panel": "new",
+                "focus": true
+            },
+            "runOptions": {
+                "reevaluateOnRerun": false
+            },
+            "problemMatcher": []
+        },
         {
             "label": "📝 Standards Automation: Format CodeQL",
             "type": "shell",
@@ -249,6 +271,7 @@
                 "Null",
                 "OperatorInvariants",
                 "Operators",
+                "OrderOfEvaluation",
                 "OutOfBounds",
                 "Pointers",
                 "Pointers1",

README.md

@@ -6,17 +6,19 @@ This repository contains CodeQL queries and libraries which support various Codi
 
 _Carnegie Mellon and CERT are registered trademarks of Carnegie Mellon University._
 
-This repository contains CodeQL queries and libraries which support various Coding Standards for the [C++14](https://www.iso.org/standard/64029.html) programming language.
+This repository contains CodeQL queries and libraries which support various Coding Standards for the [C++14](https://www.iso.org/standard/64029.html), [C99](https://www.iso.org/standard/29237.html) and [C11](https://www.iso.org/standard/57853.html) programming languages.
 
 The following coding standards are supported:
 - [AUTOSAR - Guidelines for the use of C++14 language in critical and safety-related systems (Releases R22-11, R20-11, R19-11 and R19-03)](https://www.autosar.org/fileadmin/standards/R22-11/AP/AUTOSAR_RS_CPP14Guidelines.pdf). 
-- [MISRA C++:2008](https://www.misra.org.uk) (support limited to the rules specified in AUTOSAR).
 - [SEI CERT C++ Coding Standard: Rules for Developing Safe, Reliable, and Secure Systems (2016 Edition)](https://resources.sei.cmu.edu/library/asset-view.cfm?assetID=494932)
+- [SEI CERT C Coding Standard: Rules for Developing Safe, Reliable, and Secure Systems (2016 Edition)](https://resources.sei.cmu.edu/downloads/secure-coding/assets/sei-cert-c-coding-standard-2016-v01.pdf)
+- [MISRA C 2012, 3rd Edition, 1st revision](https://www.misra.org.uk/product/misra-c2012-third-edition-first-revision/) (incoporating Amendment 1 & Technical Corrigendum 1). In addition, we support the following additional amendments and technical corrigendums:
+   - [MISRA C 2012 Amendment 2](https://misra.org.uk/app/uploads/2021/06/MISRA-C-2012-AMD2.pdf)
+   - [MISRA C 2012 Technical Corrigendum 2](https://misra.org.uk/app/uploads/2022/04/MISRA-C-2012-TC2.pdf)
 
-In addition, the following Coding Standards for the C programming language are under development:
+## :construction: Standards under development :construction:
 
-- [SEI CERT C Coding Standard: Rules for Developing Safe, Reliable, and Secure Systems (2016 Edition)](https://resources.sei.cmu.edu/downloads/secure-coding/assets/sei-cert-c-coding-standard-2016-v01.pdf)
-- [MISRA C 2012](https://www.misra.org.uk/product/misra-c2012-third-edition-first-revision/).
+- [MISRA C++ 2023](https://misra.org.uk/product/misra-cpp2023/) - under development _scheduled for release 2024 Q4_.
 
 ## How do I use the CodeQL Coding Standards Queries?
 

c/cert/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cert-c-coding-standards
-version: 2.22.0-dev
+version: 2.33.0-dev
 description: CERT C 2016
 suites: codeql-suites
 license: MIT

c/cert/src/rules/INT30-C/UnsignedIntegerOperationsWrapAround.ql

@@ -15,24 +15,11 @@
 
 import cpp
 import codingstandards.c.cert
-import codingstandards.cpp.Overflow
-import semmle.code.cpp.controlflow.Guards
-import semmle.code.cpp.valuenumbering.GlobalValueNumbering
+import codingstandards.cpp.rules.unsignedoperationwithconstantoperandswraps.UnsignedOperationWithConstantOperandsWraps
 
-from InterestingOverflowingOperation op
-where
-  not isExcluded(op, IntegerOverflowPackage::unsignedIntegerOperationsWrapAroundQuery()) and
-  op.getType().getUnderlyingType().(IntegralType).isUnsigned() and
-  // Not within a guard condition
-  not exists(GuardCondition gc | gc.getAChild*() = op) and
-  // Not guarded by a check, where the check is not an invalid overflow check
-  not op.hasValidPreCheck() and
-  // Is not checked after the operation
-  not op.hasValidPostCheck() and
-  // Permitted by exception 3
-  not op instanceof LShiftExpr and
-  // Permitted by exception 2 - zero case is handled in separate query
-  not op instanceof DivExpr and
-  not op instanceof RemExpr
-select op,
-  "Operation " + op.getOperator() + " of type " + op.getType().getUnderlyingType() + " may wrap."
+class UnsignedIntegerOperationsWrapAroundQuery extends UnsignedOperationWithConstantOperandsWrapsSharedQuery
+{
+  UnsignedIntegerOperationsWrapAroundQuery() {
+    this = IntegerOverflowPackage::unsignedIntegerOperationsWrapAroundQuery()
+  }
+}

c/cert/src/rules/INT34-C/ExprShiftedbyNegativeOrGreaterPrecisionOperand.ql

@@ -15,91 +15,8 @@ import codingstandards.c.cert
 import semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis
 import semmle.code.cpp.valuenumbering.GlobalValueNumbering
 import semmle.code.cpp.controlflow.Guards
+import codingstandards.cpp.UndefinedBehavior
 
-/*
- * Precision predicate based on a sample implementation from
- * https://wiki.sei.cmu.edu/confluence/display/c/INT35-C.+Use+correct+integer+precisions
- */
-
-/**
- * A function whose name is suggestive that it counts the number of bits set.
- */
-class PopCount extends Function {
-  PopCount() { this.getName().toLowerCase().matches("%popc%nt%") }
-}
-
-/**
- * A macro which is suggestive that it is used to determine the precision of an integer.
- */
-class PrecisionMacro extends Macro {
-  PrecisionMacro() { this.getName().toLowerCase().matches("precision") }
-}
-
-class LiteralZero extends Literal {
-  LiteralZero() { this.getValue() = "0" }
-}
-
-class BitShiftExpr extends BinaryBitwiseOperation {
-  BitShiftExpr() {
-    this instanceof LShiftExpr or
-    this instanceof RShiftExpr
-  }
-}
-
-int getPrecision(IntegralType type) {
-  type.isExplicitlyUnsigned() and result = type.getSize() * 8
-  or
-  type.isExplicitlySigned() and result = type.getSize() * 8 - 1
-}
-
-predicate isForbiddenShiftExpr(BitShiftExpr shift, string message) {
-  (
-    (
-      getPrecision(shift.getLeftOperand().getExplicitlyConverted().getUnderlyingType()) <=
-        upperBound(shift.getRightOperand()) and
-      message =
-        "The operand " + shift.getLeftOperand() + " is shifted by an expression " +
-          shift.getRightOperand() + " whose upper bound (" + upperBound(shift.getRightOperand()) +
-          ") is greater than or equal to the precision."
-      or
-      lowerBound(shift.getRightOperand()) < 0 and
-      message =
-        "The operand " + shift.getLeftOperand() + " is shifted by an expression " +
-          shift.getRightOperand() + " which may be negative."
-    ) and
-    /*
-     * Shift statement is not at a basic block where
-     * `shift_rhs < PRECISION(...)` is ensured
-     */
-
-    not exists(GuardCondition gc, BasicBlock block, Expr precisionCall, Expr lTLhs |
-      block = shift.getBasicBlock() and
-      (
-        precisionCall.(FunctionCall).getTarget() instanceof PopCount
-        or
-        precisionCall = any(PrecisionMacro pm).getAnInvocation().getExpr()
-      )
-    |
-      globalValueNumber(lTLhs) = globalValueNumber(shift.getRightOperand()) and
-      gc.ensuresLt(lTLhs, precisionCall, 0, block, true)
-    ) and
-    /*
-     * Shift statement is not at a basic block where
-     * `shift_rhs < 0` is ensured
-     */
-
-    not exists(GuardCondition gc, BasicBlock block, Expr literalZero, Expr lTLhs |
-      block = shift.getBasicBlock() and
-      literalZero instanceof LiteralZero
-    |
-      globalValueNumber(lTLhs) = globalValueNumber(shift.getRightOperand()) and
-      gc.ensuresLt(lTLhs, literalZero, 0, block, true)
-    )
-  )
-}
-
-from BinaryBitwiseOperation badShift, string message
-where
-  not isExcluded(badShift, Types1Package::exprShiftedbyNegativeOrGreaterPrecisionOperandQuery()) and
-  isForbiddenShiftExpr(badShift, message)
-select badShift, message
+from ShiftByNegativeOrGreaterPrecisionOperand badShift
+where not isExcluded(badShift, Types1Package::exprShiftedbyNegativeOrGreaterPrecisionOperandQuery())
+select badShift, badShift.getReason()