Merge remote-tracking branch 'origin/main' into toolchains

.bazelrc

@@ -7,22 +7,18 @@ build:opt --compilation_mode=opt
 build:san-common --config=dbg --strip=never --copt=-O0 --copt=-fno-omit-frame-pointer
 
 build:asan --config=san-common --copt=-fsanitize=address --linkopt=-fsanitize=address
-build:asan --copt=-DADDRESS_SANITIZER=1
 # ASAN hits ODR violations with shared linkage due to rules_proto.
 build:asan --dynamic_mode=off
 
 build:msan --config=san-common --copt=-fsanitize=memory --linkopt=-fsanitize=memory
 build:msan --copt=-fsanitize-memory-track-origins
 build:msan --copt=-fsanitize-memory-use-after-dtor
 build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1
-build:msan --copt=-DMEMORY_SANITIZER=1
 
 build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread
-build:tsan --copt=-DTHREAD_SANITIZER=1
 
 build:ubsan --config=san-common --copt=-fsanitize=undefined --linkopt=-fsanitize=undefined
 build:ubsan --action_env=UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1
-build:ubsan --copt=-DUNDEFINED_SANITIZER=1
 # Workaround for the fact that Bazel links with $CC, not $CXX
 # https://github.com/bazelbuild/bazel/issues/11122#issuecomment-613746748
 build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr
@@ -31,5 +27,6 @@ build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr
 # https://github.com/protocolbuffers/protobuf/issues/14313
 common --noenable_bzlmod
 
-# Enable proto toolchain resolution
-common --incompatible_enable_proto_toolchain_resolution
+# Important: this flag ensures that we remain compliant with the C++ layering
+# check.
+build --features=layering_check

.github/workflows/clear_caches.yml

@@ -3,7 +3,7 @@ name: Clear expensive caches to prevent unbounded growth
 on:
   schedule:
     # Run every 4 months at 10 AM UTC (2 AM PDT)
-    - cron: 0 10 1 */4 *
+    - cron: 0 10 5 */4 *
 
   # manual
   workflow_dispatch:
@@ -31,5 +31,5 @@ jobs:
       - name: Create an empty cache with a single file
         run: |
           rm -rf .repository-cache
-          mkdir -p .repository-cache'
+          mkdir -p .repository-cache
           touch .repository-cache/reset_file

.github/workflows/janitor.yml

@@ -28,3 +28,51 @@ jobs:
             echo "Closing #$pr..."
             gh pr close --comment "Auto-closing Copybara pull request" --delete-branch "$pr"
           done
+
+  stale-others:
+    name: Close stale non-copybara PRs and issues
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write # allow the action to comment on, add labels to, and close issues
+      pull-requests: write # allow the action to comment on, add labels to, and close PRs
+    steps:
+      - uses: actions/stale@b69b346013879cedbf50c69f572cd85439a41936
+        with:
+          stale-issue-message: >
+            We triage inactive PRs and issues in order to make it easier to find
+            active work. If this issue should remain active or becomes active
+            again, please add a comment.
+
+
+            This issue is labeled `inactive` because the last activity was over
+            90 days ago.
+          close-issue-message: >
+            We triage inactive PRs and issues in order to make it easier to find
+            active work. If this issue should remain active or becomes active
+            again, please reopen it.
+
+
+            This issue was closed and archived because there has been no new
+            activity in the 14 days since the `inactive` label was added.
+          stale-pr-message: >
+            We triage inactive PRs and issues in order to make it easier to find
+            active work. If this PR should remain active, please add a comment.
+
+
+            This PR is labeled `inactive` because the last activity was over 90
+            days ago. This PR will be closed and archived after 14 additional
+            days without activity.
+          close-pr-message: >
+            We triage inactive PRs and issues in order to make it easier to find
+            active work. If this PR should remain active or becomes active
+            again, please reopen it.
+
+
+            This PR was closed and archived because there has been no new
+            activity in the 14 days since the `inactive` label was added.
+          stale-issue-label: 'inactive'
+          stale-pr-label: 'inactive'
+          exempt-issue-labels: 'help wanted'
+          days-before-stale: 90
+          days-before-close: 14
+          operations-per-run: 100

.github/workflows/scorecard.yml

@@ -32,7 +32,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # v2.1.2
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
         with:
           results_file: results.sarif
           results_format: sarif

.github/workflows/staleness_check.yml

@@ -54,6 +54,7 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: staleness
           bash: >
+            set -ex;
             if [[ -z $COMMIT_TRIGGERED_RUN || -z $MAIN_RUN ]]; then
             bazel query 'attr(tags, "staleness_test", //...)' | xargs bazel test $BAZEL_FLAGS ||
             echo "Please run ./regenerate_stale_files.sh to regenerate stale files";

.github/workflows/staleness_refresh.yml

@@ -30,10 +30,6 @@ jobs:
           # failing then you may need to generate a fresh token.
           token: ${{ secrets.BOT_ACCESS_TOKEN }}
       - name: Configure name and email address in Git
-        run: git config user.name "Protobuf Team Bot" && git config user.email "[email protected]"
+        run: cd ${{ github.workspace }} && git config user.name "Protobuf Team Bot" && git config user.email "[email protected]"
       - name: Commit and push update
-        uses: protocolbuffers/protobuf-ci/bazel@v2
-        with:
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          bazel-cache: staleness
-          bash: ci/push_auto_update.sh
+        run: cd ${{ github.workspace }} && ./ci/push_auto_update.sh

.github/workflows/test_cpp.yml

@@ -27,14 +27,17 @@ jobs:
         include:
           # Set defaults
           - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:04cd765285bc52cbbf51d66c8c66d8603579cf0f19cc42df26b09d2c270541fb
-          - targets: //pkg/... //src/... @com_google_protobuf_examples//...
+          - targets: //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/...
 
           # Override cases with custom images
+          - config: { name: "Bazel7" }
+            image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.0.0-a04396cc76704d4b7c722789e9c08df18f47df53"
+            targets: "//src/... //third_party/utf8_range/..."
           - config: { name: "TCMalloc" }
             image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:bd39119d74b8a3fad4ae335d4cf5294e70384676331b7e19949459fc7a8d8328"
-            targets: "//src/..."
+            targets: "//src/... //third_party/utf8_range/..."
           - config: { name: "aarch64" }
-            targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test"
+            targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test //third_party/utf8_range/..."
             image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17"
     name: Linux ${{ matrix.config.name }}
     runs-on: ${{ matrix.config.runner || 'ubuntu-latest' }}
@@ -69,7 +72,7 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: cpp_linux/gcc-${{ matrix.version }}
-          bazel: test //pkg/... //src/... @com_google_protobuf_examples//...
+          bazel: test //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/...
 
   linux-release:
     strategy:
@@ -313,21 +316,32 @@ jobs:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         include:
-          - name: MacOS
+          - name: MacOS Bazel
             os: macos-12
             cache_key: macos-12
-            bazel: test //src/...
-          - name: MacOS Apple Silicon (build only)
+            bazel: test //src/... //third_party/utf8_range/...
+          # TODO Enable these once mac setup is working for Bazel 7
+          #- name: MacOS Bazel 7
+          #  os: macos-12
+          #  cache_key: macos-12-bazel7
+          #  bazel: test //src/... //third_party/utf8_range/...
+          #  bazel_version: '7.0.0'
+          - name: MacOS Apple Silicon (build only) Bazel
             os: macos-12
             cache_key: macos-12-arm
             # Current github runners are all Intel based, so just build/compile
             # for Apple Silicon to detect issues there.
-            bazel: build --cpu=darwin_arm64 //src/...
-          - name: Windows
-            os: windows-2019
-            cache_key: windows-2019
+            bazel: build --cpu=darwin_arm64 //src/... //third_party/utf8_range/...
+          - name: Windows Bazel
+            os: windows-2022
+            cache_key: windows-2022
             bazel: test //src/...  @com_google_protobuf_examples//... --test_tag_filters=-conformance --build_tag_filters=-conformance
-    name: ${{ matrix.name }} Bazel
+          - name: Windows Bazel 7
+            os: windows-2022
+            cache_key: windows-2022-bazel7
+            bazel: test //src/...  @com_google_protobuf_examples//... --test_tag_filters=-conformance --build_tag_filters=-conformance
+            bazel_version: '7.0.0'
+    name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout pending changes
@@ -340,55 +354,59 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel: ${{ matrix.bazel }}
           bazel-cache: cpp_${{ matrix.cache_key }}
+          version: ${{ matrix.bazel_version || '6.3.0' }}
 
   non-linux-cmake:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         include:
-          - name: MacOS CMake
-            os: macos-12
-            flags: -DCMAKE_CXX_STANDARD=14
-            cache-prefix: macos-cmake
+          # TODO: investigate and fix
+          # - name: MacOS CMake
+          #   os: macos-12
+          #   flags: -DCMAKE_CXX_STANDARD=14
+          #   cache-prefix: macos-cmake
           - name: Windows CMake
-            os: windows-2019
-            flags: >-
-              -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
-              -Dprotobuf_BUILD_SHARED_LIBS=OFF
-              -Dprotobuf_BUILD_EXAMPLES=ON
-            vsversion: '2019'
-            cache-prefix: windows-2019-cmake
-          - name: Windows CMake 2022
             os: windows-2022
             flags: >-
               -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
               -Dprotobuf_BUILD_SHARED_LIBS=OFF
               -Dprotobuf_BUILD_EXAMPLES=ON
             vsversion: '2022'
             cache-prefix: windows-2022-cmake
-          - name: Windows CMake 32-bit
+          - name: Windows CMake 2019
             os: windows-2019
             flags: >-
               -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
+              -Dprotobuf_BUILD_SHARED_LIBS=OFF
+              -Dprotobuf_BUILD_EXAMPLES=ON
             vsversion: '2019'
+            cache-prefix: windows-2019-cmake
+            # windows-2019 has python3.7 installed, which is incompatible with the latest gcloud
+            python-version: '3.8'
+          - name: Windows CMake 32-bit
+            os: windows-2022
+            flags: >-
+              -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
+            vsversion: '2022'
             windows-arch: 'win32'
-            cache-prefix: windows-2019-win32-cmake
+            cache-prefix: windows-2022-win32-cmake
           - name: Windows CMake Shared
-            os: windows-2019
+            os: windows-2022
             flags: >-
               -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
               -Dprotobuf_BUILD_SHARED_LIBS=ON
-            vsversion: '2019'
-            cache-prefix: windows-2019-cmake
+            vsversion: '2022'
+            cache-prefix: windows-2022-cmake
           - name: Windows CMake Install
-            os: windows-2019
+            os: windows-2022
             install-flags: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF
             flags: >-
               -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
               -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON
               -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF
-            vsversion: '2019'
-            cache-prefix: windows-2019-cmake
+            vsversion: '2022'
+            cache-prefix: windows-2022-cmake
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     steps:
@@ -405,6 +423,17 @@ jobs:
           arch: ${{ matrix.windows-arch || 'x64' }}
           vsversion: ${{ matrix.vsversion }}
 
+      # Workaround for incompatibility between gcloud and windows-2019 runners.
+      - name: Install Python
+        if: ${{ matrix.python-version }}
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Use custom python for gcloud
+        if: ${{ matrix.python-version }}
+        run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV
+        shell: bash
+
       - name: Setup sccache
         uses: protocolbuffers/protobuf-ci/sccache@v2
         with:

.github/workflows/test_java.yml

@@ -20,7 +20,9 @@ jobs:
           - name: OpenJDK 8
             version: '8'
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
-            targets: //java/... //java/internal:java_version
+            # TODO: b/318555165 - enable the layering check. Currently it does
+            # not work correctly with the toolchain in this Docker image.
+            targets: //java/... //java/internal:java_version --features=-layering_check
           - name: OpenJDK 11
             version: '11'
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
@@ -63,7 +65,9 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: java_linux/8
-          bazel: test --test_output=all //java:linkage_monitor --spawn_strategy=standalone
+          # TODO: b/318555165 - enable the layering check. Currently it does
+          # not work correctly with the toolchain in this Docker image.
+          bazel: test --test_output=all //java:linkage_monitor --spawn_strategy=standalone --features=-layering_check
 
   protobuf-bom:
     name: Protobuf Maven BOM

.github/workflows/test_php.yml

@@ -150,7 +150,7 @@ jobs:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
-        version: ['8.0']
+        version: ['8.2']
 
     name: MacOS PHP ${{ matrix.version }}
     runs-on: macos-12

.github/workflows/test_python.yml

@@ -17,7 +17,8 @@ jobs:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         type: [ Pure, C++]
-        version: ["3.8", "3.9", "3.10", "3.11" ]
+        # TODO: b/309627662 - Add coverage for Python 3.12.
+        version: ["3.8", "3.9", "3.10", "3.11"]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
@@ -55,7 +56,7 @@ jobs:
       matrix:
         type: [ Pure, C++]
         # TODO Consider expanding this set of versions.
-        version: [ "3.11" ]
+        version: [ "3.12" ]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
@@ -76,6 +77,7 @@ jobs:
         with:
           python-version: ${{ matrix.version }}
           cache: pip
+          cache-dependency-path: 'python/requirements.txt'
 
       - name: Validate version
         run: python3 --version | grep ${{ matrix.version }} || (echo "Invalid Python version - $(python3 --version)" && exit 1)