[Python dist] Build aarch64 in different Kokoro job

tools/bazelify_tests/dockerimage_current_versions.bzl

@@ -36,7 +36,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:004e02902825b54c7f0d60eaed0819acf6f10c24853bf8f793001114e9969abd",
     "tools/dockerfile/distribtest/csharp_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu2204_x64@sha256:a439f2ccbc666f231e511a8e58eb7f66a3de4820a4d5aded1e62275cf8ac49f0",
     "tools/dockerfile/distribtest/php8_debian12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php8_debian12_x64@sha256:42399d061f234ee0ef79b333555db0e11ca4f106bb1d49276abf99c459c104f0",
-    "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e",
+    "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:bcdfc865daa8c61dc6f9b67ddab63d47fcacb109ed57de5b9a7673dff907e02c",
     "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:1bbc6fa5b4b650d3037d089e164d364e05a6daf1ed6dd1025ba07cc127f73d7d",
     "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1",
     "tools/dockerfile/distribtest/python_bullseye_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x64@sha256:80553398f0c59c1dc186052f4f2deaf18fea582f6d1d166eec6ea298639031fb",

tools/dockerfile/distribtest/python_alpine_aarch64.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:086c6b7ba4177b5c086a4e1e36019fdd1d86a061@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e
+us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:78b5c9d459ceaedfa8dc8fa6dc52ce11666dd581@sha256:bcdfc865daa8c61dc6f9b67ddab63d47fcacb109ed57de5b9a7673dff907e02c

tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM python:3.10-alpine3.14
+# this an aarch64 image
+FROM arm64v8/python:3.10-alpine
 
 # Our test infrastructure demands bash
 RUN apk update && apk add bash

tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg

@@ -0,0 +1,31 @@
+# Copyright 2025 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "aarch64 musllinux_1_1"
+}

tools/internal_ci/linux/grpc_distribtests_python.cfg

@@ -24,3 +24,8 @@ action {
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "-e aarch64 musllinux_1_1"
+}

tools/internal_ci/linux/grpc_distribtests_python.sh

@@ -23,10 +23,14 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-# some distribtests use a pre-registered binfmt_misc hook
-# to automatically execute foreign binaries (such as aarch64)
-# under qemu emulator.
-source tools/internal_ci/helper_scripts/prepare_qemu_rc
+if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" ]]; then
+  echo "Skipping prepare_qemu_rc'"
+else
+  # some distribtests use a pre-registered binfmt_misc hook
+  # to automatically execute foreign binaries (such as aarch64)
+  # under qemu emulator.
+  source tools/internal_ci/helper_scripts/prepare_qemu_rc
+fi
 
 # configure ccache
 source tools/internal_ci/helper_scripts/prepare_ccache_rc
@@ -40,7 +44,11 @@ mkdir -p input_artifacts
 cp -r artifacts/* input_artifacts/ || true
 
 # This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/
-tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true"
+if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" ]]; then
+  tools/run_tests/task_runner.py -f package linux python ${TASK_RUNNER_EXTRA_FILTERS} -x build_packages/sponge_log.xml || FAILED="true"
+else
+  tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true"
+fi
 
 # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
 # in addition to that, preserve the contents of "artifacts" directory since we want kokoro
@@ -52,7 +60,12 @@ cp -r artifacts/* input_artifacts/ || true
 # Run all python linux distribtests
 # We run the distribtests even if some of the artifacts have failed to build, since that gives
 # a better signal about which distribtest are affected by the currently broken artifact builds.
-tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true"
+if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" ]]; then
+  # We're using alpine as tag in distribtest targets.
+  tools/run_tests/task_runner.py -f distribtest linux python aarch64 alpine -j 12 -x distribtests/sponge_log.xml || FAILED="true"
+else
+  tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true"
+fi
 
 # This step checks if any of the artifacts exceeds a per-file size limit.
 tools/internal_ci/helper_scripts/check_python_artifacts_size.sh

tools/internal_ci/linux/pull_request/grpc_examples_sanity_tests.cfg

@@ -23,3 +23,13 @@ action {
     regex: "github/grpc/reports/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "aarch64 -e x64 x86 armv7"
+}
+
+# env_vars {
+#   key: "TASK_RUNNER_EXCLUDE_FILTERS"
+#   value: "x64 x86 armv7"
+# }

tools/internal_ci/linux/release/grpc_distribtests_python.cfg

@@ -24,3 +24,8 @@ action {
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "-e aarch64 musllinux_1_1"
+}

tools/run_tests/artifacts/package_targets.py

@@ -142,9 +142,18 @@ def build_jobspec(self, inner_jobs=None):
 class PythonPackage:
     """Collects python eggs and wheels created in the artifact phase"""
 
-    def __init__(self):
+    def __init__(self, platform="", arch=""):
         self.name = "python_package"
         self.labels = ["package", "python", "linux"]
+        self.platform = platform
+        self.arch = arch
+        if self.platform:
+            self.labels.append(platform)
+            self.name += "_" + platform
+        if self.arch:
+            self.labels.append(arch)
+            self.name += "_" + arch
+
 
     def pre_build_jobspecs(self):
         return []
@@ -154,9 +163,12 @@ def build_jobspec(self, inner_jobs=None):
         # since the python package build does very little, we can use virtually
         # any image that has new-enough python, so reusing one of the images used
         # for artifact building seems natural.
+        dockerfile_dir = "tools/dockerfile/grpc_artifact_python_manylinux2014_x64"
+        if "musllinux_1_1" in self.platform and "aarch64" in self.arch:
+            dockerfile_dir = "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64"
         return create_docker_jobspec(
             self.name,
-            "tools/dockerfile/grpc_artifact_python_manylinux2014_x64",
+            dockerfile_dir,
             "tools/run_tests/artifacts/build_package_python.sh",
             environ={"PYTHON": "/opt/python/cp39-cp39/bin/python"},
         )
@@ -189,5 +201,6 @@ def targets():
         CSharpPackage("windows"),
         RubyPackage(),
         PythonPackage(),
+        PythonPackage("musllinux_1_1", "aarch64"),
         PHPPackage(),
     ]

tools/run_tests/python_utils/jobset.py

@@ -45,6 +45,7 @@ def strip_non_ascii_chars(s):
 def sanitized_environment(env):
     sanitized = {}
     for key, value in list(env.items()):
+        sys.stderr.write(f"strip_non_ascii_chars key: {key}, value: {value}\n"); sys.stderr.flush()
         sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value)
     return sanitized
 

tools/run_tests/task_runner.py

@@ -71,6 +71,14 @@ def _create_build_map():
     default=[],
     help="Filter targets to build with AND semantics.",
 )
+argp.add_argument(
+    "-e",
+    "--exclude",
+    choices=sorted(_BUILD_MAP.keys()),
+    nargs="+",
+    default=[],
+    help="Target labels to exclude from building.",
+)
 argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int)
 argp.add_argument(
     "-x",
@@ -106,6 +114,9 @@ def _create_build_map():
 # Among targets selected by -b, filter out those that don't match the filter
 targets = [t for t in targets if all(f in t.labels for f in args.filter)]
 
+# Exclude target if it has ALL of the specified exclude labels.
+targets = [t for t in targets if not all(l in args.exclude for l in t.labels)]
+
 print("Will build %d targets:" % len(targets))
 for target in targets:
     print("  %s, labels %s" % (target.name, target.labels))