ci: make dcou partitioned steps not-skewed and cached

ci/docker-run.sh

@@ -57,6 +57,12 @@ if [[ -n $CI ]]; then
       ARGS+=(
         --env "RUSTC_WRAPPER=/usr/local/cargo/bin/sccache"
       )
+      # local disk storage for sccache (experimental; only used by dcou for now)
+      mkdir -p "$HOME/.cache/sccache-for-docker"
+      CONTAINER_HOME="/"
+      ARGS+=(
+        --volume "$HOME/.cache/sccache-for-docker:$CONTAINER_HOME/.cache/sccache"
+      )
 
       # s3
       if [ -n "$AWS_ACCESS_KEY_ID" ]; then

ci/docker/Dockerfile

@@ -6,7 +6,7 @@ ARG \
   RUST_NIGHTLY_VERSION= \
   GOLANG_VERSION=1.21.3 \
   NODE_MAJOR=18 \
-  SCCACHE_VERSION=v0.8.1 \
+  SCCACHE_VERSION=v0.9.1 \
   GRCOV_VERSION=v0.8.18
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -78,7 +78,9 @@ RUN \
   rustup component add llvm-tools-preview --toolchain=$RUST_NIGHTLY_VERSION && \
   rustup target add wasm32-unknown-unknown && \
   cargo install cargo-audit && \
-  cargo install cargo-hack && \
+  # uncomment once the dcou-parition related patch is upstreamed...
+  # cargo install cargo-hack && \
+  cargo install --git https://github.com/anza-xyz/cargo-hack.git --rev 5e59c3ec6c661c02601487c0d4b2a2649fe06c9f cargo-hack && \
   cargo install cargo-sort && \
   cargo install mdbook && \
   cargo install mdbook-linkcheck && \

ci/docker/env.sh

@@ -15,7 +15,7 @@ export CI_DOCKER_ARG_RUST_VERSION="${rust_stable}"
 export CI_DOCKER_ARG_RUST_NIGHTLY_VERSION="${rust_nightly}"
 export CI_DOCKER_ARG_GOLANG_VERSION=1.21.3
 export CI_DOCKER_ARG_NODE_MAJOR=18
-export CI_DOCKER_ARG_SCCACHE_VERSION=v0.8.1
+export CI_DOCKER_ARG_SCCACHE_VERSION=v0.9.1
 export CI_DOCKER_ARG_GRCOV_VERSION=v0.8.18
 
 hash_vars=(

ci/test-dev-context-only-utils.sh

@@ -1,6 +1,29 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
+source ./ci/_
+
+(unset RUSTC_WRAPPER; cargo install --force --git https://github.com/anza-xyz/cargo-hack.git --rev 5e59c3ec6c661c02601487c0d4b2a2649fe06c9f cargo-hack)
+
+# Here, experimentally switch the sccache storage from GCS to local disk by
+# `unset`-ing gcs credentials so that sccache automatically falls back to the
+# local disk storage.
+unset SCCACHE_GCS_KEY_PATH SCCACHE_GCS_BUCKET SCCACHE_GCS_RW_MODE SCCACHE_GCS_KEY_PREFIX
+
+# sccache's default is 10G, but our boxes have far more storage. :)
+export SCCACHE_CACHE_SIZE="200G"
+
+# Disable incremental compilation as this is documented as not-compatible with
+# sccache at https://github.com/mozilla/sccache/blob/v0.9.1/README.md#rust
+# > Incrementally compiled crates cannot be cached.
+export CARGO_INCREMENTAL=0
+
+_ sccache --show-stats
 
 scripts/check-dev-context-only-utils.sh check-all-targets "$@"
 scripts/check-dev-context-only-utils.sh check-bins-and-lib "$@"
+
+# This shows final stats while stopping the sccache background server as well
+# for later normal sccache use (if any). Remember that sccache is now
+# temporarily and experimtally running with the local disk storage.
+_ sccache --stop-server

scripts/check-dev-context-only-utils.sh

@@ -151,6 +151,15 @@ fi
 # consistency with other CI steps and for the possibility of new similar lints.
 export RUSTFLAGS="-D warnings -Z threads=8 $RUSTFLAGS"
 
+# As this environment value is used by the rather deep crate of our dep graph
+# (solana-varsion), this could invalidate significant portion of caches when
+# this changes just with a new tiny commit. Technically, it's possible for
+# CI_COMMIT to affect the outcome of compilation via build.rs, but it's
+# extremely unrealistic for such diverting compilation behaviors to be desired
+# as a sane use-case. So, just unset CI_COMMIT unconditionally to increase
+# cache efficiency.
+unset CI_COMMIT
+
 if [[ $mode = "check-bins-and-lib" || $mode = "full" ]]; then
   _ cargo "+${rust_nightly}" hack "$@" check
 fi