sdk: Move out things that won't be in the sdk (#4624)

* sdk: Move things that won't be in the sdk out

#### Problem

The Solana sdk will soon have a standalone repository, but the `sdk/`
directory in the agave repo contains some things that won't move over.

#### Summary of changes

Move the following and update paths:

* cargo-build-sbf
* cargo-test-sbf
* sdk/sbf
* gen-headers

* Regenerate headers with gen-headers

* Update package paths for cargo-build-sbf tests

* Update CI scripts to look at platform-tools-sdk dir

* Move docker-solana into top-level, update paths

* Fix path in cargo-build-sbf test

Cargo.toml

@@ -72,6 +72,9 @@ members = [
     "net-utils",
     "notifier",
     "perf",
+    "platform-tools-sdk/cargo-build-sbf",
+    "platform-tools-sdk/cargo-test-sbf",
+    "platform-tools-sdk/gen-headers",
     "poh",
     "poh-bench",
     "poseidon",
@@ -115,8 +118,6 @@ members = [
     "sdk/bincode",
     "sdk/blake3-hasher",
     "sdk/borsh",
-    "sdk/cargo-build-sbf",
-    "sdk/cargo-test-sbf",
     "sdk/client-traits",
     "sdk/clock",
     "sdk/cluster-type",
@@ -138,7 +139,6 @@ members = [
     "sdk/file-download",
     "sdk/frozen-abi",
     "sdk/frozen-abi/macro",
-    "sdk/gen-headers",
     "sdk/genesis-config",
     "sdk/hard-forks",
     "sdk/hash",

cargo-build-sbf

@@ -2,15 +2,15 @@
 
 here=$(dirname "$0")
 
-maybe_sbf_sdk="--sbf-sdk $here/sdk/sbf"
+maybe_sbf_sdk="--sbf-sdk $here/platform-tools-sdk/sbf"
 for a in "$@"; do
   if [[ $a = --sbf-sdk ]]; then
     maybe_sbf_sdk=
   fi
 done
 
 set -ex
-if [[ ! -f "$here"/sdk/sbf/syscalls.txt ]]; then
+if [[ ! -f "$here"/platform-tools-sdk/sbf/syscalls.txt ]]; then
   cargo build --manifest-path "$here"/programs/bpf_loader/gen-syscall-list/Cargo.toml
 fi
-exec cargo run --manifest-path "$here"/sdk/cargo-build-sbf/Cargo.toml -- $maybe_sbf_sdk "$@"
+exec cargo run --manifest-path "$here"/platform-tools-sdk/cargo-build-sbf/Cargo.toml -- $maybe_sbf_sdk "$@"

cargo-test-sbf

@@ -2,7 +2,7 @@
 
 here=$(dirname "$0")
 
-maybe_sbf_sdk="--sbf-sdk $here/sdk/sbf"
+maybe_sbf_sdk="--sbf-sdk $here/platform-tools-sdk/sbf"
 for a in "$@"; do
   if [[ $a = --sbf-sdk ]]; then
     maybe_sbf_sdk=
@@ -11,4 +11,4 @@ done
 
 export CARGO_BUILD_SBF="$here"/cargo-build-sbf
 set -x
-exec cargo run --manifest-path "$here"/sdk/cargo-test-sbf/Cargo.toml -- $maybe_sbf_sdk "$@"
+exec cargo run --manifest-path "$here"/platform-tools-sdk/cargo-test-sbf/Cargo.toml -- $maybe_sbf_sdk "$@"

ci/buildkite-pipeline-in-disk.sh

@@ -180,6 +180,7 @@ all_test_steps() {
              ^ci/test-local-cluster.sh \
              ^core/build.rs \
              ^fetch-perf-libs.sh \
+             ^platform-tools-sdk/ \
              ^programs/ \
              ^sdk/ \
       ; then
@@ -207,6 +208,7 @@ EOF
              ^ci/test-local-cluster.sh \
              ^core/build.rs \
              ^fetch-perf-libs.sh \
+             ^platform-tools-sdk/ \
              ^programs/ \
              ^sdk/ \
              ^scripts/build-downstream-projects.sh \

ci/buildkite-pipeline.sh

@@ -217,6 +217,7 @@ all_test_steps() {
              ^ci/test-local-cluster.sh \
              ^core/build.rs \
              ^fetch-perf-libs.sh \
+             ^platform-tools-sdk/ \
              ^programs/ \
              ^sdk/ \
              cargo-build-sbf$ \
@@ -259,6 +260,7 @@ EOF
              ^ci/test-local-cluster.sh \
              ^core/build.rs \
              ^fetch-perf-libs.sh \
+             ^platform-tools-sdk/ \
              ^programs/ \
              ^sdk/ \
              cargo-build-sbf$ \

ci/buildkite-secondary.yml

@@ -20,7 +20,7 @@ steps:
     timeout_in_minutes: 5
   - wait
   - name: "publish docker"
-    command: "sdk/docker-solana/build.sh"
+    command: "docker-solana/build.sh"
     agents:
       queue: "release-build"
     timeout_in_minutes: 60

ci/buildkite-solana-private.sh

@@ -164,6 +164,7 @@ all_test_steps() {
              ^ci/test-local-cluster.sh \
              ^core/build.rs \
              ^fetch-perf-libs.sh \
+             ^platform-tools-sdk/ \
              ^programs/ \
              ^sdk/ \
       ; then
@@ -191,6 +192,7 @@ EOF
              ^ci/test-local-cluster.sh \
              ^core/build.rs \
              ^fetch-perf-libs.sh \
+             ^platform-tools-sdk/ \
              ^programs/ \
              ^sdk/ \
              ^ci/downstream-projects \

ci/nits.sh

@@ -24,10 +24,11 @@ declare print_free_tree=(
   ':ledger/src/**.rs'
   ':metrics/src/**.rs'
   ':net-utils/src/**.rs'
+  ':platform-tools-sdk/**.rs'
+  ':platform-tools-sdk/sbf/rust/rust-utils/**.rs'
+  ':^platform-tools-sdk/cargo-build-sbf/**.rs'
   ':runtime/src/**.rs'
-  ':sdk/sbf/rust/rust-utils/**.rs'
   ':sdk/**.rs'
-  ':^sdk/cargo-build-sbf/**.rs'
   ':^sdk/msg/src/lib.rs'
   ':^sdk/program-option/src/lib.rs'
   ':^sdk/pubkey/src/lib.rs'

ci/platform-tools-info.sh

@@ -9,7 +9,7 @@ here="$(dirname "$0")"
 
 SBF_TOOLS_VERSION=unknown
 
-cargo_build_sbf_main="${here}/../sdk/cargo-build-sbf/src/main.rs"
+cargo_build_sbf_main="${here}/../platform-tools-sdk/cargo-build-sbf/src/main.rs"
 if [[ -f "${cargo_build_sbf_main}" ]]; then
     version=$(sed -e 's/^.*DEFAULT_PLATFORM_TOOLS_VERSION.*=\s*"\(v[0-9.]\+\)".*/\1/;t;d' "${cargo_build_sbf_main}")
     if [[ ${version} != '' ]]; then

ci/publish-tarball.sh

@@ -102,7 +102,7 @@ MAYBE_TARBALLS=
 if [[ "$CI_OS_NAME" = linux ]]; then
   (
     set -x
-    sdk/sbf/scripts/package.sh
+    platform-tools-sdk/sbf/scripts/package.sh
     [[ -f sbf-sdk.tar.bz2 ]]
   )
   MAYBE_TARBALLS="sbf-sdk.tar.bz2"

sdk/docker-solana/build.sh → docker-solana/build.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -ex
 
-cd "$(dirname "$0")"/../..
+cd "$(dirname "$0")"/..
 eval "$(ci/channel-info.sh)"
 source ci/rust-version.sh
 source ci/docker/env.sh
@@ -21,10 +21,10 @@ fi
 
 cd "$(dirname "$0")"
 rm -rf usr/
-../../ci/docker-run-default-image.sh scripts/cargo-install-all.sh sdk/docker-solana/usr
+../ci/docker-run-default-image.sh scripts/cargo-install-all.sh docker-solana/usr
 
-cp -f ../../scripts/run.sh usr/bin/solana-run.sh
-cp -f ../../fetch-spl.sh usr/bin/
+cp -f ../scripts/run.sh usr/bin/solana-run.sh
+cp -f ../fetch-spl.sh usr/bin/
 (
   cd usr/bin
   ./fetch-spl.sh

sdk/cargo-build-sbf/src/main.rs → platform-tools-sdk/cargo-build-sbf/src/main.rs

@@ -56,7 +56,7 @@ impl Default for Config<'_> {
                 .parent()
                 .expect("Unable to get parent directory")
                 .to_path_buf()
-                .join("sdk")
+                .join("platform-tools-sdk")
                 .join("sbf"),
             sbf_out_dir: None,
             platform_tools_version: None,
@@ -340,7 +340,7 @@ fn install_if_missing(
         fs::remove_file(download_file_path).map_err(|err| err.to_string())?;
     }
     // Make a symbolic link source_path -> target_path in the
-    // sdk/sbf/dependencies directory if no valid link found.
+    // platform-tools-sdk/sbf/dependencies directory if no valid link found.
     let source_base = config.sbf_sdk.join("dependencies");
     if !source_base.exists() {
         fs::create_dir_all(&source_base).map_err(|err| err.to_string())?;

sdk/cargo-build-sbf/tests/crates.rs → platform-tools-sdk/cargo-build-sbf/tests/crates.rs

@@ -160,7 +160,7 @@ fn test_sbfv2() {
         .parent()
         .expect("Unable to get ../.. of current working dir");
     let readelf = root
-        .join("sdk")
+        .join("platform-tools-sdk")
         .join("sbf")
         .join("dependencies")
         .join("platform-tools")

sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml → platform-tools-sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml

@@ -10,14 +10,14 @@ edition = "2021"
 publish = false
 
 [dependencies]
-solana-program = { path = "../../../../program", version = "=2.2.0" }
+solana-program = { path = "../../../../../sdk/program", version = "=2.2.0" }
 
 [lib]
 crate-type = ["cdylib"]
 
 [patch.crates-io]
-solana-decode-error = { path = "../../../../decode-error" }
-solana-instruction = { path = "../../../../instruction" }
-solana-pubkey = { path = "../../../../pubkey" }
+solana-decode-error = { path = "../../../../../sdk/decode-error" }
+solana-instruction = { path = "../../../../../sdk/instruction" }
+solana-pubkey = { path = "../../../../../sdk/pubkey" }
 
 [workspace]

sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml → platform-tools-sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml

@@ -10,14 +10,14 @@ edition = "2021"
 publish = false
 
 [dependencies]
-solana-program = { path = "../../../../program", version = "=2.2.0" }
+solana-program = { path = "../../../../../sdk/program", version = "=2.2.0" }
 
 [lib]
 crate-type = ["cdylib"]
 
 [patch.crates-io]
-solana-decode-error = { path = "../../../../decode-error" }
-solana-instruction = { path = "../../../../instruction" }
-solana-pubkey = { path = "../../../../pubkey" }
+solana-decode-error = { path = "../../../../../sdk/decode-error" }
+solana-instruction = { path = "../../../../../sdk/instruction" }
+solana-pubkey = { path = "../../../../../sdk/pubkey" }
 
 [workspace]

sdk/cargo-build-sbf/tests/crates/package-metadata/Cargo.toml → platform-tools-sdk/cargo-build-sbf/tests/crates/package-metadata/Cargo.toml

@@ -14,15 +14,15 @@ tools-version = "v1.43"
 program-id = "MyProgram1111111111111111111111111111111111"
 
 [dependencies]
-solana-package-metadata = { path = "../../../../package-metadata", version = "=2.2.0" }
-solana-program = { path = "../../../../program", version = "=2.2.0" }
+solana-package-metadata = { path = "../../../../../sdk/package-metadata", version = "=2.2.0" }
+solana-program = { path = "../../../../../sdk/program", version = "=2.2.0" }
 
 [lib]
 crate-type = ["cdylib"]
 
 [patch.crates-io]
-solana-decode-error = { path = "../../../../decode-error" }
-solana-instruction = { path = "../../../../instruction" }
-solana-pubkey = { path = "../../../../pubkey" }
+solana-decode-error = { path = "../../../../../sdk/decode-error" }
+solana-instruction = { path = "../../../../../sdk/instruction" }
+solana-pubkey = { path = "../../../../../sdk/pubkey" }
 
 [workspace]

sdk/cargo-build-sbf/tests/crates/workspace-metadata/Cargo.toml → platform-tools-sdk/cargo-build-sbf/tests/crates/workspace-metadata/Cargo.toml

@@ -10,15 +10,15 @@ edition = "2021"
 publish = false
 
 [dependencies]
-solana-program = { path = "../../../../program", version = "=2.2.0" }
+solana-program = { path = "../../../../../sdk/program", version = "=2.2.0" }
 
 [lib]
 crate-type = ["cdylib"]
 
 [patch.crates-io]
-solana-decode-error = { path = "../../../../decode-error" }
-solana-instruction = { path = "../../../../instruction" }
-solana-pubkey = { path = "../../../../pubkey" }
+solana-decode-error = { path = "../../../../../sdk/decode-error" }
+solana-instruction = { path = "../../../../../sdk/instruction" }
+solana-pubkey = { path = "../../../../../sdk/pubkey" }
 
 [workspace]
 

sdk/gen-headers/src/main.rs → platform-tools-sdk/gen-headers/src/main.rs

@@ -22,7 +22,7 @@ use {
  *    used the call the syscall function.
  */
 fn main() {
-    let syscalls_inc_path = PathBuf::from("sdk/sbf/c/inc/sol/inc");
+    let syscalls_inc_path = PathBuf::from("platform-tools-sdk/sbf/c/inc/sol/inc");
 
     if syscalls_inc_path.is_dir() {
         for entry in fs::read_dir(syscalls_inc_path).expect("Can't open headers dir") {

sdk/sbf/c/inc/sol/alt_bn128.h → platform-tools-sdk/sbf/c/inc/sol/alt_bn128.h

@@ -53,7 +53,7 @@ extern "C" {
  * @param result 64 byte array to hold the result. ...
  * @return 0 if executed successfully
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/alt_bn128.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/alt_bn128.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 uint64_t sol_alt_bn128_group_op(const uint64_t, const uint8_t *, const uint64_t, uint8_t *);
 #else

sdk/sbf/c/inc/sol/alt_bn128_compression.h → platform-tools-sdk/sbf/c/inc/sol/alt_bn128_compression.h

@@ -58,7 +58,7 @@ extern "C" {
  * @param result 64 byte array to hold the result. ...
  * @return 0 if executed successfully
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/alt_bn128_compression.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/alt_bn128_compression.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 uint64_t sol_alt_bn128_compression(
         const uint64_t op,

sdk/sbf/c/inc/sol/assert.h → platform-tools-sdk/sbf/c/inc/sol/assert.h

@@ -16,7 +16,7 @@ extern "C" {
  * Prints the line number where the panic occurred and then causes
  * the SBF VM to immediately halt execution. No accounts' data are updated
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/assert.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/assert.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 void sol_panic_(const char *, uint64_t, uint64_t, uint64_t);
 #else

sdk/sbf/c/inc/sol/big_mod_exp.h → platform-tools-sdk/sbf/c/inc/sol/big_mod_exp.h

@@ -14,7 +14,7 @@ extern "C" {
  * @param result 32 byte array to hold the result
  * @return 0 if executed successfully
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/big_mod_exp.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/big_mod_exp.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 uint64_t sol_big_mod_exp(const uint8_t *, uint8_t *);
 #else

sdk/sbf/c/inc/sol/blake3.h → platform-tools-sdk/sbf/c/inc/sol/blake3.h

@@ -21,7 +21,7 @@ extern "C" {
  * @param bytes_len Number of byte arrays
  * @param result 32 byte array to hold the result
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/blake3.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/blake3.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 uint64_t sol_blake3(const SolBytes *, int, const uint8_t *);
 #else

sdk/sbf/c/inc/sol/compute_units.h → platform-tools-sdk/sbf/c/inc/sol/compute_units.h

@@ -14,7 +14,7 @@ extern "C" {
 /**
  * Prints a string to stdout
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/compute_units.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/compute_units.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 uint64_t sol_remaining_compute_units();
 #else

sdk/sbf/c/inc/sol/cpi.h → platform-tools-sdk/sbf/c/inc/sol/cpi.h

@@ -56,7 +56,7 @@ typedef struct {
 /**
  * Internal cross-program invocation function
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/cpi.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/cpi.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 uint64_t sol_invoke_signed_c(
   const SolInstruction *,

sdk/sbf/c/inc/sol/keccak.h → platform-tools-sdk/sbf/c/inc/sol/keccak.h

@@ -21,7 +21,7 @@ extern "C" {
  * @param bytes_len Number of byte arrays
  * @param result 32 byte array to hold the result
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/keccak.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/keccak.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 uint64_t sol_keccak256(const SolBytes *, int, uint8_t *);
 #else

sdk/sbf/c/inc/sol/last_restart_slot.h → platform-tools-sdk/sbf/c/inc/sol/last_restart_slot.h

@@ -12,7 +12,7 @@ extern "C" {
 /**
  * Get Last Restart Slot
  */
-/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE sdk/sbf/c/inc/sol/inc/last_restart_slot.inc AND RUN `cargo run --bin gen-headers` */
+/* DO NOT MODIFY THIS GENERATED FILE. INSTEAD CHANGE platform-tools-sdk/sbf/c/inc/sol/inc/last_restart_slot.inc AND RUN `cargo run --bin gen-headers` */
 #ifndef SOL_SBFV2
 u64 sol_get_last_restart_slot(uint8_t *result);
 #else