From 79a4d06fb0d005d9169c94ee9016f049481a8407 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 11 Dec 2024 10:02:06 +0000 Subject: [PATCH 01/34] . --- Cargo.Bazel.Fuzzing.json.lock | 190 +++++++++++++++++- Cargo.Bazel.Fuzzing.toml.lock | 31 +++ Cargo.Bazel.json.lock | 190 +++++++++++++++++- Cargo.Bazel.toml.lock | 31 +++ Cargo.lock | 32 ++- bazel/external_crates.bzl | 5 + rs/config/src/embedders.rs | 2 +- .../random_traffic_test/BUILD.bazel | 2 +- .../random_traffic_test/Cargo.toml | 2 +- 9 files changed, 479 insertions(+), 6 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index f2147abd496..5e7210f5de4 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "450ee9a1af0058d25f3cc1bc2b3906b3e4271ca79174522e2e1a71c0d9a9c85a", + "checksum": "fe04d37e772298af4879fb7b2b0213d59172328242a352853967334a9a49f3cf", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -18840,6 +18840,11 @@ "id": "ic-cdk 0.16.0", "target": "ic_cdk" }, + { + "id": "ic-cdk 0.18.0-alpha.1", + "target": "ic_cdk", + "alias": "ic_cdk_next" + }, { "id": "ic-cdk-timers 0.11.0", "target": "ic_cdk_timers" @@ -31642,6 +31647,77 @@ ], "license_file": "LICENSE" }, + "ic-cdk 0.18.0-alpha.1": { + "name": "ic-cdk", + "version": "0.18.0-alpha.1", + "package_url": "https://github.com/dfinity/cdk-rs", + "repository": { + "Git": { + "remote": "https://github.com/dfinity/cdk-rs.git", + "commitish": { + "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + }, + "strip_prefix": "ic-cdk" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ic_cdk", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "ic_cdk", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "candid 0.10.10", + "target": "candid" + }, + { + "id": "ic0 0.24.0-alpha.1", + "target": "ic0" + }, + { + "id": "serde 1.0.214", + "target": "serde" + }, + { + "id": "serde_bytes 0.11.15", + "target": "serde_bytes" + } + ], + "selects": {} + }, + "edition": "2021", + "proc_macro_deps": { + "common": [ + { + "id": "ic-cdk-macros 0.18.0-alpha.1", + "target": "ic_cdk_macros" + } + ], + "selects": {} + }, + "version": "0.18.0-alpha.1" + }, + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "license_file": "LICENSE" + }, "ic-cdk-macros 0.8.4": { "name": "ic-cdk-macros", "version": "0.8.4", @@ -31977,6 +32053,76 @@ ], "license_file": "LICENSE" }, + "ic-cdk-macros 0.18.0-alpha.1": { + "name": "ic-cdk-macros", + "version": "0.18.0-alpha.1", + "package_url": "https://github.com/dfinity/cdk-rs", + "repository": { + "Git": { + "remote": "https://github.com/dfinity/cdk-rs.git", + "commitish": { + "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + }, + "strip_prefix": "ic-cdk-macros" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "ic_cdk_macros", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "ic_cdk_macros", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "candid 0.10.10", + "target": "candid" + }, + { + "id": "proc-macro2 1.0.89", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.37", + "target": "quote" + }, + { + "id": "serde 1.0.214", + "target": "serde" + }, + { + "id": "serde_tokenstream 0.2.1", + "target": "serde_tokenstream" + }, + { + "id": "syn 2.0.87", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.18.0-alpha.1" + }, + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "license_file": "LICENSE" + }, "ic-cdk-timers 0.11.0": { "name": "ic-cdk-timers", "version": "0.11.0", @@ -33392,6 +33538,47 @@ ], "license_file": "LICENSE" }, + "ic0 0.24.0-alpha.1": { + "name": "ic0", + "version": "0.24.0-alpha.1", + "package_url": "https://github.com/dfinity/cdk-rs", + "repository": { + "Git": { + "remote": "https://github.com/dfinity/cdk-rs.git", + "commitish": { + "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + }, + "strip_prefix": "ic0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ic0", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "ic0", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.24.0-alpha.1" + }, + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "license_file": "LICENSE" + }, "ic_bls12_381 0.10.0": { "name": "ic_bls12_381", "version": "0.10.0", @@ -87742,6 +87929,7 @@ "ic-canister-sig-creation 1.0.1", "ic-cbor 2.6.0", "ic-cdk 0.16.0", + "ic-cdk 0.18.0-alpha.1", "ic-cdk-macros 0.9.0", "ic-cdk-timers 0.11.0", "ic-certificate-verification 2.6.0", diff --git a/Cargo.Bazel.Fuzzing.toml.lock b/Cargo.Bazel.Fuzzing.toml.lock index ae82144c530..8b890cb5843 100644 --- a/Cargo.Bazel.Fuzzing.toml.lock +++ b/Cargo.Bazel.Fuzzing.toml.lock @@ -3091,6 +3091,7 @@ dependencies = [ "ic-canister-sig-creation", "ic-cbor", "ic-cdk 0.16.0", + "ic-cdk 0.18.0-alpha.1", "ic-cdk-macros 0.9.0", "ic-cdk-timers", "ic-certificate-verification", @@ -5144,6 +5145,18 @@ dependencies = [ "serde_bytes", ] +[[package]] +name = "ic-cdk" +version = "0.18.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +dependencies = [ + "candid", + "ic-cdk-macros 0.18.0-alpha.1", + "ic0 0.24.0-alpha.1", + "serde", + "serde_bytes", +] + [[package]] name = "ic-cdk-macros" version = "0.8.4" @@ -5214,6 +5227,19 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "ic-cdk-macros" +version = "0.18.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +dependencies = [ + "candid", + "proc-macro2", + "quote", + "serde", + "serde_tokenstream 0.2.1", + "syn 2.0.87", +] + [[package]] name = "ic-cdk-timers" version = "0.11.0" @@ -5484,6 +5510,11 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de254dd67bbd58073e23dc1c8553ba12fa1dc610a19de94ad2bbcd0460c067f" +[[package]] +name = "ic0" +version = "0.24.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" + [[package]] name = "ic_bls12_381" version = "0.10.0" diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index d0949c64615..6ee09450320 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "b344eca479cfedf88abc9bad22dc5e3b5058749d7e97869732d8137396906468", + "checksum": "ebfd73515305a638602b4e5933b89a112d86920807987c76358925713748b777", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -18668,6 +18668,11 @@ "id": "ic-cdk 0.16.0", "target": "ic_cdk" }, + { + "id": "ic-cdk 0.18.0-alpha.1", + "target": "ic_cdk", + "alias": "ic_cdk_next" + }, { "id": "ic-cdk-timers 0.11.0", "target": "ic_cdk_timers" @@ -31497,6 +31502,77 @@ ], "license_file": "LICENSE" }, + "ic-cdk 0.18.0-alpha.1": { + "name": "ic-cdk", + "version": "0.18.0-alpha.1", + "package_url": "https://github.com/dfinity/cdk-rs", + "repository": { + "Git": { + "remote": "https://github.com/dfinity/cdk-rs.git", + "commitish": { + "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + }, + "strip_prefix": "ic-cdk" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ic_cdk", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "ic_cdk", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "candid 0.10.10", + "target": "candid" + }, + { + "id": "ic0 0.24.0-alpha.1", + "target": "ic0" + }, + { + "id": "serde 1.0.214", + "target": "serde" + }, + { + "id": "serde_bytes 0.11.15", + "target": "serde_bytes" + } + ], + "selects": {} + }, + "edition": "2021", + "proc_macro_deps": { + "common": [ + { + "id": "ic-cdk-macros 0.18.0-alpha.1", + "target": "ic_cdk_macros" + } + ], + "selects": {} + }, + "version": "0.18.0-alpha.1" + }, + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "license_file": "LICENSE" + }, "ic-cdk-macros 0.8.4": { "name": "ic-cdk-macros", "version": "0.8.4", @@ -31832,6 +31908,76 @@ ], "license_file": "LICENSE" }, + "ic-cdk-macros 0.18.0-alpha.1": { + "name": "ic-cdk-macros", + "version": "0.18.0-alpha.1", + "package_url": "https://github.com/dfinity/cdk-rs", + "repository": { + "Git": { + "remote": "https://github.com/dfinity/cdk-rs.git", + "commitish": { + "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + }, + "strip_prefix": "ic-cdk-macros" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "ic_cdk_macros", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "ic_cdk_macros", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "candid 0.10.10", + "target": "candid" + }, + { + "id": "proc-macro2 1.0.89", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.37", + "target": "quote" + }, + { + "id": "serde 1.0.214", + "target": "serde" + }, + { + "id": "serde_tokenstream 0.2.1", + "target": "serde_tokenstream" + }, + { + "id": "syn 2.0.87", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.18.0-alpha.1" + }, + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "license_file": "LICENSE" + }, "ic-cdk-timers 0.11.0": { "name": "ic-cdk-timers", "version": "0.11.0", @@ -33226,6 +33372,47 @@ ], "license_file": "LICENSE" }, + "ic0 0.24.0-alpha.1": { + "name": "ic0", + "version": "0.24.0-alpha.1", + "package_url": "https://github.com/dfinity/cdk-rs", + "repository": { + "Git": { + "remote": "https://github.com/dfinity/cdk-rs.git", + "commitish": { + "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + }, + "strip_prefix": "ic0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ic0", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "ic0", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.24.0-alpha.1" + }, + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "license_file": "LICENSE" + }, "ic_bls12_381 0.10.0": { "name": "ic_bls12_381", "version": "0.10.0", @@ -87622,6 +87809,7 @@ "ic-canister-sig-creation 1.0.1", "ic-cbor 2.6.0", "ic-cdk 0.16.0", + "ic-cdk 0.18.0-alpha.1", "ic-cdk-macros 0.9.0", "ic-cdk-timers 0.11.0", "ic-certificate-verification 2.6.0", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index 949f5b8aeee..e99dbc247f7 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -3080,6 +3080,7 @@ dependencies = [ "ic-canister-sig-creation", "ic-cbor", "ic-cdk 0.16.0", + "ic-cdk 0.18.0-alpha.1", "ic-cdk-macros 0.9.0", "ic-cdk-timers", "ic-certificate-verification", @@ -5134,6 +5135,18 @@ dependencies = [ "serde_bytes", ] +[[package]] +name = "ic-cdk" +version = "0.18.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +dependencies = [ + "candid", + "ic-cdk-macros 0.18.0-alpha.1", + "ic0 0.24.0-alpha.1", + "serde", + "serde_bytes", +] + [[package]] name = "ic-cdk-macros" version = "0.8.4" @@ -5204,6 +5217,19 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "ic-cdk-macros" +version = "0.18.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +dependencies = [ + "candid", + "proc-macro2", + "quote", + "serde", + "serde_tokenstream 0.2.1", + "syn 2.0.87", +] + [[package]] name = "ic-cdk-timers" version = "0.11.0" @@ -5474,6 +5500,11 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de254dd67bbd58073e23dc1c8553ba12fa1dc610a19de94ad2bbcd0460c067f" +[[package]] +name = "ic0" +version = "0.24.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" + [[package]] name = "ic_bls12_381" version = "0.10.0" diff --git a/Cargo.lock b/Cargo.lock index b1512375918..aae46bad45f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6406,6 +6406,18 @@ dependencies = [ "serde_bytes", ] +[[package]] +name = "ic-cdk" +version = "0.18.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +dependencies = [ + "candid", + "ic-cdk-macros 0.18.0-alpha.1", + "ic0 0.24.0-alpha.1", + "serde", + "serde_bytes", +] + [[package]] name = "ic-cdk-macros" version = "0.8.4" @@ -6476,6 +6488,19 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "ic-cdk-macros" +version = "0.18.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +dependencies = [ + "candid", + "proc-macro2", + "quote", + "serde", + "serde_tokenstream 0.2.2", + "syn 2.0.87", +] + [[package]] name = "ic-cdk-timers" version = "0.7.0" @@ -13773,6 +13798,11 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de254dd67bbd58073e23dc1c8553ba12fa1dc610a19de94ad2bbcd0460c067f" +[[package]] +name = "ic0" +version = "0.24.0-alpha.1" +source = "git+https://github.com/dfinity/cdk-rs?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" + [[package]] name = "ic_bls12_381" version = "0.10.0" @@ -18169,7 +18199,7 @@ dependencies = [ "candid", "futures", "ic-base-types", - "ic-cdk 0.16.0", + "ic-cdk 0.18.0-alpha.1", "ic-cdk-macros 0.9.0", "ic-error-types", "ic-types", diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index c644f89d273..c4e9368c43f 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -600,6 +600,11 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable "ic-cdk-macros": crate.spec( version = "^0.9.0", ), + "ic-cdk-next": crate.spec( + package = "ic-cdk", + git = "https://github.com/dfinity/cdk-rs.git", + branch = "next", + ), "ic-certified-map": crate.spec( version = "^0.3.1", ), diff --git a/rs/config/src/embedders.rs b/rs/config/src/embedders.rs index 076b675f535..1c8dbfacf12 100644 --- a/rs/config/src/embedders.rs +++ b/rs/config/src/embedders.rs @@ -121,7 +121,7 @@ impl FeatureFlags { write_barrier: FlagStatus::Disabled, wasm_native_stable_memory: FlagStatus::Enabled, wasm64: FlagStatus::Enabled, - best_effort_responses: FlagStatus::Disabled, + best_effort_responses: FlagStatus::Enabled, canister_backtrace: FlagStatus::Enabled, } } diff --git a/rs/rust_canisters/random_traffic_test/BUILD.bazel b/rs/rust_canisters/random_traffic_test/BUILD.bazel index 23eaf726e71..867e48d68f3 100644 --- a/rs/rust_canisters/random_traffic_test/BUILD.bazel +++ b/rs/rust_canisters/random_traffic_test/BUILD.bazel @@ -8,7 +8,7 @@ DEPENDENCIES = [ "//rs/types/error_types", "//rs/types/types", "@crate_index//:candid", - "@crate_index//:ic-cdk", + "@crate_index//:ic_cdk_next", "@crate_index//:futures", "@crate_index//:rand", "@crate_index//:serde", diff --git a/rs/rust_canisters/random_traffic_test/Cargo.toml b/rs/rust_canisters/random_traffic_test/Cargo.toml index 44814930525..7033946d200 100644 --- a/rs/rust_canisters/random_traffic_test/Cargo.toml +++ b/rs/rust_canisters/random_traffic_test/Cargo.toml @@ -10,7 +10,7 @@ path = "src/main.rs" [dependencies] candid = { workspace = true } ic-base-types = { path = "../../types/base_types" } -ic-cdk = { workspace = true } +ic-cdk = { git = "https://github.com/dfinity/cdk-rs", branch = "next" } ic-cdk-macros = { workspace = true } ic-error-types = { path = "../../types/error_types" } ic-types = { path = "../../types/types" } From b6ee737fd2e600c02d85fb347dde738d52493368 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 12 Dec 2024 15:26:02 +0000 Subject: [PATCH 02/34] . --- rs/messaging/src/routing/stream_handler.rs | 20 +++---- rs/messaging/tests/memory_tests.rs | 52 ++++++++++++++++++- .../random_traffic_test/src/main.rs | 46 +++++++++------- 3 files changed, 88 insertions(+), 30 deletions(-) diff --git a/rs/messaging/src/routing/stream_handler.rs b/rs/messaging/src/routing/stream_handler.rs index 62dda3c0ed7..897d5b32dc0 100644 --- a/rs/messaging/src/routing/stream_handler.rs +++ b/rs/messaging/src/routing/stream_handler.rs @@ -843,15 +843,17 @@ impl StreamHandlerImpl { return Some((reason, msg)); } RequestOrResponse::Response(response) => { - // Critical error, responses should always be inducted successfully. - error!( - self.log, - "{}: Inducting response failed: {}\n{:?}", - CRITICAL_ERROR_INDUCT_RESPONSE_FAILED, - err, - response - ); - self.metrics.critical_error_induct_response_failed.inc(); + if response.deadline == ic_types::messages::NO_DEADLINE { + // Critical error, responses should always be inducted successfully. + error!( + self.log, + "{}: Inducting response failed: {}\n{:?}", + CRITICAL_ERROR_INDUCT_RESPONSE_FAILED, + err, + response + ); + self.metrics.critical_error_induct_response_failed.inc(); + } } } } diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index cdb888ef398..724d846c658 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -27,6 +27,30 @@ const MB: u64 = KB * KB; const MAX_PAYLOAD_BYTES: u32 = MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32; + +#[test] +fn playground() { + let seeds = vec![0_u64; 3]; + let max_payload_bytes = MAX_PAYLOAD_BYTES; + let calls_per_round = 1; + let reply_weight = 1; + let call_weight = 10; + + if let Err((msg, mut dbg)) = check_guaranteed_response_message_memory_limits_are_respected_impl( + seeds.as_slice(), + max_payload_bytes, + calls_per_round as u32, + reply_weight as u32, + call_weight as u32, + ) { + let r = dbg.records.pop_first().unwrap().1; + assert!(false, "{:?}\n\n{}\n\n{:#?}", msg, r.len(), r); + } else { + unreachable!(); + } +} + + proptest! { #![proptest_config(ProptestConfig::with_cases(1))] #[test] @@ -71,7 +95,7 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( call_weight: u32, ) -> Result<(), (String, DebugInfo)> { // The number of rounds to execute while chatter is on. - const CHATTER_PHASE_ROUND_COUNT: u64 = 30; + const CHATTER_PHASE_ROUND_COUNT: u64 = 300; // The maximum number of rounds to execute after chatter is turned off. It it takes more than // this number of rounds until there are no more pending calls, the test fails. const SHUTDOWN_PHASE_MAX_ROUNDS: u64 = 300; @@ -88,6 +112,10 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( remote_max_instructions_per_round: 100_000_000, remote_message_memory_capacity: REMOTE_MESSAGE_MEMORY_CAPACITY, }); + //assert!(false, "{:?}", fixture.canisters()); + + let local_time = fixture.local_env.get_time(); + let remote_time = fixture.remote_env.get_time(); let config = CanisterConfig::try_new( fixture.canisters(), // receivers @@ -119,7 +147,16 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( REMOTE_MESSAGE_MEMORY_CAPACITY, )?; } - +/* + return fixture.failed_with_reason( + format!("{:?} / {:?}, {:?} / {:?}", + local_time, + fixture.local_env.get_time(), + remote_time, + fixture.remote_env.get_time() + ) + ); +*/ // Shut down chatter by putting a canister into `Stopping` state every 10 ticks until they are // all `Stopping` or `Stopped`. for canister in fixture.canisters().into_iter() { @@ -162,6 +199,15 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( // One extra tick to make sure everything is gc'ed. fixture.tick(); + + return fixture.failed_with_reason( + format!("{:?} / {:?}, {:?} / {:?}", + local_time, + fixture.local_env.get_time(), + remote_time, + fixture.remote_env.get_time() + ) + ); // Check the records agree on 'no pending calls'. if fixture @@ -563,6 +609,7 @@ impl Fixture { } else { self.remote_env.tick(); } + self.remote_env.advance_time(std::time::Duration::from_secs(1)); if let Ok(xnet_payload) = self.remote_env.generate_xnet_payload( self.local_env.get_subnet_id(), @@ -575,6 +622,7 @@ impl Fixture { } else { self.local_env.tick(); } + self.local_env.advance_time(std::time::Duration::from_secs(1)); } /// Migrates `canister` between `local_env` and `remote_env` (either direction). diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index ce928a0553e..f94e6005478 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -1,7 +1,8 @@ use candid::{CandidType, Encode}; use futures::future::select_all; use ic_base_types::{CanisterId, PrincipalId}; -use ic_cdk::{api, caller, id}; +use ic_cdk::{api, caller, id, setup}; +use ic_cdk::api::call::{Call, ConfigurableCall, SendableCall}; use ic_cdk_macros::{heartbeat, init, query, update}; use rand::{ distributions::{Distribution, WeightedIndex}, @@ -48,8 +49,8 @@ struct Message { call_tree_id: u32, /// The depth of the call starting from 0 and incrementing by 1 for each downstream call. call_depth: u32, - /// A payload of a certain size; it otherwise does not any contain information. - payload: Vec, + /// Optional padding, to bring the payload to the desired byte size. + padding: Vec, } impl Message { @@ -59,13 +60,13 @@ impl Message { Self { call_tree_id, call_depth, - payload: vec![0_u8; bytes_count.saturating_sub(std::mem::size_of::())], + padding: vec![0_u8; bytes_count.saturating_sub(std::mem::size_of::())], } } /// Returns the number of bytes the message consists of. fn count_bytes(&self) -> usize { - std::mem::size_of::() + self.payload.len() + std::mem::size_of::() + self.padding.len() } } @@ -176,21 +177,26 @@ fn setup_call( // Inserts a new call record at the next `index`. let index = next_call_index(); RECORDS.with_borrow_mut(|records| { - records.insert( - index, - Record { - receiver, - caller: (call_depth > 0) - .then_some(CanisterId::unchecked_from_principal(PrincipalId(caller()))), - call_tree_id, - call_depth, - sent_bytes: msg.count_bytes() as u32, - reply: None, - }, - ); + assert!(records + .insert( + index, + Record { + receiver, + caller: (call_depth > 0) + .then_some(CanisterId::unchecked_from_principal(PrincipalId(caller()))), + call_tree_id, + call_depth, + sent_bytes: msg.count_bytes() as u32, + reply: None, + }, + ) + .is_none()); }); - let future = api::call::call_raw(receiver.into(), "handle_call", Encode!(&msg).unwrap(), 0); + let future = Call::new(receiver.into(), "handle_call") + .with_raw_args(Encode!(&msg).unwrap()) + .change_timeout(100) + .call_raw(); (future, index) } @@ -305,4 +311,6 @@ fn initialize_hasher() { HASHER.with_borrow_mut(|hasher| hasher.write(id().as_slice())); } -fn main() {} +fn main() { + setup(); +} From 30dd66cd5276d7bf739c5688cf8375dccefdc72a Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Tue, 17 Dec 2024 20:49:55 +0000 Subject: [PATCH 03/34] . --- Cargo.Bazel.json.lock | 2 +- rs/messaging/tests/memory_tests.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 8c529fa697c..9a5987b4a52 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "ebfd73515305a638602b4e5933b89a112d86920807987c76358925713748b777", + "checksum": "426018d109eca1845ca4b3d2e5755c16846719bf3f009e6969cf716a8955195a", "crates": { "abnf 0.12.0": { "name": "abnf", diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 724d846c658..6c589e9e726 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -46,7 +46,7 @@ fn playground() { let r = dbg.records.pop_first().unwrap().1; assert!(false, "{:?}\n\n{}\n\n{:#?}", msg, r.len(), r); } else { - unreachable!(); + //unreachable!(); } } @@ -95,7 +95,7 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( call_weight: u32, ) -> Result<(), (String, DebugInfo)> { // The number of rounds to execute while chatter is on. - const CHATTER_PHASE_ROUND_COUNT: u64 = 300; + const CHATTER_PHASE_ROUND_COUNT: u64 = 100; // The maximum number of rounds to execute after chatter is turned off. It it takes more than // this number of rounds until there are no more pending calls, the test fails. const SHUTDOWN_PHASE_MAX_ROUNDS: u64 = 300; @@ -199,7 +199,7 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( // One extra tick to make sure everything is gc'ed. fixture.tick(); - + /* return fixture.failed_with_reason( format!("{:?} / {:?}, {:?} / {:?}", local_time, @@ -208,7 +208,7 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( fixture.remote_env.get_time() ) ); - + */ // Check the records agree on 'no pending calls'. if fixture .canisters() From bf8efd851e8a7a2974ae63460b99170cb3b801e3 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 8 Jan 2025 18:38:45 +0000 Subject: [PATCH 04/34] . --- Cargo.Bazel.Fuzzing.json.lock | 2 +- rs/messaging/tests/memory_tests.rs | 325 +++++++++--------- .../random_traffic_test/src/lib.rs | 139 +++++--- .../random_traffic_test/src/main.rs | 232 +++++++------ 4 files changed, 390 insertions(+), 308 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 6eecde3116a..5688ae1c017 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "fe04d37e772298af4879fb7b2b0213d59172328242a352853967334a9a49f3cf", + "checksum": "6712d2aabc11e75facdd5779b32d59fcecd3e8dc2806096dc111d59322e5de62", "crates": { "abnf 0.12.0": { "name": "abnf", diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 6c589e9e726..f50a66ca4b9 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -26,8 +26,7 @@ const KB: u64 = 1024; const MB: u64 = KB * KB; const MAX_PAYLOAD_BYTES: u32 = MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32; - - +/* #[test] fn playground() { let seeds = vec![0_u64; 3]; @@ -49,25 +48,57 @@ fn playground() { //unreachable!(); } } +*/ +prop_compose! { + /// Generates an arbitrary pair of weights such that w1 + w2 = total > 0. + fn arb_weights(total: u32)( + w1 in 0..=total + ) -> (u32, u32) + { + (w1, total - w1) + } +} + +prop_compose! { + /// Generates a random `CanisterConfig` using reasonable ranges of values; receivers is empty + /// and assumed to be populated manually. + fn arb_canister_config(max_payload_bytes: u32, max_calls_per_heartbeat: u32)( + max_call_bytes in 0..=max_payload_bytes, + max_reply_bytes in 0..=max_payload_bytes, + calls_per_heartbeat in 0..=max_calls_per_heartbeat, + max_timeout_secs in 10..=100_u32, + (reply_weight, downstream_call_weight) in arb_weights(3), + (best_effort_weight, guaranteed_response_weight) in arb_weights(3), + ) -> CanisterConfig { + CanisterConfig::try_new( + vec![], + 0..=max_call_bytes, + 0..=max_reply_bytes, + 0..=0, // instructions_count + 0..=max_timeout_secs, + calls_per_heartbeat, + reply_weight, + downstream_call_weight, + best_effort_weight, + guaranteed_response_weight, + ) + .expect("bad config inputs") + } +} proptest! { #![proptest_config(ProptestConfig::with_cases(1))] #[test] fn check_guaranteed_response_message_memory_limits_are_respected( seeds in proptest::collection::vec(any::().no_shrink(), 3), - max_payload_bytes in (MAX_PAYLOAD_BYTES / 4)..=MAX_PAYLOAD_BYTES, - calls_per_round in 1..=10, - reply_weight in 1..=2, - call_weight in 0..=2, - // Note: both weights zero defaults to only replies. + config in arb_canister_config(MAX_PAYLOAD_BYTES, 5), ) { prop_assert!(check_guaranteed_response_message_memory_limits_are_respected_impl( + 30, // chatter_phase_round_count + 300, // shutdown_phase_max_rounds seeds.as_slice(), - max_payload_bytes, - calls_per_round as u32, - reply_weight as u32, - call_weight as u32, + config, ).is_ok()); } } @@ -75,8 +106,9 @@ proptest! { /// Runs a state machine test with two subnets, a local subnet with 2 canisters installed and a /// remote subnet with 1 canister installed. /// -/// In the first phase a number of rounds are executed on both subnets, including XNet traffic with -/// 'chatter' enabled, i.e. the installed canisters are making random calls (including downstream calls). +/// In the first phase `chatter_phase_round_count` rounds are executed on both subnets, including XNet +/// traffic with 'chatter' enabled, i.e. the installed canisters are making random calls (including +/// downstream calls depending on `config`). /// /// For the second phase, the 'chatter' is disabled by putting a canister into `Stopping` state /// every 10 rounds. In addition to shutting down traffic altogether from that canister (including @@ -84,21 +116,18 @@ proptest! { /// canister fails to reach `Stopped` state (i.e. no pending calls), something went wrong in /// message routing, most likely a bug connected to reject signals for requests. /// -/// Checks that the guaranteed response message memory never exceeds the limit; that all calls eventually -/// receive a reply (or were rejected synchronously when issued); and that the message memory goes -/// back to 0 after all in-flight messages have been dealt with. +/// In the final phase, up to `shutdown_phase_max_rounds` additional rounds are executed after +/// 'chatter' has been turned off to conclude all calls (or else return `Err(_)` if any call fails +/// to do so). +/// +/// During all these phases, a check ensures that guaranteed response message memory never exceeds +/// the limit specified in the `FixtureConfig` used to generate the fixture used in this test. fn check_guaranteed_response_message_memory_limits_are_respected_impl( + chatter_phase_round_count: usize, + shutdown_phase_max_rounds: usize, seeds: &[u64], - max_payload_bytes: u32, - calls_per_round: u32, - reply_weight: u32, - call_weight: u32, + mut config: CanisterConfig, ) -> Result<(), (String, DebugInfo)> { - // The number of rounds to execute while chatter is on. - const CHATTER_PHASE_ROUND_COUNT: u64 = 100; - // The maximum number of rounds to execute after chatter is turned off. It it takes more than - // this number of rounds until there are no more pending calls, the test fails. - const SHUTDOWN_PHASE_MAX_ROUNDS: u64 = 300; // The amount of memory available for guaranteed response message memory on `local_env`. const LOCAL_MESSAGE_MEMORY_CAPACITY: u64 = 100 * MB; // The amount of memory available for guaranteed response message memory on `remote_env`. @@ -112,32 +141,17 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( remote_max_instructions_per_round: 100_000_000, remote_message_memory_capacity: REMOTE_MESSAGE_MEMORY_CAPACITY, }); - //assert!(false, "{:?}", fixture.canisters()); - - let local_time = fixture.local_env.get_time(); - let remote_time = fixture.remote_env.get_time(); - let config = CanisterConfig::try_new( - fixture.canisters(), // receivers - 0..=max_payload_bytes, // call_bytes - 0..=max_payload_bytes, // reply_bytes - 0..=0, // instructions_count - ) - .unwrap(); + config.receivers = fixture.canisters(); // Send configs to canisters, seed the rng. for (index, canister) in fixture.canisters().into_iter().enumerate() { fixture.set_config(canister, config.clone()); fixture.seed_rng(canister, seeds[index]); - fixture.set_reply_weight(canister, reply_weight); - fixture.set_call_weight(canister, call_weight); } - // Start chatter on all canisters. - fixture.start_chatter(calls_per_round); - // Build up backlog and keep up chatter for while. - for _ in 0..CHATTER_PHASE_ROUND_COUNT { + for _ in 0..chatter_phase_round_count { fixture.tick(); // Check message memory limits are respected. @@ -147,23 +161,11 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( REMOTE_MESSAGE_MEMORY_CAPACITY, )?; } -/* - return fixture.failed_with_reason( - format!("{:?} / {:?}, {:?} / {:?}", - local_time, - fixture.local_env.get_time(), - remote_time, - fixture.remote_env.get_time() - ) - ); -*/ + // Shut down chatter by putting a canister into `Stopping` state every 10 ticks until they are // all `Stopping` or `Stopped`. for canister in fixture.canisters().into_iter() { - // The max calls per heartbeat are set to 0 here, because the canister has to be started - // to query it's records. This is to make sure the canister doesn't start making calls - // immediately before we can get its records. - fixture.set_max_calls_per_heartbeat(canister, 0); + fixture.stop_chatter(canister); fixture.stop_canister_non_blocking(canister); for _ in 0..10 { fixture.tick(); @@ -177,50 +179,30 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( } } - // Keep ticking until all calls are answered. - for counter in 0.. { - fixture.tick(); - - // Check message memory limits are respected. + // Tick until all calls have concluded. + fixture.tick_to_conclusion(shutdown_phase_max_rounds, |fixture| { fixture.expect_guaranteed_response_message_memory_taken_at_most( - "Shutdown", + "Wrap up", LOCAL_MESSAGE_MEMORY_CAPACITY, REMOTE_MESSAGE_MEMORY_CAPACITY, - )?; - - if fixture.open_call_contexts_count().values().sum::() == 0 { - break; - } - - if counter > SHUTDOWN_PHASE_MAX_ROUNDS { - return fixture.failed_with_reason("shutdown phase hanging"); - } - } - - // One extra tick to make sure everything is gc'ed. - fixture.tick(); - /* - return fixture.failed_with_reason( - format!("{:?} / {:?}, {:?} / {:?}", - local_time, - fixture.local_env.get_time(), - remote_time, - fixture.remote_env.get_time() ) - ); - */ - // Check the records agree on 'no pending calls'. - if fixture - .canisters() - .into_iter() - .map(|canister| extract_metrics(&fixture.force_query_records(canister))) - .any(|metrics| metrics.pending_calls != 0) - { - return fixture.failed_with_reason("found pending calls in the records"); - } + }) +} - // After the fact, all memory is freed and back to 0. - fixture.expect_guaranteed_response_message_memory_taken_at_most("Final check", 0, 0) +proptest! { + #![proptest_config(ProptestConfig::with_cases(1))] + #[test] + fn check_calls_conclude_with_migrating_canister( + seed in any::().no_shrink(), + config in arb_canister_config(1 * KB as u32, 10), + ) { + prop_assert!(check_calls_conclude_with_migrating_canister_impl( + 10, // chatter_phase_round_count + 300, // shutdown_phase_max_rounds + seed, + config, + ).is_ok()); + } } /// Runs a state machine test with two subnets, a local subnet with 2 canisters installed and a @@ -241,34 +223,24 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( /// If there are pending calls after a threshold number of rounds, there is most likely a bug /// connected to reject signals for requests, specifically with the corresponding exceptions due to /// canister migration. -#[test] -fn check_calls_conclude_with_migrating_canister() { - // The number of rounds to execute while the migrating canister is making calls. - const BUILDUP_PHASE_ROUND_COUNT: u64 = 10; - // The maximum number of rounds to execute after chatter is turned off. It it takes more than - // this number of rounds until there are no more pending calls, the test fails. - const SHUTDOWN_PHASE_MAX_ROUNDS: u64 = 300; - +fn check_calls_conclude_with_migrating_canister_impl( + chatter_phase_round_count: usize, + shutdown_phase_max_rounds: usize, + seed: u64, + mut config: CanisterConfig, +) -> Result<(), (String, DebugInfo)> { let mut fixture = Fixture::new(FixtureConfig { local_canisters_count: 2, remote_canisters_count: 5, ..FixtureConfig::default() }); - let migrating_canister = *fixture.local_canisters.first().unwrap(); - let config = CanisterConfig::try_new( - fixture.canisters(), // receivers - 0..=0, // call_bytes - 0..=0, // reply_bytes - 0..=0, // instructions_count - ) - .unwrap(); - fixture.set_config(migrating_canister, config); + config.receivers = fixture.canisters(); + let migrating_canister = fixture.local_canisters.first().unwrap().clone(); - fixture.seed_rng(migrating_canister, 73); - fixture.set_reply_weight(migrating_canister, 1); - fixture.set_call_weight(migrating_canister, 0); - fixture.set_max_calls_per_heartbeat(migrating_canister, 10); + // Send config to `migrating_canister` and seed its rng. + fixture.set_config(migrating_canister, config); + fixture.seed_rng(migrating_canister, seed); // Stop all canisters except `migrating_canister`. for canister in fixture.canisters() { @@ -277,28 +249,16 @@ fn check_calls_conclude_with_migrating_canister() { } } // Make calls on `migrating_canister`. - for _ in 0..BUILDUP_PHASE_ROUND_COUNT { + for _ in 0..chatter_phase_round_count { fixture.tick(); } // Stop making calls and migrate `migrating_canister`. - fixture.set_max_calls_per_heartbeat(migrating_canister, 0); + fixture.stop_chatter(migrating_canister); fixture.migrate_canister(migrating_canister); // Tick until all calls have concluded. - for counter in 0.. { - fixture.tick(); - if fixture.open_call_contexts_count().values().sum::() == 0 { - break; - } - assert!(counter < SHUTDOWN_PHASE_MAX_ROUNDS); - } - - // Check that the records agree on 'no pending calls'. - assert_eq!( - 0, - extract_metrics(&fixture.force_query_records(migrating_canister)).pending_calls - ); + fixture.tick_to_conclusion(shutdown_phase_max_rounds, |_| Ok(())) } #[derive(Debug)] @@ -459,27 +419,6 @@ impl Fixture { self.set_canister_state(canister, "set_config", config) } - /// Sets the `max_calls_per_heartbeat` in `canister`; returns the current value. - /// - /// Panics if `canister` is not installed in `Self`. - pub fn set_max_calls_per_heartbeat(&self, canister: CanisterId, count: u32) -> u32 { - self.set_canister_state(canister, "set_max_calls_per_heartbeat", count) - } - - /// Sets the `reply_weight` in `canister`; returns the current weight. - /// - /// Panics if `canister` is not installed in `Self`. - pub fn set_reply_weight(&self, canister: CanisterId, weight: u32) -> u32 { - self.set_canister_state(canister, "set_reply_weight", weight) - } - - /// Sets the `call_weight` in `canister`. - /// - /// Panics if `canister` is not installed in `Self`. - pub fn set_call_weight(&self, canister: CanisterId, weight: u32) -> u32 { - self.set_canister_state(canister, "set_call_weight", weight) - } - /// Seeds the `Rng` in `canister`. /// /// Panics if `canister` is not installed in `Self`. @@ -490,13 +429,6 @@ impl Fixture { .unwrap(); } - /// Sets `max_calls_per_heartbeat` on all canisters to the same value. - pub fn start_chatter(&self, max_calls_per_heartbeat: u32) { - for canister in self.canisters() { - self.set_max_calls_per_heartbeat(canister, max_calls_per_heartbeat); - } - } - /// Starts `canister`. /// /// Panics if `canister` is not installed in `Self`. @@ -514,6 +446,15 @@ impl Fixture { self.get_env(&canister).stop_canister_non_blocking(canister) } + /// Calls the `stop_chatter()` function on `canister`. + /// + /// This stops the canister from making calls, downstream and from the heartbeat. + pub fn stop_chatter(&self, canister: CanisterId) { + self.get_env(&canister) + .execute_ingress(canister, "stop_chatter", candid::Encode!().unwrap()) + .unwrap(); + } + /// Queries the records from `canister`. /// /// Panics if `canister` is not installed in `Self`. @@ -558,16 +499,16 @@ impl Fixture { /// upper limit. pub fn expect_guaranteed_response_message_memory_taken_at_most( &self, - label: &str, + label: impl std::fmt::Display, local_memory_upper_limit: u64, remote_memory_upper_limit: u64, ) -> Result<(), (String, DebugInfo)> { let (local_memory, remote_memory) = self.guaranteed_response_message_memory_taken(); if local_memory > local_memory_upper_limit.into() { - return self.failed_with_reason(format!("{label}: local memory exceeds limit")); + return self.failed_with_reason(format!("{}: local memory exceeds limit", label)); } if remote_memory > remote_memory_upper_limit.into() { - return self.failed_with_reason(format!("{label}: remote memory exceeds limit")); + return self.failed_with_reason(format!("{}: remote memory exceeds limit", label)); } Ok(()) } @@ -596,6 +537,8 @@ impl Fixture { /// Executes one round on both the `local_env` and the `remote_env` by generating a XNet /// payload on one and inducting it into the other, and vice versa; if there are no XNet /// messages either way, performs a normal `tick()` on the receiving subnet. + /// + /// Advances time on each env by 1 second. pub fn tick(&self) { if let Ok(xnet_payload) = self.local_env.generate_xnet_payload( self.remote_env.get_subnet_id(), @@ -609,7 +552,8 @@ impl Fixture { } else { self.remote_env.tick(); } - self.remote_env.advance_time(std::time::Duration::from_secs(1)); + self.remote_env + .advance_time(std::time::Duration::from_secs(1)); if let Ok(xnet_payload) = self.remote_env.generate_xnet_payload( self.local_env.get_subnet_id(), @@ -622,7 +566,60 @@ impl Fixture { } else { self.local_env.tick(); } - self.local_env.advance_time(std::time::Duration::from_secs(1)); + self.local_env + .advance_time(std::time::Duration::from_secs(1)); + } + + /// Ticks until all calls have concluded; i.e. there are no more open call contexts. + /// + /// Returns `Err(_)` if + /// - `perform_checks()` fails. + /// - any call fails to conclude after `max_ticks` ticks. + /// - there is still memory reserved after all calls have concluded. + pub fn tick_to_conclusion( + &self, + max_ticks: usize, + perform_checks: F, + ) -> Result<(), (String, DebugInfo)> + where + F: Fn(&Self) -> Result<(), (String, DebugInfo)>, + { + // Keep ticking until all calls are answered. + for _ in 0..max_ticks { + self.tick(); + + perform_checks(&self)?; + + // Check for open call contexts. + if self.open_call_contexts_count().values().sum::() == 0 { + // Check the records agree on 'no pending calls'. + if self + .canisters() + .into_iter() + .map(|canister| extract_metrics(&self.force_query_records(canister))) + .any(|metrics| metrics.pending_calls != 0) + { + return self.failed_with_reason( + "no open call contexts but found pending calls in the records", + ); + } + + // One extra tick to make sure everything is gc'ed. + self.tick(); + + // After the fact, all memory is freed and back to 0. + return self.expect_guaranteed_response_message_memory_taken_at_most( + "Message memory used despite no open call contexts", + 0, + 0, + ); + } + } + + self.failed_with_reason(format!( + "failed to conclude calls after {} ticks", + max_ticks + )) } /// Migrates `canister` between `local_env` and `remote_env` (either direction). diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index e36fda86353..22c978fc0ec 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -5,18 +5,36 @@ use ic_types::messages::MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::ops::RangeInclusive; +use std::time::Duration; /// A full config for generating random calls and replies. Ranges are stored as individual u32 /// because ranges don't implement `CandidType`. #[derive(Serialize, Deserialize, Clone, Debug, CandidType, Hash)] pub struct Config { + /// A list of canister IDs, i.e. receivers for calls made by this canister. pub receivers: Vec, - pub call_bytes_min: u32, - pub call_bytes_max: u32, - pub reply_bytes_min: u32, - pub reply_bytes_max: u32, - pub instructions_count_min: u32, - pub instructions_count_max: u32, + /// `(min, max)` for the payload size in bytes included in a call. + pub call_bytes: (u32, u32), + /// `(min, max)` for the payload size in bytes included in a reply. + pub reply_bytes: (u32, u32), + /// `(min, max)` for the simulated number of instructions to generate a reply. + pub instructions_count: (u32, u32), + /// `(min, max)` for the timeout in seconds used for best-effort calls. + pub timeout_secs: (u32, u32), + /// The maximum number of calls attempted per heartbeat. + pub calls_per_heartbeat: u32, + /// The weight for making a reply used in a binominal distribution together with + /// `downstream_call_weight`. + pub reply_weight: u32, + /// The weight for making a downstream call used in a binomial distribution together with + /// `downstream_call_weight`. + pub downstream_call_weight: u32, + /// The weight for making a best-effort call used in a binomial distribution together with + /// `guaranteed_response_weight`. + pub best_effort_weight: u32, + /// The weight for making a guaranteed response call used in a binomial distribution together + /// with `best_effort_weight`. + pub guaranteed_response_weight: u32, } impl Default for Config { @@ -24,12 +42,15 @@ impl Default for Config { fn default() -> Self { Self { receivers: vec![], - call_bytes_min: 0, - call_bytes_max: MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32, - reply_bytes_min: 0, - reply_bytes_max: MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32, - instructions_count_min: 0, - instructions_count_max: 0, + call_bytes: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), + reply_bytes: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), + instructions_count: (0, 0), + timeout_secs: (10, 100), + calls_per_heartbeat: 3, + reply_weight: 1, + downstream_call_weight: 0, + best_effort_weight: 1, + guaranteed_response_weight: 0, } } } @@ -50,6 +71,12 @@ impl Config { call_bytes: RangeInclusive, reply_bytes: RangeInclusive, instructions_count: RangeInclusive, + timeout_secs: RangeInclusive, + calls_per_heartbeat: u32, + reply_weight: u32, + downstream_call_weight: u32, + best_effort_weight: u32, + guaranteed_response_weight: u32, ) -> Result { // Sanity checks. After passing these, the canister should run as intended. if call_bytes.is_empty() { @@ -67,15 +94,27 @@ impl Config { if instructions_count.is_empty() { return Err("empty instructions_count range".to_string()); } + if timeout_secs.is_empty() { + return Err("empty timeout range".to_string()); + } + if reply_weight == 0 && downstream_call_weight == 0 { + return Err("bad downstream call weights, both 0".to_string()); + } + if best_effort_weight == 0 && guaranteed_response_weight == 0 { + return Err("bad call type weights, both 0".to_string()); + } Ok(Self { receivers, - call_bytes_min: *call_bytes.start(), - call_bytes_max: *call_bytes.end(), - reply_bytes_min: *reply_bytes.start(), - reply_bytes_max: *reply_bytes.end(), - instructions_count_min: *instructions_count.start(), - instructions_count_max: *instructions_count.end(), + call_bytes: (*call_bytes.start(), *call_bytes.end()), + reply_bytes: (*reply_bytes.start(), *reply_bytes.end()), + instructions_count: (*instructions_count.start(), *instructions_count.end()), + timeout_secs: (*timeout_secs.start(), *timeout_secs.end()), + calls_per_heartbeat, + reply_weight, + downstream_call_weight, + best_effort_weight, + guaranteed_response_weight, }) } } @@ -102,38 +141,54 @@ pub struct Record { pub call_depth: u32, /// The number of bytes included in the payload. pub sent_bytes: u32, - /// The kind of reply received, i.e. a payload or a reject response. - pub reply: Option, + /// The timeout in seconds set for a best-effort call; `None` for a guaranteed response call. + pub timeout_secs: Option, + /// The kind of reply received, i.e. a payload or a reject response; and the duration after + /// which the reply was received (from when the call was made). + pub duration_and_reply: Option<(Duration, Reply)>, } /// Human readable printer. impl std::fmt::Debug for Record { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - match &self.caller { - None => write!( - f, - "{} ({:x}) | ", - &self.receiver.to_string()[..5], - self.call_tree_id, - ), - Some(caller) => write!( + write!( + f, + "{} ({:x}) ", + &self.receiver.to_string()[..5], + self.call_tree_id + )?; + + // A timeout indicates a best-effort message. + if let Some(timeout_secs) = self.timeout_secs { + write!(f, "to[s]: {} ", timeout_secs)?; + } + + // A caller indicates a downstream call. + if let Some(caller) = self.caller { + write!( f, - "{} ({:x}) (caller {} @ depth {}) | ", - &self.receiver.to_string()[..5], - self.call_tree_id, + "(caller {} @ depth {}) ", &caller.to_string()[..5], - self.call_depth, - ), - }?; + self.call_depth + )?; + } - write!(f, "sending {} bytes | ", self.sent_bytes)?; + write!(f, "| sending {} bytes | ", self.sent_bytes)?; - match &self.reply { + match &self.duration_and_reply { None => write!(f, "..."), - Some(Reply::Bytes(bytes)) => write!(f, "received {} bytes", bytes), - Some(Reply::Reject(error_code, error_msg)) => write!( + Some((call_duration, Reply::Bytes(bytes))) => { + write!( + f, + "duration[s]: {}, received {} bytes", + call_duration.as_secs(), + bytes + ) + } + Some((call_duration, Reply::Reject(error_code, error_msg))) => write!( f, - "reject({}): {error_msg}", + "duration[s]: {}, reject({}): {error_msg}", + call_duration.as_secs(), RejectCode::try_from(*error_code as u64).unwrap(), ), } @@ -164,12 +219,12 @@ pub fn extract_metrics(records: &BTreeMap) -> Metrics { metrics.downstream_calls_attempted += 1; } - match &record.reply { - Some(Reply::Bytes(received_bytes)) => { + match &record.duration_and_reply { + Some((_, Reply::Bytes(received_bytes))) => { metrics.calls_replied += 1; metrics.received_bytes += received_bytes; } - Some(Reply::Reject(_, _)) => { + Some((_, Reply::Reject(..))) => { metrics.calls_rejected += 1; metrics.rejected_bytes += record.sent_bytes; } diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index f94e6005478..3bf811cb558 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -1,8 +1,9 @@ -use candid::{CandidType, Encode}; +use candid::CandidType; +use candid::Encode; use futures::future::select_all; use ic_base_types::{CanisterId, PrincipalId}; -use ic_cdk::{api, caller, id, setup}; use ic_cdk::api::call::{Call, ConfigurableCall, SendableCall}; +use ic_cdk::{api, caller, id, setup}; use ic_cdk_macros::{heartbeat, init, query, update}; use rand::{ distributions::{Distribution, WeightedIndex}, @@ -16,29 +17,27 @@ use std::cell::{Cell, RefCell}; use std::collections::BTreeMap; use std::future::Future; use std::hash::{DefaultHasher, Hasher}; -use std::ops::RangeInclusive; +use std::time::Duration; thread_local! { - /// Random number generator used for determining payload sizes et.al. - static RNG: RefCell = RefCell::new(StdRng::seed_from_u64(13)); - /// Weight for making a reply used in a weighted binomial distribution. - static REPLY_WEIGHT: Cell = const { Cell::new(1) }; - /// Weight for making a downstream call used in a weighted binomial distribution. - static CALL_WEIGHT: Cell = const { Cell::new(0) }; /// A configuration holding parameters determining the canisters behavior, such as the range /// of payload bytes it should send. static CONFIG: RefCell = RefCell::default(); - /// The maximum number of calls each heartbeat will attempt to make. - static MAX_CALLS_PER_HEARTBEAT: Cell = Cell::default(); + /// Random number generator used for determining payload sizes et.al. + static RNG: RefCell = RefCell::new(StdRng::seed_from_u64(13)); /// A hasher used to generate unique call tree IDs. static HASHER: RefCell = RefCell::default(); /// An index for each attempted call; starts at 0 and then increments with each call. static CALL_INDEX: Cell = Cell::default(); - /// A collection of records; one record for each call. Keeps track of how each call went, + /// A collection of timestamps and records; one record for each call. Keeps track of how each call went, /// whether it was rejected or not and how many bytes were received. - static RECORDS: RefCell> = RefCell::default(); + static RECORDS: RefCell> = RefCell::default(); /// A counter for synchronous rejections. static SYNCHRONOUS_REJECTIONS_COUNT: Cell = Cell::default(); + /// A Binomial distribution used to determine whether to make a downstream call or not. + static DOWNSTREAM_CALL_DISTRIBUTION: RefCell> = RefCell::new(WeightedIndex::::new([0, 1]).unwrap()); + /// A Binomial distribution used to determine whether to make a best-effort call or not. + static BEST_EFFORT_CALL_DISTRIBUTION: RefCell> = RefCell::new(WeightedIndex::::new([1, 0]).unwrap()); } /// The intercanister message sent to `handle_call()` by the heartbeat of this canister @@ -56,11 +55,11 @@ struct Message { impl Message { /// Creates a new `Message` of size `bytes_count`; may slightly exceed the target for /// very small numbers. - fn new(call_tree_id: u32, call_depth: u32, bytes_count: usize) -> Self { + fn new(call_tree_id: u32, call_depth: u32, bytes_count: u32) -> Self { Self { call_tree_id, call_depth, - padding: vec![0_u8; bytes_count.saturating_sub(std::mem::size_of::())], + padding: vec![0_u8; (bytes_count as usize).saturating_sub(std::mem::size_of::())], } } @@ -70,18 +69,6 @@ impl Message { } } -/// Returns a random receiver if any. -fn choose_receiver() -> Option { - CONFIG.with_borrow(|config| { - RNG.with_borrow_mut(|rng| config.receivers.as_slice().choose(rng).cloned()) - }) -} - -/// Generates a random `u32` contained in `range`. -fn gen_range(range: RangeInclusive) -> u32 { - RNG.with_borrow_mut(|rng| rng.gen_range(range)) -} - /// Returns the next call index. fn next_call_index() -> u32 { CALL_INDEX.replace(CALL_INDEX.get() + 1) @@ -99,47 +86,58 @@ fn next_call_tree_id() -> u32 { } /// Returns `true` if `error_msg` corresponds to a synchronous rejection. -fn synchronous_rejection(error_msg: &str) -> bool { +fn is_synchronous_rejection(error_msg: &str) -> bool { error_msg.contains("Couldn't send message") } -/// Returns the call bytes range as defined in `CONFIG`. -fn call_bytes_range() -> RangeInclusive { - CONFIG.with_borrow(|config| config.call_bytes_min..=config.call_bytes_max) +/// Generates a random `u32` by sampling `min..=max`. +fn sample((min, max): (u32, u32)) -> u32 { + RNG.with_borrow_mut(|rng| rng.gen_range(min..=max)) } -/// Returns the reply bytes range as defined in `CONFIG`. -fn reply_bytes_range() -> RangeInclusive { - CONFIG.with_borrow(|config| config.reply_bytes_min..=config.reply_bytes_max) +/// Generates a random payload size for a call. +fn call_bytes() -> u32 { + CONFIG.with_borrow(|config| sample(config.call_bytes)) } -/// Returns the instructions count range as defined in `CONFIG`. -fn instructions_count_range() -> RangeInclusive { - CONFIG.with_borrow(|config| config.instructions_count_min..=config.instructions_count_max) +/// Generates a random payload size for a reply. +fn reply_bytes() -> u32 { + CONFIG.with_borrow(|config| sample(config.reply_bytes)) } -/// Sets the test config; returns the current config. -#[update] -fn set_config(config: Config) -> Config { - CONFIG.replace(config) +/// Generates a random number of simulated instructions for generating a reply. +fn instructions_count() -> u32 { + CONFIG.with_borrow(|config| sample(config.instructions_count)) } -/// Sets the requests per round to be sent each heart beat; returns the current value. -#[update] -fn set_max_calls_per_heartbeat(max_calls_per_heartbeat: u32) -> u32 { - MAX_CALLS_PER_HEARTBEAT.replace(max_calls_per_heartbeat) +/// Generates a timeout in seconds for a best-effort call. +fn timeout_secs() -> u32 { + CONFIG.with_borrow(|config| sample(config.timeout_secs)) } -/// Sets the reply weight; returns the current weight. -#[update] -fn set_reply_weight(reply_weight: u32) -> u32 { - REPLY_WEIGHT.replace(reply_weight) +/// Picks a random receiver from `config.receivers` if any; otherwise return own canister ID. +fn receiver() -> CanisterId { + match CONFIG.with_borrow(|config| { + RNG.with_borrow_mut(|rng| config.receivers.as_slice().choose(rng).cloned()) + }) { + Some(receiver) => receiver, + None => CanisterId::try_from(id().as_slice()).unwrap(), + } } -/// Sets the call weight; returns the current weight. +/// Sets the test config; returns the current config. #[update] -fn set_call_weight(call_weight: u32) -> u32 { - CALL_WEIGHT.replace(call_weight) +fn set_config(config: Config) -> Config { + // Update weighted distributions. + DOWNSTREAM_CALL_DISTRIBUTION.replace( + WeightedIndex::::new([config.reply_weight, config.downstream_call_weight]).unwrap(), + ); + BEST_EFFORT_CALL_DISTRIBUTION.replace( + WeightedIndex::::new([config.best_effort_weight, config.guaranteed_response_weight]) + .unwrap(), + ); + + CONFIG.replace(config) } /// Seeds `RNG`. @@ -148,10 +146,27 @@ fn seed_rng(seed: u64) { RNG.with_borrow_mut(|rng| *rng = StdRng::seed_from_u64(seed)); } +/// Sets `CONFIG` such that the canister stops making calls altogether; returns the modified +/// config. +#[update] +fn stop_chatter() -> Config { + set_config(Config { + calls_per_heartbeat: 0, + reply_weight: 1, + downstream_call_weight: 0, + ..CONFIG.take() + }) +} + /// Returns the canister records. #[query] fn records() -> BTreeMap { - RECORDS.with_borrow(|records| records.clone()) + RECORDS.with_borrow(|records| { + records + .iter() + .map(|(index, (_, record))| (*index, record.clone())) + .collect() + }) } /// Returns the number of synchronous rejections. @@ -160,45 +175,63 @@ fn synchronous_rejections_count() -> u32 { SYNCHRONOUS_REJECTIONS_COUNT.get() } +/// Determines whether to make a downstream call or reply. +fn should_make_downstream_call() -> bool { + RNG.with_borrow_mut(|rng| { + DOWNSTREAM_CALL_DISTRIBUTION.with_borrow(|distr| distr.sample(rng)) == 0 + }) +} + +/// Determines whether to make a downstream call or reply. +fn make_best_effort_call() -> bool { + RNG.with_borrow_mut(|rng| { + BEST_EFFORT_CALL_DISTRIBUTION.with_borrow(|distr| distr.sample(rng)) == 0 + }) +} + /// Generates a future for a randomized call that can be awaited; inserts a new record at `index` /// that must updated (or removed) after awaiting the call. For each call, the call index is /// incremented by 1, such that successive calls have adjacent indices. fn setup_call( - receiver: CanisterId, call_tree_id: u32, call_depth: u32, ) -> (impl Future>>, u32) { - let msg = Message::new( - call_tree_id, - call_depth, - gen_range(call_bytes_range()) as usize, - ); + let msg = Message::new(call_tree_id, call_depth, call_bytes()); + let receiver = receiver(); + let caller = + (call_depth > 0).then_some(CanisterId::unchecked_from_principal(PrincipalId(caller()))); + let timeout_secs = make_best_effort_call().then_some(timeout_secs()); + + let call = + Call::new(receiver.into(), "handle_call").with_raw_args(candid::Encode!(&msg).unwrap()); + let call = match timeout_secs { + Some(timeout_secs) => call.change_timeout(timeout_secs), + None => call.with_guaranteed_response(), + }; - // Inserts a new call record at the next `index`. + // Once the call was successfully generated, insert a call timestamp and record at `index`. let index = next_call_index(); RECORDS.with_borrow_mut(|records| { assert!(records .insert( index, - Record { - receiver, - caller: (call_depth > 0) - .then_some(CanisterId::unchecked_from_principal(PrincipalId(caller()))), - call_tree_id, - call_depth, - sent_bytes: msg.count_bytes() as u32, - reply: None, - }, + ( + api::time(), + Record { + receiver, + caller, + call_tree_id, + call_depth, + sent_bytes: msg.count_bytes() as u32, + timeout_secs, + duration_and_reply: None, + }, + ) ) .is_none()); }); - let future = Call::new(receiver.into(), "handle_call") - .with_raw_args(Encode!(&msg).unwrap()) - .change_timeout(100) - .call_raw(); - - (future, index) + (call.call_raw(), index) } /// Updates the record at `index` using the `response` to the corresponding call. @@ -211,19 +244,32 @@ fn update_record(response: &api::call::CallResult>, index: u32) { // Updates the `Reply` at `index` in `RECORDS`. let set_reply_in_call_record = move |reply: Reply| { RECORDS.with_borrow_mut(|records| { - let record = records.get_mut(&index).unwrap(); - assert!(record.reply.is_none(), "duplicate reply received"); - record.reply = Some(reply); + let (call_timestamp, record) = records.get_mut(&index).unwrap(); + assert!( + record.duration_and_reply.is_none(), + "duplicate reply received" + ); + let reply_timestamp = api::time(); + assert!(reply_timestamp >= *call_timestamp, "retrograde blocktime"); + + let call_duration = Duration::from_nanos(reply_timestamp - *call_timestamp); + record.duration_and_reply = Some((call_duration, reply)); }); }; match response { - Err((_, msg)) if synchronous_rejection(msg) => { + Err((_, msg)) if is_synchronous_rejection(msg) => { // Remove the record for synchronous rejections. SYNCHRONOUS_REJECTIONS_COUNT.set(SYNCHRONOUS_REJECTIONS_COUNT.get() + 1); RECORDS.with_borrow_mut(|records| { - assert!(records.remove(&index).unwrap().reply.is_none()) + assert!(records + .remove(&index) + .unwrap() + .1 + .duration_and_reply + .is_none()) }); } + Err((reject_code, msg)) => { set_reply_in_call_record(Reply::Reject(*reject_code as u32, msg.to_string())); } @@ -238,11 +284,8 @@ fn update_record(response: &api::call::CallResult>, index: u32) { #[heartbeat] async fn heartbeat() { let (mut futures, mut record_indices) = (Vec::new(), Vec::new()); - for _ in 0..MAX_CALLS_PER_HEARTBEAT.get() { - let Some(receiver) = choose_receiver() else { - return; - }; - let (future, index) = setup_call(receiver, next_call_tree_id(), 0); + for _ in 0..CONFIG.with_borrow(|config| config.calls_per_heartbeat) { + let (future, index) = setup_call(next_call_tree_id(), 0); futures.push(future); record_indices.push(index); } @@ -266,24 +309,12 @@ async fn heartbeat() { /// - if it tells us not to do so but the attempted downstream call fails for any reason. #[update] async fn handle_call(msg: Message) -> Vec { - // Samples a weighted binomial distribution to decide whether to make a downstream call (true) - // or reply (false). Defaults to `false` for bad weights (e.g. both 0). - fn should_make_downstream_call() -> bool { - RNG.with_borrow_mut(|rng| { - WeightedIndex::new([CALL_WEIGHT.get(), REPLY_WEIGHT.get()]) - .map_or(false, |dist| dist.sample(rng) == 0) - }) - } - // Make downstream calls until // - sampling the distribution tells us to stop. // - setting up a call fails. // - a downstream call is rejected for any reason. while should_make_downstream_call() { - let Some(receiver) = choose_receiver() else { - break; - }; - let (future, record_index) = setup_call(receiver, msg.call_tree_id, msg.call_depth + 1); + let (future, record_index) = setup_call(msg.call_tree_id, msg.call_depth + 1); let result = future.await; update_record(&result, record_index); @@ -295,11 +326,10 @@ async fn handle_call(msg: Message) -> Vec { } } - let payload_bytes = gen_range(reply_bytes_range()); - let instructions_count = gen_range(instructions_count_range()); + let payload_bytes = reply_bytes(); // Do some thinking. - let counts = api::performance_counter(0) + instructions_count as u64; + let counts = api::performance_counter(0) + instructions_count() as u64; while counts > api::performance_counter(0) {} vec![0_u8; payload_bytes as usize] From 670deb00c571990702b3fdef970ae10043f50212 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 8 Jan 2025 21:21:30 +0000 Subject: [PATCH 05/34] . --- Cargo.Bazel.json.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 83421800d0d..903694cfc40 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "426018d109eca1845ca4b3d2e5755c16846719bf3f009e6969cf716a8955195a", + "checksum": "bcfd580ac2798f1729c8a7a4e6150a3dc800c834e1596ee0e9bf495070ff83e6", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -30639,7 +30639,7 @@ "target": "ic0" }, { - "id": "serde 1.0.214", + "id": "serde 1.0.217", "target": "serde" }, { @@ -30982,7 +30982,7 @@ "target": "quote" }, { - "id": "serde 1.0.214", + "id": "serde 1.0.217", "target": "serde" }, { From 5dac2a7b6bd97ac1a414ed9896d028eb07dfd9c0 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Wed, 8 Jan 2025 21:23:10 +0000 Subject: [PATCH 06/34] Automatically updated Cargo*.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 234011c4ae8..385ece8f6ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6408,7 +6408,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream 0.2.2", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] From ae1199f1c60e4c036bf013a08f0928f5efafc09a Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 9 Jan 2025 15:33:51 +0000 Subject: [PATCH 07/34] . --- Cargo.lock | 2 +- rs/messaging/tests/memory_tests.rs | 116 ++++++++++++++---- .../src/canister_state/system_state.rs | 3 +- rs/state_machine_tests/src/lib.rs | 9 +- 4 files changed, 100 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 234011c4ae8..385ece8f6ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6408,7 +6408,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream 0.2.2", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 4ace5683cb9..368b086e578 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -29,27 +29,33 @@ const MAX_PAYLOAD_BYTES: u32 = MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32; /* #[test] fn playground() { - let seeds = vec![0_u64; 3]; - let max_payload_bytes = MAX_PAYLOAD_BYTES; - let calls_per_round = 1; - let reply_weight = 1; - let call_weight = 10; - - if let Err((msg, mut dbg)) = check_guaranteed_response_message_memory_limits_are_respected_impl( - seeds.as_slice(), - max_payload_bytes, - calls_per_round as u32, - reply_weight as u32, - call_weight as u32, + let seed = vec![17374506990209185531; 3]; + let config = CanisterConfig { + receivers: vec![], + call_bytes: (0, 131), + reply_bytes: (0, 0), + instructions_count: (0, 0), + timeout_secs: (0, 37), + calls_per_heartbeat: 2, + reply_weight: 3, + downstream_call_weight: 0, + best_effort_weight: 0, + guaranteed_response_weight: 3, + }; + + if let Err((error_msg, mut nfo)) = check_guaranteed_response_message_memory_limits_are_respected_impl( + 10, // chatter_phase_round_count + 300, // shutdown_phase_max_rounds + &seed, + config, ) { - let r = dbg.records.pop_first().unwrap().1; - assert!(false, "{:?}\n\n{}\n\n{:#?}", msg, r.len(), r); - } else { - //unreachable!(); + assert!(false, "{}\n\n{:#?}", error_msg, nfo.records.pop_first().unwrap()); + } + else { + unreachable!(); } } */ - prop_compose! { /// Generates an arbitrary pair of weights such that w1 + w2 = total > 0. fn arb_weights(total: u32)( @@ -88,7 +94,7 @@ prop_compose! { } proptest! { - #![proptest_config(ProptestConfig::with_cases(1))] + #![proptest_config(ProptestConfig::with_cases(5))] #[test] fn check_guaranteed_response_message_memory_limits_are_respected( seeds in proptest::collection::vec(any::().no_shrink(), 3), @@ -186,11 +192,13 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( LOCAL_MESSAGE_MEMORY_CAPACITY, REMOTE_MESSAGE_MEMORY_CAPACITY, ) - }) + })?; + + fixture.failed_with_reason("TEST") } proptest! { - #![proptest_config(ProptestConfig::with_cases(1))] + #![proptest_config(ProptestConfig::with_cases(5))] #[test] fn check_calls_conclude_with_migrating_canister( seed in any::().no_shrink(), @@ -204,7 +212,57 @@ proptest! { ).is_ok()); } } - +/* +#[test] +fn playground2() { + let seed = 17374506990209185531; + let config = CanisterConfig { + receivers: vec![], + call_bytes: (0, 131), + reply_bytes: (0, 0), + instructions_count: (0, 0), + timeout_secs: (0, 37), + calls_per_heartbeat: 2, + reply_weight: 3, + downstream_call_weight: 0, + best_effort_weight: 0, + guaranteed_response_weight: 3, + }; + + if let Err((error_msg, mut nfo)) = check_calls_conclude_with_migrating_canister_impl( + 10, // chatter_phase_round_count + 300, // shutdown_phase_max_rounds + seed, + config, + ) { + assert!( + false, + "{}\n\n{:#?}\n\n{:#?}", + error_msg, + nfo.records.pop_first().unwrap(), + nfo + //.local_env + .remote_env + .get_latest_state() + .canister_states + .iter() + .filter_map(|(canister_id, canister_state)| Some( + (canister_id, (canister_state.system_state.get_status(), canister_state.system_state.queues())) + )) + + //.filter_map(|(canister_id, canister_state)| canister_state.has_input().then_some( + // (canister_id, canister_state.system_state.queues()) + //)) + //.map(|(canister_id, canister_state)| (canister_id, canister_state.has_input())) + .collect::>(), + ); +// assert!(false, "{}\n\n{:#?}", error_msg, nfo.remote_env.get_latest_state().canister_states); + } + else { + unreachable!(); + } +} +*/ /// Runs a state machine test with two subnets, a local subnet with 2 canisters installed and a /// remote subnet with 5 canisters installed. All canisters, except one local canister referred to /// as `migrating_canister`, are stopped. @@ -236,8 +294,6 @@ fn check_calls_conclude_with_migrating_canister_impl( }); config.receivers = fixture.canisters(); -// config.best_effort_weight = 0; -// config.guaranteed_response_weight = 1; let migrating_canister = fixture.local_canisters.first().unwrap().clone(); @@ -248,6 +304,9 @@ fn check_calls_conclude_with_migrating_canister_impl( // Stop all canisters except `migrating_canister`. for canister in fixture.canisters() { if canister != migrating_canister { + // Make sure the canister doesn't make calls when it is + // put into running state to read its records. + fixture.stop_chatter(canister); fixture.stop_canister_non_blocking(canister); } } @@ -475,6 +534,9 @@ impl Fixture { /// Force queries the records from `canister` by first attempting to query them; if it fails, start /// the canister and try querying them again. /// + /// Note: If the canister is configured to make calls, starting it will trigger calls before + /// the records are returned. + /// /// Panics if `canister` is not installed in `Self`. pub fn force_query_records(&self, canister: CanisterId) -> BTreeMap { match self.query_records(canister) { @@ -675,8 +737,8 @@ impl Fixture { .into_iter() .map(|canister| (canister, self.force_query_records(canister))) .collect(), - latest_local_state: self.local_env.get_latest_state(), - latest_remote_state: self.remote_env.get_latest_state(), + local_env: self.local_env.clone(), + remote_env: self.remote_env.clone(), }, )) } @@ -686,8 +748,8 @@ impl Fixture { #[allow(dead_code)] struct DebugInfo { pub records: BTreeMap>, - pub latest_local_state: Arc, - pub latest_remote_state: Arc, + pub local_env: Arc, + pub remote_env: Arc, } /// Installs a 'random-traffic-test-canister' in `env`. diff --git a/rs/replicated_state/src/canister_state/system_state.rs b/rs/replicated_state/src/canister_state/system_state.rs index c00f0ddafbc..2f311822c1b 100644 --- a/rs/replicated_state/src/canister_state/system_state.rs +++ b/rs/replicated_state/src/canister_state/system_state.rs @@ -2144,7 +2144,8 @@ impl SystemState { /// following a canister migration, based on the updated set of local canisters. /// /// See [`CanisterQueues::split_input_schedules`] for further details. - pub(crate) fn split_input_schedules( + pub fn split_input_schedules( + //pub(crate) fn split_input_schedules( &mut self, own_canister_id: &CanisterId, local_canisters: &BTreeMap, diff --git a/rs/state_machine_tests/src/lib.rs b/rs/state_machine_tests/src/lib.rs index 6578db7c5c7..a5c71e3be3e 100644 --- a/rs/state_machine_tests/src/lib.rs +++ b/rs/state_machine_tests/src/lib.rs @@ -2632,7 +2632,7 @@ impl StateMachine { } } - let canister_state = ic_state_manager::checkpoint::load_canister_state( + let mut canister_state = ic_state_manager::checkpoint::load_canister_state( &tip_canister_layout, &canister_id, ic_types::Height::new(0), @@ -2649,7 +2649,14 @@ impl StateMachine { .0; let (h, mut state) = self.state_manager.take_tip(); + + // Repartition input schedules; Required step for migrating canisters. + canister_state + .system_state + .split_input_schedules(&canister_id, &state.canister_states); + state.put_canister_state(canister_state); + self.state_manager.commit_and_certify( state, h.increment(), From 541a6d5efb3588327d620cbb2398b9ec583a08a5 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 9 Jan 2025 17:03:35 +0000 Subject: [PATCH 08/34] . --- rs/replicated_state/src/canister_state/system_state.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rs/replicated_state/src/canister_state/system_state.rs b/rs/replicated_state/src/canister_state/system_state.rs index 2f311822c1b..1833b3cb99c 100644 --- a/rs/replicated_state/src/canister_state/system_state.rs +++ b/rs/replicated_state/src/canister_state/system_state.rs @@ -2145,7 +2145,6 @@ impl SystemState { /// /// See [`CanisterQueues::split_input_schedules`] for further details. pub fn split_input_schedules( - //pub(crate) fn split_input_schedules( &mut self, own_canister_id: &CanisterId, local_canisters: &BTreeMap, From 5eb450258b2b54bb370c807cdc6b07321c033937 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Fri, 10 Jan 2025 15:01:16 +0000 Subject: [PATCH 09/34] . --- rs/messaging/tests/memory_tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 91352f7baf0..ac320801a8b 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -199,7 +199,7 @@ proptest! { #[test] fn check_calls_conclude_with_migrating_canister( seed in any::().no_shrink(), - config in arb_canister_config(1 * KB as u32, 10), + config in arb_canister_config(KB as u32, 10), ) { prop_assert!(check_calls_conclude_with_migrating_canister_impl( 10, // chatter_phase_round_count @@ -292,7 +292,7 @@ fn check_calls_conclude_with_migrating_canister_impl( config.receivers = fixture.canisters(); - let migrating_canister = fixture.local_canisters.first().unwrap().clone(); + let migrating_canister = *fixture.local_canisters.first().unwrap(); // Send config to `migrating_canister` and seed its rng. fixture.set_config(migrating_canister, config); @@ -650,7 +650,7 @@ impl Fixture { for _ in 0..max_ticks { self.tick(); - perform_checks(&self)?; + perform_checks(self)?; // Check for open call contexts. if self.open_call_contexts_count().values().sum::() == 0 { From 3dbafc25bc1c987d3ca216abf295f3b607f705da Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Mon, 13 Jan 2025 09:59:03 +0000 Subject: [PATCH 10/34] . --- rs/messaging/tests/memory_tests.rs | 128 ++++-------------- .../random_traffic_test/src/lib.rs | 5 +- .../random_traffic_test/src/main.rs | 17 +-- 3 files changed, 38 insertions(+), 112 deletions(-) diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index ac320801a8b..46fb3672557 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -7,6 +7,7 @@ use ic_config::{ }; use ic_registry_routing_table::{routing_table_insert_subnet, RoutingTable}; use ic_registry_subnet_type::SubnetType; +use ic_replicated_state::ReplicatedState; use ic_state_machine_tests::{StateMachine, StateMachineBuilder, StateMachineConfig, UserError}; use ic_test_utilities_types::ids::{SUBNET_0, SUBNET_1}; use ic_types::{ @@ -25,38 +26,9 @@ const KB: u64 = 1024; const MB: u64 = KB * KB; const MAX_PAYLOAD_BYTES: u32 = MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32; -/* -#[test] -fn playground() { - let seed = vec![17374506990209185531; 3]; - let config = CanisterConfig { - receivers: vec![], - call_bytes: (0, 131), - reply_bytes: (0, 0), - instructions_count: (0, 0), - timeout_secs: (0, 37), - calls_per_heartbeat: 2, - reply_weight: 3, - downstream_call_weight: 0, - best_effort_weight: 0, - guaranteed_response_weight: 3, - }; - - if let Err((error_msg, mut nfo)) = check_guaranteed_response_message_memory_limits_are_respected_impl( - 10, // chatter_phase_round_count - 300, // shutdown_phase_max_rounds - &seed, - config, - ) { - assert!(false, "{}\n\n{:#?}", error_msg, nfo.records.pop_first().unwrap()); - } - else { - unreachable!(); - } -} -*/ + prop_compose! { - /// Generates an arbitrary pair of weights such that w1 + w2 = total > 0. + /// Generates an arbitrary pair of weights such that w1 + w2 = total. fn arb_weights(total: u32)( w1 in 0..=total ) -> (u32, u32) @@ -73,8 +45,8 @@ prop_compose! { max_reply_bytes in 0..=max_payload_bytes, calls_per_heartbeat in 0..=max_calls_per_heartbeat, max_timeout_secs in 10..=100_u32, - (reply_weight, downstream_call_weight) in arb_weights(3), - (best_effort_weight, guaranteed_response_weight) in arb_weights(3), + (reply_weight, downstream_call_weight) in arb_weights(4), + (best_effort_weight, guaranteed_response_weight) in arb_weights(4), ) -> CanisterConfig { CanisterConfig::try_new( vec![], @@ -93,18 +65,20 @@ prop_compose! { } proptest! { - #![proptest_config(ProptestConfig::with_cases(5))] + #![proptest_config(ProptestConfig::with_cases(3))] #[test] fn check_guaranteed_response_message_memory_limits_are_respected( seeds in proptest::collection::vec(any::().no_shrink(), 3), config in arb_canister_config(MAX_PAYLOAD_BYTES, 5), ) { - prop_assert!(check_guaranteed_response_message_memory_limits_are_respected_impl( + if let Err((err_msg, nfo)) = check_guaranteed_response_message_memory_limits_are_respected_impl( 30, // chatter_phase_round_count 300, // shutdown_phase_max_rounds seeds.as_slice(), config, - ).is_ok()); + ) { + unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); + } } } @@ -201,72 +175,24 @@ proptest! { seed in any::().no_shrink(), config in arb_canister_config(KB as u32, 10), ) { - prop_assert!(check_calls_conclude_with_migrating_canister_impl( + if let Err((err_msg, nfo)) = check_calls_conclude_with_migrating_canister_impl( 10, // chatter_phase_round_count 300, // shutdown_phase_max_rounds seed, config, - ).is_ok()); - } -} -/* -#[test] -fn playground2() { - let seed = 17374506990209185531; - let config = CanisterConfig { - receivers: vec![], - call_bytes: (0, 131), - reply_bytes: (0, 0), - instructions_count: (0, 0), - timeout_secs: (0, 37), - calls_per_heartbeat: 2, - reply_weight: 3, - downstream_call_weight: 0, - best_effort_weight: 0, - guaranteed_response_weight: 3, - }; - - if let Err((error_msg, mut nfo)) = check_calls_conclude_with_migrating_canister_impl( - 10, // chatter_phase_round_count - 300, // shutdown_phase_max_rounds - seed, - config, - ) { - assert!( - false, - "{}\n\n{:#?}\n\n{:#?}", - error_msg, - nfo.records.pop_first().unwrap(), - nfo - //.local_env - .remote_env - .get_latest_state() - .canister_states - .iter() - .filter_map(|(canister_id, canister_state)| Some( - (canister_id, (canister_state.system_state.get_status(), canister_state.system_state.queues())) - )) - - //.filter_map(|(canister_id, canister_state)| canister_state.has_input().then_some( - // (canister_id, canister_state.system_state.queues()) - //)) - //.map(|(canister_id, canister_state)| (canister_id, canister_state.has_input())) - .collect::>(), - ); -// assert!(false, "{}\n\n{:#?}", error_msg, nfo.remote_env.get_latest_state().canister_states); - } - else { - unreachable!(); + ) { + unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); + } } } -*/ + /// Runs a state machine test with two subnets, a local subnet with 2 canisters installed and a /// remote subnet with 5 canisters installed. All canisters, except one local canister referred to /// as `migrating_canister`, are stopped. /// /// In the first phase a number of rounds are executed on both subnets, including XNet traffic with /// the `migrating_canister` making random calls to all installed canisters (since all calls are -/// rejected except those to self, downstream calls are disabled). +/// rejected except those to self). /// /// For the second phase, `migrating_canister` stops making calls and is then migrated to the /// remote subnet. Since all other canisters are stopped, there are bound to be a number of reject @@ -455,8 +381,8 @@ impl Fixture { unreachable!(); } - /// Helper function for setting canister state elements; returns the current element before - /// setting it. + /// Helper function for update calls to `canister`; returns the current `T` as it was before + /// this call. /// /// Panics if `canister` is not installed in `Self`. fn set_canister_state(&self, canister: CanisterId, method: &str, item: T) -> T @@ -545,7 +471,14 @@ impl Fixture { } } - /// Return the number of bytes taken by guaranteed response memory (`local_env`, `remote_env`). + /// Returns the latest state `canister` is located on. + /// + /// Panics if `canister` is not installed on either env. + pub fn get_latest_state(&self, canister: &CanisterId) -> Arc { + self.get_env(canister).get_latest_state() + } + + /// Returns the number of bytes taken by guaranteed response memory (`local_env`, `remote_env`). pub fn guaranteed_response_message_memory_taken(&self) -> (NumBytes, NumBytes) { ( self.local_env @@ -583,8 +516,7 @@ impl Fixture { .into_iter() .map(|canister| { let count = self - .get_env(&canister) - .get_latest_state() + .get_latest_state(&canister) .canister_states .get(&canister) .unwrap() @@ -734,8 +666,7 @@ impl Fixture { .into_iter() .map(|canister| (canister, self.force_query_records(canister))) .collect(), - local_env: self.local_env.clone(), - remote_env: self.remote_env.clone(), + fixture: self.clone(), }, )) } @@ -745,8 +676,7 @@ impl Fixture { #[allow(dead_code)] struct DebugInfo { pub records: BTreeMap>, - pub local_env: Arc, - pub remote_env: Arc, + pub fixture: Fixture, } /// Installs a 'random-traffic-test-canister' in `env`. diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index 22c978fc0ec..904e80819dc 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -38,7 +38,6 @@ pub struct Config { } impl Default for Config { - /// Default using the full range of payloads and no delayed responses. fn default() -> Self { Self { receivers: vec![], @@ -160,7 +159,7 @@ impl std::fmt::Debug for Record { // A timeout indicates a best-effort message. if let Some(timeout_secs) = self.timeout_secs { - write!(f, "to[s]: {} ", timeout_secs)?; + write!(f, "timeout[s]: {} ", timeout_secs)?; } // A caller indicates a downstream call. @@ -173,7 +172,7 @@ impl std::fmt::Debug for Record { )?; } - write!(f, "| sending {} bytes | ", self.sent_bytes)?; + write!(f, " sending {} bytes | ", self.sent_bytes)?; match &self.duration_and_reply { None => write!(f, "..."), diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index 3bf811cb558..4f505db32a1 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -29,8 +29,8 @@ thread_local! { static HASHER: RefCell = RefCell::default(); /// An index for each attempted call; starts at 0 and then increments with each call. static CALL_INDEX: Cell = Cell::default(); - /// A collection of timestamps and records; one record for each call. Keeps track of how each call went, - /// whether it was rejected or not and how many bytes were received. + /// A collection of timestamps and records; one record for each call. Keeps track of whether it was + /// rejected or not and how many bytes were sent and received. static RECORDS: RefCell> = RefCell::default(); /// A counter for synchronous rejections. static SYNCHRONOUS_REJECTIONS_COUNT: Cell = Cell::default(); @@ -182,7 +182,7 @@ fn should_make_downstream_call() -> bool { }) } -/// Determines whether to make a downstream call or reply. +/// Determines whether to make a best-effort call or a guaranteed response call. fn make_best_effort_call() -> bool { RNG.with_borrow_mut(|rng| { BEST_EFFORT_CALL_DISTRIBUTION.with_borrow(|distr| distr.sample(rng)) == 0 @@ -190,7 +190,7 @@ fn make_best_effort_call() -> bool { } /// Generates a future for a randomized call that can be awaited; inserts a new record at `index` -/// that must updated (or removed) after awaiting the call. For each call, the call index is +/// that must be updated (or removed) after awaiting the call. For each call, the call index is /// incremented by 1, such that successive calls have adjacent indices. fn setup_call( call_tree_id: u32, @@ -279,8 +279,8 @@ fn update_record(response: &api::call::CallResult>, index: u32) { } } -/// Generates `MAX_CALLS_PER_HEARTBEAT` calls as futures. The records are updated whenever a -/// reply comes in, i.e. not only after all of them have completed. +/// Generates `calls_per_heartbeat` call futures. They are awaited; whenever a call concludes +/// its record is updated (or removed in case of a synchronous rejection). #[heartbeat] async fn heartbeat() { let (mut futures, mut record_indices) = (Vec::new(), Vec::new()); @@ -309,10 +309,7 @@ async fn heartbeat() { /// - if it tells us not to do so but the attempted downstream call fails for any reason. #[update] async fn handle_call(msg: Message) -> Vec { - // Make downstream calls until - // - sampling the distribution tells us to stop. - // - setting up a call fails. - // - a downstream call is rejected for any reason. + // Make downstream calls as long as sampling the distribution tells us to do so. while should_make_downstream_call() { let (future, record_index) = setup_call(msg.call_tree_id, msg.call_depth + 1); From b77f3dbfd9bc136723f77b976e2dad93cd654af9 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Mon, 13 Jan 2025 10:21:55 +0000 Subject: [PATCH 11/34] . --- bazel/external_crates.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index e86e4a1b365..a3f15cdef21 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -598,7 +598,7 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable "ic-cdk-macros": crate.spec( version = "^0.9.0", ), - "ic-cdk-next": crate.spec( + "ic-cdk-next": crate.spec( package = "ic-cdk", git = "https://github.com/dfinity/cdk-rs.git", branch = "next", From 924190321136d9429283c5af2489434a46f12201 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Mon, 13 Jan 2025 10:38:56 +0000 Subject: [PATCH 12/34] . --- rs/messaging/tests/memory_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 46fb3672557..00b48354b0e 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -169,7 +169,7 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( } proptest! { - #![proptest_config(ProptestConfig::with_cases(5))] + #![proptest_config(ProptestConfig::with_cases(3))] #[test] fn check_calls_conclude_with_migrating_canister( seed in any::().no_shrink(), From bc133193a4a564f712ff83b4227913f1f7b5a5b6 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Mon, 13 Jan 2025 12:57:33 +0000 Subject: [PATCH 13/34] . --- rs/config/src/embedders.rs | 2 +- rs/messaging/tests/memory_tests.rs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/rs/config/src/embedders.rs b/rs/config/src/embedders.rs index 4e599ea4c5a..9c6ef4db9d7 100644 --- a/rs/config/src/embedders.rs +++ b/rs/config/src/embedders.rs @@ -121,7 +121,7 @@ impl FeatureFlags { write_barrier: FlagStatus::Disabled, wasm_native_stable_memory: FlagStatus::Enabled, wasm64: FlagStatus::Enabled, - best_effort_responses: FlagStatus::Enabled, + best_effort_responses: FlagStatus::Disabled, canister_backtrace: FlagStatus::Enabled, } } diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 00b48354b0e..4cddae7db58 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -2,7 +2,9 @@ use candid::{Decode, Encode}; use canister_test::Project; use ic_base_types::{CanisterId, NumBytes, SubnetId}; use ic_config::{ + embedders::{Config as EmbeddersConfig, FeatureFlags}, execution_environment::Config as HypervisorConfig, + flag_status::FlagStatus, subnet_config::{CyclesAccountManagerConfig, SchedulerConfig, SubnetConfig}, }; use ic_registry_routing_table::{routing_table_insert_subnet, RoutingTable}; @@ -289,6 +291,13 @@ impl FixtureConfig { }, HypervisorConfig { subnet_message_memory_capacity: subnet_message_memory_capacity.into(), + embedders_config: EmbeddersConfig { + feature_flags: FeatureFlags { + best_effort_responses: FlagStatus::Enabled, + ..FeatureFlags::default() + }, + ..EmbeddersConfig::default() + }, ..HypervisorConfig::default() }, ) From 3507b14d5fe526f5398c2a8621631ad8ad3d2686 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Mon, 13 Jan 2025 16:55:15 +0000 Subject: [PATCH 14/34] . --- rs/messaging/tests/memory_tests.rs | 97 ++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 4cddae7db58..89689d54f28 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -13,6 +13,7 @@ use ic_replicated_state::ReplicatedState; use ic_state_machine_tests::{StateMachine, StateMachineBuilder, StateMachineConfig, UserError}; use ic_test_utilities_types::ids::{SUBNET_0, SUBNET_1}; use ic_types::{ + ingress::{IngressState, IngressStatus}, messages::{MessageId, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64}, Cycles, }; @@ -248,6 +249,102 @@ fn check_calls_conclude_with_migrating_canister_impl( fixture.tick_to_conclusion(shutdown_phase_max_rounds, |_| Ok(())) } +proptest! { + #![proptest_config(ProptestConfig::with_cases(3))] + #[test] + fn check_canister_can_be_stopped_with_remote_subnet_stalling( + seeds in proptest::collection::vec(any::().no_shrink(), 2), + config in arb_canister_config(MAX_PAYLOAD_BYTES, 5), + ) { + if let Err((err_msg, nfo)) = check_canister_can_be_stopped_with_remote_subnet_stalling_impl( + 30, // chatter_phase_round_count + 300, // shutdown_phase_max_rounds + seeds.as_slice(), + config, + ) { + unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); + } + } +} + +/// Runs a state machine test with two subnets, a local subnet with one canister installed that +/// only makes best-effort calls and a remote subnet with one canister installed that makes random +/// calls of all kinds. +/// +/// In the first phase a number of rounds are executed on both subnet, including XNet traffic +/// between both canisters. +/// +/// For the second phase the local canister is put into `Stopping` state and the remote subnet +/// stalls, i.e. no more ticks are made on it. The local canister should reject any incoming calls +/// and since it made only best-effort calls, all pending calls should be rejected or timed out +/// eventually making the transition to `Stopped` state possible even with the remote subnet stalling. +/// +/// If the local canister fails to reach `Stopped` state, there is most likely a bug with timing +/// out best-effort messages. +fn check_canister_can_be_stopped_with_remote_subnet_stalling_impl( + chatter_phase_round_count: usize, + shutdown_phase_max_rounds: usize, + seeds: &[u64], + mut config: CanisterConfig, +) -> Result<(), (String, DebugInfo)> { + let fixture = Fixture::new(FixtureConfig { + local_canisters_count: 1, + remote_canisters_count: 1, + ..FixtureConfig::default() + }); + + config.receivers = fixture.canisters(); + + let local_canister = *fixture.local_canisters.first().unwrap(); + let remote_canister = *fixture.remote_canisters.first().unwrap(); + + fixture.seed_rng(local_canister, seeds[0]); + fixture.seed_rng(remote_canister, seeds[1]); + + // Set the local `config` adapted such that only best-effort calls are made. + fixture.set_config( + local_canister, + CanisterConfig { + best_effort_weight: 1, + guaranteed_response_weight: 0, + ..config.clone() + }, + ); + // Set the remote `config` as is. + fixture.set_config(remote_canister, config); + + // Make calls on both canisters. + for _ in 0..chatter_phase_round_count { + fixture.tick(); + } + // Stop chatter on the local canister. + fixture.stop_chatter(local_canister); + + // Put local canister into `Stopping` state. + let msg_id = fixture.stop_canister_non_blocking(local_canister); + + // Tick for up to `shutdown_phase_max_rounds` times on the local subnet only + // or until the local canister has stopped. + for _ in 0..shutdown_phase_max_rounds { + match fixture.local_env.ingress_status(&msg_id) { + IngressStatus::Known { + state: IngressState::Completed(_), + .. + } => return Ok(()), + _ => { + fixture.local_env.tick(); + fixture + .local_env + .advance_time(std::time::Duration::from_secs(1)); + } + } + } + + fixture.failed_with_reason(format!( + "failed to stop local canister after {shutdown_phase_max_rounds} ticks" + )) +} + #[derive(Debug)] struct FixtureConfig { local_canisters_count: u64, From 42c892d47f5e46cc87b3dcba71ff22e0b85b8464 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 15 Jan 2025 09:31:25 +0000 Subject: [PATCH 15/34] . --- Cargo.lock | 1 + rs/messaging/BUILD.bazel | 4 ++ rs/messaging/Cargo.toml | 1 + rs/messaging/tests/memory_tests.rs | 74 +++++++++++++----------------- 4 files changed, 38 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df1b4e4942c..bf3312390df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9655,6 +9655,7 @@ dependencies = [ "serde", "slog", "tempfile", + "test-strategy 0.3.1", "tracing", "xnet-test", ] diff --git a/rs/messaging/BUILD.bazel b/rs/messaging/BUILD.bazel index ea5c1ef8500..ccaba056491 100644 --- a/rs/messaging/BUILD.bazel +++ b/rs/messaging/BUILD.bazel @@ -118,4 +118,8 @@ rust_ic_test_suite( "@crate_index//:proptest", "@crate_index//:serde", ], + proc_macro_deps = [ + # Keep sorted. + "@crate_index//:test-strategy", + ], ) diff --git a/rs/messaging/Cargo.toml b/rs/messaging/Cargo.toml index 8d3b7bbc82b..5b141022eef 100644 --- a/rs/messaging/Cargo.toml +++ b/rs/messaging/Cargo.toml @@ -70,6 +70,7 @@ rand_chacha = { workspace = true } random-traffic-test = { path = "../rust_canisters/random_traffic_test" } serde = { workspace = true } tempfile = { workspace = true } +test-strategy = "0.3.1" xnet-test = { path = "../rust_canisters/xnet_test" } [features] diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 89689d54f28..2e523c1fe5c 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -67,21 +67,18 @@ prop_compose! { } } -proptest! { - #![proptest_config(ProptestConfig::with_cases(3))] - #[test] - fn check_guaranteed_response_message_memory_limits_are_respected( - seeds in proptest::collection::vec(any::().no_shrink(), 3), - config in arb_canister_config(MAX_PAYLOAD_BYTES, 5), +#[test_strategy::proptest(ProptestConfig::with_cases(3))] +fn check_guaranteed_response_message_memory_limits_are_respected( + #[strategy(proptest::collection::vec(any::().no_shrink(), 3))] seeds: Vec, + #[strategy(arb_canister_config(MAX_PAYLOAD_BYTES, 5))] config: CanisterConfig, +) { + if let Err((err_msg, nfo)) = check_guaranteed_response_message_memory_limits_are_respected_impl( + 30, // chatter_phase_round_count + 300, // shutdown_phase_max_rounds + seeds.as_slice(), + config, ) { - if let Err((err_msg, nfo)) = check_guaranteed_response_message_memory_limits_are_respected_impl( - 30, // chatter_phase_round_count - 300, // shutdown_phase_max_rounds - seeds.as_slice(), - config, - ) { - unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); - } + unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); } } @@ -171,21 +168,17 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( }) } -proptest! { - #![proptest_config(ProptestConfig::with_cases(3))] - #[test] - fn check_calls_conclude_with_migrating_canister( - seed in any::().no_shrink(), - config in arb_canister_config(KB as u32, 10), +#[test_strategy::proptest(ProptestConfig::with_cases(3))] +fn check_calls_conclude_with_migrating_canister( + #[strategy(any::().no_shrink())] seed: u64, + #[strategy(arb_canister_config(KB as u32, 10))] config: CanisterConfig, +) { + if let Err((err_msg, nfo)) = check_calls_conclude_with_migrating_canister_impl( + 10, // chatter_phase_round_count + 300, // shutdown_phase_max_rounds + seed, config, ) { - if let Err((err_msg, nfo)) = check_calls_conclude_with_migrating_canister_impl( - 10, // chatter_phase_round_count - 300, // shutdown_phase_max_rounds - seed, - config, - ) { - unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); - } + unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); } } @@ -249,21 +242,18 @@ fn check_calls_conclude_with_migrating_canister_impl( fixture.tick_to_conclusion(shutdown_phase_max_rounds, |_| Ok(())) } -proptest! { - #![proptest_config(ProptestConfig::with_cases(3))] - #[test] - fn check_canister_can_be_stopped_with_remote_subnet_stalling( - seeds in proptest::collection::vec(any::().no_shrink(), 2), - config in arb_canister_config(MAX_PAYLOAD_BYTES, 5), +#[test_strategy::proptest(ProptestConfig::with_cases(3))] +fn check_canister_can_be_stopped_with_remote_subnet_stalling( + #[strategy(proptest::collection::vec(any::().no_shrink(), 2))] seeds: Vec, + #[strategy(arb_canister_config(MAX_PAYLOAD_BYTES, 5))] config: CanisterConfig, +) { + if let Err((err_msg, nfo)) = check_canister_can_be_stopped_with_remote_subnet_stalling_impl( + 30, // chatter_phase_round_count + 300, // shutdown_phase_max_rounds + seeds.as_slice(), + config, ) { - if let Err((err_msg, nfo)) = check_canister_can_be_stopped_with_remote_subnet_stalling_impl( - 30, // chatter_phase_round_count - 300, // shutdown_phase_max_rounds - seeds.as_slice(), - config, - ) { - unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); - } + unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); } } From 47fb84a4c7beffc62ff2e2c71cf7f41d7f7cf283 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 15 Jan 2025 10:32:01 +0000 Subject: [PATCH 16/34] . --- rs/messaging/BUILD.bazel | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/messaging/BUILD.bazel b/rs/messaging/BUILD.bazel index ccaba056491..5c3aaf362aa 100644 --- a/rs/messaging/BUILD.bazel +++ b/rs/messaging/BUILD.bazel @@ -93,6 +93,10 @@ rust_ic_test_suite( "RANDOM_TRAFFIC_TEST_CANISTER_WASM_PATH": "$(rootpath //rs/rust_canisters/random_traffic_test:random-traffic-test-canister)", "XNET_TEST_CANISTER_WASM_PATH": "$(rootpath //rs/rust_canisters/xnet_test:xnet-test-canister)", }, + proc_macro_deps = [ + # Keep sorted. + "@crate_index//:test-strategy", + ], deps = [ # Keep sorted. ":messaging", @@ -118,8 +122,4 @@ rust_ic_test_suite( "@crate_index//:proptest", "@crate_index//:serde", ], - proc_macro_deps = [ - # Keep sorted. - "@crate_index//:test-strategy", - ], ) From fbe5208df855fa7dade03a700f29e7f6b8431777 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 16 Jan 2025 22:07:15 +0000 Subject: [PATCH 17/34] . --- .../random_traffic_test/src/lib.rs | 24 ++++++------- .../random_traffic_test/src/main.rs | 36 ++++++++++--------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index 904e80819dc..6df2a173aa2 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -14,13 +14,13 @@ pub struct Config { /// A list of canister IDs, i.e. receivers for calls made by this canister. pub receivers: Vec, /// `(min, max)` for the payload size in bytes included in a call. - pub call_bytes: (u32, u32), + pub call_bytes_range: (u32, u32), /// `(min, max)` for the payload size in bytes included in a reply. - pub reply_bytes: (u32, u32), + pub reply_bytes_range: (u32, u32), /// `(min, max)` for the simulated number of instructions to generate a reply. - pub instructions_count: (u32, u32), + pub instructions_count_range: (u32, u32), /// `(min, max)` for the timeout in seconds used for best-effort calls. - pub timeout_secs: (u32, u32), + pub timeout_secs_range: (u32, u32), /// The maximum number of calls attempted per heartbeat. pub calls_per_heartbeat: u32, /// The weight for making a reply used in a binominal distribution together with @@ -41,10 +41,10 @@ impl Default for Config { fn default() -> Self { Self { receivers: vec![], - call_bytes: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), - reply_bytes: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), - instructions_count: (0, 0), - timeout_secs: (10, 100), + call_bytes_range: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), + reply_bytes_range: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), + instructions_count_range: (0, 0), + timeout_secs_range: (10, 100), calls_per_heartbeat: 3, reply_weight: 1, downstream_call_weight: 0, @@ -105,10 +105,10 @@ impl Config { Ok(Self { receivers, - call_bytes: (*call_bytes.start(), *call_bytes.end()), - reply_bytes: (*reply_bytes.start(), *reply_bytes.end()), - instructions_count: (*instructions_count.start(), *instructions_count.end()), - timeout_secs: (*timeout_secs.start(), *timeout_secs.end()), + call_bytes_range: (*call_bytes.start(), *call_bytes.end()), + reply_bytes_range: (*reply_bytes.start(), *reply_bytes.end()), + instructions_count_range: (*instructions_count.start(), *instructions_count.end()), + timeout_secs_range: (*timeout_secs.start(), *timeout_secs.end()), calls_per_heartbeat, reply_weight, downstream_call_weight, diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index 4f505db32a1..6ebdff58224 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -34,10 +34,12 @@ thread_local! { static RECORDS: RefCell> = RefCell::default(); /// A counter for synchronous rejections. static SYNCHRONOUS_REJECTIONS_COUNT: Cell = Cell::default(); - /// A Binomial distribution used to determine whether to make a downstream call or not. - static DOWNSTREAM_CALL_DISTRIBUTION: RefCell> = RefCell::new(WeightedIndex::::new([0, 1]).unwrap()); - /// A Binomial distribution used to determine whether to make a best-effort call or not. - static BEST_EFFORT_CALL_DISTRIBUTION: RefCell> = RefCell::new(WeightedIndex::::new([1, 0]).unwrap()); + /// A `COIN` that can be 'flipped' to determine whether to make a downstream call or not. + /// The default value set here will yield only 'reply'. + static DOWNSTREAM_CALL_COIN: RefCell> = RefCell::new(WeightedIndex::::new([0, 1]).unwrap()); + /// A `COIN` that can be 'flipped' to determine whether to make a best-effort call or a guaranteed response call. + /// The default value set here will yield only 'best_effort' + static BEST_EFFORT_CALL_COIN: RefCell> = RefCell::new(WeightedIndex::::new([1, 0]).unwrap()); } /// The intercanister message sent to `handle_call()` by the heartbeat of this canister @@ -97,22 +99,22 @@ fn sample((min, max): (u32, u32)) -> u32 { /// Generates a random payload size for a call. fn call_bytes() -> u32 { - CONFIG.with_borrow(|config| sample(config.call_bytes)) + CONFIG.with_borrow(|config| sample(config.call_bytes_range)) } /// Generates a random payload size for a reply. fn reply_bytes() -> u32 { - CONFIG.with_borrow(|config| sample(config.reply_bytes)) + CONFIG.with_borrow(|config| sample(config.reply_bytes_range)) } /// Generates a random number of simulated instructions for generating a reply. fn instructions_count() -> u32 { - CONFIG.with_borrow(|config| sample(config.instructions_count)) + CONFIG.with_borrow(|config| sample(config.instructions_count_range)) } /// Generates a timeout in seconds for a best-effort call. fn timeout_secs() -> u32 { - CONFIG.with_borrow(|config| sample(config.timeout_secs)) + CONFIG.with_borrow(|config| sample(config.timeout_secs_range)) } /// Picks a random receiver from `config.receivers` if any; otherwise return own canister ID. @@ -128,11 +130,11 @@ fn receiver() -> CanisterId { /// Sets the test config; returns the current config. #[update] fn set_config(config: Config) -> Config { - // Update weighted distributions. - DOWNSTREAM_CALL_DISTRIBUTION.replace( - WeightedIndex::::new([config.reply_weight, config.downstream_call_weight]).unwrap(), + // Update `COINS`. + DOWNSTREAM_CALL_COIN.replace( + WeightedIndex::::new([config.downstream_call_weight, config.reply_weight]).unwrap(), ); - BEST_EFFORT_CALL_DISTRIBUTION.replace( + BEST_EFFORT_CALL_COIN.replace( WeightedIndex::::new([config.best_effort_weight, config.guaranteed_response_weight]) .unwrap(), ); @@ -175,17 +177,19 @@ fn synchronous_rejections_count() -> u32 { SYNCHRONOUS_REJECTIONS_COUNT.get() } -/// Determines whether to make a downstream call or reply. +/// Flip the `DOWNSTREAM_CALL_COIN` to determine whether we should make a downstream call or reply +/// instead. fn should_make_downstream_call() -> bool { RNG.with_borrow_mut(|rng| { - DOWNSTREAM_CALL_DISTRIBUTION.with_borrow(|distr| distr.sample(rng)) == 0 + DOWNSTREAM_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0 }) } -/// Determines whether to make a best-effort call or a guaranteed response call. +/// Flip the `BEST_EFFORT_COIN` to determine whether we should make a best-effort call or a +/// guaranteed response call. fn make_best_effort_call() -> bool { RNG.with_borrow_mut(|rng| { - BEST_EFFORT_CALL_DISTRIBUTION.with_borrow(|distr| distr.sample(rng)) == 0 + BEST_EFFORT_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0 }) } From 6ddb2ce90025375c3cfe5acf1e305505d693174e Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 16 Jan 2025 23:03:06 +0000 Subject: [PATCH 18/34] . --- Cargo.lock | 1 + rs/messaging/tests/memory_tests.rs | 4 +- .../random_traffic_test/BUILD.bazel | 1 + .../random_traffic_test/Cargo.toml | 1 + .../random_traffic_test/src/lib.rs | 8 ++++ .../random_traffic_test/src/main.rs | 41 ++++++++----------- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33e5dd7ac91..7a197a2b632 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18106,6 +18106,7 @@ dependencies = [ "ic-types", "rand 0.8.5", "serde", + "serde_bytes", ] [[package]] diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 2e523c1fe5c..6e5485cc97a 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -158,7 +158,7 @@ fn check_guaranteed_response_message_memory_limits_are_respected_impl( } } - // Tick until all calls have concluded. + // Tick until all calls have concluded; or else fail the test. fixture.tick_to_conclusion(shutdown_phase_max_rounds, |fixture| { fixture.expect_guaranteed_response_message_memory_taken_at_most( "Wrap up", @@ -238,7 +238,7 @@ fn check_calls_conclude_with_migrating_canister_impl( fixture.stop_chatter(migrating_canister); fixture.migrate_canister(migrating_canister); - // Tick until all calls have concluded. + // Tick until all calls have concluded; or else fail the test. fixture.tick_to_conclusion(shutdown_phase_max_rounds, |_| Ok(())) } diff --git a/rs/rust_canisters/random_traffic_test/BUILD.bazel b/rs/rust_canisters/random_traffic_test/BUILD.bazel index 867e48d68f3..062f00a4aa1 100644 --- a/rs/rust_canisters/random_traffic_test/BUILD.bazel +++ b/rs/rust_canisters/random_traffic_test/BUILD.bazel @@ -12,6 +12,7 @@ DEPENDENCIES = [ "@crate_index//:futures", "@crate_index//:rand", "@crate_index//:serde", + "@crate_index//:serde_bytes", ] MACRO_DEPENDENCIES = [ diff --git a/rs/rust_canisters/random_traffic_test/Cargo.toml b/rs/rust_canisters/random_traffic_test/Cargo.toml index 7033946d200..8fcb8edb57e 100644 --- a/rs/rust_canisters/random_traffic_test/Cargo.toml +++ b/rs/rust_canisters/random_traffic_test/Cargo.toml @@ -17,3 +17,4 @@ ic-types = { path = "../../types/types" } futures = { workspace = true } rand = { workspace = true } serde = { workspace = true } +serde_bytes = { workspace = true } diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index 6df2a173aa2..cb4dbeb7c19 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -202,9 +202,11 @@ pub struct Metrics { pub downstream_calls_attempted: u32, pub calls_replied: u32, pub calls_rejected: u32, + pub calls_unknown_outcome: u32, pub sent_bytes: u32, pub received_bytes: u32, pub rejected_bytes: u32, + pub unknown_outcome_bytes: u32, } /// Extracts some basic metrics from the records. @@ -223,6 +225,12 @@ pub fn extract_metrics(records: &BTreeMap) -> Metrics { metrics.calls_replied += 1; metrics.received_bytes += received_bytes; } + Some((_, Reply::Reject(reject_code, _))) + if RejectCode::try_from(*reject_code as u64).unwrap() == RejectCode::SysUnknown => + { + metrics.calls_unknown_outcome += 1; + metrics.unknown_outcome_bytes += record.sent_bytes; + } Some((_, Reply::Reject(..))) => { metrics.calls_rejected += 1; metrics.rejected_bytes += record.sent_bytes; diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index 6ebdff58224..b61a63f39d9 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -51,6 +51,7 @@ struct Message { /// The depth of the call starting from 0 and incrementing by 1 for each downstream call. call_depth: u32, /// Optional padding, to bring the payload to the desired byte size. + #[serde(with = "serde_bytes")] padding: Vec, } @@ -180,17 +181,13 @@ fn synchronous_rejections_count() -> u32 { /// Flip the `DOWNSTREAM_CALL_COIN` to determine whether we should make a downstream call or reply /// instead. fn should_make_downstream_call() -> bool { - RNG.with_borrow_mut(|rng| { - DOWNSTREAM_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0 - }) + RNG.with_borrow_mut(|rng| DOWNSTREAM_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0) } /// Flip the `BEST_EFFORT_COIN` to determine whether we should make a best-effort call or a /// guaranteed response call. fn make_best_effort_call() -> bool { - RNG.with_borrow_mut(|rng| { - BEST_EFFORT_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0 - }) + RNG.with_borrow_mut(|rng| BEST_EFFORT_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0) } /// Generates a future for a randomized call that can be awaited; inserts a new record at `index` @@ -216,23 +213,21 @@ fn setup_call( // Once the call was successfully generated, insert a call timestamp and record at `index`. let index = next_call_index(); RECORDS.with_borrow_mut(|records| { - assert!(records - .insert( - index, - ( - api::time(), - Record { - receiver, - caller, - call_tree_id, - call_depth, - sent_bytes: msg.count_bytes() as u32, - timeout_secs, - duration_and_reply: None, - }, - ) - ) - .is_none()); + records.insert( + index, + ( + api::time(), + Record { + receiver, + caller, + call_tree_id, + call_depth, + sent_bytes: msg.count_bytes() as u32, + timeout_secs, + duration_and_reply: None, + }, + ), + ) }); (call.call_raw(), index) From f9efe00902d38276c9f1f2dd6ccaa443e5f8db4b Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 16 Jan 2025 23:13:32 +0000 Subject: [PATCH 19/34] . --- rs/messaging/tests/memory_tests.rs | 2 +- rs/rust_canisters/random_traffic_test/src/main.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 6e5485cc97a..61d28f36f7c 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -91,7 +91,7 @@ fn check_guaranteed_response_message_memory_limits_are_respected( /// /// For the second phase, the 'chatter' is disabled by putting a canister into `Stopping` state /// every 10 rounds. In addition to shutting down traffic altogether from that canister (including -/// downstream calls) this will also induce a lot asychnronous rejections for requests. If any +/// downstream calls) this will also induce a lot asynchronous rejections for requests. If any /// canister fails to reach `Stopped` state (i.e. no pending calls), something went wrong in /// message routing, most likely a bug connected to reject signals for requests. /// diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index b61a63f39d9..ea9ca86bb95 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -186,7 +186,7 @@ fn should_make_downstream_call() -> bool { /// Flip the `BEST_EFFORT_COIN` to determine whether we should make a best-effort call or a /// guaranteed response call. -fn make_best_effort_call() -> bool { +fn should_make_best_effort_call() -> bool { RNG.with_borrow_mut(|rng| BEST_EFFORT_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0) } @@ -201,7 +201,7 @@ fn setup_call( let receiver = receiver(); let caller = (call_depth > 0).then_some(CanisterId::unchecked_from_principal(PrincipalId(caller()))); - let timeout_secs = make_best_effort_call().then_some(timeout_secs()); + let timeout_secs = should_make_best_effort_call().then_some(timeout_secs()); let call = Call::new(receiver.into(), "handle_call").with_raw_args(candid::Encode!(&msg).unwrap()); From b78ec7148d29c88aed65174f88e943e39a0b23f0 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 16 Jan 2025 23:18:40 +0000 Subject: [PATCH 20/34] . --- rs/rust_canisters/random_traffic_test/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index cb4dbeb7c19..9d53605f54e 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -44,7 +44,7 @@ impl Default for Config { call_bytes_range: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), reply_bytes_range: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), instructions_count_range: (0, 0), - timeout_secs_range: (10, 100), + timeout_secs_range: (1, 100), calls_per_heartbeat: 3, reply_weight: 1, downstream_call_weight: 0, From 3c97409416c6eafdddf0b94832cb3015a075576e Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 16 Jan 2025 23:22:17 +0000 Subject: [PATCH 21/34] . --- .../random_traffic_test/src/main.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index ea9ca86bb95..dda45bc39fb 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -99,22 +99,22 @@ fn sample((min, max): (u32, u32)) -> u32 { } /// Generates a random payload size for a call. -fn call_bytes() -> u32 { +fn gen_call_bytes() -> u32 { CONFIG.with_borrow(|config| sample(config.call_bytes_range)) } /// Generates a random payload size for a reply. -fn reply_bytes() -> u32 { +fn gen_reply_bytes() -> u32 { CONFIG.with_borrow(|config| sample(config.reply_bytes_range)) } /// Generates a random number of simulated instructions for generating a reply. -fn instructions_count() -> u32 { +fn gen_instructions_count() -> u32 { CONFIG.with_borrow(|config| sample(config.instructions_count_range)) } /// Generates a timeout in seconds for a best-effort call. -fn timeout_secs() -> u32 { +fn gen_timeout_secs() -> u32 { CONFIG.with_borrow(|config| sample(config.timeout_secs_range)) } @@ -197,11 +197,11 @@ fn setup_call( call_tree_id: u32, call_depth: u32, ) -> (impl Future>>, u32) { - let msg = Message::new(call_tree_id, call_depth, call_bytes()); + let msg = Message::new(call_tree_id, call_depth, gen_call_bytes()); let receiver = receiver(); let caller = (call_depth > 0).then_some(CanisterId::unchecked_from_principal(PrincipalId(caller()))); - let timeout_secs = should_make_best_effort_call().then_some(timeout_secs()); + let timeout_secs = should_make_best_effort_call().then_some(gen_timeout_secs()); let call = Call::new(receiver.into(), "handle_call").with_raw_args(candid::Encode!(&msg).unwrap()); @@ -322,10 +322,10 @@ async fn handle_call(msg: Message) -> Vec { } } - let payload_bytes = reply_bytes(); + let payload_bytes = gen_reply_bytes(); // Do some thinking. - let counts = api::performance_counter(0) + instructions_count() as u64; + let counts = api::performance_counter(0) + gen_instructions_count() as u64; while counts > api::performance_counter(0) {} vec![0_u8; payload_bytes as usize] From be726a496af9d1c3e05af56e1b0304e5ae47452c Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Thu, 16 Jan 2025 23:43:31 +0000 Subject: [PATCH 22/34] . --- .../random_traffic_test/src/lib.rs | 28 ++++++++--------- .../random_traffic_test/src/main.rs | 31 ++++++++++--------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index 9d53605f54e..bf0b8185b24 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -7,7 +7,7 @@ use std::collections::BTreeMap; use std::ops::RangeInclusive; use std::time::Duration; -/// A full config for generating random calls and replies. Ranges are stored as individual u32 +/// A full config for generating random calls and replies. Ranges are stored as `(u32, u32)` /// because ranges don't implement `CandidType`. #[derive(Serialize, Deserialize, Clone, Debug, CandidType, Hash)] pub struct Config { @@ -120,9 +120,9 @@ impl Config { /// Records the outcome of an outgoing call. #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, CandidType)] -pub enum Reply { - /// A response including a data payload of a distinct size was received. - Bytes(u32), +pub enum Response { + /// A reply including a data payload of a distinct size was received. + Reply(u32), /// The call was rejected with a reject code and a reject message. Reject(u32, String), } @@ -142,9 +142,9 @@ pub struct Record { pub sent_bytes: u32, /// The timeout in seconds set for a best-effort call; `None` for a guaranteed response call. pub timeout_secs: Option, - /// The kind of reply received, i.e. a payload or a reject response; and the duration after - /// which the reply was received (from when the call was made). - pub duration_and_reply: Option<(Duration, Reply)>, + /// The kind of response received, i.e. a reply with a payload or a reject response; + /// and the duration after which the response was received (from when the call was made). + pub duration_and_response: Option<(Duration, Response)>, } /// Human readable printer. @@ -174,9 +174,9 @@ impl std::fmt::Debug for Record { write!(f, " sending {} bytes | ", self.sent_bytes)?; - match &self.duration_and_reply { + match &self.duration_and_response { None => write!(f, "..."), - Some((call_duration, Reply::Bytes(bytes))) => { + Some((call_duration, Response::Reply(bytes))) => { write!( f, "duration[s]: {}, received {} bytes", @@ -184,7 +184,7 @@ impl std::fmt::Debug for Record { bytes ) } - Some((call_duration, Reply::Reject(error_code, error_msg))) => write!( + Some((call_duration, Response::Reject(error_code, error_msg))) => write!( f, "duration[s]: {}, reject({}): {error_msg}", call_duration.as_secs(), @@ -220,18 +220,18 @@ pub fn extract_metrics(records: &BTreeMap) -> Metrics { metrics.downstream_calls_attempted += 1; } - match &record.duration_and_reply { - Some((_, Reply::Bytes(received_bytes))) => { + match &record.duration_and_response { + Some((_, Response::Reply(received_bytes))) => { metrics.calls_replied += 1; metrics.received_bytes += received_bytes; } - Some((_, Reply::Reject(reject_code, _))) + Some((_, Response::Reject(reject_code, _))) if RejectCode::try_from(*reject_code as u64).unwrap() == RejectCode::SysUnknown => { metrics.calls_unknown_outcome += 1; metrics.unknown_outcome_bytes += record.sent_bytes; } - Some((_, Reply::Reject(..))) => { + Some((_, Response::Reject(..))) => { metrics.calls_rejected += 1; metrics.rejected_bytes += record.sent_bytes; } diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index dda45bc39fb..c514192ff5d 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -224,7 +224,7 @@ fn setup_call( call_depth, sent_bytes: msg.count_bytes() as u32, timeout_secs, - duration_and_reply: None, + duration_and_response: None, }, ), ) @@ -233,29 +233,32 @@ fn setup_call( (call.call_raw(), index) } -/// Updates the record at `index` using the `response` to the corresponding call. +/// Updates the record at `index` using the `result` of the corresponding call. /// /// Removes the record for a synchronous rejection since those can be quite numerous when the /// subnet is at its limits. Note that since the call `index` is part of the records, removing /// the records for synchronous rejections will result in gaps in these numbers thus they are /// still included indirectly. -fn update_record(response: &api::call::CallResult>, index: u32) { - // Updates the `Reply` at `index` in `RECORDS`. - let set_reply_in_call_record = move |reply: Reply| { +fn update_record(result: &api::call::CallResult>, index: u32) { + // Updates the `Response` at `index` in `RECORDS`. + let set_reply_in_call_record = move |response: Response| { RECORDS.with_borrow_mut(|records| { let (call_timestamp, record) = records.get_mut(&index).unwrap(); assert!( - record.duration_and_reply.is_none(), + record.duration_and_response.is_none(), "duplicate reply received" ); - let reply_timestamp = api::time(); - assert!(reply_timestamp >= *call_timestamp, "retrograde blocktime"); + let response_timestamp = api::time(); + assert!( + response_timestamp >= *call_timestamp, + "retrograde blocktime" + ); - let call_duration = Duration::from_nanos(reply_timestamp - *call_timestamp); - record.duration_and_reply = Some((call_duration, reply)); + let call_duration = Duration::from_nanos(response_timestamp - *call_timestamp); + record.duration_and_response = Some((call_duration, response)); }); }; - match response { + match result { Err((_, msg)) if is_synchronous_rejection(msg) => { // Remove the record for synchronous rejections. SYNCHRONOUS_REJECTIONS_COUNT.set(SYNCHRONOUS_REJECTIONS_COUNT.get() + 1); @@ -264,16 +267,16 @@ fn update_record(response: &api::call::CallResult>, index: u32) { .remove(&index) .unwrap() .1 - .duration_and_reply + .duration_and_response .is_none()) }); } Err((reject_code, msg)) => { - set_reply_in_call_record(Reply::Reject(*reject_code as u32, msg.to_string())); + set_reply_in_call_record(Response::Reject(*reject_code as u32, msg.to_string())); } Ok(result) => { - set_reply_in_call_record(Reply::Bytes(result.len() as u32)); + set_reply_in_call_record(Response::Reply(result.len() as u32)); } } } From e55ce7198946519a75371aeb5b9ed66195fa5125 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Fri, 17 Jan 2025 10:31:14 +0000 Subject: [PATCH 23/34] . --- rs/messaging/tests/memory_tests.rs | 23 +++-------- .../random_traffic_test/src/lib.rs | 40 +++++-------------- .../random_traffic_test/src/main.rs | 30 ++++++++------ 3 files changed, 32 insertions(+), 61 deletions(-) diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 61d28f36f7c..64d296d031f 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -30,16 +30,6 @@ const MB: u64 = KB * KB; const MAX_PAYLOAD_BYTES: u32 = MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32; -prop_compose! { - /// Generates an arbitrary pair of weights such that w1 + w2 = total. - fn arb_weights(total: u32)( - w1 in 0..=total - ) -> (u32, u32) - { - (w1, total - w1) - } -} - prop_compose! { /// Generates a random `CanisterConfig` using reasonable ranges of values; receivers is empty /// and assumed to be populated manually. @@ -48,8 +38,8 @@ prop_compose! { max_reply_bytes in 0..=max_payload_bytes, calls_per_heartbeat in 0..=max_calls_per_heartbeat, max_timeout_secs in 10..=100_u32, - (reply_weight, downstream_call_weight) in arb_weights(4), - (best_effort_weight, guaranteed_response_weight) in arb_weights(4), + downstream_call_percentage in 0..=100_u32, + best_effort_call_percentage in 0..=100_u32, ) -> CanisterConfig { CanisterConfig::try_new( vec![], @@ -58,10 +48,8 @@ prop_compose! { 0..=0, // instructions_count 0..=max_timeout_secs, calls_per_heartbeat, - reply_weight, - downstream_call_weight, - best_effort_weight, - guaranteed_response_weight, + downstream_call_percentage, + best_effort_call_percentage, ) .expect("bad config inputs") } @@ -295,8 +283,7 @@ fn check_canister_can_be_stopped_with_remote_subnet_stalling_impl( fixture.set_config( local_canister, CanisterConfig { - best_effort_weight: 1, - guaranteed_response_weight: 0, + best_effort_call_percentage: 100, ..config.clone() }, ); diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index bf0b8185b24..9ab0ce492ed 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -23,18 +23,10 @@ pub struct Config { pub timeout_secs_range: (u32, u32), /// The maximum number of calls attempted per heartbeat. pub calls_per_heartbeat: u32, - /// The weight for making a reply used in a binominal distribution together with - /// `downstream_call_weight`. - pub reply_weight: u32, - /// The weight for making a downstream call used in a binomial distribution together with - /// `downstream_call_weight`. - pub downstream_call_weight: u32, - /// The weight for making a best-effort call used in a binomial distribution together with - /// `guaranteed_response_weight`. - pub best_effort_weight: u32, - /// The weight for making a guaranteed response call used in a binomial distribution together - /// with `best_effort_weight`. - pub guaranteed_response_weight: u32, + /// The probability for a making a dowstream call rather than reply in %. + pub downstream_call_percentage: u32, + /// The probability for making a best-effort call rather a guaranteed response call in %. + pub best_effort_call_percentage: u32, } impl Default for Config { @@ -46,10 +38,8 @@ impl Default for Config { instructions_count_range: (0, 0), timeout_secs_range: (1, 100), calls_per_heartbeat: 3, - reply_weight: 1, - downstream_call_weight: 0, - best_effort_weight: 1, - guaranteed_response_weight: 0, + downstream_call_percentage: 0, + best_effort_call_percentage: 100, } } } @@ -72,10 +62,8 @@ impl Config { instructions_count: RangeInclusive, timeout_secs: RangeInclusive, calls_per_heartbeat: u32, - reply_weight: u32, - downstream_call_weight: u32, - best_effort_weight: u32, - guaranteed_response_weight: u32, + downstream_call_percentage: u32, + best_effort_call_percentage: u32, ) -> Result { // Sanity checks. After passing these, the canister should run as intended. if call_bytes.is_empty() { @@ -96,12 +84,6 @@ impl Config { if timeout_secs.is_empty() { return Err("empty timeout range".to_string()); } - if reply_weight == 0 && downstream_call_weight == 0 { - return Err("bad downstream call weights, both 0".to_string()); - } - if best_effort_weight == 0 && guaranteed_response_weight == 0 { - return Err("bad call type weights, both 0".to_string()); - } Ok(Self { receivers, @@ -110,10 +92,8 @@ impl Config { instructions_count_range: (*instructions_count.start(), *instructions_count.end()), timeout_secs_range: (*timeout_secs.start(), *timeout_secs.end()), calls_per_heartbeat, - reply_weight, - downstream_call_weight, - best_effort_weight, - guaranteed_response_weight, + downstream_call_percentage, + best_effort_call_percentage, }) } } diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index c514192ff5d..622afaebceb 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -36,10 +36,10 @@ thread_local! { static SYNCHRONOUS_REJECTIONS_COUNT: Cell = Cell::default(); /// A `COIN` that can be 'flipped' to determine whether to make a downstream call or not. /// The default value set here will yield only 'reply'. - static DOWNSTREAM_CALL_COIN: RefCell> = RefCell::new(WeightedIndex::::new([0, 1]).unwrap()); + static DOWNSTREAM_CALL_COIN: RefCell> = RefCell::new(WeightedIndex::::new([0, 100]).unwrap()); /// A `COIN` that can be 'flipped' to determine whether to make a best-effort call or a guaranteed response call. /// The default value set here will yield only 'best_effort' - static BEST_EFFORT_CALL_COIN: RefCell> = RefCell::new(WeightedIndex::::new([1, 0]).unwrap()); + static BEST_EFFORT_CALL_COIN: RefCell> = RefCell::new(WeightedIndex::::new([100, 0]).unwrap()); } /// The intercanister message sent to `handle_call()` by the heartbeat of this canister @@ -131,13 +131,18 @@ fn receiver() -> CanisterId { /// Sets the test config; returns the current config. #[update] fn set_config(config: Config) -> Config { + fn to_weights(mut percentage: u32) -> [u32; 2] { + if percentage > 100 { + percentage = 100; + } + [percentage, 100 - percentage] + } + // Update `COINS`. - DOWNSTREAM_CALL_COIN.replace( - WeightedIndex::::new([config.downstream_call_weight, config.reply_weight]).unwrap(), - ); + DOWNSTREAM_CALL_COIN + .replace(WeightedIndex::::new(to_weights(config.downstream_call_percentage)).unwrap()); BEST_EFFORT_CALL_COIN.replace( - WeightedIndex::::new([config.best_effort_weight, config.guaranteed_response_weight]) - .unwrap(), + WeightedIndex::::new(to_weights(config.best_effort_call_percentage)).unwrap(), ); CONFIG.replace(config) @@ -155,8 +160,7 @@ fn seed_rng(seed: u64) { fn stop_chatter() -> Config { set_config(Config { calls_per_heartbeat: 0, - reply_weight: 1, - downstream_call_weight: 0, + downstream_call_percentage: 0, ..CONFIG.take() }) } @@ -181,13 +185,13 @@ fn synchronous_rejections_count() -> u32 { /// Flip the `DOWNSTREAM_CALL_COIN` to determine whether we should make a downstream call or reply /// instead. fn should_make_downstream_call() -> bool { - RNG.with_borrow_mut(|rng| DOWNSTREAM_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0) + RNG.with_borrow_mut(|rng| DOWNSTREAM_CALL_COIN.with_borrow(|coin| coin.sample(rng)) == 0) } /// Flip the `BEST_EFFORT_COIN` to determine whether we should make a best-effort call or a /// guaranteed response call. fn should_make_best_effort_call() -> bool { - RNG.with_borrow_mut(|rng| BEST_EFFORT_CALL_COIN.with_borrow(|distr| distr.sample(rng)) == 0) + RNG.with_borrow_mut(|rng| BEST_EFFORT_CALL_COIN.with_borrow(|coin| coin.sample(rng)) == 0) } /// Generates a future for a randomized call that can be awaited; inserts a new record at `index` @@ -307,11 +311,11 @@ async fn heartbeat() { /// Handles incoming calls; this method is called from the heartbeat method. /// /// Replies if: -/// - sampling the weighted binomial distribution tells us to do so. +/// - flipping the coin tells us to do so. /// - if it tells us not to do so but the attempted downstream call fails for any reason. #[update] async fn handle_call(msg: Message) -> Vec { - // Make downstream calls as long as sampling the distribution tells us to do so. + // Make downstream calls as long as flipping the coin tells us to do so. while should_make_downstream_call() { let (future, record_index) = setup_call(msg.call_tree_id, msg.call_depth + 1); From 377832a132d62dcb6b61e8dc1feb513a1a0188ff Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Mon, 20 Jan 2025 09:02:23 +0000 Subject: [PATCH 24/34] . --- Cargo.Bazel.json.lock | 86 ++++++++++--------- Cargo.Bazel.toml.lock | 11 +-- bazel/external_crates.bzl | 4 +- rs/messaging/tests/memory_tests.rs | 22 +++-- .../random_traffic_test/src/main.rs | 62 ++++++------- 5 files changed, 101 insertions(+), 84 deletions(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index f7d9f9b1318..f8562b0edc1 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "bc1f3a256bf2388f48d2cce47a7bfc9360d6397aee0a795ac068f90b1cd68ea4", + "checksum": "0f5103043e6989c6387ac9435bf654f8d897751690bb7f6f599e4012865e9e6e", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -10027,7 +10027,7 @@ "target": "canbench_rs" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -10122,7 +10122,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -10209,14 +10209,14 @@ ], "license_file": null }, - "candid 0.10.10": { + "candid 0.10.12": { "name": "candid", - "version": "0.10.10", + "version": "0.10.12", "package_url": "https://github.com/dfinity/candid", "repository": { "Http": { - "url": "https://static.crates.io/crates/candid/0.10.10/download", - "sha256": "6c30ee7f886f296b6422c0ff017e89dd4f831521dfdcc76f3f71aae1ce817222" + "url": "https://static.crates.io/crates/candid/0.10.12/download", + "sha256": "51e129c4051c57daf943586e01ef72faae48b04a8f692d5f646febf17a264c38" } }, "targets": [ @@ -10323,7 +10323,7 @@ ], "selects": {} }, - "version": "0.10.10" + "version": "0.10.12" }, "license": "Apache-2.0", "license_ids": [ @@ -10438,7 +10438,7 @@ "target": "anyhow" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -18333,7 +18333,7 @@ "target": "canbench_rs" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -23060,7 +23060,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -29811,7 +29811,7 @@ "target": "cached" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30282,7 +30282,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30386,7 +30386,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30473,7 +30473,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30536,7 +30536,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30604,7 +30604,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30672,7 +30672,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30716,7 +30716,7 @@ "Git": { "remote": "https://github.com/dfinity/cdk-rs.git", "commitish": { - "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + "Rev": "4e287ce51636b0e70768c193da38d2fc5324ea15" }, "strip_prefix": "ic-cdk" } @@ -30743,7 +30743,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30757,6 +30757,10 @@ { "id": "serde_bytes 0.11.15", "target": "serde_bytes" + }, + { + "id": "thiserror 2.0.3", + "target": "thiserror" } ], "selects": {} @@ -30811,7 +30815,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30878,7 +30882,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30945,7 +30949,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31012,7 +31016,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31055,7 +31059,7 @@ "Git": { "remote": "https://github.com/dfinity/cdk-rs.git", "commitish": { - "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + "Rev": "4e287ce51636b0e70768c193da38d2fc5324ea15" }, "strip_prefix": "ic-cdk-macros" } @@ -31082,7 +31086,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31220,7 +31224,7 @@ "target": "cached" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31495,7 +31499,7 @@ "target": "base64" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31574,7 +31578,7 @@ "target": "bytes" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31750,7 +31754,7 @@ "target": "base64" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31947,7 +31951,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32006,7 +32010,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32090,7 +32094,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32184,7 +32188,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32462,7 +32466,7 @@ "target": "anyhow" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32537,7 +32541,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32678,7 +32682,7 @@ "Git": { "remote": "https://github.com/dfinity/cdk-rs.git", "commitish": { - "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + "Rev": "4e287ce51636b0e70768c193da38d2fc5324ea15" }, "strip_prefix": "ic0" } @@ -32919,7 +32923,7 @@ "target": "anyhow" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32990,7 +32994,7 @@ "target": "anyhow" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -49383,7 +49387,7 @@ "target": "base64" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -87409,7 +87413,7 @@ "cached 0.49.2", "canbench 0.1.8", "canbench-rs 0.1.8", - "candid 0.10.10", + "candid 0.10.12", "candid_parser 0.1.2", "cargo_metadata 0.14.2", "cc 1.0.83", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index 2c2a7c2cbba..fb71d41e765 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -1746,9 +1746,9 @@ dependencies = [ [[package]] name = "candid" -version = "0.10.10" +version = "0.10.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c30ee7f886f296b6422c0ff017e89dd4f831521dfdcc76f3f71aae1ce817222" +checksum = "51e129c4051c57daf943586e01ef72faae48b04a8f692d5f646febf17a264c38" dependencies = [ "anyhow", "binread", @@ -5114,13 +5114,14 @@ dependencies = [ [[package]] name = "ic-cdk" version = "0.18.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs.git?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" dependencies = [ "candid", "ic-cdk-macros 0.18.0-alpha.1", "ic0 0.24.0-alpha.1", "serde", "serde_bytes", + "thiserror 2.0.3", ] [[package]] @@ -5182,7 +5183,7 @@ dependencies = [ [[package]] name = "ic-cdk-macros" version = "0.18.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs.git?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" dependencies = [ "candid", "proc-macro2", @@ -5498,7 +5499,7 @@ checksum = "8de254dd67bbd58073e23dc1c8553ba12fa1dc610a19de94ad2bbcd0460c067f" [[package]] name = "ic0" version = "0.24.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs.git?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" [[package]] name = "ic_bls12_381" diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index a3f15cdef21..001d20897e0 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -340,7 +340,7 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable version = "^0.1.8", ), "candid": crate.spec( - version = "^0.10.6", + version = "^0.10.12", ), "cargo_metadata": crate.spec( version = "^0.14.2", @@ -601,7 +601,7 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable "ic-cdk-next": crate.spec( package = "ic-cdk", git = "https://github.com/dfinity/cdk-rs.git", - branch = "next", + rev = "4e287ce51636b0e70768c193da38d2fc5324ea15", ), "ic-certified-map": crate.spec( version = "^0.3.1", diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index 64d296d031f..a81f53bbd15 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -13,7 +13,7 @@ use ic_replicated_state::ReplicatedState; use ic_state_machine_tests::{StateMachine, StateMachineBuilder, StateMachineConfig, UserError}; use ic_test_utilities_types::ids::{SUBNET_0, SUBNET_1}; use ic_types::{ - ingress::{IngressState, IngressStatus}, + ingress::{IngressState, IngressStatus, WasmResult}, messages::{MessageId, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64}, Cycles, }; @@ -166,6 +166,8 @@ fn check_calls_conclude_with_migrating_canister( 300, // shutdown_phase_max_rounds seed, config, ) { + // unreachable!("{:?}\n\n{:?}", nfo.fixture.local_canisters, nfo.fixture.remote_canisters); + // unreachable!("{:#?}", nfo.fixture.set_config(*nfo.fixture.local_canisters.first().unwrap(), CanisterConfig::default())); unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); } } @@ -213,7 +215,10 @@ fn check_calls_conclude_with_migrating_canister_impl( if canister != migrating_canister { // Make sure the canister doesn't make calls when it is // put into running state to read its records. - fixture.stop_chatter(canister); + let config = fixture.stop_chatter(canister); + // assert!(false, "{:?}", migrating_canister); + // let config = fixture.set_config(canister, config); + // assert!(false, "{:#?}", config); fixture.stop_canister_non_blocking(canister); } } @@ -517,10 +522,15 @@ impl Fixture { /// Calls the `stop_chatter()` function on `canister`. /// /// This stops the canister from making calls, downstream and from the heartbeat. - pub fn stop_chatter(&self, canister: CanisterId) { - self.get_env(&canister) - .execute_ingress(canister, "stop_chatter", candid::Encode!().unwrap()) - .unwrap(); + pub fn stop_chatter(&self, canister: CanisterId) -> CanisterConfig { + match self.get_env(&canister).execute_ingress( + canister, + "stop_chatter", + candid::Encode!().unwrap(), + ) { + Ok(WasmResult::Reply(reply)) => candid::Decode!(&reply, CanisterConfig).unwrap(), + _ => unreachable!(), + } } /// Queries the records from `canister`. diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index 622afaebceb..8775e0a66b0 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -1,9 +1,11 @@ use candid::CandidType; -use candid::Encode; use futures::future::select_all; use ic_base_types::{CanisterId, PrincipalId}; -use ic_cdk::api::call::{Call, ConfigurableCall, SendableCall}; -use ic_cdk::{api, caller, id, setup}; +use ic_cdk::{ + api, + call::{Call, CallError, CallResult, ConfigurableCall, SendableCall}, + setup, +}; use ic_cdk_macros::{heartbeat, init, query, update}; use rand::{ distributions::{Distribution, WeightedIndex}, @@ -88,11 +90,6 @@ fn next_call_tree_id() -> u32 { }) } -/// Returns `true` if `error_msg` corresponds to a synchronous rejection. -fn is_synchronous_rejection(error_msg: &str) -> bool { - error_msg.contains("Couldn't send message") -} - /// Generates a random `u32` by sampling `min..=max`. fn sample((min, max): (u32, u32)) -> u32 { RNG.with_borrow_mut(|rng| rng.gen_range(min..=max)) @@ -124,7 +121,7 @@ fn receiver() -> CanisterId { RNG.with_borrow_mut(|rng| config.receivers.as_slice().choose(rng).cloned()) }) { Some(receiver) => receiver, - None => CanisterId::try_from(id().as_slice()).unwrap(), + None => CanisterId::try_from(api::canister_self().as_slice()).unwrap(), } } @@ -154,15 +151,15 @@ fn seed_rng(seed: u64) { RNG.with_borrow_mut(|rng| *rng = StdRng::seed_from_u64(seed)); } -/// Sets `CONFIG` such that the canister stops making calls altogether; returns the modified -/// config. +/// Sets `CONFIG` such that the canister stops making calls altogether; returns the config +/// prior to modifying. #[update] fn stop_chatter() -> Config { - set_config(Config { + set_config(CONFIG.with_borrow(|config| Config { calls_per_heartbeat: 0, downstream_call_percentage: 0, - ..CONFIG.take() - }) + ..config.clone() + })) } /// Returns the canister records. @@ -200,15 +197,16 @@ fn should_make_best_effort_call() -> bool { fn setup_call( call_tree_id: u32, call_depth: u32, -) -> (impl Future>>, u32) { +) -> (impl Future>, u32) { let msg = Message::new(call_tree_id, call_depth, gen_call_bytes()); + let sent_bytes = msg.count_bytes() as u32; let receiver = receiver(); - let caller = - (call_depth > 0).then_some(CanisterId::unchecked_from_principal(PrincipalId(caller()))); + let caller = (call_depth > 0).then_some(CanisterId::unchecked_from_principal(PrincipalId( + api::msg_caller(), + ))); let timeout_secs = should_make_best_effort_call().then_some(gen_timeout_secs()); - let call = - Call::new(receiver.into(), "handle_call").with_raw_args(candid::Encode!(&msg).unwrap()); + let call = Call::new(receiver.into(), "handle_call").with_arg(msg); let call = match timeout_secs { Some(timeout_secs) => call.change_timeout(timeout_secs), None => call.with_guaranteed_response(), @@ -226,7 +224,7 @@ fn setup_call( caller, call_tree_id, call_depth, - sent_bytes: msg.count_bytes() as u32, + sent_bytes, timeout_secs, duration_and_response: None, }, @@ -234,7 +232,7 @@ fn setup_call( ) }); - (call.call_raw(), index) + (call.call(), index) } /// Updates the record at `index` using the `result` of the corresponding call. @@ -243,7 +241,7 @@ fn setup_call( /// subnet is at its limits. Note that since the call `index` is part of the records, removing /// the records for synchronous rejections will result in gaps in these numbers thus they are /// still included indirectly. -fn update_record(result: &api::call::CallResult>, index: u32) { +fn update_record(result: &CallResult, index: u32) { // Updates the `Response` at `index` in `RECORDS`. let set_reply_in_call_record = move |response: Response| { RECORDS.with_borrow_mut(|records| { @@ -263,7 +261,7 @@ fn update_record(result: &api::call::CallResult>, index: u32) { }); }; match result { - Err((_, msg)) if is_synchronous_rejection(msg) => { + Err(CallError::CallRejected(rejection)) if rejection.is_sync() => { // Remove the record for synchronous rejections. SYNCHRONOUS_REJECTIONS_COUNT.set(SYNCHRONOUS_REJECTIONS_COUNT.get() + 1); RECORDS.with_borrow_mut(|records| { @@ -275,12 +273,16 @@ fn update_record(result: &api::call::CallResult>, index: u32) { .is_none()) }); } - - Err((reject_code, msg)) => { - set_reply_in_call_record(Response::Reject(*reject_code as u32, msg.to_string())); + Err(CallError::CallRejected(rejection)) => { + set_reply_in_call_record(Response::Reject( + rejection.reject_code().into(), + rejection.reject_message().to_string(), + )); } - Ok(result) => { - set_reply_in_call_record(Response::Reply(result.len() as u32)); + Err(CallError::CandidDecodeFailed(..)) => unreachable!(), + + Ok(msg) => { + set_reply_in_call_record(Response::Reply(msg.count_bytes() as u32)); } } } @@ -292,7 +294,7 @@ async fn heartbeat() { let (mut futures, mut record_indices) = (Vec::new(), Vec::new()); for _ in 0..CONFIG.with_borrow(|config| config.calls_per_heartbeat) { let (future, index) = setup_call(next_call_tree_id(), 0); - futures.push(future); + futures.push(Box::pin(future)); record_indices.push(index); } @@ -341,7 +343,7 @@ async fn handle_call(msg: Message) -> Vec { /// Initializes the `HASHER` by hashing our own canister ID. #[init] fn initialize_hasher() { - HASHER.with_borrow_mut(|hasher| hasher.write(id().as_slice())); + HASHER.with_borrow_mut(|hasher| hasher.write(api::canister_self().as_slice())); } fn main() { From 97bc3d19fdc5ddcdb266c0f4acbd08b6540df757 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Mon, 20 Jan 2025 09:13:51 +0000 Subject: [PATCH 25/34] Automatically updated Cargo*.lock --- Cargo.Bazel.Fuzzing.json.lock | 90 ++++++++++++++++++----------------- Cargo.Bazel.Fuzzing.toml.lock | 11 +++-- Cargo.Bazel.json.lock | 2 +- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index e91c051ad81..00f8c1fb4c8 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "6712d2aabc11e75facdd5779b32d59fcecd3e8dc2806096dc111d59322e5de62", + "checksum": "07b96ead5f35f753cd65d3d30e79397e2d19015b5d1333c3d9a81c6aefccac6e", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -10110,7 +10110,7 @@ "target": "canbench_rs" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -10205,7 +10205,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -10292,14 +10292,14 @@ ], "license_file": null }, - "candid 0.10.10": { + "candid 0.10.12": { "name": "candid", - "version": "0.10.10", + "version": "0.10.12", "package_url": "https://github.com/dfinity/candid", "repository": { "Http": { - "url": "https://static.crates.io/crates/candid/0.10.10/download", - "sha256": "6c30ee7f886f296b6422c0ff017e89dd4f831521dfdcc76f3f71aae1ce817222" + "url": "https://static.crates.io/crates/candid/0.10.12/download", + "sha256": "51e129c4051c57daf943586e01ef72faae48b04a8f692d5f646febf17a264c38" } }, "targets": [ @@ -10427,7 +10427,7 @@ ], "selects": {} }, - "version": "0.10.10" + "version": "0.10.12" }, "license": "Apache-2.0", "license_ids": [ @@ -10542,7 +10542,7 @@ "target": "anyhow" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -18528,7 +18528,7 @@ "target": "canbench_rs" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -23240,7 +23240,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -29987,7 +29987,7 @@ "target": "cached" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30458,7 +30458,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30562,7 +30562,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30649,7 +30649,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30712,7 +30712,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30780,7 +30780,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30848,7 +30848,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30892,7 +30892,7 @@ "Git": { "remote": "https://github.com/dfinity/cdk-rs.git", "commitish": { - "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + "Rev": "4e287ce51636b0e70768c193da38d2fc5324ea15" }, "strip_prefix": "ic-cdk" } @@ -30919,7 +30919,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -30927,12 +30927,16 @@ "target": "ic0" }, { - "id": "serde 1.0.214", + "id": "serde 1.0.217", "target": "serde" }, { "id": "serde_bytes 0.11.15", "target": "serde_bytes" + }, + { + "id": "thiserror 2.0.3", + "target": "thiserror" } ], "selects": {} @@ -30987,7 +30991,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31054,7 +31058,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31121,7 +31125,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31188,7 +31192,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31231,7 +31235,7 @@ "Git": { "remote": "https://github.com/dfinity/cdk-rs.git", "commitish": { - "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + "Rev": "4e287ce51636b0e70768c193da38d2fc5324ea15" }, "strip_prefix": "ic-cdk-macros" } @@ -31258,7 +31262,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31270,7 +31274,7 @@ "target": "quote" }, { - "id": "serde 1.0.214", + "id": "serde 1.0.217", "target": "serde" }, { @@ -31396,7 +31400,7 @@ "target": "cached" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31671,7 +31675,7 @@ "target": "base64" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31750,7 +31754,7 @@ "target": "bytes" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -31926,7 +31930,7 @@ "target": "base64" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32144,7 +32148,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32203,7 +32207,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32287,7 +32291,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32381,7 +32385,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32659,7 +32663,7 @@ "target": "anyhow" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32734,7 +32738,7 @@ "deps": { "common": [ { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -32875,7 +32879,7 @@ "Git": { "remote": "https://github.com/dfinity/cdk-rs.git", "commitish": { - "Rev": "605abcade44d8fa4888c564c6fab5c715d1c33c8" + "Rev": "4e287ce51636b0e70768c193da38d2fc5324ea15" }, "strip_prefix": "ic0" } @@ -33116,7 +33120,7 @@ "target": "anyhow" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -33187,7 +33191,7 @@ "target": "anyhow" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -49612,7 +49616,7 @@ "target": "base64" }, { - "id": "candid 0.10.10", + "id": "candid 0.10.12", "target": "candid" }, { @@ -87650,7 +87654,7 @@ "cached 0.49.2", "canbench 0.1.8", "canbench-rs 0.1.8", - "candid 0.10.10", + "candid 0.10.12", "candid_parser 0.1.2", "cargo_metadata 0.14.2", "cc 1.1.37", diff --git a/Cargo.Bazel.Fuzzing.toml.lock b/Cargo.Bazel.Fuzzing.toml.lock index 6730f5568e6..28caa184cf8 100644 --- a/Cargo.Bazel.Fuzzing.toml.lock +++ b/Cargo.Bazel.Fuzzing.toml.lock @@ -1745,9 +1745,9 @@ dependencies = [ [[package]] name = "candid" -version = "0.10.10" +version = "0.10.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c30ee7f886f296b6422c0ff017e89dd4f831521dfdcc76f3f71aae1ce817222" +checksum = "51e129c4051c57daf943586e01ef72faae48b04a8f692d5f646febf17a264c38" dependencies = [ "anyhow", "binread", @@ -5127,13 +5127,14 @@ dependencies = [ [[package]] name = "ic-cdk" version = "0.18.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs.git?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" dependencies = [ "candid", "ic-cdk-macros 0.18.0-alpha.1", "ic0 0.24.0-alpha.1", "serde", "serde_bytes", + "thiserror 2.0.3", ] [[package]] @@ -5195,7 +5196,7 @@ dependencies = [ [[package]] name = "ic-cdk-macros" version = "0.18.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs.git?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" dependencies = [ "candid", "proc-macro2", @@ -5511,7 +5512,7 @@ checksum = "8de254dd67bbd58073e23dc1c8553ba12fa1dc610a19de94ad2bbcd0460c067f" [[package]] name = "ic0" version = "0.24.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs.git?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs.git?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" [[package]] name = "ic_bls12_381" diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index e4632e1dc57..bd7a801f799 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "0f5103043e6989c6387ac9435bf654f8d897751690bb7f6f599e4012865e9e6e", + "checksum": "1e65184579336e8e32f98c87ea66a02736faf39eb650f2d387c7c34b8d0cb58f", "crates": { "abnf 0.12.0": { "name": "abnf", From b3195c71095ba82e302208fb109dc3a9f619a8cb Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Tue, 21 Jan 2025 20:01:39 +0000 Subject: [PATCH 26/34] . --- Cargo.Bazel.json.lock | 2 +- rs/messaging/tests/memory_tests.rs | 69 +++++++++---------- .../random_traffic_test/src/lib.rs | 3 +- .../random_traffic_test/src/main.rs | 40 +++++------ 4 files changed, 54 insertions(+), 60 deletions(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index e4632e1dc57..bd7a801f799 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "0f5103043e6989c6387ac9435bf654f8d897751690bb7f6f599e4012865e9e6e", + "checksum": "1e65184579336e8e32f98c87ea66a02736faf39eb650f2d387c7c34b8d0cb58f", "crates": { "abnf 0.12.0": { "name": "abnf", diff --git a/rs/messaging/tests/memory_tests.rs b/rs/messaging/tests/memory_tests.rs index a81f53bbd15..7f6899cf1f6 100644 --- a/rs/messaging/tests/memory_tests.rs +++ b/rs/messaging/tests/memory_tests.rs @@ -13,7 +13,7 @@ use ic_replicated_state::ReplicatedState; use ic_state_machine_tests::{StateMachine, StateMachineBuilder, StateMachineConfig, UserError}; use ic_test_utilities_types::ids::{SUBNET_0, SUBNET_1}; use ic_types::{ - ingress::{IngressState, IngressStatus, WasmResult}, + ingress::{IngressState, IngressStatus}, messages::{MessageId, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64}, Cycles, }; @@ -166,8 +166,6 @@ fn check_calls_conclude_with_migrating_canister( 300, // shutdown_phase_max_rounds seed, config, ) { - // unreachable!("{:?}\n\n{:?}", nfo.fixture.local_canisters, nfo.fixture.remote_canisters); - // unreachable!("{:#?}", nfo.fixture.set_config(*nfo.fixture.local_canisters.first().unwrap(), CanisterConfig::default())); unreachable!("\nerr_msg: {err_msg}\n{:#?}", nfo.records); } } @@ -215,10 +213,7 @@ fn check_calls_conclude_with_migrating_canister_impl( if canister != migrating_canister { // Make sure the canister doesn't make calls when it is // put into running state to read its records. - let config = fixture.stop_chatter(canister); - // assert!(false, "{:?}", migrating_canister); - // let config = fixture.set_config(canister, config); - // assert!(false, "{:#?}", config); + fixture.stop_chatter(canister); fixture.stop_canister_non_blocking(canister); } } @@ -472,7 +467,7 @@ impl Fixture { /// Helper function for update calls to `canister`; returns the current `T` as it was before /// this call. /// - /// Panics if `canister` is not installed in `Self`. + /// Panics if `canister` is not installed in `self`. fn set_canister_state(&self, canister: CanisterId, method: &str, item: T) -> T where T: candid::CandidType + for<'a> candid::Deserialize<'a>, @@ -487,14 +482,14 @@ impl Fixture { /// Sets the `CanisterConfig` in `canister`; returns the current config. /// - /// Panics if `canister` is not installed in `Self`. + /// Panics if `canister` is not installed in `self`. pub fn set_config(&self, canister: CanisterId, config: CanisterConfig) -> CanisterConfig { self.set_canister_state(canister, "set_config", config) } /// Seeds the `Rng` in `canister`. /// - /// Panics if `canister` is not installed in `Self`. + /// Panics if `canister` is not installed in `self`. pub fn seed_rng(&self, canister: CanisterId, seed: u64) { let msg = candid::Encode!(&seed).unwrap(); self.get_env(&canister) @@ -504,7 +499,7 @@ impl Fixture { /// Starts `canister`. /// - /// Panics if `canister` is not installed in `Self`. + /// Panics if `canister` is not installed in `self`. pub fn start_canister(&self, canister: CanisterId) { self.get_env(&canister).start_canister(canister).unwrap(); } @@ -514,7 +509,7 @@ impl Fixture { /// This function is asynchronous. It returns the ID of the ingress message /// that can be awaited later with [await_ingress]. /// - /// Panics if `canister` is not installed in `Self`. + /// Panics if `canister` is not installed in `self`. pub fn stop_canister_non_blocking(&self, canister: CanisterId) -> MessageId { self.get_env(&canister).stop_canister_non_blocking(canister) } @@ -523,42 +518,40 @@ impl Fixture { /// /// This stops the canister from making calls, downstream and from the heartbeat. pub fn stop_chatter(&self, canister: CanisterId) -> CanisterConfig { - match self.get_env(&canister).execute_ingress( - canister, - "stop_chatter", - candid::Encode!().unwrap(), - ) { - Ok(WasmResult::Reply(reply)) => candid::Decode!(&reply, CanisterConfig).unwrap(), - _ => unreachable!(), - } + let reply = self + .get_env(&canister) + .execute_ingress(canister, "stop_chatter", candid::Encode!().unwrap()) + .unwrap(); + candid::Decode!(&reply.bytes(), CanisterConfig).unwrap() } - /// Queries the records from `canister`. + /// Queries `canister` for `method`. /// - /// Panics if `canister` is not installed in `Self`. - pub fn query_records( + /// Panics if `canister` is not installed in `self`. + pub fn query candid::Deserialize<'a>>( &self, canister: CanisterId, - ) -> Result, UserError> { - let dummy_msg = candid::Encode!().unwrap(); + method: &str, + ) -> Result { let reply = self .get_env(&canister) - .query(canister, "records", dummy_msg)?; - Ok(candid::Decode!(&reply.bytes(), BTreeMap).unwrap()) + .query(canister, method, candid::Encode!().unwrap())?; + Ok(candid::Decode!(&reply.bytes(), T).unwrap()) } - /// Force queries the records from `canister` by first attempting to query them; if it fails, start - /// the canister and try querying them again. - /// - /// Note: If the canister is configured to make calls, starting it will trigger calls before - /// the records are returned. + /// Force queries `canister` for `method` by first attempting to a normal query; if it fails, start + /// the canister and try again. /// - /// Panics if `canister` is not installed in `Self`. - pub fn force_query_records(&self, canister: CanisterId) -> BTreeMap { - match self.query_records(canister) { + /// Panics if `canister` is not installed in `self`. + pub fn force_query candid::Deserialize<'a>>( + &self, + canister: CanisterId, + method: &str, + ) -> T { + match self.query::(canister, method) { Err(_) => { self.start_canister(canister); - self.query_records(canister).unwrap() + self.query::(canister, method).unwrap() } Ok(records) => records, } @@ -683,7 +676,7 @@ impl Fixture { if self .canisters() .into_iter() - .map(|canister| extract_metrics(&self.force_query_records(canister))) + .map(|canister| extract_metrics(&self.force_query(canister, "records"))) .any(|metrics| metrics.pending_calls != 0) { return self.failed_with_reason( @@ -757,7 +750,7 @@ impl Fixture { records: self .canisters() .into_iter() - .map(|canister| (canister, self.force_query_records(canister))) + .map(|canister| (canister, self.force_query(canister, "records"))) .collect(), fixture: self.clone(), }, diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index 9ab0ce492ed..b099843d0e0 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -37,7 +37,8 @@ impl Default for Config { reply_bytes_range: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), instructions_count_range: (0, 0), timeout_secs_range: (1, 100), - calls_per_heartbeat: 3, + // calls_per_heartbeat: 3, + calls_per_heartbeat: 0, downstream_call_percentage: 0, best_effort_call_percentage: 100, } diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index 8775e0a66b0..13e27c64062 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -1,9 +1,9 @@ -use candid::CandidType; +use candid::{CandidType, Encode}; use futures::future::select_all; use ic_base_types::{CanisterId, PrincipalId}; use ic_cdk::{ api, - call::{Call, CallError, CallResult, ConfigurableCall, SendableCall}, + call::{Call, ConfigurableCall, SendableCall, SystemResult}, setup, }; use ic_cdk_macros::{heartbeat, init, query, update}; @@ -67,13 +67,12 @@ impl Message { padding: vec![0_u8; (bytes_count as usize).saturating_sub(std::mem::size_of::())], } } - - /// Returns the number of bytes the message consists of. - fn count_bytes(&self) -> usize { - std::mem::size_of::() + self.padding.len() - } } +/// Wrapper around the reply from `handle_call()` such that `serde_bytes` can be used. +#[derive(Serialize, Deserialize, CandidType)] +struct Reply(#[serde(with = "serde_bytes")] Vec); + /// Returns the next call index. fn next_call_index() -> u32 { CALL_INDEX.replace(CALL_INDEX.get() + 1) @@ -197,16 +196,17 @@ fn should_make_best_effort_call() -> bool { fn setup_call( call_tree_id: u32, call_depth: u32, -) -> (impl Future>, u32) { - let msg = Message::new(call_tree_id, call_depth, gen_call_bytes()); - let sent_bytes = msg.count_bytes() as u32; +) -> (impl Future>>, u32) { + let bytes = candid::Encode!(&Message::new(call_tree_id, call_depth, gen_call_bytes())).unwrap(); + let sent_bytes = bytes.len() as u32; let receiver = receiver(); let caller = (call_depth > 0).then_some(CanisterId::unchecked_from_principal(PrincipalId( api::msg_caller(), ))); let timeout_secs = should_make_best_effort_call().then_some(gen_timeout_secs()); - let call = Call::new(receiver.into(), "handle_call").with_arg(msg); + let call = Call::new(receiver.into(), "handle_call"); + let call = call.with_raw_args(bytes); let call = match timeout_secs { Some(timeout_secs) => call.change_timeout(timeout_secs), None => call.with_guaranteed_response(), @@ -232,7 +232,7 @@ fn setup_call( ) }); - (call.call(), index) + (call.call_raw(), index) } /// Updates the record at `index` using the `result` of the corresponding call. @@ -241,7 +241,7 @@ fn setup_call( /// subnet is at its limits. Note that since the call `index` is part of the records, removing /// the records for synchronous rejections will result in gaps in these numbers thus they are /// still included indirectly. -fn update_record(result: &CallResult, index: u32) { +fn update_record(result: &SystemResult>, index: u32) { // Updates the `Response` at `index` in `RECORDS`. let set_reply_in_call_record = move |response: Response| { RECORDS.with_borrow_mut(|records| { @@ -260,8 +260,9 @@ fn update_record(result: &CallResult, index: u32) { record.duration_and_response = Some((call_duration, response)); }); }; + match result { - Err(CallError::CallRejected(rejection)) if rejection.is_sync() => { + Err(rejection) if rejection.is_sync() => { // Remove the record for synchronous rejections. SYNCHRONOUS_REJECTIONS_COUNT.set(SYNCHRONOUS_REJECTIONS_COUNT.get() + 1); RECORDS.with_borrow_mut(|records| { @@ -273,16 +274,15 @@ fn update_record(result: &CallResult, index: u32) { .is_none()) }); } - Err(CallError::CallRejected(rejection)) => { + Err(rejection) => { set_reply_in_call_record(Response::Reject( rejection.reject_code().into(), rejection.reject_message().to_string(), )); } - Err(CallError::CandidDecodeFailed(..)) => unreachable!(), - Ok(msg) => { - set_reply_in_call_record(Response::Reply(msg.count_bytes() as u32)); + Ok(bytes) => { + set_reply_in_call_record(Response::Reply(bytes.len() as u32)); } } } @@ -316,7 +316,7 @@ async fn heartbeat() { /// - flipping the coin tells us to do so. /// - if it tells us not to do so but the attempted downstream call fails for any reason. #[update] -async fn handle_call(msg: Message) -> Vec { +async fn handle_call(msg: Message) -> Reply { // Make downstream calls as long as flipping the coin tells us to do so. while should_make_downstream_call() { let (future, record_index) = setup_call(msg.call_tree_id, msg.call_depth + 1); @@ -337,7 +337,7 @@ async fn handle_call(msg: Message) -> Vec { let counts = api::performance_counter(0) + gen_instructions_count() as u64; while counts > api::performance_counter(0) {} - vec![0_u8; payload_bytes as usize] + Reply(vec![0_u8; payload_bytes as usize]) } /// Initializes the `HASHER` by hashing our own canister ID. From 18466eb4db5affbbdf57b9fdf8b5bd18883a8702 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Tue, 21 Jan 2025 20:13:11 +0000 Subject: [PATCH 27/34] Automatically updated Cargo*.lock --- Cargo.Bazel.Fuzzing.json.lock | 2 +- Cargo.Bazel.json.lock | 2 +- Cargo.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 3084f095934..d6d698bcf82 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "07b96ead5f35f753cd65d3d30e79397e2d19015b5d1333c3d9a81c6aefccac6e", + "checksum": "31d108b4a4982572672c954576cb1de774d95b6efbedede950aed738c4911526", "crates": { "abnf 0.12.0": { "name": "abnf", diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 84d5102fcf9..459198d03cd 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "1e65184579336e8e32f98c87ea66a02736faf39eb650f2d387c7c34b8d0cb58f", + "checksum": "a6900533493381c6741d42a9829f03bc3632ff16b30648bfc27e3d63cb57ad96", "crates": { "abnf 0.12.0": { "name": "abnf", diff --git a/Cargo.lock b/Cargo.lock index 9827412f40d..5c6c0576998 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6453,7 +6453,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream 0.2.2", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] From 7aa284894770e89488cdf908121aadda83d85b64 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Tue, 21 Jan 2025 20:31:32 +0000 Subject: [PATCH 28/34] . --- Cargo.Bazel.json.lock | 2 +- Cargo.lock | 2 +- .../random_traffic_test/src/main.rs | 49 ++++++++----------- 3 files changed, 22 insertions(+), 31 deletions(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 84d5102fcf9..459198d03cd 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "1e65184579336e8e32f98c87ea66a02736faf39eb650f2d387c7c34b8d0cb58f", + "checksum": "a6900533493381c6741d42a9829f03bc3632ff16b30648bfc27e3d63cb57ad96", "crates": { "abnf 0.12.0": { "name": "abnf", diff --git a/Cargo.lock b/Cargo.lock index 9827412f40d..5c6c0576998 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6453,7 +6453,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream 0.2.2", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] diff --git a/rs/rust_canisters/random_traffic_test/src/main.rs b/rs/rust_canisters/random_traffic_test/src/main.rs index c257e5445cf..90ffa7e0669 100644 --- a/rs/rust_canisters/random_traffic_test/src/main.rs +++ b/rs/rust_canisters/random_traffic_test/src/main.rs @@ -1,9 +1,9 @@ -use candid::{CandidType, Encode}; +use candid::CandidType; use futures::future::select_all; use ic_base_types::{CanisterId, PrincipalId}; use ic_cdk::{ api, - call::{Call, ConfigurableCall, SendableCall, SystemResult}, + call::{Call, CallError, CallResult, ConfigurableCall, SendableCall}, setup, }; use ic_cdk_macros::{heartbeat, init, query, update}; @@ -67,6 +67,11 @@ impl Message { padding: vec![0_u8; (bytes_count as usize).saturating_sub(std::mem::size_of::())], } } + + /// Returns the number of bytes the message consists of. + fn count_bytes(&self) -> usize { + std::mem::size_of::() + self.padding.len() + } } /// Wrapper around the reply from `handle_call()` such that `serde_bytes` can be used. @@ -196,9 +201,9 @@ fn should_make_best_effort_call() -> bool { fn setup_call( call_tree_id: u32, call_depth: u32, -) -> (impl Future>>, u32) { - let bytes = candid::Encode!(&Message::new(call_tree_id, call_depth, gen_call_bytes())).unwrap(); - let sent_bytes = bytes.len() as u32; +) -> (impl Future>, u32) { + let msg = Message::new(call_tree_id, call_depth, gen_call_bytes()); + let sent_bytes = msg.count_bytes() as u32; let receiver = receiver(); let caller = (call_depth > 0).then_some(CanisterId::unchecked_from_principal(PrincipalId( api::msg_caller(), @@ -206,7 +211,7 @@ fn setup_call( let timeout_secs = should_make_best_effort_call().then_some(gen_timeout_secs()); let call = Call::new(receiver.into(), "handle_call"); - let call = call.with_raw_args(bytes); + let call = call.with_arg(msg); let call = match timeout_secs { Some(timeout_secs) => call.change_timeout(timeout_secs), None => call.with_guaranteed_response(), @@ -232,7 +237,7 @@ fn setup_call( ) }); - (call.call_raw(), index) + (call.call(), index) } /// Updates the record at `index` using the `result` of the corresponding call. @@ -241,7 +246,7 @@ fn setup_call( /// subnet is at its limits. Note that since the call `index` is part of the records, removing /// the records for synchronous rejections will result in gaps in these numbers thus they are /// still included indirectly. -fn update_record(result: &SystemResult>, index: u32) { +fn update_record(result: &CallResult, index: u32) { // Updates the `Response` at `index` in `RECORDS`. let set_reply_in_call_record = move |response: Response| { RECORDS.with_borrow_mut(|records| { @@ -262,7 +267,7 @@ fn update_record(result: &SystemResult>, index: u32) { }; match result { - Err(rejection) if rejection.is_sync() => { + Err(CallError::CallRejected(rejection)) if rejection.is_sync() => { // Remove the record for synchronous rejections. SYNCHRONOUS_REJECTIONS_COUNT.set(SYNCHRONOUS_REJECTIONS_COUNT.get() + 1); RECORDS.with_borrow_mut(|records| { @@ -274,15 +279,18 @@ fn update_record(result: &SystemResult>, index: u32) { .is_none()) }); } - Err(rejection) => { + Err(CallError::CallRejected(rejection)) => { set_reply_in_call_record(Response::Reject( rejection.reject_code().into(), rejection.reject_message().to_string(), )); } + Err(CallError::CandidDecodeFailed(error)) => { + unreachable!("{error}"); + } - Ok(bytes) => { - set_reply_in_call_record(Response::Reply(bytes.len() as u32)); + Ok(reply) => { + set_reply_in_call_record(Response::Reply(reply.0.len() as u32)); } } } @@ -316,25 +324,8 @@ async fn heartbeat() { /// - flipping the coin tells us to do so. /// - if it tells us not to do so but the attempted downstream call fails for any reason. #[update] -<<<<<<< HEAD async fn handle_call(msg: Message) -> Reply { // Make downstream calls as long as flipping the coin tells us to do so. -======= -async fn handle_call(msg: Message) -> Vec { - // Samples a weighted binomial distribution to decide whether to make a downstream call (true) - // or reply (false). Defaults to `false` for bad weights (e.g. both 0). - fn should_make_downstream_call() -> bool { - RNG.with_borrow_mut(|rng| { - WeightedIndex::new([CALL_WEIGHT.get(), REPLY_WEIGHT.get()]) - .is_ok_and(|dist| dist.sample(rng) == 0) - }) - } - - // Make downstream calls until - // - sampling the distribution tells us to stop. - // - setting up a call fails. - // - a downstream call is rejected for any reason. ->>>>>>> 9e2784a0d668048a457f90c593269161e7cbb2a2 while should_make_downstream_call() { let (future, record_index) = setup_call(msg.call_tree_id, msg.call_depth + 1); From 9cf615645041cc83ff6f2666b119128daa4a1bcc Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 22 Jan 2025 08:44:46 +0000 Subject: [PATCH 29/34] . --- rs/rust_canisters/random_traffic_test/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rs/rust_canisters/random_traffic_test/src/lib.rs b/rs/rust_canisters/random_traffic_test/src/lib.rs index b099843d0e0..9577671b003 100644 --- a/rs/rust_canisters/random_traffic_test/src/lib.rs +++ b/rs/rust_canisters/random_traffic_test/src/lib.rs @@ -37,7 +37,6 @@ impl Default for Config { reply_bytes_range: (0, MAX_INTER_CANISTER_PAYLOAD_IN_BYTES_U64 as u32), instructions_count_range: (0, 0), timeout_secs_range: (1, 100), - // calls_per_heartbeat: 3, calls_per_heartbeat: 0, downstream_call_percentage: 0, best_effort_call_percentage: 100, From 22e34c9f3c46ef789d8d293438f7b9191db55490 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 22 Jan 2025 09:52:19 +0000 Subject: [PATCH 30/34] . --- rs/rust_canisters/random_traffic_test/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/rust_canisters/random_traffic_test/Cargo.toml b/rs/rust_canisters/random_traffic_test/Cargo.toml index 8fcb8edb57e..7b585c1ef9e 100644 --- a/rs/rust_canisters/random_traffic_test/Cargo.toml +++ b/rs/rust_canisters/random_traffic_test/Cargo.toml @@ -10,7 +10,7 @@ path = "src/main.rs" [dependencies] candid = { workspace = true } ic-base-types = { path = "../../types/base_types" } -ic-cdk = { git = "https://github.com/dfinity/cdk-rs", branch = "next" } +ic-cdk = { git = "https://github.com/dfinity/cdk-rs", rev = "4e287ce51636b0e70768c193da38d2fc5324ea15" } ic-cdk-macros = { workspace = true } ic-error-types = { path = "../../types/error_types" } ic-types = { path = "../../types/types" } From 3c80963c8bbe9cc019c21be0cba737d185669ecb Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Wed, 22 Jan 2025 09:53:44 +0000 Subject: [PATCH 31/34] Automatically updated Cargo*.lock --- Cargo.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c6c0576998..94ce06540ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6364,13 +6364,14 @@ dependencies = [ [[package]] name = "ic-cdk" version = "0.18.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" dependencies = [ "candid", "ic-cdk-macros 0.18.0-alpha.1", "ic0 0.24.0-alpha.1", "serde", "serde_bytes", + "thiserror 2.0.11", ] [[package]] @@ -6446,7 +6447,7 @@ dependencies = [ [[package]] name = "ic-cdk-macros" version = "0.18.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" dependencies = [ "candid", "proc-macro2", @@ -13760,7 +13761,7 @@ checksum = "8de254dd67bbd58073e23dc1c8553ba12fa1dc610a19de94ad2bbcd0460c067f" [[package]] name = "ic0" version = "0.24.0-alpha.1" -source = "git+https://github.com/dfinity/cdk-rs?branch=next#605abcade44d8fa4888c564c6fab5c715d1c33c8" +source = "git+https://github.com/dfinity/cdk-rs?rev=4e287ce51636b0e70768c193da38d2fc5324ea15#4e287ce51636b0e70768c193da38d2fc5324ea15" [[package]] name = "ic_bls12_381" From 25d55d32640bcbba423a8129c0e61195513dca4e Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 22 Jan 2025 12:37:00 +0000 Subject: [PATCH 32/34] . --- Cargo.Bazel.json.lock | 8 +++++++- Cargo.Bazel.toml.lock | 3 ++- bazel/external_crates.bzl | 5 +++++ rs/rust_canisters/random_traffic_test/BUILD.bazel | 2 +- rs/rust_canisters/random_traffic_test/Cargo.toml | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 459198d03cd..41080f21f89 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "a6900533493381c6741d42a9829f03bc3632ff16b30648bfc27e3d63cb57ad96", + "checksum": "b68f88e7aabce820556a6b7f8b0373dfb1e55e46fa3f4ebece0092b0c634e5e3", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -19510,6 +19510,11 @@ "id": "ic-cdk-macros 0.9.0", "target": "ic_cdk_macros" }, + { + "id": "ic-cdk-macros 0.18.0-alpha.1", + "target": "ic_cdk_macros", + "alias": "ic_cdk_macros_next" + }, { "id": "indoc 1.0.9", "target": "indoc" @@ -87557,6 +87562,7 @@ "ic-cdk 0.16.0", "ic-cdk 0.18.0-alpha.1", "ic-cdk-macros 0.9.0", + "ic-cdk-macros 0.18.0-alpha.1", "ic-cdk-timers 0.11.0", "ic-certificate-verification 3.0.2", "ic-certification 3.0.2", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index 7e913bc71ea..2d78c613af6 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "abnf" @@ -3070,6 +3070,7 @@ dependencies = [ "ic-cbor", "ic-cdk 0.16.0", "ic-cdk 0.18.0-alpha.1", + "ic-cdk-macros 0.18.0-alpha.1", "ic-cdk-macros 0.9.0", "ic-cdk-timers", "ic-certificate-verification", diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index 6b21ddcd7ea..9d04b8104c0 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -592,6 +592,11 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable "ic-cdk-macros": crate.spec( version = "^0.9.0", ), + "ic-cdk-macros-next": crate.spec( + package = "ic-cdk-macros", + git = "https://github.com/dfinity/cdk-rs.git", + rev = "4e287ce51636b0e70768c193da38d2fc5324ea15", + ), "ic-cdk-next": crate.spec( package = "ic-cdk", git = "https://github.com/dfinity/cdk-rs.git", diff --git a/rs/rust_canisters/random_traffic_test/BUILD.bazel b/rs/rust_canisters/random_traffic_test/BUILD.bazel index 062f00a4aa1..356e46469df 100644 --- a/rs/rust_canisters/random_traffic_test/BUILD.bazel +++ b/rs/rust_canisters/random_traffic_test/BUILD.bazel @@ -16,7 +16,7 @@ DEPENDENCIES = [ ] MACRO_DEPENDENCIES = [ - "@crate_index//:ic-cdk-macros", + "@crate_index//:ic_cdk_macros_next", ] rust_library( diff --git a/rs/rust_canisters/random_traffic_test/Cargo.toml b/rs/rust_canisters/random_traffic_test/Cargo.toml index 7b585c1ef9e..58bf565c329 100644 --- a/rs/rust_canisters/random_traffic_test/Cargo.toml +++ b/rs/rust_canisters/random_traffic_test/Cargo.toml @@ -11,7 +11,7 @@ path = "src/main.rs" candid = { workspace = true } ic-base-types = { path = "../../types/base_types" } ic-cdk = { git = "https://github.com/dfinity/cdk-rs", rev = "4e287ce51636b0e70768c193da38d2fc5324ea15" } -ic-cdk-macros = { workspace = true } +ic-cdk-macros = { git = "https://github.com/dfinity/cdk-rs", rev = "4e287ce51636b0e70768c193da38d2fc5324ea15" } ic-error-types = { path = "../../types/error_types" } ic-types = { path = "../../types/types" } futures = { workspace = true } From 3441b437eb117e39fbda7baa1c12dd1db377737e Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Wed, 22 Jan 2025 12:39:04 +0000 Subject: [PATCH 33/34] Automatically updated Cargo*.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c79d9e7d113..fe69c30577a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18170,7 +18170,7 @@ dependencies = [ "futures", "ic-base-types", "ic-cdk 0.18.0-alpha.1", - "ic-cdk-macros 0.9.0", + "ic-cdk-macros 0.18.0-alpha.1", "ic-error-types", "ic-types", "rand 0.8.5", From 81bf7cfc30bcdf7c1a648d3f169096b4fa226883 Mon Sep 17 00:00:00 2001 From: Christian Stieger Date: Wed, 22 Jan 2025 13:46:55 +0000 Subject: [PATCH 34/34] . --- .../governance/canbench/canbench_results.yml | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/rs/nns/governance/canbench/canbench_results.yml b/rs/nns/governance/canbench/canbench_results.yml index 7fc2d61ada9..911234a3a49 100644 --- a/rs/nns/governance/canbench/canbench_results.yml +++ b/rs/nns/governance/canbench/canbench_results.yml @@ -13,55 +13,55 @@ benches: scopes: {} add_neuron_inactive_maximum: total: - instructions: 112624643 + instructions: 112624637 heap_increase: 1 stable_memory_increase: 0 scopes: {} add_neuron_inactive_typical: total: - instructions: 8497304 + instructions: 8497298 heap_increase: 0 stable_memory_increase: 0 scopes: {} cascading_vote_all_heap: total: - instructions: 35002536 + instructions: 35004550 heap_increase: 0 stable_memory_increase: 128 scopes: {} cascading_vote_heap_neurons_stable_index: total: - instructions: 61137575 + instructions: 61139589 heap_increase: 0 stable_memory_increase: 128 scopes: {} cascading_vote_stable_everything: total: - instructions: 188621982 + instructions: 188616472 heap_increase: 0 stable_memory_increase: 128 scopes: {} cascading_vote_stable_neurons_with_heap_index: total: - instructions: 162353547 + instructions: 162348037 heap_increase: 0 stable_memory_increase: 128 scopes: {} centralized_following_all_stable: total: - instructions: 78268135 + instructions: 78266563 heap_increase: 0 stable_memory_increase: 128 scopes: {} compute_ballots_for_new_proposal_with_stable_neurons: total: - instructions: 2169168 + instructions: 2223231 heap_increase: 0 stable_memory_increase: 0 scopes: {} draw_maturity_from_neurons_fund_heap: total: - instructions: 7598030 + instructions: 7597898 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -85,7 +85,7 @@ benches: scopes: {} list_neurons_heap: total: - instructions: 4763239 + instructions: 4708588 heap_increase: 9 stable_memory_increase: 0 scopes: {} @@ -103,13 +103,13 @@ benches: scopes: {} list_neurons_stable: total: - instructions: 113374022 + instructions: 113290234 heap_increase: 5 stable_memory_increase: 0 scopes: {} list_proposals: total: - instructions: 168095717 + instructions: 168095016 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -127,13 +127,13 @@ benches: scopes: {} neuron_data_validation_heap: total: - instructions: 406864991 + instructions: 406862191 heap_increase: 0 stable_memory_increase: 0 scopes: {} neuron_data_validation_stable: total: - instructions: 362661286 + instructions: 362659179 heap_increase: 0 stable_memory_increase: 0 scopes: {} @@ -145,19 +145,19 @@ benches: scopes: {} neuron_metrics_calculation_stable: total: - instructions: 3027095 + instructions: 3026795 heap_increase: 0 stable_memory_increase: 0 scopes: {} range_neurons_performance: total: - instructions: 56448740 + instructions: 56447340 heap_increase: 0 stable_memory_increase: 0 scopes: {} single_vote_all_stable: total: - instructions: 2805838 + instructions: 2803120 heap_increase: 0 stable_memory_increase: 128 scopes: {}