diff --git a/Cargo.lock b/Cargo.lock index 5084e9dd..2386a6ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -831,7 +831,7 @@ version = "0.69.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cexpr", "clang-sys", "lazy_static", @@ -853,9 +853,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-buffer" @@ -1338,8 +1338,7 @@ dependencies = [ [[package]] name = "containerd-shim-wasm" -version = "0.4.0" -source = "git+https://github.com/containerd/runwasi?rev=c768e5b0919ca02903a301bf82a390489437dabe#c768e5b0919ca02903a301bf82a390489437dabe" +version = "0.5.0" dependencies = [ "anyhow", "caps", @@ -1355,14 +1354,18 @@ dependencies = [ "log", "nix 0.27.1", "oci-spec", + "prost-types 0.11.9", "protobuf 3.2.0", "serde", "serde_json", + "sha256", "thiserror", "tokio", + "tokio-stream", "ttrpc", + "wasmparser 0.121.2", "wat", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2167,9 +2170,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2182,9 +2185,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -2192,15 +2195,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2210,9 +2213,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -2244,9 +2247,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -2255,21 +2258,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -2298,7 +2301,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "debugid", "fxhash", "serde", @@ -3200,15 +3203,15 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libcgroups" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a5d56267f6ee2386e6d49a0333eaf20eb04fca611e94644af7505bace5fd7f" +checksum = "aae95afce340512934ea70c9cc8125b100b4146120633d57cdf4174b611b672c" dependencies = [ "fixedbitset 0.4.2", "nix 0.27.1", @@ -3221,11 +3224,11 @@ dependencies = [ [[package]] name = "libcontainer" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63ffa9cc1f2e58ff183ab0a523d491ff0591a9191a74fcff283c9c1a6d11186" +checksum = "7d2dcd1a969a82f9c021d6e4cc9ec2a93f4df7686637bd0124dda92374fbcff3" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "caps", "chrono", "fastrand 2.0.1", @@ -3233,6 +3236,7 @@ dependencies = [ "libc", "libcgroups", "libseccomp", + "nc", "nix 0.27.1", "oci-spec", "once_cell", @@ -3294,7 +3298,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "redox_syscall 0.4.1", ] @@ -3325,7 +3329,7 @@ checksum = "43adbef635c87aaf72870e0a1a8cb39eefcc2c0b0386c75a9436ba6048548f07" dependencies = [ "async-trait", "base64 0.21.5", - "bitflags 2.4.1", + "bitflags 2.4.2", "bytes", "fallible-iterator 0.3.0", "futures", @@ -3347,7 +3351,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "600b1fc036f15466a4293adbf82d1c3ac7a22b865b5d501db325adeb8a116063" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cc", "fallible-iterator 0.3.0", "indexmap 2.1.0", @@ -3740,7 +3744,7 @@ checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef" dependencies = [ "base64 0.21.5", "bindgen", - "bitflags 2.4.1", + "bitflags 2.4.2", "btoi", "byteorder", "bytes", @@ -3783,6 +3787,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nc" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98c2a2fe6028696ccf8ec0213734da32b901e50b23d8fd3fa953b3498eb3842f" +dependencies = [ + "cc", +] + [[package]] name = "nix" version = "0.25.1" @@ -3814,7 +3827,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if 1.0.0", "libc", "memoffset 0.9.0", @@ -3989,7 +4002,7 @@ version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if 1.0.0", "foreign-types", "libc", @@ -4554,7 +4567,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "chrono", "flate2", "hex", @@ -4569,7 +4582,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "chrono", "hex", ] @@ -5087,7 +5100,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "fallible-iterator 0.2.0", "fallible-streaming-iterator", "hashlink", @@ -5182,7 +5195,7 @@ version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "itoa", "libc", @@ -5542,6 +5555,19 @@ dependencies = [ "digest", ] +[[package]] +name = "sha256" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18278f6a914fa3070aa316493f7d2ddfb9ac86ebc06fa3b83bffda487e9065b0" +dependencies = [ + "async-trait", + "bytes", + "hex", + "sha2", + "tokio", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -6288,7 +6314,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0682e006dd35771e392a6623ac180999a9a854b1d4a6c12fb2e804941c2b1f58" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cap-fs-ext", "cap-std", "fd-lock", @@ -6484,9 +6510,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.34.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -6754,7 +6780,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "base64 0.21.5", - "bitflags 2.4.1", + "bitflags 2.4.2", "bytes", "futures-core", "futures-util", @@ -7193,7 +7219,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "449d17849e3c83a931374442fe2deee4d6bd1ebf469719ef44192e9e82e19c89" dependencies = [ "anyhow", - "bitflags 2.4.1", + "bitflags 2.4.2", "cap-rand", "cap-std", "io-extras", @@ -7390,7 +7416,18 @@ version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9148127f39cbffe43efee8d5442b16ecdba21567785268daa1ec9e134389705" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", + "indexmap 2.1.0", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.121.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dbe55c8f9d0dbd25d9447a5a889ff90c0cc3feaa7395310d3d826b2c703eaab" +dependencies = [ + "bitflags 2.4.2", "indexmap 2.1.0", "semver", ] @@ -7685,7 +7722,7 @@ checksum = "902cc299b73655c36679b77efdfce4bb5971992f1a4a8a436dd3809a6848ff0e" dependencies = [ "anyhow", "async-trait", - "bitflags 2.4.1", + "bitflags 2.4.2", "bytes", "cap-fs-ext", "cap-net-ext", @@ -7866,7 +7903,7 @@ checksum = "737728db69a7657a5f6a7bac445c02d8564d603d62c46c95edf928554e67d072" dependencies = [ "anyhow", "async-trait", - "bitflags 2.4.1", + "bitflags 2.4.2", "thiserror", "tracing", "wasmtime", @@ -8113,7 +8150,7 @@ version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9643b83820c0cd246ecabe5fa454dd04ba4fa67996369466d0747472d337346" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "windows-sys 0.52.0", ] @@ -8124,7 +8161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65672b7a81f9c7a4420af2ad8d0de608e27b520a6d4b68f29f5146e060a86ee4" dependencies = [ "anyhow", - "bitflags 2.4.1", + "bitflags 2.4.2", "indexmap 2.1.0", "log", "serde", diff --git a/containerd-shim-spin/Cargo.toml b/containerd-shim-spin/Cargo.toml index 8e661282..015a8813 100644 --- a/containerd-shim-spin/Cargo.toml +++ b/containerd-shim-spin/Cargo.toml @@ -11,7 +11,8 @@ Containerd shim for running Spin workloads. """ [dependencies] -containerd-shim-wasm = { git = "https://github.com/containerd/runwasi", rev = "c768e5b0919ca02903a301bf82a390489437dabe" } +# containerd-shim-wasm = { git = "https://github.com/containerd/runwasi", rev = "c768e5b0919ca02903a301bf82a390489437dabe" } +containerd-shim-wasm = { path = "../../../containerd/runwasi/crates/containerd-shim-wasm" } log = "0.4" spin-app = { git = "https://github.com/fermyon/spin", tag = "v2.2.0" } spin-core = { git = "https://github.com/fermyon/spin", tag = "v2.2.0" } diff --git a/containerd-shim-spin/src/engine.rs b/containerd-shim-spin/src/engine.rs index 2eed36b7..50e957c1 100644 --- a/containerd-shim-spin/src/engine.rs +++ b/containerd-shim-spin/src/engine.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, ensure, Context, Result}; use containerd_shim_wasm::container::{Engine, RuntimeContext, Stdio}; +use containerd_shim_wasm::sandbox::WasmLayer; use log::info; use oci_spec::image::MediaType; use spin_app::locked::LockedApp; @@ -10,9 +11,11 @@ use spin_redis_engine::RedisTrigger; use spin_trigger::TriggerHooks; use spin_trigger::{loader, RuntimeConfig, TriggerExecutor, TriggerExecutorBuilder}; use spin_trigger_http::HttpTrigger; +use std::collections::hash_map::DefaultHasher; use std::collections::HashSet; use std::env; use std::fs::File; +use std::hash::{Hash, Hasher}; use std::io::Write; use std::net::SocketAddr; use std::net::ToSocketAddrs; @@ -27,8 +30,18 @@ const SPIN_ADDR: &str = "0.0.0.0:80"; /// root `/` of the container. const RUNTIME_CONFIG_PATH: &str = "/runtime-config.toml"; -#[derive(Clone, Default)] -pub struct SpinEngine; +#[derive(Clone)] +pub struct SpinEngine { + pub(crate) wasmtime_engine: wasmtime::Engine, +} + +impl Default for SpinEngine { + fn default() -> Self { + Self { + wasmtime_engine: wasmtime::Engine::new(&wasmtime::Config::default()).unwrap(), + } + } +} struct StdioTriggerHook; impl TriggerHooks for StdioTriggerHook { @@ -64,16 +77,22 @@ impl std::fmt::Debug for AppSource { impl SpinEngine { async fn app_source(&self, ctx: &impl RuntimeContext, cache: &Cache) -> Result { - match ctx.wasm_layers() { - [] => Ok(AppSource::File( - spin_common::paths::resolve_manifest_file_path("/spin.toml")?, - )), - layers => { - info!( - " >>> configuring spin oci application {}", - ctx.wasm_layers().len() + match ctx.entrypoint().source { + containerd_shim_wasm::container::Source::File(f) => { + log::warn!( + ">>> attempting to load a spin application from a local file {:?}", + f ); + Ok(AppSource::File(f)) + } + containerd_shim_wasm::container::Source::Oci(layers) => { + info!(" >>> configuring spin oci application {}", layers.len()); + + for layer in layers { + log::debug!("<<< layer config: {:?}", layer.config); + } + for artifact in layers { match artifact.config.media_type() { MediaType::Other(name) @@ -85,14 +104,13 @@ impl SpinEngine { .context("failed to create spin.json")? .write_all(&artifact.layer) .context("failed to write spin.json")?; - let lockfile = std::str::from_utf8(&artifact.layer)?; - log::info!("lockfile: {:?}", lockfile); + log::debug!("<<< loading Spin lockfile"); } MediaType::Other(name) if name == "application/vnd.wasm.content.layer.v1+wasm" => { - log::info!( - "writing artifact config to cache, near {:?}", + log::debug!( + "<<< writing artifact config to cache, near {:?}", cache.manifests_dir() ); cache @@ -102,15 +120,20 @@ impl SpinEngine { MediaType::Other(name) if name == "application/vnd.wasm.content.layer.v1+data" => { - log::info!( - "writing data layer to cache, near {:?}", + log::debug!( + "<<< writing data layer to cache, near {:?}", cache.manifests_dir() ); cache .write_data(&artifact.layer, &artifact.config.digest()) .await?; } - _ => {} + _ => { + log::debug!( + "<<< unknown media type {:?}", + artifact.config.media_type() + ); + } } } Ok(AppSource::Oci) @@ -277,6 +300,40 @@ impl Engine for SpinEngine { fn can_handle(&self, _ctx: &impl RuntimeContext) -> Result<()> { Ok(()) } + + fn supported_layers_types() -> &'static [&'static str] { + &[ + "application/vnd.wasm.content.layer.v1+wasm", + "application/vnd.wasm.content.layer.v1+data", + "application/vnd.fermyon.spin.application.v1+config", + ] + } + + fn precompile(&self, layer: &WasmLayer) -> Option>> { + log::info!( + "Precompiling layer with Spin Engine: {:?}", + layer.config.digest() + ); + + match layer.config.media_type() { + MediaType::Other(name) => { + if name == "application/vnd.wasm.content.layer.v1+wasm" { + Some(self.wasmtime_engine.precompile_module(&layer.layer)) + } else { + None + } + } + _ => None, + } + } + + fn can_precompile(&self) -> Option { + let mut hasher = DefaultHasher::new(); + self.wasmtime_engine + .precompile_compatibility_hash() + .hash(&mut hasher); + Some(hasher.finish().to_string()) + } } fn parse_addr(addr: &str) -> Result {