From e74895b2dad862f19aa04fcab8e73f3e773a9117 Mon Sep 17 00:00:00 2001 From: itowlson Date: Thu, 12 Sep 2024 15:39:18 +1200 Subject: [PATCH] WIT-based environment syntax Signed-off-by: itowlson --- Cargo.lock | 81 +++++++++++ crates/environments/Cargo.toml | 5 + .../src/environment_definition.rs | 127 ++++++++++++---- crates/environments/src/lib.rs | 136 +++++++----------- 4 files changed, 237 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cdb9ba50b4..83f71d413f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7314,7 +7314,10 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "async-trait", + "bytes", "futures", + "futures-util", + "id-arena", "indexmap 2.2.6", "miette 7.2.0", "oci-distribution 0.11.0 (git+https://github.com/fermyon/oci-distribution?rev=7e4ce9be9bcd22e78a28f06204931f10c44402ba)", @@ -7333,6 +7336,8 @@ dependencies = [ "wac-resolver", "wac-types", "wasm-pkg-loader", + "wit-component 0.217.0", + "wit-parser 0.217.0", ] [[package]] @@ -9757,6 +9762,16 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasm-encoder" +version = "0.217.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b88b0814c9a2b323a9b46c687e726996c255ac8b64aa237dd11c81ed4854760" +dependencies = [ + "leb128", + "wasmparser 0.217.0", +] + [[package]] name = "wasm-metadata" version = "0.10.20" @@ -9821,6 +9836,22 @@ dependencies = [ "wasmparser 0.209.1", ] +[[package]] +name = "wasm-metadata" +version = "0.217.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a146bf9a60e9264f0548a2599aa9656dba9a641eff9ab88299dc2a637e483c" +dependencies = [ + "anyhow", + "indexmap 2.2.6", + "serde 1.0.197", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder 0.217.0", + "wasmparser 0.217.0", +] + [[package]] name = "wasm-pkg-common" version = "0.4.1" @@ -9941,6 +9972,19 @@ dependencies = [ "serde 1.0.197", ] +[[package]] +name = "wasmparser" +version = "0.217.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca917a21307d3adf2b9857b94dd05ebf8496bdcff4437a9b9fb3899d3e6c74e7" +dependencies = [ + "ahash", + "bitflags 2.5.0", + "hashbrown 0.14.3", + "indexmap 2.2.6", + "semver", +] + [[package]] name = "wasmprinter" version = "0.2.80" @@ -10898,6 +10942,25 @@ dependencies = [ "wit-parser 0.209.1", ] +[[package]] +name = "wit-component" +version = "0.217.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7117809905e49db716d81e794f79590c052bf2fdbbcda1731ca0fb28f6f3ddf" +dependencies = [ + "anyhow", + "bitflags 2.5.0", + "indexmap 2.2.6", + "log", + "serde 1.0.197", + "serde_derive", + "serde_json", + "wasm-encoder 0.217.0", + "wasm-metadata 0.217.0", + "wasmparser 0.217.0", + "wit-parser 0.217.0", +] + [[package]] name = "wit-parser" version = "0.13.2" @@ -10969,6 +11032,24 @@ dependencies = [ "wasmparser 0.209.1", ] +[[package]] +name = "wit-parser" +version = "0.217.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb893dcd6d370cfdf19a0d9adfcd403efb8e544e1a0ea3a8b81a21fe392eaa78" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.2.6", + "log", + "semver", + "serde 1.0.197", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.217.0", +] + [[package]] name = "witx" version = "0.9.1" diff --git a/crates/environments/Cargo.toml b/crates/environments/Cargo.toml index 350599bda4..daea8209ed 100644 --- a/crates/environments/Cargo.toml +++ b/crates/environments/Cargo.toml @@ -7,7 +7,10 @@ edition = { workspace = true } [dependencies] anyhow = { workspace = true } async-trait = "0.1" +bytes = "1.1" futures = "0.3" +futures-util = "0.3" +id-arena = "2" indexmap = "2.2.6" miette = "7.2.0" oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "7e4ce9be9bcd22e78a28f06204931f10c44402ba" } @@ -26,6 +29,8 @@ wac-parser = "0.6.0" wac-resolver = "0.6.0" wac-types = "0.6.0" wasm-pkg-loader = "0.4.1" +wit-component = "0.217.0" +wit-parser = "0.217.0" [lints] workspace = true diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs index 79813ecac3..5e576b2cdd 100644 --- a/crates/environments/src/environment_definition.rs +++ b/crates/environments/src/environment_definition.rs @@ -1,37 +1,112 @@ -use wasm_pkg_loader::PackageRef; +use anyhow::Context; -#[derive(Debug, serde::Deserialize)] -pub struct TargetEnvironment { - pub name: String, - pub environments: std::collections::HashMap, -} +pub async fn load_environment(env_id: &str) -> anyhow::Result { + use futures_util::TryStreamExt; + + let (pkg_name, pkg_ver) = env_id.split_once('@').unwrap(); + + let mut client = wasm_pkg_loader::Client::with_global_defaults()?; + + let package = pkg_name.to_owned().try_into().context("pkg ref parse")?; + let version = wasm_pkg_loader::Version::parse(pkg_ver).context("pkg ver parse")?; + + let release = client + .get_release(&package, &version) + .await + .context("get release")?; + let stm = client + .stream_content(&package, &release) + .await + .context("stream content")?; + let bytes = stm + .try_collect::() + .await + .context("collect stm")? + .to_vec(); -#[derive(Debug, Eq, Hash, PartialEq, serde::Deserialize)] -pub struct TargetWorld { - wit_package: PackageRef, - package_ver: String, // TODO: tidy to semver::Version - world_name: WorldNames, + TargetEnvironment::new(env_id.to_owned(), bytes) } -#[derive(Debug, Eq, Hash, PartialEq, serde::Deserialize)] -#[serde(untagged)] -enum WorldNames { - Exactly(String), - AnyOf(Vec), +pub struct TargetEnvironment { + name: String, + decoded: wit_parser::decoding::DecodedWasm, + package: wit_parser::Package, // saves unwrapping it every time + package_id: id_arena::Id, + package_bytes: Vec, } -impl TargetWorld { - fn versioned_name(&self, world_name: &str) -> String { - format!("{}/{}@{}", self.wit_package, world_name, self.package_ver) +impl TargetEnvironment { + fn new(name: String, bytes: Vec) -> anyhow::Result { + let decoded = wit_component::decode(&bytes).context("decode wasm")?; + let package_id = decoded.package(); + let package = decoded + .resolve() + .packages + .get(package_id) + .context("should had a package")? + .clone(); + + Ok(Self { + name, + decoded, + package, + package_id, + package_bytes: bytes, + }) + } + + pub fn is_world_for(&self, trigger_type: &TriggerType, world: &wit_parser::World) -> bool { + world.name.starts_with(&format!("trigger-{trigger_type}")) + && world.package.is_some_and(|p| p == self.package_id) + } + + pub fn supports_trigger_type(&self, trigger_type: &TriggerType) -> bool { + self.decoded + .resolve() + .worlds + .iter() + .any(|(_, world)| self.is_world_for(trigger_type, world)) + } + + pub fn worlds(&self, trigger_type: &TriggerType) -> Vec { + self.decoded + .resolve() + .worlds + .iter() + .filter(|(_, world)| self.is_world_for(trigger_type, world)) + .map(|(_, world)| self.world_qname(world)) + .collect() + } + + /// Fully qualified world name (e.g. fermyon:spin/http-trigger@2.0.0) + fn world_qname(&self, world: &wit_parser::World) -> String { + let version_suffix = match self.package_version() { + Some(version) => format!("@{version}"), + None => "".to_owned(), + }; + format!( + "{}/{}{version_suffix}", + self.package_namespaced_name(), + world.name, + ) + } + + /// The environment name for UI purposes + pub fn name(&self) -> &str { + &self.name + } + + /// Namespaced but unversioned package name (e.g. spin:cli) + pub fn package_namespaced_name(&self) -> String { + format!("{}:{}", self.package.name.namespace, self.package.name.name) + } + + pub fn package_version(&self) -> Option<&semver::Version> { + self.package.name.version.as_ref() } - pub fn versioned_names(&self) -> Vec { - match &self.world_name { - WorldNames::Exactly(name) => vec![self.versioned_name(name)], - WorldNames::AnyOf(names) => { - names.iter().map(|name| self.versioned_name(name)).collect() - } - } + pub fn package_bytes(&self) -> &[u8] { + &self.package_bytes } } diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 09e2ef0757..2997d96417 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -1,9 +1,9 @@ -use anyhow::{anyhow, Context}; +use anyhow::anyhow; mod environment_definition; mod loader; -use environment_definition::{TargetEnvironment, TargetWorld, TriggerType}; +use environment_definition::{load_environment, TargetEnvironment, TriggerType}; pub use loader::ResolutionContext; use loader::{load_and_resolve_all, ComponentToValidate}; @@ -12,43 +12,11 @@ pub async fn validate_application_against_environment_ids( app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { - let envs = join_all_result(env_ids.map(resolve_environment_id)).await?; + let envs = join_all_result(env_ids.map(load_environment)).await?; validate_application_against_environments(&envs, app, resolution_context).await } -async fn resolve_environment_id(id: &str) -> anyhow::Result { - let (name, ver) = id.split_once('@').ok_or(anyhow!( - "Target environment '{id}' does not specify a version" - ))?; - let client = oci_distribution::Client::default(); - let auth = oci_distribution::secrets::RegistryAuth::Anonymous; - let env_def_ref = - oci_distribution::Reference::try_from(format!("ghcr.io/itowlson/spinenvs/{name}:{ver}"))?; - let (man, _digest) = client - .pull_manifest(&env_def_ref, &auth) - .await - .with_context(|| format!("Failed to find environment '{id}' in registry"))?; - let im = match man { - oci_distribution::manifest::OciManifest::Image(im) => im, - oci_distribution::manifest::OciManifest::ImageIndex(_ind) => { - anyhow::bail!("Environment '{id}' definition is unusable - stored in registry in incorrect format") - } - }; - let the_layer = &im.layers[0]; - let mut out = Vec::with_capacity(the_layer.size.try_into().unwrap_or_default()); - client - .pull_blob(&env_def_ref, the_layer, &mut out) - .await - .with_context(|| { - format!("Failed to download environment '{id}' definition from registry") - })?; - let te = serde_json::from_slice(&out).with_context(|| { - format!("Failed to load environment '{id}' definition - invalid JSON schema") - })?; - Ok(te) -} - -pub async fn validate_application_against_environments( +async fn validate_application_against_environments( envs: &[TargetEnvironment], app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, @@ -56,13 +24,10 @@ pub async fn validate_application_against_environments( use futures::FutureExt; for trigger_type in app.triggers.keys() { - if let Some(env) = envs - .iter() - .find(|e| !e.environments.contains_key(trigger_type)) - { + if let Some(env) = envs.iter().find(|e| !e.supports_trigger_type(trigger_type)) { anyhow::bail!( "Environment {} does not support trigger type {trigger_type}", - env.name + env.name() ); } } @@ -87,28 +52,9 @@ async fn validate_component_against_environments( trigger_type: &TriggerType, component: &ComponentToValidate<'_>, ) -> anyhow::Result<()> { - let worlds = envs - .iter() - .map(|e| { - e.environments - .get(trigger_type) - .ok_or(anyhow!( - "Environment '{}' doesn't support trigger type {trigger_type}", - e.name - )) - .map(|w| (e.name.as_str(), w)) - }) - .collect::, _>>()?; - validate_component_against_worlds(worlds.into_iter(), component).await?; - Ok(()) -} - -async fn validate_component_against_worlds( - target_worlds: impl Iterator, - component: &ComponentToValidate<'_>, -) -> anyhow::Result<()> { - for (env_name, target_world) in target_worlds { - validate_wasm_against_any_world(env_name, target_world, component).await?; + for env in envs { + let worlds = env.worlds(trigger_type); + validate_wasm_against_any_world(env, &worlds, component).await?; } tracing::info!( @@ -120,21 +66,21 @@ async fn validate_component_against_worlds( } async fn validate_wasm_against_any_world( - env_name: &str, - target_world: &TargetWorld, + env: &TargetEnvironment, + world_names: &[String], component: &ComponentToValidate<'_>, ) -> anyhow::Result<()> { let mut result = Ok(()); - for target_str in target_world.versioned_names() { - tracing::info!( - "Trying component {} {} against target world {target_str}", + for target_world in world_names { + tracing::debug!( + "Trying component {} {} against target world {target_world}", component.id(), component.source_description(), ); - match validate_wasm_against_world(env_name, &target_str, component).await { + match validate_wasm_against_world(env, target_world, component).await { Ok(()) => { tracing::info!( - "Validated component {} {} against target world {target_str}", + "Validated component {} {} against target world {target_world}", component.id(), component.source_description(), ); @@ -143,7 +89,7 @@ async fn validate_wasm_against_any_world( Err(e) => { // Record the error, but continue in case a different world succeeds tracing::info!( - "Rejecting component {} {} for target world {target_str} because {e:?}", + "Rejecting component {} {} for target world {target_world} because {e:?}", component.id(), component.source_description(), ); @@ -155,34 +101,52 @@ async fn validate_wasm_against_any_world( } async fn validate_wasm_against_world( - env_name: &str, - target_str: &str, + env: &TargetEnvironment, + target_world: &str, component: &ComponentToValidate<'_>, ) -> anyhow::Result<()> { - let comp_name = "root:component"; + // Because we are abusing a composition tool to do validation, we have to + // provide a name by which to refer to the component in the dummy composition. + let component_name = "root:component"; + let component_key = wac_types::BorrowedPackageKey::from_name_and_version(component_name, None); + + // wac is going to get the world from the environment package bytes. + // This constructs a key for that mapping. + let env_pkg_name = env.package_namespaced_name(); + let env_pkg_key = + wac_types::BorrowedPackageKey::from_name_and_version(&env_pkg_name, env.package_version()); + + let env_name = env.name(); let wac_text = format!( r#" - package validate:component@1.0.0 targets {target_str}; - let c = new {comp_name} {{ ... }}; + package validate:component@1.0.0 targets {target_world}; + let c = new {component_name} {{ ... }}; export c...; "# ); let doc = wac_parser::Document::parse(&wac_text)?; - let compkey = wac_types::BorrowedPackageKey::from_name_and_version(comp_name, None); + // TODO: if we end up needing the registry, we need to do this dance + // for things we are providing separately, or the registry will try to + // hoover them up and will fail. + // let mut refpkgs = wac_resolver::packages(&doc)?; + // refpkgs.shift_remove(&env_pkg_key); + // refpkgs.shift_remove(&component_key); - let mut refpkgs = wac_resolver::packages(&doc)?; - refpkgs.retain(|k, _| k != &compkey); + // TODO: determine if this is needed in circumstances other than the simple test + // let reg_resolver = wac_resolver::RegistryPackageResolver::new(Some("wa.dev"), None).await?; + // let mut packages = reg_resolver + // .resolve(&refpkgs) + // .await + // .context("reg_resolver.resolve failed")?; - let reg_resolver = wac_resolver::RegistryPackageResolver::new(Some("wa.dev"), None).await?; - let mut packages = reg_resolver - .resolve(&refpkgs) - .await - .context("reg_resolver.resolve failed")?; + let mut packages: indexmap::IndexMap> = + Default::default(); - packages.insert(compkey, component.wasm_bytes().to_vec()); + packages.insert(env_pkg_key, env.package_bytes().to_vec()); + packages.insert(component_key, component.wasm_bytes().to_vec()); match doc.resolve(packages) { Ok(_) => Ok(()), @@ -195,7 +159,7 @@ async fn validate_wasm_against_world( } Err(wac_parser::resolution::Error::PackageMissingExport { export, .. }) => { // TODO: The export here seems wrong - it seems to contain the world name rather than the interface name - Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_str} requires an export named {export}, which the component does not provide", component.id(), component.source_description())) + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_world} requires an export named {export}, which the component does not provide", component.id(), component.source_description())) } Err(wac_parser::resolution::Error::ImportNotInTarget { name, world, .. }) => { Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} does not provide an import named {name}, which the component requires", component.id(), component.source_description()))