feat(hydroflow_plus)!: minimize dependencies pulled into trybuild b…

…uilds (#1611)

We don't need `hydroflow_lang` in the runtime builds, do some feature
flagging to avoid it.

hydroflow/Cargo.toml

@@ -11,8 +11,9 @@ description = "Hydro's low-level dataflow runtime and IR"
 workspace = true
 
 [features]
-default = [ "macros", "debugging" ]
+default = [ "macros", "debugging", "meta" ]
 
+meta = [ "dep:hydroflow_lang" ]
 macros = [ "hydroflow_macro", "hydroflow_datalog" ]
 hydroflow_macro = [ "dep:hydroflow_macro" ]
 hydroflow_datalog = [ "dep:hydroflow_datalog" ]
@@ -35,7 +36,7 @@ bytes = "1.1.0"
 futures = "0.3.0"
 hydroflow_deploy_integration = { optional = true, path = "../hydro_deploy/hydroflow_deploy_integration", version = "^0.10.0" }
 hydroflow_datalog = { optional = true, path = "../hydroflow_datalog", version = "^0.10.0" }
-hydroflow_lang = { path = "../hydroflow_lang", version = "^0.10.0" }
+hydroflow_lang = { path = "../hydroflow_lang", version = "^0.10.0", optional = true }
 hydroflow_macro = { optional = true, path = "../hydroflow_macro", version = "^0.10.0" }
 itertools = "0.10.0"
 lattices = { path = "../lattices", version = "^0.5.8", features = [ "serde" ] }

hydroflow/src/lib.rs

@@ -20,12 +20,14 @@ pub mod compiled;
 pub mod scheduled;
 pub mod util;
 
+#[cfg(feature = "meta")]
+pub use hydroflow_lang as lang;
 #[cfg(feature = "python")]
 pub use pyo3;
 pub use variadics::{self, var_args, var_expr, var_type};
 pub use {
-    bincode, bytes, futures, hydroflow_lang as lang, itertools, lattices, pusherator, rustc_hash,
-    serde, serde_json, tokio, tokio_stream, tokio_util, tracing, web_time,
+    bincode, bytes, futures, itertools, lattices, pusherator, rustc_hash, serde, serde_json, tokio,
+    tokio_stream, tokio_util, tracing, web_time,
 };
 
 /// `#[macro_use]` automagically brings the declarative macro export to the crate-level.

hydroflow/src/scheduled/graph.rs

@@ -6,7 +6,9 @@ use std::cell::Cell;
 use std::future::Future;
 use std::marker::PhantomData;
 
+#[cfg(feature = "meta")]
 use hydroflow_lang::diagnostic::{Diagnostic, SerdeSpan};
+#[cfg(feature = "meta")]
 use hydroflow_lang::graph::HydroflowGraph;
 use ref_cast::RefCast;
 use smallvec::SmallVec;
@@ -31,8 +33,11 @@ pub struct Hydroflow<'a> {
 
     handoffs: Vec<HandoffData>,
 
+    #[cfg(feature = "meta")]
     /// See [`Self::meta_graph()`].
     meta_graph: Option<HydroflowGraph>,
+
+    #[cfg(feature = "meta")]
     /// See [`Self::diagnostics()`].
     diagnostics: Option<Vec<Diagnostic<SerdeSpan>>>,
 }
@@ -131,36 +136,44 @@ impl<'a> Hydroflow<'a> {
 
     /// Assign the `HydroflowGraph` via JSON string.
     #[doc(hidden)]
-    pub fn __assign_meta_graph(&mut self, meta_graph_json: &str) {
-        let mut meta_graph: HydroflowGraph =
-            serde_json::from_str(meta_graph_json).expect("Failed to deserialize graph.");
-
-        let mut op_inst_diagnostics = Vec::new();
-        meta_graph.insert_node_op_insts_all(&mut op_inst_diagnostics);
-        assert!(
-            op_inst_diagnostics.is_empty(),
-            "Expected no diagnostics, got: {:#?}",
-            op_inst_diagnostics
-        );
+    pub fn __assign_meta_graph(&mut self, _meta_graph_json: &str) {
+        #[cfg(feature = "meta")]
+        {
+            let mut meta_graph: HydroflowGraph =
+                serde_json::from_str(_meta_graph_json).expect("Failed to deserialize graph.");
+
+            let mut op_inst_diagnostics = Vec::new();
+            meta_graph.insert_node_op_insts_all(&mut op_inst_diagnostics);
+            assert!(
+                op_inst_diagnostics.is_empty(),
+                "Expected no diagnostics, got: {:#?}",
+                op_inst_diagnostics
+            );
 
-        assert!(self.meta_graph.replace(meta_graph).is_none());
+            assert!(self.meta_graph.replace(meta_graph).is_none());
+        }
     }
     /// Assign the diagnostics via JSON string.
     #[doc(hidden)]
-    pub fn __assign_diagnostics(&mut self, diagnostics_json: &'static str) {
-        let diagnostics: Vec<Diagnostic<SerdeSpan>> =
-            serde_json::from_str(diagnostics_json).expect("Failed to deserialize diagnostics.");
+    pub fn __assign_diagnostics(&mut self, _diagnostics_json: &'static str) {
+        #[cfg(feature = "meta")]
+        {
+            let diagnostics: Vec<Diagnostic<SerdeSpan>> = serde_json::from_str(_diagnostics_json)
+                .expect("Failed to deserialize diagnostics.");
 
-        assert!(self.diagnostics.replace(diagnostics).is_none());
+            assert!(self.diagnostics.replace(diagnostics).is_none());
+        }
     }
 
+    #[cfg(feature = "meta")]
     /// Return a handle to the meta `HydroflowGraph` if set. The `HydroflowGraph is a
     /// representation of all the operators, subgraphs, and handoffs in this `Hydroflow` instance.
     /// Will only be set if this graph was constructed using a surface syntax macro.
     pub fn meta_graph(&self) -> Option<&HydroflowGraph> {
         self.meta_graph.as_ref()
     }
 
+    #[cfg(feature = "meta")]
     /// Returns any diagnostics generated by the surface syntax macro. Each diagnostic is a pair of
     /// (1) a `Diagnostic` with span info reset and (2) the `ToString` version of the diagnostic
     /// with original span info.

hydroflow_plus/Cargo.toml

@@ -14,27 +14,27 @@ workspace = true
 path = "src/lib.rs"
 
 [features]
-default = ["deploy_runtime"]
+default = []
 stageleft_devel = []
-deploy_runtime = [ "hydroflow/deploy_integration" ]
-deploy = [ "deploy_runtime", "dep:hydro_deploy", "dep:trybuild-internals-api", "dep:toml", "dep:prettyplease" ]
+deploy = [ "build", "dep:hydro_deploy", "dep:trybuild-internals-api", "dep:toml", "dep:prettyplease", "dep:sha2", "dep:stageleft_tool", "dep:nameof" ]
+build = [ "dep:hydroflow_lang" ]
 
 [dependencies]
 bincode = "1.3.1"
 hydro_deploy = { path = "../hydro_deploy/core", version = "^0.10.0", optional = true }
-hydroflow = { path = "../hydroflow", version = "^0.10.0", default-features = false }
-hydroflow_lang = { path = "../hydroflow_lang", version = "^0.10.0" }
+hydroflow = { path = "../hydroflow", version = "^0.10.0", default-features = false, features = ["deploy_integration"] }
+hydroflow_lang = { path = "../hydroflow_lang", version = "^0.10.0", optional = true }
 match_box = "0.0.2"
-nameof = "1.0.0"
+nameof = { version = "1.0.0", optional = true }
 prettyplease = { version = "0.2.0", features = [ "verbatim" ], optional = true }
 proc-macro-crate = "1.0.0"
 proc-macro2 = "1.0.74"
 quote = "1.0.35"
 sealed = "0.6.0"
 serde = { version = "1.0.197", features = [ "derive" ] }
-sha2 = "0.10.0"
+sha2 = { version = "0.10.0", optional = true }
 stageleft = { path = "../stageleft", version = "^0.5.0" }
-stageleft_tool = { path = "../stageleft_tool", version = "^0.4.0" }
+stageleft_tool = { path = "../stageleft_tool", version = "^0.4.0", optional = true }
 syn = { version = "2.0.46", features = [ "parsing", "extra-traits", "visit-mut" ] }
 tokio = { version = "1.29.0", features = [ "full" ] }
 toml = { version = "0.8.0", optional = true }

hydroflow_plus/src/builder/mod.rs

@@ -3,17 +3,23 @@ use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::rc::Rc;
 
+#[cfg(feature = "build")]
 use compiled::CompiledFlow;
+#[cfg(feature = "build")]
 use deploy::{DeployFlow, DeployResult};
 use stageleft::*;
 
+#[cfg(feature = "build")]
 use crate::deploy::{ClusterSpec, Deploy, ExternalSpec, IntoProcessSpec, LocalDeploy};
 use crate::ir::HfPlusLeaf;
 use crate::location::{Cluster, ExternalProcess, Process};
 use crate::staging_util::Invariant;
 
+#[cfg(feature = "build")]
 pub mod built;
+#[cfg(feature = "build")]
 pub mod compiled;
+#[cfg(feature = "build")]
 pub mod deploy;
 
 pub struct FlowStateInner {
@@ -89,6 +95,7 @@ impl<'a> FlowBuilder<'a> {
         }
     }
 
+    #[cfg(feature = "build")]
     pub fn finalize(mut self) -> built::BuiltFlow<'a> {
         self.finalized = true;
 
@@ -101,10 +108,12 @@ impl<'a> FlowBuilder<'a> {
         }
     }
 
+    #[cfg(feature = "build")]
     pub fn with_default_optimize<D: LocalDeploy<'a>>(self) -> DeployFlow<'a, D> {
         self.finalize().with_default_optimize()
     }
 
+    #[cfg(feature = "build")]
     pub fn optimize_with(
         self,
         f: impl FnOnce(Vec<HfPlusLeaf>) -> Vec<HfPlusLeaf>,
@@ -158,6 +167,7 @@ impl<'a> FlowBuilder<'a> {
         }
     }
 
+    #[cfg(feature = "build")]
     pub fn with_process<P, D: LocalDeploy<'a>>(
         self,
         process: &Process<P>,
@@ -166,6 +176,7 @@ impl<'a> FlowBuilder<'a> {
         self.with_default_optimize().with_process(process, spec)
     }
 
+    #[cfg(feature = "build")]
     pub fn with_external<P, D: LocalDeploy<'a>>(
         self,
         process: &ExternalProcess<P>,
@@ -174,6 +185,7 @@ impl<'a> FlowBuilder<'a> {
         self.with_default_optimize().with_external(process, spec)
     }
 
+    #[cfg(feature = "build")]
     pub fn with_cluster<C, D: LocalDeploy<'a>>(
         self,
         cluster: &Cluster<C>,
@@ -182,14 +194,17 @@ impl<'a> FlowBuilder<'a> {
         self.with_default_optimize().with_cluster(cluster, spec)
     }
 
+    #[cfg(feature = "build")]
     pub fn compile<D: Deploy<'a>>(self, env: &D::CompileEnv) -> CompiledFlow<'a, D::GraphId> {
         self.with_default_optimize::<D>().compile(env)
     }
 
+    #[cfg(feature = "build")]
     pub fn compile_no_network<D: LocalDeploy<'a>>(self) -> CompiledFlow<'a, D::GraphId> {
         self.with_default_optimize::<D>().compile_no_network()
     }
 
+    #[cfg(feature = "build")]
     pub fn deploy<D: Deploy<'a, CompileEnv = ()>>(
         self,
         env: &mut D::InstantiateEnv,

hydroflow_plus/src/deploy/deploy_graph.rs

@@ -15,17 +15,17 @@ use hydro_deploy::hydroflow_crate::HydroflowCrateService;
 use hydro_deploy::{CustomService, Deployment, Host, HydroflowCrate};
 use hydroflow::bytes::Bytes;
 use hydroflow::futures::{Sink, SinkExt, Stream, StreamExt};
-use hydroflow::lang::graph::HydroflowGraph;
 use hydroflow::util::deploy::{ConnectedSink, ConnectedSource};
+use hydroflow_lang::graph::HydroflowGraph;
 use nameof::name_of;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 use stageleft::{QuotedWithContext, RuntimeData};
 use tokio::sync::RwLock;
 
-use super::deploy_runtime::*;
 use super::trybuild::create_graph_trybuild;
 use super::{ClusterSpec, Deploy, ExternalSpec, IntoProcessSpec, Node, ProcessSpec, RegisterPort};
+use crate::deploy_runtime::*;
 
 pub struct HydroDeploy {}
 

hydroflow_plus/src/deploy/macro_runtime.rs

@@ -9,8 +9,8 @@ use hydroflow::util::deploy::DeployPorts;
 use hydroflow_lang::graph::HydroflowGraph;
 use stageleft::{QuotedWithContext, RuntimeData};
 
-use super::HydroflowPlusMeta;
 use crate::deploy::{ClusterSpec, Deploy, ExternalSpec, Node, ProcessSpec, RegisterPort};
+use crate::deploy_runtime::HydroflowPlusMeta;
 
 pub struct DeployRuntime {}
 
@@ -35,7 +35,7 @@ impl<'a> Deploy<'a> for DeployRuntime {
         }
     }
 
-    fn trivail_cluster(_id: usize) -> Self::Cluster {
+    fn trivial_cluster(_id: usize) -> Self::Cluster {
         DeployRuntimeCluster {
             next_port: Rc::new(RefCell::new(0)),
         }
@@ -60,7 +60,7 @@ impl<'a> Deploy<'a> for DeployRuntime {
         _p2: &Self::Process,
         p2_port: &Self::Port,
     ) -> (syn::Expr, syn::Expr) {
-        super::deploy_runtime::deploy_o2o(*env, p1_port.as_str(), p2_port.as_str())
+        crate::deploy_runtime::deploy_o2o(*env, p1_port.as_str(), p2_port.as_str())
     }
 
     fn o2o_connect(
@@ -79,7 +79,7 @@ impl<'a> Deploy<'a> for DeployRuntime {
         _c2: &Self::Cluster,
         c2_port: &Self::Port,
     ) -> (syn::Expr, syn::Expr) {
-        super::deploy_runtime::deploy_o2m(*env, p1_port.as_str(), c2_port.as_str())
+        crate::deploy_runtime::deploy_o2m(*env, p1_port.as_str(), c2_port.as_str())
     }
 
     fn o2m_connect(
@@ -98,7 +98,7 @@ impl<'a> Deploy<'a> for DeployRuntime {
         _p2: &Self::Process,
         p2_port: &Self::Port,
     ) -> (syn::Expr, syn::Expr) {
-        super::deploy_runtime::deploy_m2o(*env, c1_port.as_str(), p2_port.as_str())
+        crate::deploy_runtime::deploy_m2o(*env, c1_port.as_str(), p2_port.as_str())
     }
 
     fn m2o_connect(
@@ -117,7 +117,7 @@ impl<'a> Deploy<'a> for DeployRuntime {
         _c2: &Self::Cluster,
         c2_port: &Self::Port,
     ) -> (syn::Expr, syn::Expr) {
-        super::deploy_runtime::deploy_m2m(*env, c1_port.as_str(), c2_port.as_str())
+        crate::deploy_runtime::deploy_m2m(*env, c1_port.as_str(), c2_port.as_str())
     }
 
     fn m2m_connect(
@@ -171,11 +171,11 @@ impl<'a> Deploy<'a> for DeployRuntime {
         env: &Self::CompileEnv,
         of_cluster: usize,
     ) -> impl QuotedWithContext<'a, &'a Vec<u32>, ()> + Copy + 'a {
-        super::deploy_runtime::cluster_members(*env, of_cluster)
+        crate::deploy_runtime::cluster_members(*env, of_cluster)
     }
 
     fn cluster_self_id(env: &Self::CompileEnv) -> impl QuotedWithContext<'a, u32, ()> + Copy + 'a {
-        super::deploy_runtime::cluster_self_id(*env)
+        crate::deploy_runtime::cluster_self_id(*env)
     }
 }
 

hydroflow_plus/src/deploy/mod.rs

@@ -9,8 +9,8 @@ use serde::de::DeserializeOwned;
 use serde::Serialize;
 use stageleft::QuotedWithContext;
 
-#[cfg(feature = "deploy_runtime")]
 pub mod macro_runtime;
+pub use macro_runtime::*;
 
 #[cfg(feature = "deploy")]
 pub(crate) mod trybuild;
@@ -20,15 +20,9 @@ pub(crate) mod trybuild;
 #[doc(hidden)]
 pub mod trybuild_rewriters;
 
-pub use macro_runtime::*;
 #[cfg(feature = "deploy")]
 pub use trybuild::init_test;
 
-#[cfg(feature = "deploy_runtime")]
-pub mod deploy_runtime;
-#[cfg(feature = "deploy_runtime")]
-pub use deploy_runtime::HydroflowPlusMeta;
-
 #[cfg(feature = "deploy")]
 pub mod deploy_graph;
 
@@ -81,7 +75,7 @@ pub trait Deploy<'a> {
         panic!("No trivial process")
     }
 
-    fn trivail_cluster(_id: usize) -> Self::Cluster {
+    fn trivial_cluster(_id: usize) -> Self::Cluster {
         panic!("No trivial cluster")
     }
 
@@ -205,7 +199,7 @@ impl<
     }
 
     fn trivial_cluster(id: usize) -> Self::Cluster {
-        <T as Deploy<'a>>::trivail_cluster(id)
+        <T as Deploy<'a>>::trivial_cluster(id)
     }
 }
 

hydroflow_plus/src/deploy/trybuild.rs

@@ -104,7 +104,7 @@ pub fn compile_graph_trybuild(graph: HydroflowGraph, extra_stmts: Vec<syn::Stmt>
         use hydroflow_plus::*;
 
         #[allow(unused)]
-        fn __hfplus_runtime<'a>(__hydroflow_plus_trybuild_cli: &'a hydroflow_plus::hydroflow::util::deploy::DeployPorts<hydroflow_plus::deploy::HydroflowPlusMeta>) -> hydroflow_plus::hydroflow::scheduled::graph::Hydroflow<'a> {
+        fn __hfplus_runtime<'a>(__hydroflow_plus_trybuild_cli: &'a hydroflow_plus::hydroflow::util::deploy::DeployPorts<hydroflow_plus::deploy_runtime::HydroflowPlusMeta>) -> hydroflow_plus::hydroflow::scheduled::graph::Hydroflow<'a> {
             #(#extra_stmts)*
             #tokens
         }