From 6ab625273d822812e83a333e928c3dea1c3c9ccb Mon Sep 17 00:00:00 2001 From: Mingwei Samuel Date: Sun, 22 Dec 2024 11:55:06 -0800 Subject: [PATCH] docs: cleanups for the rename, fixing links (#1624) --- .github/gh-pages/index.md | 9 ++-- .github/workflows/template.yml | 4 +- CITATION.cff | 34 +++++--------- CONTRIBUTING.md | 46 +++++++++---------- LICENSE | 2 +- README.md | 14 +++--- RELEASING.md | 2 +- dfir_datalog/Cargo.toml | 2 +- dfir_lang/Cargo.toml | 2 +- dfir_lang/src/graph/ops/demux.rs | 6 +-- dfir_lang/src/graph/ops/dest_sink.rs | 2 +- dfir_lang/src/graph/ops/mod.rs | 4 +- dfir_lang/src/graph/ops/source_interval.rs | 2 +- dfir_macro/build.rs | 6 +-- dfir_rs/README.md | 34 +++++++------- docs/docs/deploy/your-first-deploy.md | 8 ++-- docs/docs/dfir/architecture/index.mdx | 12 +++-- docs/docs/dfir/lattices_crate/index.mdx | 2 +- .../dfir/quickstart/example_1_simplest.mdx | 4 +- docs/docs/dfir/syntax/surface_embedding.mdx | 2 +- docs/docs/dfir/syntax/surface_flows.mdx | 2 +- docs/docs/hydro/index.mdx | 2 +- docs/docusaurus.config.js | 14 +++--- docs/sidebars.js | 2 +- docs/src/pages/playground.js | 10 ++-- docs/src/pages/research.js | 6 +-- hydro_lang/Cargo.toml | 2 +- hydro_std/Cargo.toml | 2 +- lattices/README.md | 2 +- template/dfir/Cargo.toml | 2 +- website_playground/src/lib.rs | 24 +++++----- website_playground/tests/node.rs | 2 + 32 files changed, 130 insertions(+), 137 deletions(-) diff --git a/.github/gh-pages/index.md b/.github/gh-pages/index.md index 1cdab731f890..17518baec969 100644 --- a/.github/gh-pages/index.md +++ b/.github/gh-pages/index.md @@ -1,8 +1,7 @@ --- --- -- [The Hydroflow Book]({{ "/book/" | prepend: site.github.url | replace: 'http://', '//' }}) -- [Rust Docs]({{ "/doc/hydroflow/" | prepend: site.github.url | replace: 'http://', '//' }}) -- [Design Docs]({{ "/design_docs/" | prepend: site.github.url | replace: 'http://', '//' }}) -- [Benchmark History]({{ "/bench/" | prepend: site.github.url | replace: 'http://', '//' }}) -- [Latest Benchmarks]({{ "/criterion/report/" | prepend: site.github.url | replace: 'http://', '//' }}) +- [Main website (hydro.run) đź”—](https://hydro.run) - [Github Repository đź”—](https://github.com/hydro-project/hydroflow) +- [Design Docs]({{ "/design_docs/" | prepend: site.github.url | replace: 'https://', '//' }}) +- [Benchmark History]({{ "/bench/" | prepend: site.github.url | replace: 'https://', '//' }}) +- [Latest Benchmarks]({{ "/criterion/report/" | prepend: site.github.url | replace: 'https://', '//' }}) diff --git a/.github/workflows/template.yml b/.github/workflows/template.yml index f6982de2eb4c..65fea45e5bc6 100644 --- a/.github/workflows/template.yml +++ b/.github/workflows/template.yml @@ -84,8 +84,8 @@ jobs: echo "$ECHO_STRING" >client-input - "${{ runner.temp }}/generated/target/debug/hydroflow-template" --role server --address 127.0.0.100:2048 & - "${{ runner.temp }}/generated/target/debug/hydroflow-template" --role client --address 127.0.0.100:2048 client-output & + "${{ runner.temp }}/generated/target/debug/dfir-template" --role server --address 127.0.0.100:2048 & + "${{ runner.temp }}/generated/target/debug/dfir-template" --role client --address 127.0.0.100:2048 client-output & sleep 1 diff --git a/CITATION.cff b/CITATION.cff index f3fda3242cf3..2d4464f12997 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ # Visit https://bit.ly/cffinit to generate yours today! cff-version: 1.2.0 -title: Hydroflow +title: Hydro message: >- If you use this software, please cite it using the metadata from this file. @@ -13,10 +13,6 @@ authors: email: mingwei@shv.com affiliation: Sutter Hill Ventures orcid: 'https://orcid.org/0009-0004-9873-6266' - - given-names: Justin - family-names: Jaffray - email: justin@shv.com - affiliation: Sutter Hill Ventures - given-names: Shadaj family-names: Laddad email: shadaj@berkeley.edu @@ -27,17 +23,18 @@ authors: email: hellerstein@berkeley.edu orcid: 'https://orcid.org/0000-0002-7712-4306' affiliation: UC Berkeley + - given-names: Rohit + family-names: Kulshreshtha + email: rohit@shv.com + affiliation: Sutter Hill Ventures - given-names: Lucky family-names: Katahanas email: lucky@shv.com affiliation: Sutter Hill Ventures - - given-names: Tyler - family-names: Hou - affiliation: UC Berkeley - - given-names: Alex - family-names: Rasmussen - email: alex@bitsondisk.com - affiliation: Bits on Disk + - given-names: Justin + family-names: Jaffray + email: justin@shv.com + affiliation: Sutter Hill Ventures - given-names: David family-names: Chu affiliation: UC Berkeley @@ -47,18 +44,11 @@ authors: family-names: Power orcid: 'https://orcid.org/0000-0002-0660-5110' affiliation: UC Berkeley - - given-names: Amrita - family-names: Rajan - affiliation: UC Berkeley - - given-names: Rithvik - family-names: Panchapakesan - email: rithvik@berkeley.edu - affiliation: UC Berkeley repository-code: 'https://github.com/hydro-project/hydroflow/' -url: 'https://hydro.run/hydroflow' +url: 'https://hydro.run/' abstract: >- - Hydroflow is the lattice-based dataflow runtime for the - Hydro language stack for programming distributed systems + Hydro is a language stack for programming distributed systems, + buil on top of DFIR, a lattice-aware dataflow system. keywords: - dataflow - distributed systems diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3424f4ca8170..6add204d32f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,28 +1,28 @@ -# Contributing to Hydroflow +# Contributing to Hydro -Thanks for your interest in contributing to Hydroflow! This is an experimental, research-driven +Thanks for your interest in contributing to Hydro! This is an experimental, research-driven project which can make getting started a bit tricky. This guide will explain the project structure, code style, commit messages, testing setups, and more to help you get started. ## Repository Structure -The Hydroflow repo is set up as a monorepo and [Cargo workspace](https://doc.rust-lang.org/book/ch14-03-cargo-workspaces.html). +The Hydro repo is set up as a monorepo and [Cargo workspace](https://doc.rust-lang.org/book/ch14-03-cargo-workspaces.html). Relative to the repository root: -* `hydroflow` is the main Hydroflow package, containing the Hydroflow runtime. It re-exports the - surface syntax macros in `dfir_macro` and `dfir_lang`. The runtime is the "scheduled - layer" while the surface syntax compiler is the "compiled layer". -* `hydro_lang` and related (hydro_*) packages contain Hydro, which is a functional syntax built on - top of `hydroflow`. -* `dfir_datalog` provides a datalog compiler, based on top of the Hydroflow surface syntax. +* `dfir_rs` is the main DFIR package, containing the DFIR runtime. It re-exports the + flow syntax macros in `dfir_macro` and `dfir_lang`. The runtime is the "scheduled + layer" while the flow syntax compiler is the "compiled layer". +* `hydro_lang` and related (`hydro_*`) packages contain Hydro, which is a functional syntax built on + top of `DFIR`. +* `dfir_datalog` provides a datalog compiler, based on top of the DFIR flow syntax. * `docs` is the [Hydro.run](https://hydro.run/) website. `website_playground` contains the - playground portion of the website, used for compiling Hydroflow in-browser via WASM. -* `benches` contains some microbenchmarks for Hydroflow and other frameworks. -* `design_docs` contains old point-in-time design docs for Hydroflow's architecture. + playground portion of the website, used for compiling DFIR in-browser via WASM. +* `benches` contains some microbenchmarks for DFIR and other frameworks. +* `design_docs` contains old point-in-time design docs for DFIR's architecture. -There are several subpackages included that are used by Hydroflow but are more general-purpose: +There are several subpackages included that are used by Hydro but are more general-purpose: -* `stageleft` is a framework for staged programming in Rust, used by `hydroflow_plus`. +* `stageleft` is a framework for staged programming in Rust, used by `hydro_lang`. * `lattices` is a abstract algebra library, originally for lattice types. * `variadics` is a crate for emulating variadic generics using tuple lists. * `pusherator` is a rudimentary library providing push-based iterators. @@ -37,13 +37,13 @@ There are auxillary repositories as well: ## Rust -Hydroflow should build on latest stable releases of Rust. However we develop on a pinned nightly -version, bumped up every month or two. The version is in `rust-toolchain.toml` which is +Hydro should build on latest stable releases of Rust. However we develop on a pinned nightly +version, which is updated every month or two. The version is in `rust-toolchain.toml` which is automatically detected by `cargo`, so no special setup should be needed. ## Python -Some parts of the Hydroflow repo require a relatively recent version of Python 3, maybe 3.10 or +Some parts of the Hydro repo require a relatively recent version of Python 3, maybe 3.10 or later. On Mac, installing directly from python.org may work if `brew install` doesn't. ### `wasm-bindgen` @@ -70,7 +70,7 @@ git checkout -b $BRANCH_NAME `feature/$FEATURE_NAME` .. make changes .. git add ... # Add all changes git commit # Commit changes -git push origin HEAD +git push origin HEAD ``` ### Commit Messages @@ -90,13 +90,13 @@ failing-- you should double-check that the snapshot diff matches what you expect ## Snapshot Testing -Hydroflow uses two types of snapshot testing: [`insta`](https://insta.rs/) and [`trybuild`](https://github.com/dtolnay/trybuild). -Insta provides general snapshot testing in Rust, and we mainly use it to test the Hydroflow graphs -generated from the surface syntax. These snapshots are of the [Mermaid](https://mermaid.js.org/) or +Hydro uses two types of snapshot testing: [`insta`](https://insta.rs/) and [`trybuild`](https://github.com/dtolnay/trybuild). +Insta provides general snapshot testing in Rust, and we mainly use it to test the DFIR graphs +generated from the flow syntax. These snapshots are of the [Mermaid](https://mermaid.js.org/) or [DOT](https://graphviz.org/) graph visualizations rather than the graph datastructures themselves; -see `hydroflow/tests/snapshots`. The snapshots can be useful not just to track changes but also as +see `dfir_rs/tests/snapshots`. The snapshots can be useful not just to track changes but also as a quick reference to view the visualizations (i.e. by pasting into [mermaid.live](https://mermaid.live/)). -`trybuild` is used to test the error messages in Hydroflow's surface syntax; see `hydroflow/tests/compile-fail`. +`trybuild` is used to test the error messages in DFIR's flow syntax; see `dfir_rs/tests/compile-fail`. `insta` provides a CLI, `cargo insta` to run tests and review changes: ```shell diff --git a/LICENSE b/LICENSE index fda85432a8b5..3e47e1356859 100644 --- a/LICENSE +++ b/LICENSE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2022 Hydroflow Contributors + Copyright 2025 Hydro Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 7b5ed81dd76d..5ea4c91d8da8 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ Docs.rs

-Hydro is a novel distributed programming library for standard Rust. Hydro allows developers to build distributed systems that are efficient, scalable, and correct. +Hydro is a novel distributed programming library for standard Rust. Hydro allows developers to build distributed systems that are efficient, scalable, and correct. -Hydro integrates naturally into standard Rust constructs and IDEs, providing types and programming constructs for ensuring distributed correctness. Under the covers it provides a metaprogrammed compiler that optimizes for cross-node issues of scaling and data movement while leveraging Rust and LLVM for per-node performance. +Hydro integrates naturally into standard Rust constructs and IDEs, providing types and programming constructs for ensuring distributed correctness. Under the covers it provides a metaprogrammed compiler that optimizes for cross-node issues of scaling and data movement while leveraging Rust and LLVM for per-node performance. + +We often describe Hydro via a metaphor: *LLVM for the cloud*. Like LLVM, Hydro is a layered compilation framework with a low-level Internal Representation language. In contrast to LLVM, Hydro focuses on distributed aspects of modern software. -We often describe Hydro via a metaphor: *LLVM for the cloud*. Like LLVM, Hydro is a layered compilation framework with a low-level Internal Representation language. In contrast to LLVM, Hydro focuses on distributed aspects of modern software. -
Image description
@@ -43,14 +43,14 @@ What's wanted, we believe, is a proper language stack addressing distributed con - **Distributed Typechecking**: The type system of the language should enforce distributed correctness in the compiler, in ways that are visible in an IDE at time of authoring. For example, the types of data items should include their abstract locations, so that two items materialized at different nodes cannot be referenced together without an intervening construct for (async) communication. - **Distributed Optimizations**: The compiler should be able to correctly optimize (transform) programs for distributed concerns: removing bottlenecks by flexibly assigning compute and data to different processes or clusters, while preserving program semantics. - **Modularity**: The standard modularity of traditional programming—e.g. function calling abstractions—should work for distributed logic. For example, it should be possible to wrap a common cross-node construct like "heartbeats and timeouts", and invoke it as simply as one invokes a sequential function. -- **Native Testing Tools** for correctness. In today's standard practice, formal specification languages for testing (e.g. [TLA+](https://en.wikipedia.org/wiki/TLA%2B)) are separate from languages of implementation. We believe it should be possible to perform many kinds of formal testing (e.g. model checking) on the same code that is used in deployment. +- **Native Testing Tools** for correctness. In today's standard practice, formal specification languages for testing (e.g. [TLA+](https://en.wikipedia.org/wiki/TLA%2B)) are separate from languages of implementation. We believe it should be possible to perform many kinds of formal testing (e.g. model checking) on the same code that is used in deployment. -[Hydro](https://hydro.run) is a Rust library for distributed programming that is designed to address these goals. +[Hydro](https://hydro.run) is a Rust framework for distributed programming that is designed to address these goals. # Learning More The Hydro project's main website is at [https://hydro.run](https://hydro.run). -- **Docs**: There are docs for the [high-level Hydro language](https://hydro.run/docs/hydro/) and the low-level dataflow IR, [DFIR](https://hydro.run/docs/dfir), as well as the [HydroDeploy](https://hydro.run/docs/deploy) framework for launching Hydroflow+ programs. +- **Docs**: There are docs for the [high-level Hydro language](https://hydro.run/docs/hydro/) and the low-level dataflow IR, [DFIR](https://hydro.run/docs/dfir), as well as the [Hydro Deploy](https://hydro.run/docs/deploy) framework for launching Hydro programs. - **Research Papers**: Our [research publications](https://hydro.run/research) are available on the project website. Some notable selections: - The original Hydro vision paper from CIDR 2021: [New Directions in Cloud Programming](https://hydro.run/papers/new-directions.pdf) diff --git a/RELEASING.md b/RELEASING.md index ba82493c5ed9..aa9300fd6908 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,6 +1,6 @@ # Releasing Guide -This is a guide on how to create releases for Hydroflow and all the other crates in this workspace. +This is a guide on how to create releases for all Hydro crates in this workspace. We use the [`cargo-smart-release` crate](https://github.com/Byron/cargo-smart-release) for our release workflow. Originally, cargo-smart-release [was part of gitoxide](https://github.com/Byron/gitoxide/pull/998) diff --git a/dfir_datalog/Cargo.toml b/dfir_datalog/Cargo.toml index 5f4ff294842e..8e4bbfae17d5 100644 --- a/dfir_datalog/Cargo.toml +++ b/dfir_datalog/Cargo.toml @@ -5,7 +5,7 @@ version = "0.10.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/dfir_datalog/" -description = "Datalog proc-macro for Hydroflow" +description = "Datalog proc-macro for DFIR" [lints] workspace = true diff --git a/dfir_lang/Cargo.toml b/dfir_lang/Cargo.toml index 3f7b2d7daaa6..d9984ea15cb4 100644 --- a/dfir_lang/Cargo.toml +++ b/dfir_lang/Cargo.toml @@ -5,7 +5,7 @@ version = "0.10.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/dfir_lang/" -description = "Hydro's Data Flow Intermediate Representation (DFIR) implementation" +description = "Hydro's Dataflow Intermediate Representation (DFIR) implementation" [lints] workspace = true diff --git a/dfir_lang/src/graph/ops/demux.rs b/dfir_lang/src/graph/ops/demux.rs index f76412851470..115e473b3eba 100644 --- a/dfir_lang/src/graph/ops/demux.rs +++ b/dfir_lang/src/graph/ops/demux.rs @@ -14,7 +14,7 @@ use crate::pretty_span::PrettySpan; // TODO(mingwei): Preprocess rustdoc links in mdbook or in the `operator_docgen` macro. /// > Arguments: A Rust closure, the first argument is a received item and the -/// > second argument is a variadic [`var_args!` tuple list](https://hydro-project.github.io/hydroflow/doc/hydroflow/macro.var_args.html) +/// > second argument is a variadic [`var_args!` tuple list](https://hydro.run/rustdoc/dfir_rs/macro.var_args.html) /// > where each item name is an output port. /// /// Takes the input stream and allows the user to determine which items to @@ -22,8 +22,8 @@ use crate::pretty_span::PrettySpan; /// /// > Note: Downstream operators may need explicit type annotations. /// -/// > Note: The [`Pusherator`](https://hydro-project.github.io/hydroflow/doc/pusherator/trait.Pusherator.html) -/// > trait is automatically imported to enable the [`.give(...)` method](https://hydro-project.github.io/hydroflow/doc/pusherator/trait.Pusherator.html#tymethod.give). +/// > Note: The [`Pusherator`](https://hydro.run/rustdoc/pusherator/trait.Pusherator.html) +/// > trait is automatically imported to enable the [`.give(...)` method](https://hydro.run/rustdoc/pusherator/trait.Pusherator.html#tymethod.give). /// /// > Note: The closure has access to the [`context` object](surface_flows.mdx#the-context-object). /// diff --git a/dfir_lang/src/graph/ops/dest_sink.rs b/dfir_lang/src/graph/ops/dest_sink.rs index 05a1428f6c33..27b0e70c3043 100644 --- a/dfir_lang/src/graph/ops/dest_sink.rs +++ b/dfir_lang/src/graph/ops/dest_sink.rs @@ -52,7 +52,7 @@ use super::{ /// /// If we only want to write a stream of bytes without delineation we can use the [`BytesCodec`](https://docs.rs/tokio-util/latest/tokio_util/codec/struct.BytesCodec.html). /// -/// In this example we use a [`duplex`](crate::tokio::io::duplex) as our `AsyncWrite` with a +/// In this example we use a [`duplex`](https://docs.rs/tokio/latest/tokio/io/fn.duplex.html) as our `AsyncWrite` with a /// `BytesCodec`. /// /// ```rustbook diff --git a/dfir_lang/src/graph/ops/mod.rs b/dfir_lang/src/graph/ops/mod.rs index 176b9bb09b2c..44c80f67aaad 100644 --- a/dfir_lang/src/graph/ops/mod.rs +++ b/dfir_lang/src/graph/ops/mod.rs @@ -340,10 +340,10 @@ pub struct WriteContextArgs<'a> { /// `dfir` crate name for `use #root::something`. pub root: &'a TokenStream, /// `context` ident, the name of the provided - /// [`dfir_rs::scheduled::Context`](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/context/struct.Context.html). + /// [`dfir_rs::scheduled::Context`](https://hydro.run/rustdoc/dfir_rs/scheduled/context/struct.Context.html). pub context: &'a Ident, /// `df` ident, the name of the - /// [`dfir_rs::scheduled::graph::Dfir`](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/graph/struct.Hydroflow.html) + /// [`dfir_rs::scheduled::graph::Dfir`](https://hydro.run/rustdoc/dfir_rs/scheduled/graph/struct.Dfir.html) /// instance. pub hydroflow: &'a Ident, /// Subgraph ID in which this operator is contained. diff --git a/dfir_lang/src/graph/ops/source_interval.rs b/dfir_lang/src/graph/ops/source_interval.rs index adcc831a8378..879501dc8af6 100644 --- a/dfir_lang/src/graph/ops/source_interval.rs +++ b/dfir_lang/src/graph/ops/source_interval.rs @@ -14,7 +14,7 @@ use super::{ /// be scheduled as soon as possible. /// /// Note that this requires the dfir instance be run within a [Tokio `Runtime`](https://docs.rs/tokio/1/tokio/runtime/struct.Runtime.html). -/// The easiest way to do this is with a [`#[dfir_rs::main]`](https://hydro-project.github.io/hydroflow/doc/hydroflow/macro.hydroflow_main.html) +/// The easiest way to do this is with a [`#[dfir_rs::main]`](https://hydro.run/rustdoc/dfir_rs/attr.main) /// annotation on `async fn main() { ... }` as in the example below. /// /// ```rustbook diff --git a/dfir_macro/build.rs b/dfir_macro/build.rs index 4a745b661468..f710e2947de8 100644 --- a/dfir_macro/build.rs +++ b/dfir_macro/build.rs @@ -228,8 +228,8 @@ const PREFIX: &str = "\ sidebar_position: 4 --- -# Hydroflow's Operators +# DFIR Operators -In our previous examples we made use of some of Hydroflow's operators. -Here we document each operator in more detail. Most of these operators +In our previous examples we made use of some of DFIR's operators. +Here we document each operator in more detail. Many of these operators are based on the Rust equivalents for iterators; see the [Rust documentation](https://doc.rust-lang.org/std/iter/trait.Iterator.html)."; diff --git a/dfir_rs/README.md b/dfir_rs/README.md index 8aa98d34cd24..504cc15497b0 100644 --- a/dfir_rs/README.md +++ b/dfir_rs/README.md @@ -1,23 +1,23 @@ -# Hydroflow +# DFIR -[Hydroflow](https://github.com/hydro-project/hydroflow) is a small language and compiler for low-latency -dataflow programs, written in Rust. Hydroflow serves as the runtime library for the -[Hydro language stack](https://hydro.run/docs/hydroflow/ecosystem), which is under development +[DFIR](https://github.com/hydro-project/hydroflow) (Dataflow Intermediate Representation) is a small language and compiler for low-latency +dataflow programs, written in Rust. DFIR serves as dataflow representation and runtime execution library for the +[Hydro language stack](https://hydro.run/docs/dfir/ecosystem), which is under development as a complete compiler stack for distributed programming languages. -Hydroflow is designed with two use cases in mind: - - Expert developers can program directly in the Hydroflow language to build individual components that can interoperate in a distributed program or service. - - Higher levels of the Hydro stack will offer higher-level abstractions and DSLs, and treat Hydroflow as a compiler target. +DFIR is designed with two use cases in mind: + - Expert developers can program directly in the DFIR language to build individual components that can interoperate in a distributed program or service. + - Higher levels of the Hydro stack will offer higher-level abstractions and DSLs, and treat DFIR as a compiler target. -Hydroflow is targeted at supporting the following unique features: - 1. A type system that helps developers reason about progress and consistency guarantees in a distributed program. This includes an emphasis on [lattice types](https://hydro.run/docs/hydroflow/lattices_crate/) that can allow for consistent outcomes in the face of network messages that may be interleaved, reordered, batched, and resent. - 2. A [dataflow programming model](https://hydro.run/docs/hydroflow/syntax/surface_flows), capturing the message- and data-driven nature of many distributed services. - 3. Extremely low-latency handling of asynchronously-arriving data/messages, via aggressive exploitation of Rust's [monomorphization](https://rustc-dev-guide.rust-lang.org/backend/monomorph.html) techniques. - 4. Dataflow optimizations, both to optimize single-node Hydroflow flows, and to enable distributed optimizations across multiple flows. - -Hydroflow's language—the Hydroflow *surface syntax*—is embedded in Rust, which compiles Hydroflow code to high-efficiency machine code. -As the lowest level of the Hydro stack, Hydroflow requires some knowledge of Rust to use. +DFIR is targeted at supporting the following unique features: + 1. An easy-to-read flow syntax, embeddedable in Rust. + 2. Dataflow programming model, capturing the streaming message/data-driven nature of distributed computational services. + 3. Reactive programming model with cumulative state, capturing the nature of both stateful distributed services and front-end frameworks. + 4. Extremely low-latency execution via [Rust monomorphization](https://rustc-dev-guide.rust-lang.org/backend/monomorph.html), while maintaining high throughput. -The most recent release of the [Hydroflow Book docs](https://hydro.run/docs/hydroflow/#this-book) are online, providing documentation and numerous annotated examples. +DFIR's language—the DFIR *flow syntax*—is embeddedable in Rust, which compiles DFIR code to high-efficiency machine code. +As the lowest level of the Hydro stack, DFIR requires some knowledge of Rust to use. -You can also check out the [Hydroflow Playground](https://hydro.run/playground) to see Hydroflow's surface syntax in action! +The most recent release of the [DFIR Book docs](https://hydro.run/docs/dfir/#this-book) are online, providing documentation and numerous annotated examples. + +You can also check out the [DFIR Playground](https://hydro.run/playground) to see DFIR's flow syntax in action! diff --git a/docs/docs/deploy/your-first-deploy.md b/docs/docs/deploy/your-first-deploy.md index 8372d262e17a..74c372f5cb5b 100644 --- a/docs/docs/deploy/your-first-deploy.md +++ b/docs/docs/deploy/your-first-deploy.md @@ -5,7 +5,7 @@ sidebar_position: 3 # Your First Deploy Now that we have Hydro Deploy installed, let's deploy our first app. We'll start with a simple app that echoes packets. -First, we need to write the Hydroflow application, which will intergrate with Hydro Deploy to initialize connections to other services. We can create a project using the Cargo template: +First, we need to write the DFIR application, which will intergrate with Hydro Deploy to initialize connections to other services. We can create a project using the Cargo template: ```bash # if you don't already have cargo-generate installed @@ -13,7 +13,7 @@ First, we need to write the Hydroflow application, which will intergrate with Hy cargo install --locked cargo-generate #shell-command-next-line -cargo generate gh:hydro-project/hydroflow template/hydroflow +cargo generate gh:hydro-project/hydroflow template/dfir ``` `cd` into the generated folder, ensure the correct nightly version of rust is installed, and test the generated project: @@ -67,14 +67,14 @@ Next, we pick out the host we want to run the service on (localhost for now), an ) ``` -Now, we need to wire up the ports. Hydro Deploy uses _named ports_, which can then be loaded in our Hydroflow logic. In our example, each echo service will have an "input" and "output" port. We can wire them up using the `send_to` method: +Now, we need to wire up the ports. Hydro Deploy uses _named ports_, which can then be loaded in our DFIR logic. In our example, each echo service will have an "input" and "output" port. We can wire them up using the `send_to` method: ```python echo_service_1.ports.output.send_to(echo_service_2.ports.input) echo_service_2.ports.output.send_to(echo_service_1.ports.input) ``` -Returning briefly to our Hydroflow code, we can then load these ports and use them to send and receive packets: +Returning briefly to our DFIR code, we can then load these ports and use them to send and receive packets: ```rust use hydroflow_deploy_integration::ConnectedDirect; diff --git a/docs/docs/dfir/architecture/index.mdx b/docs/docs/dfir/architecture/index.mdx index b8b5273f7b2f..ee16213b6fbf 100644 --- a/docs/docs/dfir/architecture/index.mdx +++ b/docs/docs/dfir/architecture/index.mdx @@ -74,9 +74,9 @@ In the reachability example above, the pivot is the `flat_map()` operator: it pu See [Subgraph In-Out Trees](./in-out_trees) for more, including how to convert a graph into in-out trees. -## Surface Syntax and APIs +## Flow Syntax and APIs -DFIR _Surface Syntax_ hides +DFIR _Flow Syntax_ hides the distinction between these two layers. It offers a natural `Iterator`-like chaining syntax for building graphs that get parsed and compiled into a scheduled graph of one or more compiled subgraphs. Please see the [Surface Syntax](../syntax/) docs for more information. @@ -84,9 +84,11 @@ Alternatively, the _Core API_ allows you to interact with handoffs directly at a level. It doesn't provide any notion of chainable operators. You can use Rust `Iterator`s or any other arbitrary Rust code to implement the operators. -The Core API lives in [`dfir_rs::scheduled`](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/index.html), -mainly in methods on the [`DFIR` struct](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/graph/struct.Dfir.html). Compiled push-based iterators live in [`dfir_rs::compiled`](https://hydro-project.github.io/hydroflow/doc/hydroflow/compiled/index.html). We intend users to use the Surface Syntax as it is much more friendly, but as +The Core API lives in [`dfir_rs::scheduled`](https://hydro.run/rustdoc/dfir_rs/scheduled/index.html), +mainly in methods on the [`DFIR` struct](https://hydro.run/rustdoc/dfir_rs/scheduled/graph/struct.Dfir.html). +Compiled push-based iterators live in [`dfir_rs::compiled`](https://hydro.run/rustdoc/dfir_rs/compiled/index.html). +When working with DFIR, we intend users to use the Flow Syntax as it is much more friendly, but as DFIR is in active development some operators might not be available in -the Surface Syntax, in which case the Core API can be used instead. If you find +the Flow Syntax, in which case the Core API can be used instead. If you find yourself in this situation be sure to [submit an issue](https://github.com/hydro-project/hydroflow/issues/new)! diff --git a/docs/docs/dfir/lattices_crate/index.mdx b/docs/docs/dfir/lattices_crate/index.mdx index e043a670affa..4e147ec7cbad 100644 --- a/docs/docs/dfir/lattices_crate/index.mdx +++ b/docs/docs/dfir/lattices_crate/index.mdx @@ -11,4 +11,4 @@ import MacroDocs from '../../../../lattices_macro/README.md' Take a look at [Lattice Math](./lattice_math.md) for a simple explanation and summary of what lattices are exactly. -See the [`lattice` rustdocs here](https://hydro-project.github.io/hydroflow/doc/lattices/index.html). +See the [`lattice` rustdocs here](https://hydro.run/rustdoc/lattices/index.html). diff --git a/docs/docs/dfir/quickstart/example_1_simplest.mdx b/docs/docs/dfir/quickstart/example_1_simplest.mdx index 35f993d04da4..818265429c12 100644 --- a/docs/docs/dfir/quickstart/example_1_simplest.mdx +++ b/docs/docs/dfir/quickstart/example_1_simplest.mdx @@ -72,7 +72,7 @@ item passed in. The DFIR surface syntax is merely a *specification*; it does not actually do anything until we run it. -We can run this flow from within Rust via the [`run_available()` method](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/graph/struct.Hydroflow.html#method.run_available). +We can run this flow from within Rust via the [`run_available()` method](https://hydro.run/rustdoc/dfir_rs/scheduled/graph/struct.Dfir#method.run_available). {getLines(exampleCode, 'run')} @@ -82,7 +82,7 @@ more work will *ever* be available. In future examples we will use external inpu network ingress, in which case more work might appear at any time. In server applications that use network ingress for inputs, new work can appear at any time. In these application -we may need a different method than `run_available()`, e.g. the [`run_async()`](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/graph/struct.Dfir.html#method.run_async) method, check [the networking +we may need a different method than `run_available()`, e.g. the [`run_async()`](https://hydro.run/rustdoc/dfir_rs/scheduled/graph/struct.Dfir.html#method.run_async) method, check [the networking examples](./example_7_networking) for more. ### A Note on Project Structure diff --git a/docs/docs/dfir/syntax/surface_embedding.mdx b/docs/docs/dfir/syntax/surface_embedding.mdx index 4d72246cfcb1..ad747deffbb0 100644 --- a/docs/docs/dfir/syntax/surface_embedding.mdx +++ b/docs/docs/dfir/syntax/surface_embedding.mdx @@ -12,4 +12,4 @@ The surface syntax is embedded into Rust via a macro as follows {getLines(exampleCode, 3, 10)} -The resulting `flow` object is of type [`Dfir`](https://hydro-project.github.io/hydroflow/doc/dfir/scheduled/graph/struct.Dfir.html). +The resulting `flow` object is of type [`Dfir`](https://hydro.run/rustdoc/dfir_rs/scheduled/graph/struct.Dfir.html). diff --git a/docs/docs/dfir/syntax/surface_flows.mdx b/docs/docs/dfir/syntax/surface_flows.mdx index cb91ddcd527d..6957c53bd713 100644 --- a/docs/docs/dfir/syntax/surface_flows.mdx +++ b/docs/docs/dfir/syntax/surface_flows.mdx @@ -52,7 +52,7 @@ of [in-out trees](../architecture/in-out_trees.md). Closures inside surface syntax operators have access to a special `context` object which provides access to scheduling, timing, and state APIs. The object is accessible as a shared reference (`&Context`) via the special name `context`. -[Here is the full API documentation for `Context`](https://hydro-project.github.io/hydroflow/doc/dfir/scheduled/context/struct.Context.html). +[Here is the full API documentation for `Context`](https://hydro.run/rustdoc/dfir_rs/scheduled/context/struct.Context.html). {getLines(example4Code, 5, 6)} Output: diff --git a/docs/docs/hydro/index.mdx b/docs/docs/hydro/index.mdx index 7fd0bf0748ca..23d0ab7704e6 100644 --- a/docs/docs/hydro/index.mdx +++ b/docs/docs/hydro/index.mdx @@ -11,6 +11,6 @@ Hydro has been used to write a variety of high-performance distributed systems, :::caution -The docs for Hydro are still a work in progress. If you have any questions or run into bugs, please file an issue on the [Hydroflow GitHub repository](https://github.com/hydro-project/hydroflow). +The docs for Hydro are still a work in progress. If you have any questions or run into bugs, please file an issue on the [Hydro GitHub repository](https://github.com/hydro-project/hydroflow). ::: diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index dcd00f4f7a04..213494fb793c 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -118,11 +118,6 @@ const config = { type: 'dropdown', label: 'Docs', items: [ - { - type: 'docSidebar', - sidebarId: 'dfirSidebar', - label: 'Hydroflow', - }, { type: 'docSidebar', sidebarId: 'hydroSidebar', @@ -134,9 +129,14 @@ const config = { label: 'Hydro Deploy', }, { - href: 'pathname:///rustdoc/hydroflow/', + href: 'pathname:///rustdoc/hydro/', label: 'Rustdoc', - } + }, + { + type: 'docSidebar', + sidebarId: 'dfirSidebar', + label: 'DFIR', + }, ] }, { diff --git a/docs/sidebars.js b/docs/sidebars.js index 60536092ef8a..aeae757dbf7a 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -14,9 +14,9 @@ /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ const sidebars = { // By default, Docusaurus generates a sidebar from the docs folder structure - dfirSidebar: [{type: 'autogenerated', dirName: 'dfir'}], hydroSidebar: [{type: 'autogenerated', dirName: 'hydro'}], deploySidebar: [{type: 'autogenerated', dirName: 'deploy'}], + dfirSidebar: [{type: 'autogenerated', dirName: 'dfir'}], }; module.exports = sidebars; diff --git a/docs/src/pages/playground.js b/docs/src/pages/playground.js index d81f30d22156..65d33f4787a3 100644 --- a/docs/src/pages/playground.js +++ b/docs/src/pages/playground.js @@ -51,7 +51,7 @@ function MermaidGraph({ id, source }) { }} dangerouslySetInnerHTML={svg}>; } -const DFIRExamples = { +const DfirExamples = { "Simplest": `\ // https://hydro.run/docs/dfir/quickstart/example_1_simplest source_iter(0..10) -> for_each(|n| println!("Hello {}", n));`, @@ -177,8 +177,8 @@ reachable(x) :- seed_reachable(x) reachable(y) :- reachable(x), edges(x, y)` }; -export function DFIRSurfaceDemo() { - return +export function DfirSurfaceDemo() { + return } export function DatalogDemo() { return @@ -310,7 +310,7 @@ export function EditorDemo({ compileFn, examples, mermaidId }) { export default function Playground() { return ( + description="Playground for the DFIR compiler">
DFIR - +

Datalog

diff --git a/docs/src/pages/research.js b/docs/src/pages/research.js index f7f7afa46be5..5270d26891d3 100644 --- a/docs/src/pages/research.js +++ b/docs/src/pages/research.js @@ -135,12 +135,12 @@ const papers = [ }, { title: "Hydroflow: A Model and Runtime for Distributed Systems Programming", - pdf: "pathname:///papers/dfir_rs-thesis.pdf", + pdf: "pathname:///papers/hydroflow-thesis.pdf", thumb: require("./img/papers/hydroflow-thesis.png"), authors: <>Mingwei Samuel, Alvin Cheung, Joseph M. Hellerstein, description: [ - <>In this paper we present our ongoing work on DFIR, a new cloud programming model used to create constructively correct distributed systems. The model is a refinement and unification of the existing dataflow and reactive programming models., - <>DFIR is primarily a low-level compilation target for future declarative cloud programming languages, but developers can use it directly to precisely control program execution or fine-tune and debug compiled programs. + <>In this paper we present our ongoing work on Hydroflow, a new cloud programming model used to create constructively correct distributed systems. The model is a refinement and unification of the existing dataflow and reactive programming models., + <>Hydroflow is primarily a low-level compilation target for future declarative cloud programming languages, but developers can use it directly to precisely control program execution or fine-tune and debug compiled programs. ], conf: "UC Berkeley Technical Report", links: <>PDF / GitHub diff --git a/hydro_lang/Cargo.toml b/hydro_lang/Cargo.toml index 760109ff526a..b61673d1b3a6 100644 --- a/hydro_lang/Cargo.toml +++ b/hydro_lang/Cargo.toml @@ -5,7 +5,7 @@ version = "0.10.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/hydro_lang/" -description = "Functional programming API for hydroflow" +description = "Choreographic programming for Hydro, built on DFIR" [lints] workspace = true diff --git a/hydro_std/Cargo.toml b/hydro_std/Cargo.toml index 50f61d6646ce..12e3a81ff01a 100644 --- a/hydro_std/Cargo.toml +++ b/hydro_std/Cargo.toml @@ -4,7 +4,7 @@ publish = true version = "0.10.0" edition = "2021" license = "Apache-2.0" -documentation = "https://docs.rs/hydroflow_plus_std/" +documentation = "https://docs.rs/hydro_plus/" description = "Standard library of distributed systems building blocks for Hydro" [lints] diff --git a/lattices/README.md b/lattices/README.md index 98e37a6f2b39..60c2dd4d6c69 100644 --- a/lattices/README.md +++ b/lattices/README.md @@ -21,7 +21,7 @@ For more information on the underlying mathematics of lattices and monotonicity, [Lattice Math section of the Hydroflow Book](https://hydro.run/docs/dfir/lattices_crate/lattice_math) and Section 2 of the [Hydroflow Thesis (2021)](https://hydro.run/papers/hydroflow-thesis.pdf). -Take a look at the [`lattice` rustdocs](https://hydro-project.github.io/hydroflow/doc/lattices/index.html). +Take a look at the [`lattice` rustdocs](https://hydro.run/rustdoc/lattices/index.html). ## Lattices diff --git a/template/dfir/Cargo.toml b/template/dfir/Cargo.toml index 79cac797950e..8b4b061d4b74 100644 --- a/template/dfir/Cargo.toml +++ b/template/dfir/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "hydroflow-template" +name = "dfir-template" version = "0.1.0" edition = "2021" diff --git a/website_playground/src/lib.rs b/website_playground/src/lib.rs index 0ba9c6381f59..0cdea5332eb1 100644 --- a/website_playground/src/lib.rs +++ b/website_playground/src/lib.rs @@ -87,12 +87,12 @@ impl From for JsDiagnostic { } #[derive(Serialize, Deserialize)] -pub struct HydroflowResult { - pub output: Option, +pub struct DfirResult { + pub output: Option, pub diagnostics: Vec, } #[derive(Serialize, Deserialize)] -pub struct HydroflowOutput { +pub struct DfirOutput { pub compiled: String, pub mermaid: String, } @@ -129,14 +129,14 @@ pub fn compile_dfir( } }; let compiled = prettyplease::unparse(&file); - HydroflowOutput { mermaid, compiled } + DfirOutput { mermaid, compiled } }); - HydroflowResult { + DfirResult { output, diagnostics: diagnostics.into_iter().map(Into::into).collect(), } } - Err(errors) => HydroflowResult { + Err(errors) => DfirResult { output: None, diagnostics: errors .into_iter() @@ -191,7 +191,7 @@ pub fn compile_datalog( } }; - Some(HydroflowOutput { + Some(DfirOutput { compiled: prettyplease::unparse(&file), mermaid: part_graph.to_mermaid(&write_config), }) @@ -201,17 +201,17 @@ pub fn compile_datalog( None } }; - HydroflowResult { + DfirResult { output, diagnostics: diagnostics.into_iter().map(Into::into).collect(), } } - Err(diagnostics) => HydroflowResult { + Err(diagnostics) => DfirResult { output: None, diagnostics: diagnostics.into_iter().map(Into::into).collect(), }, }, - Err(err) => HydroflowResult { + Err(err) => DfirResult { output: None, diagnostics: vec![Diagnostic { span: Span::call_site(), @@ -225,13 +225,13 @@ pub fn compile_datalog( serde_wasm_bindgen::to_value(&out).unwrap() } -struct HydroflowInstance<'a, In, Out> { +struct DfirInstance<'a, In, Out> { dfir: Dfir<'a>, input: tokio::sync::mpsc::UnboundedSender, output: tokio::sync::mpsc::UnboundedReceiver, } -type DatalogBooleanDemoInstance = HydroflowInstance<'static, (i32,), (i32,)>; +type DatalogBooleanDemoInstance = DfirInstance<'static, (i32,), (i32,)>; thread_local! { static DATALOG_BOOLEAN_DEMO_INSTANCES: RefCell> = diff --git a/website_playground/tests/node.rs b/website_playground/tests/node.rs index 4b96098d30fd..f318b8c4c528 100644 --- a/website_playground/tests/node.rs +++ b/website_playground/tests/node.rs @@ -1,4 +1,6 @@ //! Test suite for Node. +//! +//! TODO(mingwei): update this test! #![cfg(target_arch = "wasm32")]