From 5b8ed8424ef494e1c4d3df02d3e25244232bd474 Mon Sep 17 00:00:00 2001 From: Mingwei Samuel Date: Wed, 4 Sep 2024 14:35:33 -0700 Subject: [PATCH 1/6] add working test harness site --- Cargo.lock | 12 + Cargo.toml | 1 + docs/src/pages/playground.js | 2 + wasm_test_site/.github/dependabot.yml | 8 + wasm_test_site/.gitignore | 6 + wasm_test_site/Cargo.toml | 27 + wasm_test_site/README.md | 88 ++ wasm_test_site/src/lib.rs | 31 + wasm_test_site/tests/web.rs | 13 + wasm_test_site/www/.gitignore | 1 + wasm_test_site/www/README.md | 79 ++ wasm_test_site/www/index.html | 10 + wasm_test_site/www/index.ts | 4 + wasm_test_site/www/package-lock.json | 1093 +++++++++++++++++++++++++ wasm_test_site/www/package.json | 30 + wasm_test_site/www/tsconfig.json | 12 + wasm_test_site/www/vite.config.ts | 8 + 17 files changed, 1425 insertions(+) create mode 100644 wasm_test_site/.github/dependabot.yml create mode 100644 wasm_test_site/.gitignore create mode 100644 wasm_test_site/Cargo.toml create mode 100644 wasm_test_site/README.md create mode 100644 wasm_test_site/src/lib.rs create mode 100644 wasm_test_site/tests/web.rs create mode 100644 wasm_test_site/www/.gitignore create mode 100644 wasm_test_site/www/README.md create mode 100644 wasm_test_site/www/index.html create mode 100644 wasm_test_site/www/index.ts create mode 100644 wasm_test_site/www/package-lock.json create mode 100644 wasm_test_site/www/package.json create mode 100644 wasm_test_site/www/tsconfig.json create mode 100644 wasm_test_site/www/vite.config.ts diff --git a/Cargo.lock b/Cargo.lock index 28c15037ea84..ec00cea1d0b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3769,6 +3769,18 @@ dependencies = [ "syn 2.0.75", ] +[[package]] +name = "wasm_test_site" +version = "0.1.0" +dependencies = [ + "console_error_panic_hook", + "hydroflow", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.206.0" diff --git a/Cargo.toml b/Cargo.toml index 53af7d2e836a..28330f318888 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ members = [ "stageleft_tool", "topolotree", "variadics", + "wasm_test_site", "website_playground", ] diff --git a/docs/src/pages/playground.js b/docs/src/pages/playground.js index e1f706d3f5a1..b1225ab49c37 100644 --- a/docs/src/pages/playground.js +++ b/docs/src/pages/playground.js @@ -30,6 +30,8 @@ if (siteConfig.customFields.LOAD_PLAYGROUND === '1') { } playgroundJS.init(); + + window.test_hydroflow = playgroundJS.test_hydroflow; } import mermaid from "mermaid"; diff --git a/wasm_test_site/.github/dependabot.yml b/wasm_test_site/.github/dependabot.yml new file mode 100644 index 000000000000..7377d37597f7 --- /dev/null +++ b/wasm_test_site/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: +- package-ecosystem: cargo + directory: "/" + schedule: + interval: daily + time: "08:00" + open-pull-requests-limit: 10 diff --git a/wasm_test_site/.gitignore b/wasm_test_site/.gitignore new file mode 100644 index 000000000000..4e301317e55e --- /dev/null +++ b/wasm_test_site/.gitignore @@ -0,0 +1,6 @@ +/target +**/*.rs.bk +Cargo.lock +bin/ +pkg/ +wasm-pack.log diff --git a/wasm_test_site/Cargo.toml b/wasm_test_site/Cargo.toml new file mode 100644 index 000000000000..9bf32fb4c69c --- /dev/null +++ b/wasm_test_site/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "wasm_test_site" +version = "0.1.0" +authors = ["Mingwei Samuel "] +edition = "2018" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +hydroflow = { path = "../hydroflow" } +wasm-bindgen = "0.2.84" +wasm-bindgen-futures = "0.4.43" +web-sys = { version = "0.3.51", features = [ "console" ] } + +# The `console_error_panic_hook` crate provides better debugging of panics by +# logging them with `console.error`. This is great for development, but requires +# all the `std::fmt` and `std::panicking` infrastructure, so isn't great for +# code size when deploying. +console_error_panic_hook = "0.1.7" + +[dev-dependencies] +wasm-bindgen-test = "0.3.34" + +[profile.release] +# Tell `rustc` to optimize for small code size. +opt-level = "s" diff --git a/wasm_test_site/README.md b/wasm_test_site/README.md new file mode 100644 index 000000000000..6279fd688ffa --- /dev/null +++ b/wasm_test_site/README.md @@ -0,0 +1,88 @@ +# Go to `www/README.md` + +--- + +
+ +

wasm-pack-template

+ + A template for kick starting a Rust and WebAssembly project using wasm-pack. + +

+ Build Status +

+ +

+ Tutorial + | + Chat +

+ + Built with 🦀🕸 by The Rust and WebAssembly Working Group +
+ +## About + +[**📚 Read this template tutorial! 📚**][template-docs] + +This template is designed for compiling Rust libraries into WebAssembly and +publishing the resulting package to NPM. + +Be sure to check out [other `wasm-pack` tutorials online][tutorials] for other +templates and usages of `wasm-pack`. + +[tutorials]: https://rustwasm.github.io/docs/wasm-pack/tutorials/index.html +[template-docs]: https://rustwasm.github.io/docs/wasm-pack/tutorials/npm-browser-packages/index.html + +## 🚴 Usage + +### 🐑 Use `cargo generate` to Clone this Template + +[Learn more about `cargo generate` here.](https://github.com/ashleygwilliams/cargo-generate) + +``` +cargo generate --git https://github.com/rustwasm/wasm-pack-template.git --name my-project +cd my-project +``` + +### 🛠️ Build with `wasm-pack build` + +``` +wasm-pack build +``` + +### 🔬 Test in Headless Browsers with `wasm-pack test` + +``` +wasm-pack test --headless --firefox +``` + +### 🎁 Publish to NPM with `wasm-pack publish` + +``` +wasm-pack publish +``` + +## 🔋 Batteries Included + +* [`wasm-bindgen`](https://github.com/rustwasm/wasm-bindgen) for communicating + between WebAssembly and JavaScript. +* [`console_error_panic_hook`](https://github.com/rustwasm/console_error_panic_hook) + for logging panic messages to the developer console. +* `LICENSE-APACHE` and `LICENSE-MIT`: most Rust projects are licensed this way, so these are included for you + +## License + +Licensed under either of + +* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +* MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally +submitted for inclusion in the work by you, as defined in the Apache-2.0 +license, shall be dual licensed as above, without any additional terms or +conditions. diff --git a/wasm_test_site/src/lib.rs b/wasm_test_site/src/lib.rs new file mode 100644 index 000000000000..4ccae8151197 --- /dev/null +++ b/wasm_test_site/src/lib.rs @@ -0,0 +1,31 @@ +use wasm_bindgen::prelude::*; + +fn log(string: impl AsRef) { + web_sys::console::log_1(&JsValue::from_str(string.as_ref())); +} + +#[wasm_bindgen] +pub fn greet() { + console_error_panic_hook::set_once(); + + log("Hello, wasm_test_site!"); +} + +#[wasm_bindgen] +pub fn test_hydroflow() -> web_sys::js_sys::Promise { + console_error_panic_hook::set_once(); + + let mut df = hydroflow::hydroflow_syntax! { + // https://hydro.run/docs/hydroflow/quickstart/example_1_simplest + source_iter(0..10) -> for_each(|n| log(format!("Hello {}", n))); + }; + + wasm_bindgen_futures::future_to_promise(async move { + let work_done = df.run_available_async().await; + Ok(if work_done { + JsValue::TRUE + } else { + JsValue::FALSE + }) + }) +} diff --git a/wasm_test_site/tests/web.rs b/wasm_test_site/tests/web.rs new file mode 100644 index 000000000000..de5c1dafefb6 --- /dev/null +++ b/wasm_test_site/tests/web.rs @@ -0,0 +1,13 @@ +//! Test suite for the Web and headless browsers. + +#![cfg(target_arch = "wasm32")] + +extern crate wasm_bindgen_test; +use wasm_bindgen_test::*; + +wasm_bindgen_test_configure!(run_in_browser); + +#[wasm_bindgen_test] +fn pass() { + assert_eq!(1 + 1, 2); +} diff --git a/wasm_test_site/www/.gitignore b/wasm_test_site/www/.gitignore new file mode 100644 index 000000000000..096746c1480d --- /dev/null +++ b/wasm_test_site/www/.gitignore @@ -0,0 +1 @@ +/node_modules/ \ No newline at end of file diff --git a/wasm_test_site/www/README.md b/wasm_test_site/www/README.md new file mode 100644 index 000000000000..e0bb5ed4374b --- /dev/null +++ b/wasm_test_site/www/README.md @@ -0,0 +1,79 @@ +Run once to init: +``` +npm ci +``` +Run to rebuild: +``` +npm run wasm +npm run dev +``` + +--- + +
+ +

create-wasm-app

+ + An npm init template for kick starting a project that uses NPM packages containing Rust-generated WebAssembly and bundles them with Webpack. + +

+ Build Status +

+ +

+ Usage + | + Chat +

+ + Built with 🦀🕸 by The Rust and WebAssembly Working Group +
+ +## About + +This template is designed for depending on NPM packages that contain +Rust-generated WebAssembly and using them to create a Website. + +* Want to create an NPM package with Rust and WebAssembly? [Check out + `wasm-pack-template`.](https://github.com/rustwasm/wasm-pack-template) +* Want to make a monorepo-style Website without publishing to NPM? Check out + [`rust-webpack-template`](https://github.com/rustwasm/rust-webpack-template) + and/or + [`rust-parcel-template`](https://github.com/rustwasm/rust-parcel-template). + +## 🚴 Usage + +``` +npm init wasm-app +``` + +## 🔋 Batteries Included + +- `.gitignore`: ignores `node_modules` +- `LICENSE-APACHE` and `LICENSE-MIT`: most Rust projects are licensed this way, so these are included for you +- `README.md`: the file you are reading now! +- `index.html`: a bare bones html document that includes the webpack bundle +- `index.js`: example js file with a comment showing how to import and use a wasm pkg +- `package.json` and `package-lock.json`: + - pulls in devDependencies for using webpack: + - [`webpack`](https://www.npmjs.com/package/webpack) + - [`webpack-cli`](https://www.npmjs.com/package/webpack-cli) + - [`webpack-dev-server`](https://www.npmjs.com/package/webpack-dev-server) + - defines a `start` script to run `webpack-dev-server` +- `webpack.config.js`: configuration file for bundling your js with webpack + +## License + +Licensed under either of + +* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +* MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally +submitted for inclusion in the work by you, as defined in the Apache-2.0 +license, shall be dual licensed as above, without any additional terms or +conditions. diff --git a/wasm_test_site/www/index.html b/wasm_test_site/www/index.html new file mode 100644 index 000000000000..adf9ab2f9ac1 --- /dev/null +++ b/wasm_test_site/www/index.html @@ -0,0 +1,10 @@ + + + + + Hello wasm-pack! + + + + + diff --git a/wasm_test_site/www/index.ts b/wasm_test_site/www/index.ts new file mode 100644 index 000000000000..31168be73a8a --- /dev/null +++ b/wasm_test_site/www/index.ts @@ -0,0 +1,4 @@ +import init, { greet, test_hydroflow } from "wasm_test_site"; +let { memory } = await init(); +greet(); +test_hydroflow(); diff --git a/wasm_test_site/www/package-lock.json b/wasm_test_site/www/package-lock.json new file mode 100644 index 000000000000..a6fa21bb31f0 --- /dev/null +++ b/wasm_test_site/www/package-lock.json @@ -0,0 +1,1093 @@ +{ + "name": "create-wasm-app", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "create-wasm-app", + "version": "0.1.0", + "license": "(MIT OR Apache-2.0)", + "dependencies": { + "wasm_test_site": "file:../pkg" + }, + "devDependencies": { + "typescript": "^4.9.5", + "vite": "^4.1.4", + "vite-plugin-html": "^3.2.0", + "vite-plugin-wasm-pack": "^0.1.12" + } + }, + "../pkg": { + "name": "wasm_test_site", + "version": "0.1.0" + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.16.17", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.17", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "3.1.0", + "@jridgewell/sourcemap-codec": "1.4.14" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@rollup/pluginutils": { + "version": "4.2.1", + "dev": true, + "license": "MIT", + "dependencies": { + "estree-walker": "^2.0.1", + "picomatch": "^2.2.2" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/@types/node": { + "version": "18.14.6", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, + "node_modules/acorn": { + "version": "8.8.2", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/async": { + "version": "3.2.4", + "dev": true, + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "dev": true, + "license": "MIT" + }, + "node_modules/boolbase": { + "version": "1.0.0", + "dev": true, + "license": "ISC" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "dev": true, + "license": "MIT" + }, + "node_modules/camel-case": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "dependencies": { + "pascal-case": "^3.1.2", + "tslib": "^2.0.3" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/clean-css": { + "version": "5.3.2", + "dev": true, + "license": "MIT", + "dependencies": { + "source-map": "~0.6.0" + }, + "engines": { + "node": ">= 10.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "dev": true, + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.19", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "8.3.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/connect-history-api-fallback": { + "version": "1.6.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/consola": { + "version": "2.15.3", + "dev": true, + "license": "MIT" + }, + "node_modules/css-select": { + "version": "4.3.0", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", + "domutils": "^2.8.0", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/dom-serializer": { + "version": "1.4.1", + "dev": true, + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "4.3.1", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "2.8.0", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-case": { + "version": "3.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/dotenv": { + "version": "16.0.3", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/dotenv-expand": { + "version": "8.0.3", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/ejs": { + "version": "3.1.8", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/entities": { + "version": "2.2.0", + "dev": true, + "license": "BSD-2-Clause", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/esbuild": { + "version": "0.16.17", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.16.17", + "@esbuild/android-arm64": "0.16.17", + "@esbuild/android-x64": "0.16.17", + "@esbuild/darwin-arm64": "0.16.17", + "@esbuild/darwin-x64": "0.16.17", + "@esbuild/freebsd-arm64": "0.16.17", + "@esbuild/freebsd-x64": "0.16.17", + "@esbuild/linux-arm": "0.16.17", + "@esbuild/linux-arm64": "0.16.17", + "@esbuild/linux-ia32": "0.16.17", + "@esbuild/linux-loong64": "0.16.17", + "@esbuild/linux-mips64el": "0.16.17", + "@esbuild/linux-ppc64": "0.16.17", + "@esbuild/linux-riscv64": "0.16.17", + "@esbuild/linux-s390x": "0.16.17", + "@esbuild/linux-x64": "0.16.17", + "@esbuild/netbsd-x64": "0.16.17", + "@esbuild/openbsd-x64": "0.16.17", + "@esbuild/sunos-x64": "0.16.17", + "@esbuild/win32-arm64": "0.16.17", + "@esbuild/win32-ia32": "0.16.17", + "@esbuild/win32-x64": "0.16.17" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.2.12", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.15.0", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/filelist": { + "version": "1.0.4", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/brace-expansion": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.6", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs-extra": { + "version": "10.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "dev": true, + "license": "MIT" + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "dev": true, + "license": "ISC" + }, + "node_modules/has": { + "version": "1.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/he": { + "version": "1.2.0", + "dev": true, + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/html-minifier-terser": { + "version": "6.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "camel-case": "^4.1.2", + "clean-css": "^5.2.2", + "commander": "^8.3.0", + "he": "^1.2.0", + "param-case": "^3.0.4", + "relateurl": "^0.2.7", + "terser": "^5.10.0" + }, + "bin": { + "html-minifier-terser": "cli.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/is-core-module": { + "version": "2.11.0", + "dev": true, + "license": "MIT", + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/jake": { + "version": "10.8.5", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.1", + "minimatch": "^3.0.4" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/lower-case": { + "version": "2.0.2", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.3" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.5", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/nanoid": { + "version": "3.3.4", + "dev": true, + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/narrowing": { + "version": "1.5.0", + "dev": true, + "license": "MIT" + }, + "node_modules/no-case": { + "version": "3.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "lower-case": "^2.0.2", + "tslib": "^2.0.3" + } + }, + "node_modules/node-html-parser": { + "version": "5.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "css-select": "^4.2.1", + "he": "1.2.0" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/param-case": { + "version": "3.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "dot-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/pascal-case": { + "version": "3.1.2", + "dev": true, + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "dev": true, + "license": "MIT" + }, + "node_modules/pathe": { + "version": "0.2.0", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.0.0", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.4.21", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.4", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/relateurl": { + "version": "0.2.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/resolve": { + "version": "1.22.1", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "3.18.0", + "dev": true, + "license": "MIT", + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=14.18.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.0.2", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/terser": { + "version": "5.16.6", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@jridgewell/source-map": "^0.3.2", + "acorn": "^8.5.0", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser/node_modules/commander": { + "version": "2.20.3", + "dev": true, + "license": "MIT" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/tslib": { + "version": "2.5.0", + "dev": true, + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "4.9.5", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/universalify": { + "version": "2.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/vite": { + "version": "4.1.4", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.16.14", + "postcss": "^8.4.21", + "resolve": "^1.22.1", + "rollup": "^3.10.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + }, + "peerDependencies": { + "@types/node": ">= 14", + "less": "*", + "sass": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vite-plugin-html": { + "version": "3.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@rollup/pluginutils": "^4.2.0", + "colorette": "^2.0.16", + "connect-history-api-fallback": "^1.6.0", + "consola": "^2.15.3", + "dotenv": "^16.0.0", + "dotenv-expand": "^8.0.2", + "ejs": "^3.1.6", + "fast-glob": "^3.2.11", + "fs-extra": "^10.0.1", + "html-minifier-terser": "^6.1.0", + "node-html-parser": "^5.3.3", + "pathe": "^0.2.0" + }, + "peerDependencies": { + "vite": ">=2.0.0" + } + }, + "node_modules/vite-plugin-wasm-pack": { + "version": "0.1.12", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2", + "fs-extra": "^10.0.0", + "narrowing": "^1.4.0" + } + }, + "node_modules/wasm_test_site": { + "resolved": "../pkg", + "link": true + } + } +} diff --git a/wasm_test_site/www/package.json b/wasm_test_site/www/package.json new file mode 100644 index 000000000000..0b2a75af9d98 --- /dev/null +++ b/wasm_test_site/www/package.json @@ -0,0 +1,30 @@ +{ + "name": "create-wasm-app", + "version": "0.1.0", + "description": "create an app to consume rust-generated wasm packages", + "main": "index.js", + "type": "module", + "scripts": { + "build": "vite build", + "dev": "vite serve", + "preview": "vite preview", + "wasm": "wasm-pack build ../ --target web" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/rustwasm/create-wasm-app.git" + }, + "keywords": [ + "webassembly", + "wasm", + "rust", + "webpack" + ], + "license": "(MIT OR Apache-2.0)", + "devDependencies": { + "typescript": "^4.9.5", + "vite": "^4.1.4", + "vite-plugin-html": "^3.2.0", + "vite-plugin-wasm-pack": "^0.1.12" + } +} diff --git a/wasm_test_site/www/tsconfig.json b/wasm_test_site/www/tsconfig.json new file mode 100644 index 000000000000..a0b6b0c55bb5 --- /dev/null +++ b/wasm_test_site/www/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "outDir": "./dist/", + "noImplicitAny": true, + "module": "esnext", + "target": "es2017", + "jsx": "react", + "allowJs": true, + "moduleResolution": "node", + "esModuleInterop": true + } +} diff --git a/wasm_test_site/www/vite.config.ts b/wasm_test_site/www/vite.config.ts new file mode 100644 index 000000000000..76b29b842a20 --- /dev/null +++ b/wasm_test_site/www/vite.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vite"; +import wasmPack from "vite-plugin-wasm-pack"; +import path from "node:path"; + +export default defineConfig({ + // pass your local crate path to the plugin + plugins: [wasmPack(path.resolve("../"))], +}); From 871263528d33a39b18c89e3baa63513fad600931 Mon Sep 17 00:00:00 2001 From: Mingwei Samuel Date: Thu, 5 Sep 2024 10:29:09 -0700 Subject: [PATCH 2/6] call global javascript function from rust (ugly) --- wasm_test_site/src/lib.rs | 8 +++++++- wasm_test_site/www/index.ts | 6 ++++++ wasm_test_site/www/package-lock.json | 10 ++-------- wasm_test_site/www/package.json | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/wasm_test_site/src/lib.rs b/wasm_test_site/src/lib.rs index 4ccae8151197..2384e595a19e 100644 --- a/wasm_test_site/src/lib.rs +++ b/wasm_test_site/src/lib.rs @@ -4,6 +4,12 @@ fn log(string: impl AsRef) { web_sys::console::log_1(&JsValue::from_str(string.as_ref())); } +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(js_namespace = window)] + fn writeToDom(s: &str); +} + #[wasm_bindgen] pub fn greet() { console_error_panic_hook::set_once(); @@ -17,7 +23,7 @@ pub fn test_hydroflow() -> web_sys::js_sys::Promise { let mut df = hydroflow::hydroflow_syntax! { // https://hydro.run/docs/hydroflow/quickstart/example_1_simplest - source_iter(0..10) -> for_each(|n| log(format!("Hello {}", n))); + source_iter(0..10) -> for_each(|n| writeToDom(&format!("Hello {}", n))); }; wasm_bindgen_futures::future_to_promise(async move { diff --git a/wasm_test_site/www/index.ts b/wasm_test_site/www/index.ts index 31168be73a8a..fcc5ed436434 100644 --- a/wasm_test_site/www/index.ts +++ b/wasm_test_site/www/index.ts @@ -1,4 +1,10 @@ import init, { greet, test_hydroflow } from "wasm_test_site"; let { memory } = await init(); + +(window as any).writeToDom = function(str: string) { + document.body.appendChild(document.createTextNode(str)); + document.body.append(document.createElement('br')); +}; + greet(); test_hydroflow(); diff --git a/wasm_test_site/www/package-lock.json b/wasm_test_site/www/package-lock.json index a6fa21bb31f0..4d37c875cc22 100644 --- a/wasm_test_site/www/package-lock.json +++ b/wasm_test_site/www/package-lock.json @@ -8,9 +8,6 @@ "name": "create-wasm-app", "version": "0.1.0", "license": "(MIT OR Apache-2.0)", - "dependencies": { - "wasm_test_site": "file:../pkg" - }, "devDependencies": { "typescript": "^4.9.5", "vite": "^4.1.4", @@ -20,7 +17,8 @@ }, "../pkg": { "name": "wasm_test_site", - "version": "0.1.0" + "version": "0.1.0", + "extraneous": true }, "node_modules/@esbuild/win32-x64": { "version": "0.16.17", @@ -1084,10 +1082,6 @@ "fs-extra": "^10.0.0", "narrowing": "^1.4.0" } - }, - "node_modules/wasm_test_site": { - "resolved": "../pkg", - "link": true } } } diff --git a/wasm_test_site/www/package.json b/wasm_test_site/www/package.json index 0b2a75af9d98..51c26c38cab0 100644 --- a/wasm_test_site/www/package.json +++ b/wasm_test_site/www/package.json @@ -27,4 +27,4 @@ "vite-plugin-html": "^3.2.0", "vite-plugin-wasm-pack": "^0.1.12" } -} +} \ No newline at end of file From 9897ec2e087544721eb9b0c22a71000e8d02e257 Mon Sep 17 00:00:00 2001 From: Rohit Kulshreshtha Date: Thu, 5 Sep 2024 15:35:05 -0700 Subject: [PATCH 3/6] Checkpointing websocket server. --- Cargo.lock | 48 +++++++++++++++-- hydroflow/Cargo.toml | 1 + hydroflow/examples/chat/main.rs | 17 +++--- hydroflow/examples/chat/server.rs | 25 ++++----- hydroflow/examples/echoserver/main.rs | 8 ++- hydroflow/src/util/mod.rs | 2 + hydroflow/src/util/websocket.rs | 77 +++++++++++++++++++++++++++ 7 files changed, 146 insertions(+), 32 deletions(-) create mode 100644 hydroflow/src/util/websocket.rs diff --git a/Cargo.lock b/Cargo.lock index 28c15037ea84..71f50015b24f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1228,6 +1228,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "httparse" version = "1.9.4" @@ -1274,7 +1285,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tokio-tungstenite", + "tokio-tungstenite 0.20.1", ] [[package]] @@ -1346,6 +1357,7 @@ dependencies = [ "time", "tokio", "tokio-stream", + "tokio-tungstenite 0.23.1", "tokio-util", "tracing", "tracing-subscriber", @@ -3236,7 +3248,19 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite", + "tungstenite 0.20.1", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.23.0", ] [[package]] @@ -3550,7 +3574,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 0.2.12", "httparse", "log", "rand", @@ -3560,6 +3584,24 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.1.0", + "httparse", + "log", + "rand", + "sha1", + "thiserror", + "utf-8", +] + [[package]] name = "typenum" version = "1.17.0" diff --git a/hydroflow/Cargo.toml b/hydroflow/Cargo.toml index 3d4400e9deb3..58c11d27726f 100644 --- a/hydroflow/Cargo.toml +++ b/hydroflow/Cargo.toml @@ -59,6 +59,7 @@ serde_json = "1.0.115" slotmap = "1.0.0" smallvec = "1.6.1" tokio-stream = { version = "0.1.3", default-features = false, features = [ "time", "io-util", "sync" ] } +tokio-tungstenite = "0.23.1" tracing = "0.1.37" variadics = { path = "../variadics", version = "^0.0.6" } web-time = "1.0.0" diff --git a/hydroflow/examples/chat/main.rs b/hydroflow/examples/chat/main.rs index afc61ce57360..b6c6461a67ae 100644 --- a/hydroflow/examples/chat/main.rs +++ b/hydroflow/examples/chat/main.rs @@ -49,17 +49,18 @@ async fn main() { let opts = Opts::parse(); match opts.role { - Role::Client => { - run_client(opts).await; - } + // Role::Client => { + // run_client(opts).await; + // } Role::Server => { run_server(opts).await; } - Role::GossipingServer1 - | Role::GossipingServer2 - | Role::GossipingServer3 - | Role::GossipingServer4 - | Role::GossipingServer5 => run_gossiping_server(opts).await, + // Role::GossipingServer1 + // | Role::GossipingServer2 + // | Role::GossipingServer3 + // | Role::GossipingServer4 + // | Role::GossipingServer5 => run_gossiping_server(opts).await, + _ => unimplemented!(), } } diff --git a/hydroflow/examples/chat/server.rs b/hydroflow/examples/chat/server.rs index f4cc07fea06f..c6dc4645cfdd 100644 --- a/hydroflow/examples/chat/server.rs +++ b/hydroflow/examples/chat/server.rs @@ -1,6 +1,6 @@ use hydroflow::hydroflow_syntax; use hydroflow::scheduled::graph::Hydroflow; -use hydroflow::util::bind_udp_bytes; +use hydroflow::util::{bind_udp_bytes, bind_websocket}; use crate::protocol::{Message, MessageWithAddr}; use crate::{default_server_address, Opts}; @@ -14,27 +14,20 @@ pub(crate) async fn run_server(opts: Opts) { println!("Starting server on {:?}", server_address); - let (outbound, inbound, actual_server_addr) = bind_udp_bytes(server_address).await; + let (outbound, inbound, actual_server_addr) = bind_websocket(server_address).await.unwrap(); println!("Server is live! Listening on {:?}", actual_server_addr); let mut hf: Hydroflow = hydroflow_syntax! { // Define shared inbound and outbound channels - outbound_chan = union() -> dest_sink_serde(outbound); - inbound_chan = source_stream_serde(inbound) + inbound_chan = source_stream(inbound) -> map(Result::unwrap) - -> map(|(msg, addr)| MessageWithAddr::from_message(msg, addr)) - -> demux_enum::(); - clients = inbound_chan[ConnectRequest] -> map(|(addr,)| addr) -> tee(); - inbound_chan[ConnectResponse] -> for_each(|(addr,)| println!("Received unexpected `ConnectResponse` as server from addr {}.", addr)); - - // Pipeline 1: Acknowledge client connections - clients[0] -> map(|addr| (Message::ConnectResponse, addr)) -> [0]outbound_chan; - - // Pipeline 2: Broadcast messages to all clients - inbound_chan[ChatMsg] -> map(|(_addr, nickname, message, ts)| Message::ChatMsg { nickname, message, ts }) -> [0]broadcast; - clients[1] -> [1]broadcast; - broadcast = cross_join::<'tick, 'static>() -> [1]outbound_chan; + -> tee(); + + inbound_chan -> map(|(msg, addr)| addr) -> [1]broadcast; + inbound_chan -> map(|(msg, addr)| msg) -> [0]broadcast; + + broadcast = cross_join::<'tick, 'static>() -> dest_sink(outbound); }; #[cfg(feature = "debugging")] diff --git a/hydroflow/examples/echoserver/main.rs b/hydroflow/examples/echoserver/main.rs index 6734c790289d..c6f964f420cf 100644 --- a/hydroflow/examples/echoserver/main.rs +++ b/hydroflow/examples/echoserver/main.rs @@ -2,7 +2,7 @@ use std::net::SocketAddr; use clap::{Parser, ValueEnum}; use client::run_client; -use hydroflow::util::{bind_udp_bytes, ipv4_resolve}; +use hydroflow::util::{bind_udp_bytes, bind_websocket, ipv4_resolve}; use server::run_server; mod client; @@ -35,16 +35,14 @@ async fn main() { .unwrap_or_else(|| ipv4_resolve("localhost:0").unwrap()); // allocate `outbound` sink and `inbound` stream - let (outbound, inbound, addr) = bind_udp_bytes(addr).await; + let (outbound, inbound, addr) = bind_websocket(addr).await.unwrap(); println!("Listening on {:?}", addr); match opts.role { Role::Server => { run_server(outbound, inbound, opts).await; } - Role::Client => { - run_client(outbound, inbound, opts).await; - } + _ => panic!("Unsupported!") } } diff --git a/hydroflow/src/util/mod.rs b/hydroflow/src/util/mod.rs index 4dd2332d4f36..2a637d88db62 100644 --- a/hydroflow/src/util/mod.rs +++ b/hydroflow/src/util/mod.rs @@ -29,6 +29,8 @@ pub use socket::*; #[cfg(feature = "deploy_integration")] pub mod deploy; +mod websocket; +pub use websocket::*; use std::io::Read; use std::net::SocketAddr; diff --git a/hydroflow/src/util/websocket.rs b/hydroflow/src/util/websocket.rs new file mode 100644 index 000000000000..23bbb65bff04 --- /dev/null +++ b/hydroflow/src/util/websocket.rs @@ -0,0 +1,77 @@ +use std::cell::RefCell; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::pin::pin; +use std::rc::Rc; +use futures::{SinkExt, StreamExt}; +use tokio::net::{TcpListener}; +use tokio::task::spawn_local; +use tokio_tungstenite::tungstenite::{Error, Message}; +use crate::util::unsync::mpsc::{Receiver, Sender}; +use crate::util::unsync_channel; + +pub async fn bind_websocket(endpoint: SocketAddr) -> Result<(Sender<(Message, SocketAddr)>, Receiver>, SocketAddr), std::io::Error>{ + let listener = TcpListener::bind(endpoint).await.unwrap(); + + let bound_endpoint = listener.local_addr()?; + + let (tx_egress, mut rx_egress) = unsync_channel(None); + let (tx_ingress, rx_ingress) = unsync_channel(None); + + let clients = Rc::new(RefCell::new(HashMap::new())); + + spawn_local({ + let clients = clients.clone(); + + async move { + while let Some((payload, addr)) = rx_egress.next().await { + let client = clients.borrow_mut().remove(&addr); + + if let Some(mut sender) = client { + let _ = SinkExt::send(&mut sender, payload).await; + clients.borrow_mut().insert(addr, sender); + } + } + } + }); + + // Spawn the listener + spawn_local(async move { + loop { + let (stream, peer_addr) = if let Ok((stream, _)) = listener.accept().await { + if let Ok(peer_addr) = stream.peer_addr() { + (stream, peer_addr) + } else { + continue; + } + } else { + continue; + }; + + // Perform the websocket handshake + let ws_stream = tokio_tungstenite::accept_async(stream) + .await + .expect("Error during the websocket handshake occurred"); + + // Split the stream into incoming and outgoing + let (outgoing, incoming) = ws_stream.split(); + let mut tx_ingress = tx_ingress.clone(); + + clients.borrow_mut().insert(peer_addr, outgoing); + + spawn_local({ + let clients = clients.clone(); + async move { + let mapped = incoming.map(|x| Ok(x.map(|x| (x, peer_addr)))); + let _ = tx_ingress.send_all(&mut pin!(mapped)).await; + + clients.borrow_mut().remove(&peer_addr); + } + }); + } + + }); + + Ok((tx_egress, rx_ingress, bound_endpoint)) +} + From a64a0a6cd8f98eb2ca9ab5c30859324216eaa2ac Mon Sep 17 00:00:00 2001 From: Rohit Kulshreshtha Date: Fri, 6 Sep 2024 14:52:57 -0700 Subject: [PATCH 4/6] Checkpointing: have a chat UI setup. --- Cargo.lock | 229 ++++++++++++++++++++++++++++- hydroflow/Cargo.toml | 3 +- hydroflow/src/scheduled/context.rs | 1 + hydroflow/src/util/mod.rs | 3 + hydroflow/src/util/websocket.rs | 1 + wasm_test_site/Cargo.toml | 4 +- wasm_test_site/src/lib.rs | 106 +++++++++---- wasm_test_site/www/index.html | 61 ++++++-- wasm_test_site/www/index.ts | 8 - 9 files changed, 370 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58802edcd112..28ba9a7349d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1150,6 +1150,190 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "gloo" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d15282ece24eaf4bd338d73ef580c6714c8615155c4190c781290ee3fa0fd372" +dependencies = [ + "gloo-console", + "gloo-dialogs", + "gloo-events", + "gloo-file", + "gloo-history", + "gloo-net", + "gloo-render", + "gloo-storage", + "gloo-timers", + "gloo-utils", + "gloo-worker", +] + +[[package]] +name = "gloo-console" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a17868f56b4a24f677b17c8cb69958385102fa879418052d60b50bc1727e261" +dependencies = [ + "gloo-utils", + "js-sys", + "serde", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-dialogs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4748e10122b01435750ff530095b1217cf6546173459448b83913ebe7815df" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-events" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c26fb45f7c385ba980f5fa87ac677e363949e065a083722697ef1b2cc91e41" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-file" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97563d71863fb2824b2e974e754a81d19c4a7ec47b09ced8a0e6656b6d54bd1f" +dependencies = [ + "futures-channel", + "gloo-events", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-history" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "903f432be5ba34427eac5e16048ef65604a82061fe93789f2212afc73d8617d6" +dependencies = [ + "getrandom", + "gloo-events", + "gloo-utils", + "serde", + "serde-wasm-bindgen 0.6.5", + "serde_urlencoded", + "thiserror", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-net" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43aaa242d1239a8822c15c645f02166398da4f8b5c4bae795c1f5b44e9eee173" +dependencies = [ + "futures-channel", + "futures-core", + "futures-sink", + "gloo-utils", + "http 0.2.12", + "js-sys", + "pin-project", + "serde", + "serde_json", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "gloo-render" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56008b6744713a8e8d98ac3dcb7d06543d5662358c9c805b4ce2167ad4649833" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-storage" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc8031e8c92758af912f9bc08fbbadd3c6f3cfcbf6b64cdf3d6a81f0139277a" +dependencies = [ + "gloo-utils", + "js-sys", + "serde", + "serde_json", + "thiserror", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "gloo-utils" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5555354113b18c547c1d3a98fbf7fb32a9ff4f6fa112ce823a21641a0ba3aa" +dependencies = [ + "js-sys", + "serde", + "serde_json", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-worker" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "085f262d7604911c8150162529cefab3782e91adb20202e8658f7275d2aefe5d" +dependencies = [ + "bincode", + "futures", + "gloo-utils", + "gloo-worker-macros", + "js-sys", + "pinned", + "serde", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "gloo-worker-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "956caa58d4857bc9941749d55e4bd3000032d8212762586fa5705632967140e7" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.75", +] + [[package]] name = "half" version = "2.4.1" @@ -1363,6 +1547,7 @@ dependencies = [ "tracing-subscriber", "trybuild", "variadics", + "wasm-bindgen-futures", "wasm-bindgen-test", "web-time", "zipf", @@ -2143,6 +2328,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pinned" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a829027bd95e54cfe13e3e258a1ae7b645960553fb82b75ff852c29688ee595b" +dependencies = [ + "futures", + "rustversion", + "thiserror", +] + [[package]] name = "piper" version = "0.2.4" @@ -2647,6 +2843,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + [[package]] name = "ryu" version = "1.0.18" @@ -2736,6 +2938,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_derive" version = "1.0.208" @@ -2769,6 +2982,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -3816,6 +4041,8 @@ name = "wasm_test_site" version = "0.1.0" dependencies = [ "console_error_panic_hook", + "futures", + "gloo", "hydroflow", "wasm-bindgen", "wasm-bindgen-futures", @@ -3887,7 +4114,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "serde-wasm-bindgen", + "serde-wasm-bindgen 0.4.5", "syn 2.0.75", "tokio", "wasm-bindgen", diff --git a/hydroflow/Cargo.toml b/hydroflow/Cargo.toml index 58c11d27726f..e0d268cde68b 100644 --- a/hydroflow/Cargo.toml +++ b/hydroflow/Cargo.toml @@ -59,7 +59,7 @@ serde_json = "1.0.115" slotmap = "1.0.0" smallvec = "1.6.1" tokio-stream = { version = "0.1.3", default-features = false, features = [ "time", "io-util", "sync" ] } -tokio-tungstenite = "0.23.1" +tokio-tungstenite = { version = "0.23.1", optional = true } tracing = "0.1.37" variadics = { path = "../variadics", version = "^0.0.6" } web-time = "1.0.0" @@ -74,6 +74,7 @@ tokio-util = { version = "0.7.5", features = [ "net", "codec" ] } [target.'cfg(target_arch = "wasm32")'.dependencies] tokio = { version = "1.29.0", features = [ "rt" , "sync", "macros", "io-util", "time" ] } tokio-util = { version = "0.7.5", features = [ "codec" ] } +wasm-bindgen-futures = "0.4.43" # We depend on getrandom transitively through rand. To compile getrandom to # WASM, we need to enable its "js" feature. However, rand does not expose a # passthrough to enable "js" on getrandom. As a workaround, we enable the diff --git a/hydroflow/src/scheduled/context.rs b/hydroflow/src/scheduled/context.rs index 029f43736ba4..cb059b64ec75 100644 --- a/hydroflow/src/scheduled/context.rs +++ b/hydroflow/src/scheduled/context.rs @@ -190,6 +190,7 @@ impl Context { for task in self.tasks_to_spawn.drain(..) { self.task_join_handles.push(tokio::task::spawn_local(task)); } + } /// Aborts all tasks spawned with [`Self::spawn_tasks`]. diff --git a/hydroflow/src/util/mod.rs b/hydroflow/src/util/mod.rs index 2a637d88db62..efd5a3b708a7 100644 --- a/hydroflow/src/util/mod.rs +++ b/hydroflow/src/util/mod.rs @@ -29,7 +29,10 @@ pub use socket::*; #[cfg(feature = "deploy_integration")] pub mod deploy; + +#[cfg(not(target_arch = "wasm32"))] mod websocket; +#[cfg(not(target_arch = "wasm32"))] pub use websocket::*; use std::io::Read; diff --git a/hydroflow/src/util/websocket.rs b/hydroflow/src/util/websocket.rs index 23bbb65bff04..b3c25c6c5238 100644 --- a/hydroflow/src/util/websocket.rs +++ b/hydroflow/src/util/websocket.rs @@ -10,6 +10,7 @@ use tokio_tungstenite::tungstenite::{Error, Message}; use crate::util::unsync::mpsc::{Receiver, Sender}; use crate::util::unsync_channel; + pub async fn bind_websocket(endpoint: SocketAddr) -> Result<(Sender<(Message, SocketAddr)>, Receiver>, SocketAddr), std::io::Error>{ let listener = TcpListener::bind(endpoint).await.unwrap(); diff --git a/wasm_test_site/Cargo.toml b/wasm_test_site/Cargo.toml index 9bf32fb4c69c..b21c94669bf0 100644 --- a/wasm_test_site/Cargo.toml +++ b/wasm_test_site/Cargo.toml @@ -8,16 +8,18 @@ edition = "2018" crate-type = ["cdylib", "rlib"] [dependencies] +gloo = { version = "0.11.0", features = ["futures"] } hydroflow = { path = "../hydroflow" } wasm-bindgen = "0.2.84" wasm-bindgen-futures = "0.4.43" -web-sys = { version = "0.3.51", features = [ "console" ] } +web-sys = { version = "0.3.51", features = [ "console", "Window", "Document", "Element", "HtmlElement", "HtmlInputElement", "EventTarget"] } # The `console_error_panic_hook` crate provides better debugging of panics by # logging them with `console.error`. This is great for development, but requires # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for # code size when deploying. console_error_panic_hook = "0.1.7" +futures = "0.3.30" [dev-dependencies] wasm-bindgen-test = "0.3.34" diff --git a/wasm_test_site/src/lib.rs b/wasm_test_site/src/lib.rs index 2384e595a19e..6436eb8b930b 100644 --- a/wasm_test_site/src/lib.rs +++ b/wasm_test_site/src/lib.rs @@ -1,37 +1,91 @@ use wasm_bindgen::prelude::*; +use wasm_bindgen::JsCast; +use wasm_bindgen_futures::spawn_local; +use web_sys::{window, HtmlElement, HtmlInputElement}; +use futures::channel::mpsc; +use futures::stream::StreamExt; +use gloo::timers::future::TimeoutFuture; // Use the correct future module from gloo -fn log(string: impl AsRef) { - web_sys::console::log_1(&JsValue::from_str(string.as_ref())); -} +#[wasm_bindgen(start)] +pub fn start() -> Result<(), JsValue> { + console_error_panic_hook::set_once(); -#[wasm_bindgen] -extern "C" { - #[wasm_bindgen(js_namespace = window)] - fn writeToDom(s: &str); -} + // Create a pair of channels for sending messages to the server + let (send_tx, send_rx) = mpsc::unbounded::(); -#[wasm_bindgen] -pub fn greet() { - console_error_panic_hook::set_once(); + // Create a pair of channels for receiving messages from the server + let (recv_tx, mut recv_rx) = mpsc::unbounded::(); - log("Hello, wasm_test_site!"); -} + // Spawn an async task to handle receiving messages from the server + spawn_local(async move { + // Get the document and chat container + let window = window().expect("should have a window in this context"); + let document = window.document().expect("window should have a document"); + let chat_container = document.get_element_by_id("chat-container") + .expect("document should have a chat container with id 'chat-container'") + .dyn_into::().unwrap(); -#[wasm_bindgen] -pub fn test_hydroflow() -> web_sys::js_sys::Promise { - console_error_panic_hook::set_once(); + // Process each message received from the server (via recv_rx) + while let Some(message) = recv_rx.next().await { + let new_message_element = document.create_element("div").unwrap(); + new_message_element.set_text_content(Some(&message)); + chat_container.append_child(&new_message_element).unwrap(); + chat_container.set_scroll_top(chat_container.scroll_height()); // Scroll to bottom + } + }); + + // Set up the event listener for the Send button to send messages to the server + setup_send_button(send_tx)?; let mut df = hydroflow::hydroflow_syntax! { - // https://hydro.run/docs/hydroflow/quickstart/example_1_simplest - source_iter(0..10) -> for_each(|n| writeToDom(&format!("Hello {}", n))); + + outbound = source_stream(send_rx); + inbound = dest_sink(recv_tx); + + outbound -> inbound; }; - wasm_bindgen_futures::future_to_promise(async move { - let work_done = df.run_available_async().await; - Ok(if work_done { - JsValue::TRUE - } else { - JsValue::FALSE - }) - }) + spawn_local(async move { + let work_done = df.run_async().await; + }); + + Ok(()) } + +// Set up the Send button to send messages to the server +fn setup_send_button(send_tx: mpsc::UnboundedSender) -> Result<(), JsValue> { + // Get the window and document objects + let window = window().expect("should have a window in this context"); + let document = window.document().expect("window should have a document"); + + // Get the send button and message input box + let send_button = document.get_element_by_id("send-button") + .expect("document should have a send button with id 'send-button'") + .dyn_into::()?; + let input_box = document.get_element_by_id("new-message") + .expect("document should have an input box with id 'new-message'") + .dyn_into::()?; + + // Clone the input_box and send_tx for use inside the closure + let input_box_clone = input_box.clone(); + let send_tx_clone = send_tx.clone(); + + // Create the closure that will send a message when the button is clicked + let closure = Closure::wrap(Box::new(move || { + let message = input_box_clone.value(); + if !message.is_empty() { + // Send the message to the channel for server dispatching + send_tx_clone.unbounded_send(message.clone()).unwrap(); + // Clear the input box after sending the message + input_box_clone.set_value(""); + } + }) as Box); + + // Attach the event listener to the send button + send_button.add_event_listener_with_callback("click", closure.as_ref().unchecked_ref())?; + + // Keep the closure alive + closure.forget(); + + Ok(()) +} \ No newline at end of file diff --git a/wasm_test_site/www/index.html b/wasm_test_site/www/index.html index adf9ab2f9ac1..8f266a874569 100644 --- a/wasm_test_site/www/index.html +++ b/wasm_test_site/www/index.html @@ -1,10 +1,53 @@ - - - - - Hello wasm-pack! - - - - + + + + + WASM Chat + + + +
+
+ + +
+ + + + diff --git a/wasm_test_site/www/index.ts b/wasm_test_site/www/index.ts index fcc5ed436434..8db9935d5654 100644 --- a/wasm_test_site/www/index.ts +++ b/wasm_test_site/www/index.ts @@ -1,10 +1,2 @@ import init, { greet, test_hydroflow } from "wasm_test_site"; let { memory } = await init(); - -(window as any).writeToDom = function(str: string) { - document.body.appendChild(document.createTextNode(str)); - document.body.append(document.createElement('br')); -}; - -greet(); -test_hydroflow(); From 6ed3aa619d3e9669335f25f1518640f1a639a453 Mon Sep 17 00:00:00 2001 From: Mingwei Samuel Date: Thu, 19 Sep 2024 12:49:40 -0700 Subject: [PATCH 5/6] basic echo chat working --- .vscode/settings.json | 2 +- Cargo.lock | 59 ++++++++++++++++++- hydroflow/Cargo.toml | 5 ++ hydroflow/examples/echoserver/main.rs | 2 +- .../examples/echoserver_websocket/main.rs | 50 ++++++++++++++++ hydroflow/src/util/mod.rs | 4 +- wasm_test_site/Cargo.toml | 2 + wasm_test_site/src/lib.rs | 48 +++++++++------ wasm_test_site/www/index.ts | 2 +- 9 files changed, 151 insertions(+), 23 deletions(-) create mode 100644 hydroflow/examples/echoserver_websocket/main.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index 1057a431c935..1806fb4c8ec7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,6 +17,6 @@ "**/target": true }, "rust-analyzer.cargo.features": [ - "python" + "python", "websocket", ] } diff --git a/Cargo.lock b/Cargo.lock index 28ba9a7349d4..e0496695dd9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,6 +277,17 @@ dependencies = [ "syn 2.0.75", ] +[[package]] +name = "async_io_stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" +dependencies = [ + "futures", + "pharos", + "rustc_version 0.4.1", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -2296,6 +2307,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures", + "rustc_version 0.4.1", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -2458,7 +2479,7 @@ dependencies = [ "byteorder", "libc", "nom", - "rustc_version", + "rustc_version 0.2.3", ] [[package]] @@ -2830,6 +2851,15 @@ dependencies = [ "semver 0.9.0", ] +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver 1.0.23", +] + [[package]] name = "rustix" version = "0.38.34" @@ -2912,6 +2942,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + [[package]] name = "seq-macro" version = "0.2.2" @@ -4044,10 +4080,12 @@ dependencies = [ "futures", "gloo", "hydroflow", + "pharos", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", "web-sys", + "ws_stream_wasm", ] [[package]] @@ -4394,6 +4432,25 @@ dependencies = [ "memchr", ] +[[package]] +name = "ws_stream_wasm" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7999f5f4217fe3818726b66257a4475f71e74ffd190776ad053fa159e50737f5" +dependencies = [ + "async_io_stream", + "futures", + "js-sys", + "log", + "pharos", + "rustc_version 0.4.1", + "send_wrapper", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "zerocopy" version = "0.7.35" diff --git a/hydroflow/Cargo.toml b/hydroflow/Cargo.toml index e0d268cde68b..d471fed6d46b 100644 --- a/hydroflow/Cargo.toml +++ b/hydroflow/Cargo.toml @@ -20,6 +20,7 @@ hydroflow_datalog = [ "dep:hydroflow_datalog" ] deploy_integration = [ "dep:hydroflow_deploy_integration" ] python = [ "dep:pyo3" ] debugging = [ "hydroflow_lang/debugging" ] +websocket = [ "dep:tokio-tungstenite" ] [[example]] name = "kvs_bench" @@ -37,6 +38,10 @@ required-features = [ "debugging" ] name = "modules_triple_cross_join" required-features = [ "debugging" ] +[[example]] +name = "echoserver_websocket" +required-features = [ "websocket" ] + [dependencies] bincode = "1.3.1" byteorder = "1.3.2" diff --git a/hydroflow/examples/echoserver/main.rs b/hydroflow/examples/echoserver/main.rs index c6f964f420cf..7de8f8cae6bc 100644 --- a/hydroflow/examples/echoserver/main.rs +++ b/hydroflow/examples/echoserver/main.rs @@ -35,7 +35,7 @@ async fn main() { .unwrap_or_else(|| ipv4_resolve("localhost:0").unwrap()); // allocate `outbound` sink and `inbound` stream - let (outbound, inbound, addr) = bind_websocket(addr).await.unwrap(); + let (outbound, inbound, addr) = bind_udp_bytes(addr).await.unwrap(); println!("Listening on {:?}", addr); match opts.role { diff --git a/hydroflow/examples/echoserver_websocket/main.rs b/hydroflow/examples/echoserver_websocket/main.rs new file mode 100644 index 000000000000..17b4d7343fd2 --- /dev/null +++ b/hydroflow/examples/echoserver_websocket/main.rs @@ -0,0 +1,50 @@ +use std::net::SocketAddr; + +use chrono::Utc; +use clap::{Parser, ValueEnum}; +use hydroflow::hydroflow_syntax; +use hydroflow::scheduled::graph::Hydroflow; +use hydroflow::util::{bind_websocket, ipv4_resolve}; +use tokio_tungstenite::tungstenite::Message; + +#[derive(Clone, ValueEnum, Debug)] +enum Role { + Client, + Server, +} + +#[derive(Parser, Debug)] +struct Opts { + #[clap(long, value_parser = ipv4_resolve)] + addr: Option, +} + +#[hydroflow::main] +async fn main() { + // parse command line arguments + let opts = Opts::parse(); + // if no addr was provided, we ask the OS to assign a local port by passing in "localhost:0" + let addr = opts + .addr + .unwrap_or_else(|| ipv4_resolve("localhost:0").unwrap()); + + // allocate `outbound` sink and `inbound` stream + let (outbound, inbound, addr) = bind_websocket(addr).await.unwrap(); + println!("Listening on {:?}", addr); + + let mut flow: Hydroflow = hydroflow_syntax! { + // Define a shared inbound channel + inbound_chan = source_stream(inbound) -> map(Result::unwrap) -> tee(); + + // Print all messages for debugging purposes + inbound_chan[0] + -> for_each(|(msg, addr): (Message, SocketAddr)| println!("{}: Got {:?} from {:?}", Utc::now(), msg, addr)); + + // Echo back the Echo messages with updated timestamp + inbound_chan[1] + -> map(|(msg, addr)| (msg, addr) ) -> dest_sink(outbound); + }; + + // run the server + flow.run_async().await; +} diff --git a/hydroflow/src/util/mod.rs b/hydroflow/src/util/mod.rs index efd5a3b708a7..e4ac4c73b73c 100644 --- a/hydroflow/src/util/mod.rs +++ b/hydroflow/src/util/mod.rs @@ -30,9 +30,9 @@ pub use socket::*; #[cfg(feature = "deploy_integration")] pub mod deploy; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(all(feature = "websocket", not(target_arch = "wasm32")))] mod websocket; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(all(feature = "websocket", not(target_arch = "wasm32")))] pub use websocket::*; use std::io::Read; diff --git a/wasm_test_site/Cargo.toml b/wasm_test_site/Cargo.toml index b21c94669bf0..ea6677007c66 100644 --- a/wasm_test_site/Cargo.toml +++ b/wasm_test_site/Cargo.toml @@ -10,9 +10,11 @@ crate-type = ["cdylib", "rlib"] [dependencies] gloo = { version = "0.11.0", features = ["futures"] } hydroflow = { path = "../hydroflow" } +pharos = "0.5" wasm-bindgen = "0.2.84" wasm-bindgen-futures = "0.4.43" web-sys = { version = "0.3.51", features = [ "console", "Window", "Document", "Element", "HtmlElement", "HtmlInputElement", "EventTarget"] } +ws_stream_wasm = "0.7.0" # The `console_error_panic_hook` crate provides better debugging of panics by # logging them with `console.error`. This is great for development, but requires diff --git a/wasm_test_site/src/lib.rs b/wasm_test_site/src/lib.rs index 6436eb8b930b..deab51cb566c 100644 --- a/wasm_test_site/src/lib.rs +++ b/wasm_test_site/src/lib.rs @@ -1,10 +1,13 @@ +use futures::channel::mpsc; +use futures::stream::StreamExt; +use pharos::Observable; +use pharos::ObserveConfig; use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; use wasm_bindgen_futures::spawn_local; use web_sys::{window, HtmlElement, HtmlInputElement}; -use futures::channel::mpsc; -use futures::stream::StreamExt; -use gloo::timers::future::TimeoutFuture; // Use the correct future module from gloo +use ws_stream_wasm::WsMessage; +use ws_stream_wasm::WsMeta; // Use the correct future module from gloo #[wasm_bindgen(start)] pub fn start() -> Result<(), JsValue> { @@ -21,9 +24,11 @@ pub fn start() -> Result<(), JsValue> { // Get the document and chat container let window = window().expect("should have a window in this context"); let document = window.document().expect("window should have a document"); - let chat_container = document.get_element_by_id("chat-container") + let chat_container = document + .get_element_by_id("chat-container") .expect("document should have a chat container with id 'chat-container'") - .dyn_into::().unwrap(); + .dyn_into::() + .unwrap(); // Process each message received from the server (via recv_rx) while let Some(message) = recv_rx.next().await { @@ -37,16 +42,23 @@ pub fn start() -> Result<(), JsValue> { // Set up the event listener for the Send button to send messages to the server setup_send_button(send_tx)?; - let mut df = hydroflow::hydroflow_syntax! { - - outbound = source_stream(send_rx); - inbound = dest_sink(recv_tx); - - outbound -> inbound; - }; - spawn_local(async move { - let work_done = df.run_async().await; + let (_ws_meta, ws_sink_stream) = WsMeta::connect("ws://127.0.0.1:59063", None).await.unwrap(); + let (ws_sink, ws_stream) = ws_sink_stream.split(); + // let evts = ws.observe(ObserveConfig::default()).await.unwrap(); + + let mut df = hydroflow::hydroflow_syntax! { + + source_stream(send_rx) -> map(|s| WsMessage::Text(s)) -> dest_sink(ws_sink); + source_stream(ws_stream) -> map(|msg| match msg { + WsMessage::Text(s) => s, + WsMessage::Binary(bytes) => format!("{:?}", bytes), + }) -> dest_sink(recv_tx); + }; + + let local = hydroflow::tokio::task::LocalSet::new(); + local.spawn_local(async move { df.run_async().await }); + local.await }); Ok(()) @@ -59,10 +71,12 @@ fn setup_send_button(send_tx: mpsc::UnboundedSender) -> Result<(), JsVal let document = window.document().expect("window should have a document"); // Get the send button and message input box - let send_button = document.get_element_by_id("send-button") + let send_button = document + .get_element_by_id("send-button") .expect("document should have a send button with id 'send-button'") .dyn_into::()?; - let input_box = document.get_element_by_id("new-message") + let input_box = document + .get_element_by_id("new-message") .expect("document should have an input box with id 'new-message'") .dyn_into::()?; @@ -88,4 +102,4 @@ fn setup_send_button(send_tx: mpsc::UnboundedSender) -> Result<(), JsVal closure.forget(); Ok(()) -} \ No newline at end of file +} diff --git a/wasm_test_site/www/index.ts b/wasm_test_site/www/index.ts index 8db9935d5654..a7af23a8ecce 100644 --- a/wasm_test_site/www/index.ts +++ b/wasm_test_site/www/index.ts @@ -1,2 +1,2 @@ -import init, { greet, test_hydroflow } from "wasm_test_site"; +import init from "wasm_test_site"; let { memory } = await init(); From a6c01c0545eb3f7243e1380a38a51b8c398ce867 Mon Sep 17 00:00:00 2001 From: Mingwei Samuel Date: Thu, 19 Sep 2024 14:51:44 -0700 Subject: [PATCH 6/6] working chat --- hydroflow/Cargo.toml | 2 +- .../main.rs | 9 ++++++--- hydroflow/examples/echoserver/main.rs | 2 +- hydroflow/src/util/websocket.rs | 18 +++++++++++++----- 4 files changed, 21 insertions(+), 10 deletions(-) rename hydroflow/examples/{echoserver_websocket => chat_websocket}/main.rs (80%) diff --git a/hydroflow/Cargo.toml b/hydroflow/Cargo.toml index d471fed6d46b..d2fcbca653b1 100644 --- a/hydroflow/Cargo.toml +++ b/hydroflow/Cargo.toml @@ -39,7 +39,7 @@ name = "modules_triple_cross_join" required-features = [ "debugging" ] [[example]] -name = "echoserver_websocket" +name = "chat_websocket" required-features = [ "websocket" ] [dependencies] diff --git a/hydroflow/examples/echoserver_websocket/main.rs b/hydroflow/examples/chat_websocket/main.rs similarity index 80% rename from hydroflow/examples/echoserver_websocket/main.rs rename to hydroflow/examples/chat_websocket/main.rs index 17b4d7343fd2..9d18a25e32cd 100644 --- a/hydroflow/examples/echoserver_websocket/main.rs +++ b/hydroflow/examples/chat_websocket/main.rs @@ -40,9 +40,12 @@ async fn main() { inbound_chan[0] -> for_each(|(msg, addr): (Message, SocketAddr)| println!("{}: Got {:?} from {:?}", Utc::now(), msg, addr)); - // Echo back the Echo messages with updated timestamp - inbound_chan[1] - -> map(|(msg, addr)| (msg, addr) ) -> dest_sink(outbound); + clients = inbound_chan[1] -> map(|(_msg, addr)| addr) -> unique::<'static>(); + messages = inbound_chan[2] -> map(|(msg, _addr)| msg); + + messages -> [0]cj; + clients -> [1]cj; + cj = cross_join::<'tick, 'static>() -> inspect(|msg| println!("SEND {:?}", msg)) -> dest_sink(outbound); }; // run the server diff --git a/hydroflow/examples/echoserver/main.rs b/hydroflow/examples/echoserver/main.rs index 7de8f8cae6bc..d483385f251b 100644 --- a/hydroflow/examples/echoserver/main.rs +++ b/hydroflow/examples/echoserver/main.rs @@ -35,7 +35,7 @@ async fn main() { .unwrap_or_else(|| ipv4_resolve("localhost:0").unwrap()); // allocate `outbound` sink and `inbound` stream - let (outbound, inbound, addr) = bind_udp_bytes(addr).await.unwrap(); + let (outbound, inbound, addr) = bind_udp_bytes(addr).await; println!("Listening on {:?}", addr); match opts.role { diff --git a/hydroflow/src/util/websocket.rs b/hydroflow/src/util/websocket.rs index b3c25c6c5238..410628e7fbe4 100644 --- a/hydroflow/src/util/websocket.rs +++ b/hydroflow/src/util/websocket.rs @@ -3,15 +3,25 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::pin::pin; use std::rc::Rc; + use futures::{SinkExt, StreamExt}; -use tokio::net::{TcpListener}; +use tokio::net::TcpListener; use tokio::task::spawn_local; use tokio_tungstenite::tungstenite::{Error, Message}; + use crate::util::unsync::mpsc::{Receiver, Sender}; use crate::util::unsync_channel; - -pub async fn bind_websocket(endpoint: SocketAddr) -> Result<(Sender<(Message, SocketAddr)>, Receiver>, SocketAddr), std::io::Error>{ +pub async fn bind_websocket( + endpoint: SocketAddr, +) -> Result< + ( + Sender<(Message, SocketAddr)>, + Receiver>, + SocketAddr, + ), + std::io::Error, +> { let listener = TcpListener::bind(endpoint).await.unwrap(); let bound_endpoint = listener.local_addr()?; @@ -70,9 +80,7 @@ pub async fn bind_websocket(endpoint: SocketAddr) -> Result<(Sender<(Message, So } }); } - }); Ok((tx_egress, rx_ingress, bound_endpoint)) } -