Generate stream items from expressions

Cargo.toml

@@ -30,6 +30,7 @@ pkg-fmt = "zip"
 members = [
     "components/config",
     "components/factory",
+    "components/test-cli-core",
     "crates/componentized-constants",
     "crates/test-harness",
 ]

Makefile

@@ -17,12 +17,13 @@ RUST_TOOLCHAIN := $(shell sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.t
 CARGO_INSTALL := $(if $(shell command -v cargo-binstall 2> /dev/null),cargo binstall --no-confirm --disable-telemetry,cargo install)
 
 COMPONENTS = $(sort $(foreach file,$(wildcard $(addprefix components/*/,wit/*.constants.wit *.properties *.wac *.wkg Cargo.toml)),$(word 2,$(subst /, ,$(file)))))
-TOOLS := static-config wac-cli wasm-opt wasm-tools wkg
+TOOLS := static-config tree-sitter-cli wac-cli wasm-opt wasm-tools wkg
 
 # the tools are run by path, make runs simple commands itself rather than with a shell, finding them
 # on the PATH make was started with, not the PATH exported above
 CONSTANTS := $(TOOLS_DIR)/bin/constants
 STATIC_CONFIG := $(TOOLS_DIR)/bin/static-config
+TREE_SITTER := $(TOOLS_DIR)/bin/tree-sitter
 WAC := $(TOOLS_DIR)/bin/wac
 WASM_OPT := $(TOOLS_DIR)/bin/wasm-opt
 WASM_TOOLS := $(TOOLS_DIR)/bin/wasm-tools
@@ -58,7 +59,7 @@ clean-wit:
 	rm -rf wit/deps components/wit/deps components/*/wit/deps
 
 .PHONY: test
-test: components
+test: components test-grammar
 	cargo test --workspace
 
 
@@ -84,6 +85,14 @@ endef
 
 $(foreach name,$(TOOLS),$(eval $(call INSTALL_TOOL,$(name))))
 
+# the tree-sitter grammar for `@expression` closures, see grammars/constants-expression/README.md. The parser is generated
+# with tree-sitter's own JavaScript runtime rather than node, then built and tested against its corpus.
+.PHONY: test-grammar ## Test the tree-sitter grammar for expressions
+test-grammar: | $(call tool,tree-sitter-cli)
+	cd grammars/constants-expression && \
+		TREE_SITTER_JS_RUNTIME=native $(TREE_SITTER) generate && \
+		$(TREE_SITTER) test
+
 $(CONSTANTS): Cargo.toml Cargo.lock rust-toolchain.toml $(shell find src crates/componentized-constants -type f)
 	@# forced, make only installs when the sources change, and the binary may belong to another package
 	cargo +$(RUST_TOOLCHAIN) install --force --locked --root $(TOOLS_DIR) --path .

README.md

@@ -7,6 +7,7 @@ WIT defines the shape of a component, but it can't define values, even ones that
 - [Usage](#usage)
   - [Overrides](#overrides)
   - [Supported types](#supported-types)
+    - [Generated streams](#generated-streams)
   - [Values](#values)
 - [Factory component](#factory-component)
 - [Build](#build)
@@ -97,7 +98,7 @@ Every exported function needs a value from either its `@value` tag or an overrid
 
 ### Supported types
 
-Every exported function must be synchronous, take no parameters, and return a value. Values can be expressed for:
+Every exported function must take no parameters, unless it generates a stream from its arguments, and return a value. Functions may be synchronous or `async`. Values can be expressed for:
 
 - `bool`, `s8`, `s16`, `s32`, `s64`, `u8`, `u16`, `u32`, `u64`, `f32`, `f64`, `char` and `string`
 - `list<T>` and fixed-length `list<T, N>`
@@ -111,10 +112,58 @@ Every exported function must be synchronous, take no parameters, and return a va
 
 and type aliases of any of these.
 
-Values can't be expressed for `own`, `borrow`, `stream`, `future` or `error-context`. Result types may still include them, as long as the value doesn't reach them: an `option<own<file>>` can be `none`, a `list<future<u8>>` can be `[]`, a `result<u8, stream<u8>>` can be `ok(1)`, and a variant with a case whose payload is one of these types can use any other case. A value that does reach one is an error.
+A function may also return a `stream<T>` of any of these types, written as a list of the stream's items. A stream with items must be returned by an `async` function, which writes the items as the caller reads them, then closes the stream. An empty stream, `[]`, may be returned by any function. Calling an `async` function that returns a stream needs a runtime with support for the component model's async features, which Wasmtime enables by default.
+
+```wit
+/// @value [1, 1, 2, 3, 5, 8]
+fibonacci: async func() -> stream<u32>;
+```
+
+Values can't be expressed for `own`, `borrow`, `future` or `error-context`, or for a `stream` anywhere but the function's result. Result types may still include them, as long as the value doesn't reach them: an `option<own<file>>` can be `none`, a `list<future<u8>>` can be `[]`, a `result<u8, stream<u8>>` can be `ok(1)`, and a variant with a case whose payload is one of these types can use any other case. A value that does reach one is an error.
 
 `map` isn't supported until WAVE defines a syntax for map values.
 
+#### Generated streams
+
+A stream of integers, floats or `bool` can generate its items with an `@expression`, a Rust closure returning the next item. Its parameters are the stream's previous items, starting from the items `@value` lists:
+
+```wit
+/// @value [1, 1]
+/// @expression |a, b| a + b
+fibonacci: async func() -> stream<u64>;
+```
+
+A function can take arguments, which the expression reads by name, along with `$i`, the position in the stream of the item being generated:
+
+```wit
+/// @expression || n * r ** $i
+geometric: async func(n: u64, r: u64) -> stream<u64>;
+```
+
+Expressions can bind values with `let`, loop with `while` and branch with `if`, e.g. to find each prime by trial division:
+
+```wit
+/// @value [2]
+/// @expression |v| {
+///     let mut n = v + 1;
+///     let mut d: u64 = 2;
+///     while d * d <= n {
+///         if n % d == 0 {
+///             n += 1;
+///             d = 2;
+///         } else {
+///             d += 1;
+///         }
+///     }
+///     n
+/// }
+prime: async func() -> stream<u64>;
+```
+
+A stream ends when its next item can't be represented, as Rust's checked arithmetic would fail, so `fibonacci` ends at the largest Fibonacci number a `u64` holds. Otherwise it's unbounded, writing items until the reader closes it.
+
+[docs/expressions.md](docs/expressions.md) guides writing expressions: the ways to declare a generated stream, the built-in variables, a summary of the syntax, and how types work. [grammars/constants-expression](grammars/constants-expression/README.md) specifies expressions in full, with a tree-sitter grammar for editors and language servers.
+
 ### Values
 
 Values, whether in `@value` tags or overrides, are written in WAVE. `//` line comments are allowed.
@@ -153,10 +202,10 @@ The WIT can be given as `text`, with any packages the world depends on defined i
 
 `extract` also accepts components, and so can the factory: when the `parsed` WIT was extracted from a component rather than a WIT package, the factory implements that component's world unless `world` names another one. Components don't carry doc comments, so there are no `@value` tags; every value must come from `overrides`.
 
-The factory imports `componentized:component/types` and `componentized:component/wit`, but only for their types, not their functions, so hosts can satisfy them with empty instances. Hosts must support the component model async ABI and maps; with `wasmtime run`, enable them with `-W component-model-async=y,component-model-map=y`:
+The factory imports `componentized:component/types` and `componentized:component/wit`, but only for their types, not their functions, so hosts can satisfy them with empty instances. Hosts must support the component model async ABI, which Wasmtime enables by default, and maps; with `wasmtime run`, enable maps with `-W component-model-map=y`:
 
 ```sh
-wasmtime run -W component-model-async=y,component-model-map=y \
+wasmtime run -W component-model-map=y \
   --invoke 'create(text("package a:b; world w { /// @value 42\n export answer: func() -> u32; }"), none, none)' \
   lib/factory.wasm
 ```

components/atomics/README.md

@@ -0,0 +1,3 @@
+# `atomics`
+
+lock-free, thread-safe operations on streams

components/atomics/wit/atomics.constants.wit

@@ -0,0 +1,5 @@
+package componentized:constants-component;
+
+world atomics {
+    export componentized:constants/atomics@0.2.0-dev;
+}

components/atomics/wkg.lock

@@ -0,0 +1,21 @@
+# This file is automatically generated.
+# It is not intended for manual editing.
+version = 1
+
+[[packages]]
+name = "componentized:component"
+registry = "componentized.dev"
+
+[[packages.versions]]
+requirement = "=0.0.0-dev"
+version = "0.0.0-dev"
+digest = "sha256:2d7677a89ef49bef5d8d20848c431c76208753858f4871929066397d62542af7"
+
+[[packages]]
+name = "wasi:config"
+registry = "wasi.dev"
+
+[[packages.versions]]
+requirement = "=0.2.0-rc.1"
+version = "0.2.0-rc.1"
+digest = "sha256:1b7f1b0fd07bb4cede16c6a6ec8852815dfb924639a78735fc7bdffdc164485d"

components/atomics/wkg.toml

@@ -0,0 +1,2 @@
+[overrides]
+"componentized:constants" = { path = "../../wit" }

components/factory/tests/factory.rs

@@ -4,7 +4,7 @@
 
 use anyhow::{Context, Result, anyhow, bail};
 use std::sync::OnceLock;
-use test_harness::{built_component, call, workspace_dir};
+use test_harness::{built_component, call, read_streams, read_streams_with, workspace_dir};
 use wasmtime::{
     Config, Engine, Store,
     component::{Component, Linker, Val},
@@ -240,6 +240,59 @@ fn it_creates_components_from_wit_text() -> Result<()> {
     Ok(())
 }
 
+#[test]
+fn it_creates_components_returning_streams() -> Result<()> {
+    let wit = "package example:streams;
+        world streams {
+            /// @value [\"a\", \"bc\"]
+            export words: async func() -> stream<string>;
+        }";
+    let component = create(Wit::Wit(wit), None, None)?.map_err(|e| anyhow!(e))?;
+    let words = read_streams::<String>(&component, ("", "words"), 1, 1)?;
+    assert_eq!(words, vec![vec!["a".to_string(), "bc".to_string()]; 2]);
+    Ok(())
+}
+
+#[test]
+fn it_creates_components_generating_streams() -> Result<()> {
+    let wit = "package example:streams;
+        world streams {
+            /// @value [1, 1]
+            /// @expression |a, b| a + b
+            export fibonacci: async func() -> stream<u8>;
+        }";
+    let component = create(Wit::Wit(wit), None, None)?.map_err(|e| anyhow!(e))?;
+    let items = read_streams::<u8>(&component, ("", "fibonacci"), 4, 1)?;
+    let expected = vec![1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233];
+    assert_eq!(items, vec![expected; 2]);
+    Ok(())
+}
+
+#[test]
+fn it_creates_components_generating_streams_from_arguments() -> Result<()> {
+    let wit = "package example:streams;
+        world streams {
+            /// @expression |a, b| a + b
+            export fibonacci: async func(a: u8, b: u8) -> stream<u8>;
+        }";
+    for wit in [Wit::Wit(wit), Wit::Parsed(extract(encode(wit)?)?)] {
+        let component = create(wit, None, None)?.map_err(|e| anyhow!(e))?;
+        let items = read_streams_with::<_, u8>(
+            &component,
+            ("", "fibonacci"),
+            (2u8, 3u8),
+            4,
+            1,
+            usize::MAX,
+        )?;
+        assert_eq!(
+            items,
+            vec![vec![2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]; 2]
+        );
+    }
+    Ok(())
+}
+
 #[test]
 fn it_creates_components_from_encoded_wit() -> Result<()> {
     let component = create(Wit::Wasm(encode(WIT)?), Some("other"), Some("{answer: 7}"))?

components/random/README.md

@@ -0,0 +1,3 @@
+# `rnadom`
+
+Pseudorandom number generators.

components/random/wit/random.constants.wit

@@ -0,0 +1,5 @@
+package componentized:constants-component;
+
+world random {
+    export componentized:constants/random@0.2.0-dev;
+}

components/random/wkg.lock

@@ -0,0 +1,21 @@
+# This file is automatically generated.
+# It is not intended for manual editing.
+version = 1
+
+[[packages]]
+name = "componentized:component"
+registry = "componentized.dev"
+
+[[packages.versions]]
+requirement = "=0.0.0-dev"
+version = "0.0.0-dev"
+digest = "sha256:2d7677a89ef49bef5d8d20848c431c76208753858f4871929066397d62542af7"
+
+[[packages]]
+name = "wasi:config"
+registry = "wasi.dev"
+
+[[packages.versions]]
+requirement = "=0.2.0-rc.1"
+version = "0.2.0-rc.1"
+digest = "sha256:1b7f1b0fd07bb4cede16c6a6ec8852815dfb924639a78735fc7bdffdc164485d"

components/random/wkg.toml

@@ -0,0 +1,2 @@
+[overrides]
+"componentized:constants" = { path = "../../wit" }

components/test-cli-core/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "test-cli-core"
+version = "0.1.0"
+edition = "2024"
+license = "Apache-2.0"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+wit-bindgen = { workspace = true }
+
+[build-dependencies]
+wit-parser = { workspace = true }
+
+[dev-dependencies]
+anyhow = { workspace = true }
+test-harness = { workspace = true }
+tokio = { version = "1", features = ["rt", "macros"] }
+wasmtime = { workspace = true }
+wasmtime-wasi = "49"

components/test-cli-core/README.md

@@ -0,0 +1,21 @@
+# `test-cli-core`
+
+A command calling the functions of the constants data interfaces, for testing.
+
+```sh
+test-cli <interface> <function> [arguments...]
+```
+
+The arguments are parsed as the function's parameters, and its result is printed. A stream's first item is printed, then items are read as stdin's lines ask for them: an empty line reads the next item, a number reads that many. The command ends when the stream ends or stdin closes. When stdin is a terminal, whose echo of each line's newline already ends the line, an item isn't followed by a newline of its own, so items aren't separated by blank lines. Without arguments, it lists the functions it can call.
+
+It imports `math`, `atomics`, `random` and the numeric type interfaces, `u8` to `f64`. [`test-cli`](../test-cli) composes it with the components implementing them, so it runs with `wasmtime run`:
+
+```sh
+make components/test-cli
+wasmtime run target/components/test-cli/test-cli.wasm math pi
+printf '\n\n3\n' | wasmtime run target/components/test-cli/test-cli.wasm atomics incrementer 10
+```
+
+Its dispatch is generated from the `test-cli-core` world in [`components/wit/worlds.wit`](../wit/worlds.wit) by [`build.rs`](./build.rs), so it calls each function of each imported interface.
+
+It exports WASIp3's `wasi:cli/run@0.3.0`, whose `run` is async, so the command can wait for a stream's items, which a synchronous `run` can't, the component model traps instead. It imports the other WASIp3 `wasi:cli` interfaces it needs: `environment` for its arguments, `stdin`, read from a stream, and `stdout` and `stderr`, written to streams. It's built like the other rust components, for `wasm32-unknown-unknown`.