diff --git a/Cargo.lock b/Cargo.lock index 5d3fdd7..dfc766b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "anstream" version = "1.0.0" @@ -68,7 +74,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -79,7 +85,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -141,6 +147,30 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cap-primitives" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + [[package]] name = "cc" version = "1.5.1" @@ -157,6 +187,17 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core", +] + [[package]] name = "clap" version = "4.6.7" @@ -203,7 +244,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror", + "thiserror 2.0.21", ] [[package]] @@ -269,6 +310,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "cranelift-assembler-x64" version = "0.136.2" @@ -500,7 +550,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -556,6 +606,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "futures" version = "0.3.34" @@ -642,6 +703,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + [[package]] name = "gimli" version = "0.33.0" @@ -799,6 +872,34 @@ dependencies = [ "serde_core", ] +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.60.2", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -820,6 +921,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "leb128" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c83bff1d572d6b9aeef67ddfc8448e4a3737909cb28e81f97c791b9018703e52" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -905,6 +1012,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "memchr" version = "2.8.3" @@ -930,6 +1043,17 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -1044,6 +1168,29 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "regalloc2" version = "0.15.2" @@ -1098,7 +1245,17 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", ] [[package]] @@ -1173,7 +1330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1210,6 +1367,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "spdx" version = "0.13.5" @@ -1279,21 +1446,55 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "test-cli-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "test-harness", + "tokio", + "wasmtime", + "wasmtime-wasi", + "wit-bindgen", + "wit-parser 0.261.0", +] + [[package]] name = "test-harness" version = "0.1.0" dependencies = [ "anyhow", + "futures", "wasmtime", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1317,12 +1518,69 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "topological-sort" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.20.1" @@ -1365,6 +1623,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasm-encoder" version = "0.258.3" @@ -1445,7 +1709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c3077ed029568755b5d3557bc27688360f7daa19cb182aa32ff2a1035c818d0" dependencies = [ "logos", - "thiserror", + "thiserror 2.0.21", ] [[package]] @@ -1506,6 +1770,7 @@ dependencies = [ "async-trait", "bitflags", "bumpalo", + "bytes", "cc", "encoding_rs", "futures", @@ -1534,7 +1799,7 @@ dependencies = [ "wasmtime-internal-jit-icache-coherence", "wasmtime-internal-unwinder", "wasmtime-internal-versioned-export-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1619,7 +1884,7 @@ dependencies = [ "pulley-interpreter", "smallvec", "target-lexicon", - "thiserror", + "thiserror 2.0.21", "wasmparser 0.258.3", "wasmtime-environ", "wasmtime-internal-core", @@ -1638,7 +1903,7 @@ dependencies = [ "rustix", "wasmtime-environ", "wasmtime-internal-versioned-export-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1659,7 +1924,7 @@ checksum = "5dead9a2d7b5d835698a2e1a32006e7d0b5711f4697b24bab1564bb2488e3d7a" dependencies = [ "libc", "wasmtime-internal-core", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1699,13 +1964,100 @@ dependencies = [ "wit-parser 0.258.3", ] +[[package]] +name = "wasmtime-wasi" +version = "49.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac947c8478d8ea84ef4120d04f984012a514e493fe16fa5bb72906d7e55addb" +dependencies = [ + "async-trait", + "bitflags", + "bytes", + "cap-primitives", + "futures", + "rand", + "rustix", + "rustix-linux-procfs", + "thiserror 2.0.21", + "tokio", + "tracing", + "url", + "wasmtime", + "wasmtime-wasi-io", + "wiggle", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "wasmtime-wasi-io" +version = "49.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e65d70efbe70631acbb4c6cf46eea543c75908f01e2fd09550e5a89631693304" +dependencies = [ + "async-trait", + "bytes", + "futures", + "tracing", + "wasmtime", +] + +[[package]] +name = "wast" +version = "35.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef140f1b49946586078353a453a1d28ba90adfc54dde75710bc1931de204d68" +dependencies = [ + "leb128", +] + +[[package]] +name = "wiggle" +version = "49.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7c3a2e8ec4789bb82e4bf04b0dba44b02ce05beba6cbc6eb992b8f0b93c9cd" +dependencies = [ + "bitflags", + "thiserror 2.0.21", + "tracing", + "wasmtime", + "wasmtime-environ", + "wiggle-macro", +] + +[[package]] +name = "wiggle-generate" +version = "49.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761bf518bf5972dd0ddb8c3c693b6a52ed826e25d10bfae89b1368327c86030d" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasmtime-environ", + "witx", +] + +[[package]] +name = "wiggle-macro" +version = "49.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bdaa435787bc4bb662020aea37538745fc60b4629d7f37864a09195cec0e1eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "wiggle-generate", +] + [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1714,6 +2066,24 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1723,6 +2093,145 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags", + "windows-sys 0.59.0", +] + [[package]] name = "wit-bindgen" version = "0.62.0" @@ -1890,6 +2399,18 @@ dependencies = [ "wasmparser 0.261.0", ] +[[package]] +name = "witx" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e366f27a5cabcddb2706a78296a40b8fcc451e1a6aba2fc1d94b4a01bdaaef4b" +dependencies = [ + "anyhow", + "log", + "thiserror 1.0.69", + "wast", +] + [[package]] name = "writeable" version = "0.6.4" diff --git a/Cargo.toml b/Cargo.toml index 5d34ed2..5c85fdb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ pkg-fmt = "zip" members = [ "components/config", "components/factory", + "components/test-cli-core", "crates/componentized-constants", "crates/test-harness", ] diff --git a/Makefile b/Makefile index 2c1487b..b8831e2 100644 --- a/Makefile +++ b/Makefile @@ -17,12 +17,13 @@ RUST_TOOLCHAIN := $(shell sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.t CARGO_INSTALL := $(if $(shell command -v cargo-binstall 2> /dev/null),cargo binstall --no-confirm --disable-telemetry,cargo install) COMPONENTS = $(sort $(foreach file,$(wildcard $(addprefix components/*/,wit/*.constants.wit *.properties *.wac *.wkg Cargo.toml)),$(word 2,$(subst /, ,$(file))))) -TOOLS := static-config wac-cli wasm-opt wasm-tools wkg +TOOLS := static-config tree-sitter-cli wac-cli wasm-opt wasm-tools wkg # the tools are run by path, make runs simple commands itself rather than with a shell, finding them # on the PATH make was started with, not the PATH exported above CONSTANTS := $(TOOLS_DIR)/bin/constants STATIC_CONFIG := $(TOOLS_DIR)/bin/static-config +TREE_SITTER := $(TOOLS_DIR)/bin/tree-sitter WAC := $(TOOLS_DIR)/bin/wac WASM_OPT := $(TOOLS_DIR)/bin/wasm-opt WASM_TOOLS := $(TOOLS_DIR)/bin/wasm-tools @@ -58,7 +59,7 @@ clean-wit: rm -rf wit/deps components/wit/deps components/*/wit/deps .PHONY: test -test: components +test: components test-grammar cargo test --workspace @@ -84,6 +85,14 @@ endef $(foreach name,$(TOOLS),$(eval $(call INSTALL_TOOL,$(name)))) +# the tree-sitter grammar for `@expression` closures, see grammars/constants-expression/README.md. The parser is generated +# with tree-sitter's own JavaScript runtime rather than node, then built and tested against its corpus. +.PHONY: test-grammar ## Test the tree-sitter grammar for expressions +test-grammar: | $(call tool,tree-sitter-cli) + cd grammars/constants-expression && \ + TREE_SITTER_JS_RUNTIME=native $(TREE_SITTER) generate && \ + $(TREE_SITTER) test + $(CONSTANTS): Cargo.toml Cargo.lock rust-toolchain.toml $(shell find src crates/componentized-constants -type f) @# forced, make only installs when the sources change, and the binary may belong to another package cargo +$(RUST_TOOLCHAIN) install --force --locked --root $(TOOLS_DIR) --path . diff --git a/README.md b/README.md index 4b209ef..123d8e1 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ WIT defines the shape of a component, but it can't define values, even ones that - [Usage](#usage) - [Overrides](#overrides) - [Supported types](#supported-types) + - [Generated streams](#generated-streams) - [Values](#values) - [Factory component](#factory-component) - [Build](#build) @@ -97,7 +98,7 @@ Every exported function needs a value from either its `@value` tag or an overrid ### Supported types -Every exported function must be synchronous, take no parameters, and return a value. Values can be expressed for: +Every exported function must take no parameters, unless it generates a stream from its arguments, and return a value. Functions may be synchronous or `async`. Values can be expressed for: - `bool`, `s8`, `s16`, `s32`, `s64`, `u8`, `u16`, `u32`, `u64`, `f32`, `f64`, `char` and `string` - `list` and fixed-length `list` @@ -111,10 +112,58 @@ Every exported function must be synchronous, take no parameters, and return a va and type aliases of any of these. -Values can't be expressed for `own`, `borrow`, `stream`, `future` or `error-context`. Result types may still include them, as long as the value doesn't reach them: an `option>` can be `none`, a `list>` can be `[]`, a `result>` can be `ok(1)`, and a variant with a case whose payload is one of these types can use any other case. A value that does reach one is an error. +A function may also return a `stream` of any of these types, written as a list of the stream's items. A stream with items must be returned by an `async` function, which writes the items as the caller reads them, then closes the stream. An empty stream, `[]`, may be returned by any function. Calling an `async` function that returns a stream needs a runtime with support for the component model's async features, which Wasmtime enables by default. + +```wit +/// @value [1, 1, 2, 3, 5, 8] +fibonacci: async func() -> stream; +``` + +Values can't be expressed for `own`, `borrow`, `future` or `error-context`, or for a `stream` anywhere but the function's result. Result types may still include them, as long as the value doesn't reach them: an `option>` can be `none`, a `list>` can be `[]`, a `result>` can be `ok(1)`, and a variant with a case whose payload is one of these types can use any other case. A value that does reach one is an error. `map` isn't supported until WAVE defines a syntax for map values. +#### Generated streams + +A stream of integers, floats or `bool` can generate its items with an `@expression`, a Rust closure returning the next item. Its parameters are the stream's previous items, starting from the items `@value` lists: + +```wit +/// @value [1, 1] +/// @expression |a, b| a + b +fibonacci: async func() -> stream; +``` + +A function can take arguments, which the expression reads by name, along with `$i`, the position in the stream of the item being generated: + +```wit +/// @expression || n * r ** $i +geometric: async func(n: u64, r: u64) -> stream; +``` + +Expressions can bind values with `let`, loop with `while` and branch with `if`, e.g. to find each prime by trial division: + +```wit +/// @value [2] +/// @expression |v| { +/// let mut n = v + 1; +/// let mut d: u64 = 2; +/// while d * d <= n { +/// if n % d == 0 { +/// n += 1; +/// d = 2; +/// } else { +/// d += 1; +/// } +/// } +/// n +/// } +prime: async func() -> stream; +``` + +A stream ends when its next item can't be represented, as Rust's checked arithmetic would fail, so `fibonacci` ends at the largest Fibonacci number a `u64` holds. Otherwise it's unbounded, writing items until the reader closes it. + +[docs/expressions.md](docs/expressions.md) guides writing expressions: the ways to declare a generated stream, the built-in variables, a summary of the syntax, and how types work. [grammars/constants-expression](grammars/constants-expression/README.md) specifies expressions in full, with a tree-sitter grammar for editors and language servers. + ### Values Values, whether in `@value` tags or overrides, are written in WAVE. `//` line comments are allowed. @@ -153,10 +202,10 @@ The WIT can be given as `text`, with any packages the world depends on defined i `extract` also accepts components, and so can the factory: when the `parsed` WIT was extracted from a component rather than a WIT package, the factory implements that component's world unless `world` names another one. Components don't carry doc comments, so there are no `@value` tags; every value must come from `overrides`. -The factory imports `componentized:component/types` and `componentized:component/wit`, but only for their types, not their functions, so hosts can satisfy them with empty instances. Hosts must support the component model async ABI and maps; with `wasmtime run`, enable them with `-W component-model-async=y,component-model-map=y`: +The factory imports `componentized:component/types` and `componentized:component/wit`, but only for their types, not their functions, so hosts can satisfy them with empty instances. Hosts must support the component model async ABI, which Wasmtime enables by default, and maps; with `wasmtime run`, enable maps with `-W component-model-map=y`: ```sh -wasmtime run -W component-model-async=y,component-model-map=y \ +wasmtime run -W component-model-map=y \ --invoke 'create(text("package a:b; world w { /// @value 42\n export answer: func() -> u32; }"), none, none)' \ lib/factory.wasm ``` diff --git a/components/atomics/README.md b/components/atomics/README.md new file mode 100644 index 0000000..7838ac0 --- /dev/null +++ b/components/atomics/README.md @@ -0,0 +1,3 @@ +# `atomics` + +lock-free, thread-safe operations on streams diff --git a/components/atomics/wit/atomics.constants.wit b/components/atomics/wit/atomics.constants.wit new file mode 100644 index 0000000..d019b44 --- /dev/null +++ b/components/atomics/wit/atomics.constants.wit @@ -0,0 +1,5 @@ +package componentized:constants-component; + +world atomics { + export componentized:constants/atomics@0.2.0-dev; +} diff --git a/components/atomics/wkg.lock b/components/atomics/wkg.lock new file mode 100644 index 0000000..9254e46 --- /dev/null +++ b/components/atomics/wkg.lock @@ -0,0 +1,21 @@ +# This file is automatically generated. +# It is not intended for manual editing. +version = 1 + +[[packages]] +name = "componentized:component" +registry = "componentized.dev" + +[[packages.versions]] +requirement = "=0.0.0-dev" +version = "0.0.0-dev" +digest = "sha256:2d7677a89ef49bef5d8d20848c431c76208753858f4871929066397d62542af7" + +[[packages]] +name = "wasi:config" +registry = "wasi.dev" + +[[packages.versions]] +requirement = "=0.2.0-rc.1" +version = "0.2.0-rc.1" +digest = "sha256:1b7f1b0fd07bb4cede16c6a6ec8852815dfb924639a78735fc7bdffdc164485d" diff --git a/components/atomics/wkg.toml b/components/atomics/wkg.toml new file mode 100644 index 0000000..3ef28a5 --- /dev/null +++ b/components/atomics/wkg.toml @@ -0,0 +1,2 @@ +[overrides] +"componentized:constants" = { path = "../../wit" } diff --git a/components/factory/tests/factory.rs b/components/factory/tests/factory.rs index 9f96764..fdb30bc 100644 --- a/components/factory/tests/factory.rs +++ b/components/factory/tests/factory.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result, anyhow, bail}; use std::sync::OnceLock; -use test_harness::{built_component, call, workspace_dir}; +use test_harness::{built_component, call, read_streams, read_streams_with, workspace_dir}; use wasmtime::{ Config, Engine, Store, component::{Component, Linker, Val}, @@ -240,6 +240,59 @@ fn it_creates_components_from_wit_text() -> Result<()> { Ok(()) } +#[test] +fn it_creates_components_returning_streams() -> Result<()> { + let wit = "package example:streams; + world streams { + /// @value [\"a\", \"bc\"] + export words: async func() -> stream; + }"; + let component = create(Wit::Wit(wit), None, None)?.map_err(|e| anyhow!(e))?; + let words = read_streams::(&component, ("", "words"), 1, 1)?; + assert_eq!(words, vec![vec!["a".to_string(), "bc".to_string()]; 2]); + Ok(()) +} + +#[test] +fn it_creates_components_generating_streams() -> Result<()> { + let wit = "package example:streams; + world streams { + /// @value [1, 1] + /// @expression |a, b| a + b + export fibonacci: async func() -> stream; + }"; + let component = create(Wit::Wit(wit), None, None)?.map_err(|e| anyhow!(e))?; + let items = read_streams::(&component, ("", "fibonacci"), 4, 1)?; + let expected = vec![1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]; + assert_eq!(items, vec![expected; 2]); + Ok(()) +} + +#[test] +fn it_creates_components_generating_streams_from_arguments() -> Result<()> { + let wit = "package example:streams; + world streams { + /// @expression |a, b| a + b + export fibonacci: async func(a: u8, b: u8) -> stream; + }"; + for wit in [Wit::Wit(wit), Wit::Parsed(extract(encode(wit)?)?)] { + let component = create(wit, None, None)?.map_err(|e| anyhow!(e))?; + let items = read_streams_with::<_, u8>( + &component, + ("", "fibonacci"), + (2u8, 3u8), + 4, + 1, + usize::MAX, + )?; + assert_eq!( + items, + vec![vec![2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]; 2] + ); + } + Ok(()) +} + #[test] fn it_creates_components_from_encoded_wit() -> Result<()> { let component = create(Wit::Wasm(encode(WIT)?), Some("other"), Some("{answer: 7}"))? diff --git a/components/random/README.md b/components/random/README.md new file mode 100644 index 0000000..b0b7462 --- /dev/null +++ b/components/random/README.md @@ -0,0 +1,3 @@ +# `rnadom` + +Pseudorandom number generators. diff --git a/components/random/wit/random.constants.wit b/components/random/wit/random.constants.wit new file mode 100644 index 0000000..83b1e5b --- /dev/null +++ b/components/random/wit/random.constants.wit @@ -0,0 +1,5 @@ +package componentized:constants-component; + +world random { + export componentized:constants/random@0.2.0-dev; +} diff --git a/components/random/wkg.lock b/components/random/wkg.lock new file mode 100644 index 0000000..9254e46 --- /dev/null +++ b/components/random/wkg.lock @@ -0,0 +1,21 @@ +# This file is automatically generated. +# It is not intended for manual editing. +version = 1 + +[[packages]] +name = "componentized:component" +registry = "componentized.dev" + +[[packages.versions]] +requirement = "=0.0.0-dev" +version = "0.0.0-dev" +digest = "sha256:2d7677a89ef49bef5d8d20848c431c76208753858f4871929066397d62542af7" + +[[packages]] +name = "wasi:config" +registry = "wasi.dev" + +[[packages.versions]] +requirement = "=0.2.0-rc.1" +version = "0.2.0-rc.1" +digest = "sha256:1b7f1b0fd07bb4cede16c6a6ec8852815dfb924639a78735fc7bdffdc164485d" diff --git a/components/random/wkg.toml b/components/random/wkg.toml new file mode 100644 index 0000000..3ef28a5 --- /dev/null +++ b/components/random/wkg.toml @@ -0,0 +1,2 @@ +[overrides] +"componentized:constants" = { path = "../../wit" } diff --git a/components/test-cli-core/Cargo.toml b/components/test-cli-core/Cargo.toml new file mode 100644 index 0000000..db48b0f --- /dev/null +++ b/components/test-cli-core/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "test-cli-core" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = { workspace = true } + +[build-dependencies] +wit-parser = { workspace = true } + +[dev-dependencies] +anyhow = { workspace = true } +test-harness = { workspace = true } +tokio = { version = "1", features = ["rt", "macros"] } +wasmtime = { workspace = true } +wasmtime-wasi = "49" diff --git a/components/test-cli-core/README.md b/components/test-cli-core/README.md new file mode 100644 index 0000000..40f27d4 --- /dev/null +++ b/components/test-cli-core/README.md @@ -0,0 +1,21 @@ +# `test-cli-core` + +A command calling the functions of the constants data interfaces, for testing. + +```sh +test-cli [arguments...] +``` + +The arguments are parsed as the function's parameters, and its result is printed. A stream's first item is printed, then items are read as stdin's lines ask for them: an empty line reads the next item, a number reads that many. The command ends when the stream ends or stdin closes. When stdin is a terminal, whose echo of each line's newline already ends the line, an item isn't followed by a newline of its own, so items aren't separated by blank lines. Without arguments, it lists the functions it can call. + +It imports `math`, `atomics`, `random` and the numeric type interfaces, `u8` to `f64`. [`test-cli`](../test-cli) composes it with the components implementing them, so it runs with `wasmtime run`: + +```sh +make components/test-cli +wasmtime run target/components/test-cli/test-cli.wasm math pi +printf '\n\n3\n' | wasmtime run target/components/test-cli/test-cli.wasm atomics incrementer 10 +``` + +Its dispatch is generated from the `test-cli-core` world in [`components/wit/worlds.wit`](../wit/worlds.wit) by [`build.rs`](./build.rs), so it calls each function of each imported interface. + +It exports WASIp3's `wasi:cli/run@0.3.0`, whose `run` is async, so the command can wait for a stream's items, which a synchronous `run` can't, the component model traps instead. It imports the other WASIp3 `wasi:cli` interfaces it needs: `environment` for its arguments, `stdin`, read from a stream, and `stdout` and `stderr`, written to streams. It's built like the other rust components, for `wasm32-unknown-unknown`. diff --git a/components/test-cli-core/build.rs b/components/test-cli-core/build.rs new file mode 100644 index 0000000..df3670e --- /dev/null +++ b/components/test-cli-core/build.rs @@ -0,0 +1,214 @@ +//! Generates the cli's dispatch from the test-cli-core world, a match arm calling +//! each function of each imported interface, so the cli follows the WIT. + +use std::{env, fmt::Write, fs, path::PathBuf}; +use wit_parser::{Function, Resolve, Type, TypeDefKind, WorldItem}; + +fn main() { + let wit = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("../wit"); + println!("cargo::rerun-if-changed={}", wit.display()); + let mut resolve = Resolve::default(); + let (pkg, _) = resolve + .push_path(&wit) + .expect("components/wit, run `make wit`"); + let world = resolve + .select_world(&[pkg], Some("test-cli-core")) + .expect("test-cli-core world"); + + let mut arms = String::new(); + let mut functions = String::new(); + for (key, item) in &resolve.worlds[world].imports { + let WorldItem::Interface { id, .. } = item else { + continue; + }; + let interface = &resolve.interfaces[*id]; + let Some(name) = &interface.name else { + continue; + }; + let package = &resolve.packages[interface.package.expect("a package")].name; + // only the constants interfaces, not WASI's + if (package.namespace.as_str(), package.name.as_str()) != ("componentized", "constants") { + continue; + } + let module = format!( + "{}::{}::{}", + ident(&package.namespace), + ident(&package.name), + ident(name) + ); + let _ = key; + for func in interface.functions.values() { + let signature = signature(&resolve, func); + writeln!(functions, " ({name:?}, {:?}, {signature:?}),", func.name).unwrap(); + writeln!(arms, " ({name:?}, {:?}) => {{", func.name).unwrap(); + writeln!( + arms, + " expect_args(args, {})?;", + func.params.len() + ) + .unwrap(); + let mut call_args = vec![]; + let mut parsable = true; + for (index, param) in func.params.iter().enumerate() { + match rust_type(&resolve, ¶m.ty) { + Some(ty) => { + writeln!( + arms, + " let a{index}: {ty} = parse(&args[{index}], {:?})?;", + param.name + ) + .unwrap(); + call_args.push(format!("a{index}")); + } + None => parsable = false, + } + } + if !parsable { + writeln!( + arms, + " Err(format!(\"`{}` takes parameters the cli can't parse\"))\n }}", + func.name + ) + .unwrap(); + continue; + } + let mut call = format!("{module}::{}({})", ident(&func.name), call_args.join(", ")); + if func.kind.is_async() { + call = format!("{call}.await"); + } + let stream = func + .result + .as_ref() + .is_some_and(|ty| matches!(kind(&resolve, ty), Some(TypeDefKind::Stream(_)))); + match (stream, &func.result) { + (true, _) => writeln!(arms, " read_stream(io, {call}).await;").unwrap(), + (false, Some(_)) => { + writeln!(arms, " print_value(io, &{call}).await;").unwrap() + } + (false, None) => writeln!(arms, " {call};").unwrap(), + } + writeln!(arms, " Ok(())\n }}").unwrap(); + } + } + + let code = format!( + "/// Each function: its interface, its name and its signature.\n\ + pub const FUNCTIONS: &[(&str, &str, &str)] = &[\n{functions}];\n\n\ + /// Calls `function` of `interface` with `args` parsed as its parameters.\n\ + pub async fn dispatch(io: &mut Io, interface: &str, function: &str, args: &[String]) -> Result<(), String> {{\n \ + match (interface, function) {{\n{arms} \ + _ => Err(format!(\"unknown function `{{function}}` of interface `{{interface}}`\")),\n \ + }}\n}}\n" + ); + let out = PathBuf::from(env::var("OUT_DIR").unwrap()).join("dispatch.rs"); + fs::write(out, code).unwrap(); +} + +/// A WIT name as wit-bindgen names it in Rust. +fn ident(name: &str) -> String { + let name = name.replace('-', "_"); + match name.as_str() { + "as" | "async" | "await" | "break" | "const" | "continue" | "crate" | "dyn" | "else" + | "enum" | "extern" | "false" | "fn" | "for" | "if" | "impl" | "in" | "let" | "loop" + | "match" | "mod" | "move" | "mut" | "pub" | "ref" | "return" | "self" | "static" + | "struct" | "super" | "trait" | "true" | "type" | "unsafe" | "use" | "where" | "while" => { + format!("{name}_") + } + _ => name, + } +} + +fn kind<'a>(resolve: &'a Resolve, ty: &Type) -> Option<&'a TypeDefKind> { + match ty { + Type::Id(id) => match &resolve.types[*id].kind { + TypeDefKind::Type(ty) => kind(resolve, ty), + kind => Some(kind), + }, + _ => None, + } +} + +/// The Rust type a parameter is parsed into, for the types the cli parses. +fn rust_type(resolve: &Resolve, ty: &Type) -> Option<&'static str> { + Some(match ty { + Type::Bool => "bool", + Type::U8 => "u8", + Type::U16 => "u16", + Type::U32 => "u32", + Type::U64 => "u64", + Type::S8 => "i8", + Type::S16 => "i16", + Type::S32 => "i32", + Type::S64 => "i64", + Type::F32 => "f32", + Type::F64 => "f64", + Type::Char => "char", + Type::String => "String", + Type::Id(id) => match &resolve.types[*id].kind { + TypeDefKind::Type(ty) => return rust_type(resolve, ty), + _ => return None, + }, + Type::ErrorContext => return None, + }) +} + +/// The function's WIT signature, for the usage. +fn signature(resolve: &Resolve, func: &Function) -> String { + let params: Vec = func + .params + .iter() + .map(|p| format!("{}: {}", p.name, type_name(resolve, &p.ty))) + .collect(); + let prefix = if func.kind.is_async() { + "async func" + } else { + "func" + }; + match &func.result { + Some(ty) => format!( + "{prefix}({}) -> {}", + params.join(", "), + type_name(resolve, ty) + ), + None => format!("{prefix}({})", params.join(", ")), + } +} + +fn type_name(resolve: &Resolve, ty: &Type) -> String { + match ty { + Type::Id(id) => { + let def = &resolve.types[*id]; + match (&def.name, &def.kind) { + (Some(name), _) => name.clone(), + (None, TypeDefKind::Stream(Some(ty))) => { + format!("stream<{}>", type_name(resolve, ty)) + } + (None, TypeDefKind::List(ty)) => format!("list<{}>", type_name(resolve, ty)), + (None, TypeDefKind::Option(ty)) => format!("option<{}>", type_name(resolve, ty)), + (None, TypeDefKind::Tuple(t)) => format!( + "tuple<{}>", + t.types + .iter() + .map(|ty| type_name(resolve, ty)) + .collect::>() + .join(", ") + ), + (None, kind) => kind.as_str().to_string(), + } + } + Type::Bool => "bool".into(), + Type::U8 => "u8".into(), + Type::U16 => "u16".into(), + Type::U32 => "u32".into(), + Type::U64 => "u64".into(), + Type::S8 => "s8".into(), + Type::S16 => "s16".into(), + Type::S32 => "s32".into(), + Type::S64 => "s64".into(), + Type::F32 => "f32".into(), + Type::F64 => "f64".into(), + Type::Char => "char".into(), + Type::String => "string".into(), + Type::ErrorContext => "error-context".into(), + } +} diff --git a/components/test-cli-core/src/lib.rs b/components/test-cli-core/src/lib.rs new file mode 100644 index 0000000..3180331 --- /dev/null +++ b/components/test-cli-core/src/lib.rs @@ -0,0 +1,237 @@ +//! A command calling the functions of the constants data interfaces, for +//! testing: +//! +//! ```sh +//! test-cli [arguments...] +//! ``` +//! +//! The arguments are parsed as the function's parameters, and its result is +//! printed. A stream's first item is printed, then items are read as stdin's +//! lines ask for them: an empty line reads the next item, a number that many. +//! The command ends when the stream ends or stdin closes. +//! +//! The command exports WASIp3's async `wasi:cli/run`, so it can wait for a +//! stream's items, which a synchronous entry point can't. Its arguments, stdin, +//! stdout and stderr are WASIp3's too, stdin read from a stream, and stdout and +//! stderr written to streams. + +use std::{fmt::Debug, str::FromStr}; +use wit_bindgen::{FutureReader, StreamReader, StreamResult, StreamWriter}; + +use crate::wasi::cli::{environment, stderr, stdin, stdout, terminal_stdin, types::ErrorCode}; + +wit_bindgen::generate!({ + path: "../wit", + world: "test-cli-core", + generate_all +}); + +include!(concat!(env!("OUT_DIR"), "/dispatch.rs")); + +struct Cli; + +impl exports::wasi::cli::run::Guest for Cli { + async fn run() -> Result<(), ()> { + let args: Vec = environment::get_arguments().into_iter().skip(1).collect(); + let mut io = Io::new(); + let result = match args.as_slice() { + [interface, function, args @ ..] => dispatch(&mut io, interface, function, args).await, + _ => Err(usage()), + }; + if let Err(message) = &result { + io.stderr.write(format!("{message}\n")).await; + } + io.close().await; + result.map_err(|_| ()) + } +} + +export!(Cli); + +/// The command's stdin, stdout and stderr. +pub struct Io { + stdin: Lines, + stdout: Output, + stderr: Output, + /// Whether stdin is a terminal, which echoes the newline each line ends + /// with. + terminal: bool, + /// Whether stdout's last line is a stream item not yet ended by a newline. + unended: bool, +} + +impl Io { + fn new() -> Self { + let (stdin, _) = stdin::read_via_stream(); + Self { + stdin: Lines { + stream: stdin, + buffer: vec![], + closed: false, + }, + stdout: Output::new(stdout::write_via_stream), + stderr: Output::new(stderr::write_via_stream), + terminal: terminal_stdin::get_terminal_stdin().is_some(), + unended: false, + } + } + + /// Prints a stream's item. A terminal's echo of the newline asking for the + /// next item ends the item's line, so the item isn't followed by a blank + /// line, a newline only separates items printed for the same line. + async fn print_item(&mut self, item: String) { + if !self.terminal { + return self.print(item).await; + } + if self.unended { + self.stdout.write("\n".into()).await; + } + self.stdout.write(item).await; + self.unended = true; + } + + /// The next line of stdin, whose echo ends any item printed last. + async fn next_line(&mut self) -> Option { + let line = self.stdin.next().await; + if line.is_some() { + self.unended = false; + } + line + } + + async fn print(&mut self, line: String) { + self.stdout.write(line + "\n").await; + } + + async fn error(&mut self, line: String) { + self.stderr.write(line + "\n").await; + } + + /// Closes stdout and stderr, once everything written has been taken, + /// ending the last item's line. + async fn close(mut self) { + if self.unended { + self.stdout.write("\n".into()).await; + } + self.stdout.close().await; + self.stderr.close().await; + } +} + +/// Bytes written to stdout or stderr through a stream. +struct Output { + stream: StreamWriter, + done: FutureReader>, +} + +impl Output { + fn new(write_via_stream: fn(StreamReader) -> FutureReader>) -> Self { + let (stream, reader) = wit_stream::new(); + Self { + stream, + done: write_via_stream(reader), + } + } + + async fn write(&mut self, text: String) { + self.stream.write_all(text.into_bytes()).await; + } + + async fn close(self) { + drop(self.stream); + let _ = self.done.await; + } +} + +/// stdin's lines, read from its stream. +struct Lines { + stream: StreamReader, + buffer: Vec, + closed: bool, +} + +impl Lines { + /// The next line, without its newline, or `None` once stdin closes. + async fn next(&mut self) -> Option { + loop { + if let Some(end) = self.buffer.iter().position(|b| *b == b'\n') { + let line: Vec = self.buffer.drain(..=end).collect(); + return Some(String::from_utf8_lossy(&line[..end]).into()); + } + if self.closed { + return match self.buffer.is_empty() { + true => None, + false => { + Some(String::from_utf8_lossy(&std::mem::take(&mut self.buffer)).into()) + } + }; + } + let (result, bytes) = self.stream.read(Vec::with_capacity(4096)).await; + self.buffer.extend(bytes); + if !matches!(result, StreamResult::Complete(_)) { + self.closed = true; + } + } + } +} + +fn usage() -> String { + let mut usage = String::from( + "usage: test-cli [arguments...]\n\n\ + A stream's first item is printed, then an empty line on stdin reads the next \ + item, a number reads that many.\n\nfunctions:\n", + ); + for (interface, function, signature) in FUNCTIONS { + usage.push_str(&format!(" {interface} {function}: {signature}\n")); + } + usage +} + +fn expect_args(args: &[String], count: usize) -> Result<(), String> { + match args.len() == count { + true => Ok(()), + false => Err(format!("expected {count} arguments, found {}", args.len())), + } +} + +fn parse(arg: &str, name: &str) -> Result +where + T::Err: std::fmt::Display, +{ + arg.parse() + .map_err(|e| format!("invalid argument `{name}`, {arg:?}: {e}")) +} + +async fn print_value(io: &mut Io, value: &impl Debug) { + io.print(format!("{value:?}")).await; +} + +/// Prints the stream's first item, then items as stdin's lines ask for them, +/// until the stream ends or stdin closes, which closes the stream. +async fn read_stream(io: &mut Io, mut stream: StreamReader) { + match stream.next().await { + Some(item) => io.print_item(format!("{item:?}")).await, + None => return, + } + while let Some(line) = io.next_line().await { + let count = match line.trim() { + "" => 1, + count => match count.parse::() { + Ok(count) => count, + Err(_) => { + io.error(format!( + "expected an empty line or a number of items, found {count:?}" + )) + .await; + continue; + } + }, + }; + for _ in 0..count { + match stream.next().await { + Some(item) => io.print_item(format!("{item:?}")).await, + None => return, + } + } + } +} diff --git a/components/test-cli-core/tests/test_cli.rs b/components/test-cli-core/tests/test_cli.rs new file mode 100644 index 0000000..98dcad3 --- /dev/null +++ b/components/test-cli-core/tests/test_cli.rs @@ -0,0 +1,234 @@ +//! Runs `target/components/test-cli`, built by make, as `wasmtime run` +//! would, with WASIp3's async `wasi:cli/run`. + +use anyhow::{Result, anyhow}; +use std::sync::OnceLock; +use test_harness::built_component; +use tokio::io::{AsyncRead, AsyncWrite}; +use wasmtime::{ + Config, Engine, Store, + component::{Component, Linker, ResourceTable}, +}; +use wasmtime_wasi::{ + WasiCtx, WasiCtxView, WasiView, + cli::{AsyncStdinStream, IsTerminal, StdinStream, StdoutStream}, + p2::pipe::{MemoryInputPipe, MemoryOutputPipe}, + p3::bindings::Command, +}; + +struct State { + ctx: WasiCtx, + table: ResourceTable, +} + +impl WasiView for State { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.ctx, + table: &mut self.table, + } + } +} + +/// stdout or stderr, written straight to memory, so everything the command +/// wrote is there once it returns. +#[derive(Clone)] +struct Captured(MemoryOutputPipe); + +impl IsTerminal for Captured { + fn is_terminal(&self) -> bool { + false + } +} + +impl StdoutStream for Captured { + fn async_stream(&self) -> Box { + Box::new(self.0.clone()) + } +} + +/// stdin from memory, claiming to be a terminal. +struct Terminal(AsyncStdinStream); + +impl IsTerminal for Terminal { + fn is_terminal(&self) -> bool { + true + } +} + +impl StdinStream for Terminal { + fn async_stream(&self) -> Box { + self.0.async_stream() + } +} + +/// What a run printed, and whether it succeeded. +struct Output { + ok: bool, + stdout: String, + stderr: String, +} + +/// The engine and the command, compiled once for every test. +fn command() -> Result<&'static (Engine, Component)> { + static COMMAND: OnceLock> = OnceLock::new(); + COMMAND + .get_or_init(|| { + let compile = || -> Result<_> { + let mut config = Config::new(); + config.wasm_component_model_async(true); + let engine = Engine::new(&config)?; + let component = Component::from_file(&engine, built_component("test-cli")?)?; + Ok((engine, component)) + }; + compile().map_err(|e| format!("{e:#}")) + }) + .as_ref() + .map_err(|e| anyhow!("{e}")) +} + +/// Runs the command with `args`, and `stdin` as its input. +async fn run(args: &[&str], stdin: &str) -> Result { + run_with(args, stdin, false).await +} + +/// Like [`run`], with stdin a terminal when `terminal`. +async fn run_with(args: &[&str], stdin: &str, terminal: bool) -> Result { + let (engine, component) = command()?; + let mut linker = Linker::::new(engine); + wasmtime_wasi::p3::add_to_linker(&mut linker)?; + + let stdout = MemoryOutputPipe::new(1 << 20); + let stderr = MemoryOutputPipe::new(1 << 20); + let mut builder = WasiCtx::builder(); + builder + .args(&[&["test-cli"], args].concat()) + .stdout(Captured(stdout.clone())) + .stderr(Captured(stderr.clone())); + let input = AsyncStdinStream::new(MemoryInputPipe::new(stdin.to_string())); + match terminal { + true => builder.stdin(Terminal(input)), + false => builder.stdin(input), + }; + let mut store = Store::new( + engine, + State { + ctx: builder.build(), + table: ResourceTable::default(), + }, + ); + let command = Command::instantiate_async(&mut store, component, &linker).await?; + let result = store + .run_concurrent(async move |store| command.wasi_cli_run().call_run(store).await) + .await??; + Ok(Output { + ok: result.is_ok(), + stdout: String::from_utf8_lossy(&stdout.contents()).into(), + stderr: String::from_utf8_lossy(&stderr.contents()).into(), + }) +} + +fn lines(output: &str) -> Vec<&str> { + output.lines().collect() +} + +#[tokio::test(flavor = "current_thread")] +async fn it_prints_values() -> Result<()> { + for (args, expected) in [ + (["math", "pi"], "3.141592653589793"), + (["u8", "max"], "255"), + (["s64", "min"], "-9223372036854775808"), + (["f32", "epsilon"], "1.1920929e-7"), + ] { + let output = run(&args, "").await?; + assert!(output.ok, "{args:?}: {}", output.stderr); + assert_eq!(output.stdout.trim(), expected, "{args:?}"); + } + Ok(()) +} + +#[tokio::test(flavor = "current_thread")] +async fn it_reads_stream_items_as_stdin_asks() -> Result<()> { + // the first item, then an empty line reads one item, a number that many + let output = run(&["atomics", "incrementer", "10"], "\n\n3\n").await?; + assert!(output.ok, "{}", output.stderr); + assert_eq!(lines(&output.stdout), ["10", "11", "12", "13", "14", "15"]); + let output = run(&["atomics", "incrementer", "10"], "").await?; + assert_eq!(lines(&output.stdout), ["10"]); + + // the command ends with the stream + let output = run(&["atomics", "decrementer", "2"], "\n\n\n\n").await?; + assert_eq!(lines(&output.stdout), ["2", "1", "0"]); + let output = run(&["math", "fibonacci"], "1000\n").await?; + assert_eq!(output.stdout.lines().last(), Some("12200160415121876738")); + + // or when stdin closes + let output = run(&["math", "odd"], "2\n").await?; + assert_eq!(lines(&output.stdout), ["1", "3", "5"]); + + let output = run(&["random", "seeded-random-bytes", "42"], "8\n").await?; + assert_eq!( + lines(&output.stdout), + ["149", "110", "235", "47", "38", "50", "215", "189", "3"] + ); + + let output = run(&["math", "even"], "x\n\n").await?; + assert_eq!(lines(&output.stdout), ["0", "2"]); + assert!( + output.stderr.contains("expected an empty line or a number"), + "{}", + output.stderr + ); + Ok(()) +} + +#[tokio::test(flavor = "current_thread")] +async fn it_reports_errors() -> Result<()> { + for (args, expected) in [ + ( + vec!["math", "nope"], + "unknown function `nope` of interface `math`", + ), + ( + vec!["atomics", "incrementer", "abc"], + "invalid argument `seed`", + ), + ( + vec!["atomics", "incrementer"], + "expected 1 arguments, found 0", + ), + (vec![], "usage: test-cli "), + ] { + let output = run(&args, "").await?; + assert!(!output.ok, "{args:?}"); + assert!( + output.stderr.contains(expected), + "{args:?}: {}", + output.stderr + ); + } + // the usage lists the functions + let output = run(&[], "").await?; + assert!( + output + .stderr + .contains(" random seeded-random: async func(seed: u64) -> stream") + ); + Ok(()) +} + +#[tokio::test(flavor = "current_thread")] +async fn it_leaves_a_terminal_to_end_each_item_line() -> Result<()> { + // a terminal echoes the newline ending each line of stdin, which ends the + // line of the item printed for it, so the items printed for a line are + // only separated by newlines, and the last item's line is ended on exit + let output = run_with(&["atomics", "incrementer", "10"], "\n\n3\n", true).await?; + assert!(output.ok, "{}", output.stderr); + assert_eq!(output.stdout, "10111213\n14\n15\n"); + // the line asking for an item after the stream's last ends its line + let output = run_with(&["atomics", "decrementer", "1"], "\n\n", true).await?; + assert_eq!(output.stdout, "10"); + let output = run_with(&["atomics", "decrementer", "1"], "2\n", true).await?; + assert_eq!(output.stdout, "10\n"); + Ok(()) +} diff --git a/components/test-cli/README.md b/components/test-cli/README.md new file mode 100644 index 0000000..a535868 --- /dev/null +++ b/components/test-cli/README.md @@ -0,0 +1,16 @@ +# `test-cli` + +A command calling the functions of the constants data interfaces, for testing: [`test-cli-core`](../test-cli-core), composed with the `math`, `atomics`, `random` and `types` components implementing the interfaces it imports. It only imports WASI, so it runs with `wasmtime run`: + +```sh +make components/test-cli +wasmtime run target/components/test-cli/test-cli.wasm math pi + +# will continue running until stdin closes +printf '\n\n\n' | wasmtime run target/components/test-cli/test-cli.wasm atomics incrementer 10 + +# a different seed will produce a different set of numbers +printf '\n\n\n' | wasmtime run target/components/test-cli/test-cli.wasm random seeded-random 42 +``` + +See [`test-cli-core`](../test-cli-core) for its arguments and how it reads streams. diff --git a/components/test-cli/test-cli.wac b/components/test-cli/test-cli.wac new file mode 100644 index 0000000..24ce4f8 --- /dev/null +++ b/components/test-cli/test-cli.wac @@ -0,0 +1,17 @@ +// test-cli-core, with each data interface it imports provided by its component +package componentized:test-cli; + +let math = new local:math { ... }; +let atomics = new local:atomics { ... }; +let random = new local:random { ... }; +let types = new local:types { ... }; + +let cli = new local:test-cli-core { + ...math, + ...atomics, + ...random, + ...types, + ... +}; + +export cli...; diff --git a/components/wit/worlds.wit b/components/wit/worlds.wit index ee56e80..4a1cd30 100644 --- a/components/wit/worlds.wit +++ b/components/wit/worlds.wit @@ -7,3 +7,30 @@ world config { world factory { export componentized:constants/factory@0.2.0-dev; } + +/// A command calling the functions of the data interfaces, for testing, see +/// components/test-cli-core. +world test-cli-core { + import componentized:constants/math@0.2.0-dev; + import componentized:constants/atomics@0.2.0-dev; + import componentized:constants/random@0.2.0-dev; + import componentized:constants/%u8@0.2.0-dev; + import componentized:constants/%u16@0.2.0-dev; + import componentized:constants/%u32@0.2.0-dev; + import componentized:constants/%u64@0.2.0-dev; + import componentized:constants/%s8@0.2.0-dev; + import componentized:constants/%s16@0.2.0-dev; + import componentized:constants/%s32@0.2.0-dev; + import componentized:constants/%s64@0.2.0-dev; + import componentized:constants/%f32@0.2.0-dev; + import componentized:constants/%f64@0.2.0-dev; + + import wasi:cli/environment@0.3.0; + import wasi:cli/stdin@0.3.0; + import wasi:cli/terminal-stdin@0.3.0; + import wasi:cli/stdout@0.3.0; + import wasi:cli/stderr@0.3.0; + + // an async entry point, so the command can wait for a stream's items + export wasi:cli/run@0.3.0; +} diff --git a/components/wkg.lock b/components/wkg.lock index 9254e46..7b034ac 100644 --- a/components/wkg.lock +++ b/components/wkg.lock @@ -11,6 +11,15 @@ requirement = "=0.0.0-dev" version = "0.0.0-dev" digest = "sha256:2d7677a89ef49bef5d8d20848c431c76208753858f4871929066397d62542af7" +[[packages]] +name = "wasi:cli" +registry = "wasi.dev" + +[[packages.versions]] +requirement = "=0.3.0" +version = "0.3.0" +digest = "sha256:3a2d58743b67a057f7210b9a73d3b21b34ec7895ffe34d1cb092927307156e40" + [[packages]] name = "wasi:config" registry = "wasi.dev" diff --git a/crates/componentized-constants/src/docs.rs b/crates/componentized-constants/src/docs.rs index fab73fc..fb59a78 100644 --- a/crates/componentized-constants/src/docs.rs +++ b/crates/componentized-constants/src/docs.rs @@ -1,12 +1,19 @@ +use anyhow::{Result, bail}; use wit_parser::Docs; /// Doc comment tag introducing a function's value. pub const TAG: &str = "@value"; +/// Doc comment tag introducing the expression generating the items of a +/// stream that follow its `@value`. +pub const EXPRESSION_TAG: &str = "@expression"; + +const TAGS: [&str; 2] = [TAG, EXPRESSION_TAG]; + /// Returns the WAVE value following the `@value` tag in a doc comment, if any. /// /// The tag must start a line. Everything after it, including any following -/// lines, is the value, so a value may span multiple lines: +/// lines up to another tag, is the value, so a value may span multiple lines: /// /// ```wit /// /// The origin of the plane. @@ -18,18 +25,61 @@ pub const TAG: &str = "@value"; /// origin: func() -> point; /// ``` pub fn value(docs: &Docs) -> Option<&str> { + tag(docs, TAG) +} + +/// Returns the expression following the `@expression` tag in a doc comment, +/// if any, see [`value`]. +pub fn expression(docs: &Docs) -> Option<&str> { + tag(docs, EXPRESSION_TAG) +} + +/// Ensures a doc comment has at most one of each tag. +pub fn check(docs: &Docs) -> Result<()> { + let tags: Vec<&str> = lines(docs).filter_map(|(_, _, tag)| tag).collect(); + for (i, tag) in tags.iter().enumerate() { + if tags[..i].contains(tag) { + bail!("duplicate `{tag}` tag, a doc comment may have one of each tag"); + } + } + Ok(()) +} + +/// The contents following `tag`, up to the next line starting with a tag. +fn tag<'a>(docs: &'a Docs, tag: &str) -> Option<&'a str> { let contents = docs.contents.as_deref()?; - let mut offset = 0; - for line in contents.split_inclusive('\n') { - let indent = line.len() - line.trim_start().len(); - if let Some(rest) = line[indent..].strip_prefix(TAG) { - if rest.is_empty() || rest.starts_with(char::is_whitespace) { - return Some(&contents[offset + indent + TAG.len()..]); - } + let mut start = None; + for (offset, indent, found) in lines(docs) { + match (start, found) { + (None, Some(t)) if t == tag => start = Some(offset + indent + t.len()), + (Some(start), Some(_)) => return Some(&contents[start..offset]), + _ => {} } - offset += line.len(); } - None + start.map(|start| &contents[start..]) +} + +/// Each line of a doc comment's contents: its offset, its indent, and the tag +/// it starts with, if any. A tag must be followed by whitespace or the end of +/// the line. +fn lines(docs: &Docs) -> impl Iterator)> + '_ { + let contents = docs.contents.as_deref().unwrap_or_default(); + contents + .split_inclusive('\n') + .scan(0, |offset, line| { + let start = *offset; + *offset += line.len(); + Some((start, line)) + }) + .map(|(offset, line)| { + let indent = line.len() - line.trim_start().len(); + let found = TAGS.into_iter().find(|t| { + line[indent..] + .strip_prefix(t) + .is_some_and(|rest| rest.is_empty() || rest.starts_with(char::is_whitespace)) + }); + (offset, indent, found) + }) } #[cfg(test)] @@ -57,4 +107,35 @@ mod tests { assert_eq!(value_of("See @value 1"), None); assert_eq!(value_of("@values 1"), None); } + + #[test] + fn it_rejects_duplicate_tags() { + let docs = |contents: &str| Docs { + contents: Some(contents.into()), + }; + assert!(check(&Docs::default()).is_ok()); + assert!(check(&docs("@value 1\n@expression |n| n\nSee @value 2")).is_ok()); + for contents in [ + "@value 1\n@value 2", + "@expression |n| n\n@value [1]\n @expression |n| n", + ] { + let err = check(&docs(contents)).expect_err(contents).to_string(); + assert!(err.starts_with("duplicate `@"), "{err}"); + } + } + + #[test] + fn it_ends_values_at_the_next_tag() { + let docs = Docs { + contents: Some("Counts.\n@value [\n 0,\n]\n@expression |n| n + 1\n".into()), + }; + assert_eq!(value(&docs), Some(" [\n 0,\n]\n")); + assert_eq!(expression(&docs), Some(" |n| n + 1\n")); + + let docs = Docs { + contents: Some("@expression |n| n\n@value [0]".into()), + }; + assert_eq!(value(&docs), Some(" [0]")); + assert_eq!(expression(&docs), Some(" |n| n\n")); + } } diff --git a/crates/componentized-constants/src/expr.rs b/crates/componentized-constants/src/expr.rs new file mode 100644 index 0000000..24ddd32 --- /dev/null +++ b/crates/componentized-constants/src/expr.rs @@ -0,0 +1,2954 @@ +//! Expressions generating the items of a stream, written as a subset of Rust's +//! closure syntax, e.g. `|a, b| a + b`. See +//! grammars/constants-expression/README.md for the full specification. +//! +//! The closure takes the previous items of the stream as parameters and returns +//! the next item. Besides its parameters, an expression may read `$init`, a +//! tuple of the function's arguments, and `$i`, the position in the stream of +//! the item being generated. +//! +//! Supported, following Rust's precedence and semantics: +//! +//! - integer, float and `true`/`false` literals, typed by their context +//! - the closure's parameters, `$init.0`, `$init.1`, …, and `$i` +//! - `-` and `!`, `!` is a bitwise not for integers +//! - `*`, `/`, `%`, `+`, `-`, `<<`, `>>`, `&`, `^`, `|` +//! - `==`, `!=`, `<`, `<=`, `>`, `>=`, `&&` and `||` +//! - `as` casts between numeric types, and from `bool` to integers +//! - `wrapping_add`, `wrapping_sub`, `wrapping_mul`, `rotate_left` and +//! `rotate_right` methods on integers +//! - parentheses, `if c { a } else { b }`, including `else if`, and blocks +//! with `let` bindings, e.g. `{ let z = a ^ b; z ^ z >> 3 }` +//! +//! Where Rust's checked arithmetic fails, e.g. overflow, division by zero or a +//! shift by at least the type's bits, or a float operation's result isn't +//! finite, the item can't be generated and the stream ends. + +use anyhow::{Result, anyhow, bail}; +use wasm_encoder::{BlockType, Function, InstructionSink, MemArg, ValType}; +use wit_parser::{Resolve, Type}; + +use crate::streams::{INDEX, SEEDS}; +use crate::types::dealias; +use crate::values::position; + +/// The types expressions support. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Prim { + Bool, + U8, + S8, + U16, + S16, + U32, + S32, + U64, + S64, + F32, + F64, +} + +impl Prim { + pub fn from_type(resolve: &Resolve, ty: &Type) -> Option { + Some(match dealias(resolve, *ty) { + Type::Bool => Prim::Bool, + Type::U8 => Prim::U8, + Type::S8 => Prim::S8, + Type::U16 => Prim::U16, + Type::S16 => Prim::S16, + Type::U32 => Prim::U32, + Type::S32 => Prim::S32, + Type::U64 => Prim::U64, + Type::S64 => Prim::S64, + Type::F32 => Prim::F32, + Type::F64 => Prim::F64, + _ => return None, + }) + } + + /// A type named in an expression, by its Rust name, or its WIT name for + /// signed integers. + fn from_name(name: &str) -> Option { + Some(match name { + "bool" => Prim::Bool, + "u8" => Prim::U8, + "i8" | "s8" => Prim::S8, + "u16" => Prim::U16, + "i16" | "s16" => Prim::S16, + "u32" => Prim::U32, + "i32" | "s32" => Prim::S32, + "u64" => Prim::U64, + "i64" | "s64" => Prim::S64, + "f32" => Prim::F32, + "f64" => Prim::F64, + _ => return None, + }) + } + + pub(crate) fn name(self) -> &'static str { + match self { + Prim::Bool => "bool", + Prim::U8 => "u8", + Prim::S8 => "s8", + Prim::U16 => "u16", + Prim::S16 => "s16", + Prim::U32 => "u32", + Prim::S32 => "s32", + Prim::U64 => "u64", + Prim::S64 => "s64", + Prim::F32 => "f32", + Prim::F64 => "f64", + } + } + + fn is_float(self) -> bool { + matches!(self, Prim::F32 | Prim::F64) + } + + fn is_int(self) -> bool { + !self.is_float() && self != Prim::Bool + } + + fn signed(self) -> bool { + matches!(self, Prim::S8 | Prim::S16 | Prim::S32 | Prim::S64) + } + + /// Integers of at most 32 bits are i32s, sign or zero extended. + fn narrow(self) -> bool { + self.is_int() && self.bits() <= 32 + } + + fn wide(self) -> bool { + self.val_type() == ValType::I64 + } + + fn bits(self) -> u32 { + match self { + Prim::Bool | Prim::U8 | Prim::S8 => 8, + Prim::U16 | Prim::S16 => 16, + Prim::U32 | Prim::S32 | Prim::F32 => 32, + Prim::U64 | Prim::S64 | Prim::F64 => 64, + } + } + + fn range(self) -> (i128, i128) { + let bits = self.bits(); + match self.signed() { + true => (-(1 << (bits - 1)), (1 << (bits - 1)) - 1), + false => (0, (1 << bits) - 1), + } + } + + pub(crate) fn val_type(self) -> ValType { + match self { + Prim::U64 | Prim::S64 => ValType::I64, + Prim::F32 => ValType::F32, + Prim::F64 => ValType::F64, + _ => ValType::I32, + } + } + + /// The size of an item of this type in memory. + pub fn size(self) -> u32 { + match self { + Prim::Bool => 1, + _ => self.bits() / 8, + } + } + + fn mem_arg(self, offset: u32) -> MemArg { + MemArg { + offset: offset.into(), + align: self.size().trailing_zeros(), + memory_index: 0, + } + } + + fn load(self, i: &mut InstructionSink, offset: u32) { + let m = self.mem_arg(offset); + match self { + Prim::Bool | Prim::U8 => i.i32_load8_u(m), + Prim::S8 => i.i32_load8_s(m), + Prim::U16 => i.i32_load16_u(m), + Prim::S16 => i.i32_load16_s(m), + Prim::U32 | Prim::S32 => i.i32_load(m), + Prim::U64 | Prim::S64 => i.i64_load(m), + Prim::F32 => i.f32_load(m), + Prim::F64 => i.f64_load(m), + }; + } + + pub(crate) fn store(self, i: &mut InstructionSink, offset: u32) { + let m = self.mem_arg(offset); + match self { + Prim::Bool | Prim::U8 | Prim::S8 => i.i32_store8(m), + Prim::U16 | Prim::S16 => i.i32_store16(m), + Prim::U32 | Prim::S32 => i.i32_store(m), + Prim::U64 | Prim::S64 => i.i64_store(m), + Prim::F32 => i.f32_store(m), + Prim::F64 => i.f64_store(m), + }; + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Op { + WrapMul, + WrapDiv, + WrapRem, + WrapAdd, + WrapSub, + WrapShl, + WrapShr, + /// `**`, exponentiation. + Pow, + /// `**%`, wrapping exponentiation. + WrapPow, + Mul, + Div, + Rem, + Add, + Sub, + Shl, + Shr, + BitAnd, + BitXor, + BitOr, + Eq, + Ne, + Lt, + Le, + Gt, + Ge, + And, + Or, +} + +impl Op { + fn symbol(self) -> &'static str { + match self { + Op::WrapMul => "*%", + Op::WrapDiv => "/%", + Op::WrapRem => "%%", + Op::WrapAdd => "+%", + Op::WrapSub => "-%", + Op::WrapShl => "<<%", + Op::WrapShr => ">>%", + Op::Pow => "**", + Op::WrapPow => "**%", + Op::Mul => "*", + Op::Div => "/", + Op::Rem => "%", + Op::Add => "+", + Op::Sub => "-", + Op::Shl => "<<", + Op::Shr => ">>", + Op::BitAnd => "&", + Op::BitXor => "^", + Op::BitOr => "|", + Op::Eq => "==", + Op::Ne => "!=", + Op::Lt => "<", + Op::Le => "<=", + Op::Gt => ">", + Op::Ge => ">=", + Op::And => "&&", + Op::Or => "||", + } + } + + /// The wrapping arithmetic operators, e.g. `+%`, but not the shifts. + fn is_wrapping(self) -> bool { + matches!( + self, + Op::WrapMul | Op::WrapDiv | Op::WrapRem | Op::WrapAdd | Op::WrapSub + ) + } + + fn is_shift(self) -> bool { + matches!(self, Op::Shl | Op::Shr | Op::WrapShl | Op::WrapShr) + } + + fn is_comparison(self) -> bool { + matches!(self, Op::Eq | Op::Ne | Op::Lt | Op::Le | Op::Gt | Op::Ge) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Method { + WrappingAdd, + WrappingSub, + WrappingMul, + RotateLeft, + RotateRight, +} + +impl Method { + fn from_name(name: &str) -> Option { + Some(match name { + "wrapping_add" => Method::WrappingAdd, + "wrapping_sub" => Method::WrappingSub, + "wrapping_mul" => Method::WrappingMul, + "rotate_left" => Method::RotateLeft, + "rotate_right" => Method::RotateRight, + _ => return None, + }) + } + + fn name(self) -> &'static str { + match self { + Method::WrappingAdd => "wrapping_add", + Method::WrappingSub => "wrapping_sub", + Method::WrappingMul => "wrapping_mul", + Method::RotateLeft => "rotate_left", + Method::RotateRight => "rotate_right", + } + } + + fn is_rotate(self) -> bool { + matches!(self, Method::RotateLeft | Method::RotateRight) + } +} + +enum Expr { + Int(i128), + Float(f64), + Bool(bool), + Param(u32), + Local(u32), + Seed(u32), + /// `$init.name`, a field by the function's parameter's name, and where it + /// is, for errors, resolved to its index before types are checked. + SeedName(String, String), + Index, + Neg(Box), + /// `-%e`, wrapping negation. + WrapNeg(Box), + Not(Box), + Binary(Op, Box, Box), + If(Box, Box, Box), + Block(Vec, Box), + Cast(Box, Prim), + Method(Method, Box, Box), + /// `[a, b, c]`, only as the closure's result, each element an item. + Array(Vec), + /// `x.to_le_bytes()`, or `to_be_bytes` when big endian, only as the + /// closure's result, each byte an item. + Bytes(Box, bool), +} + +/// A statement in a block, before the block's value, if any. +enum Stmt { + Let(Let), + /// `x = value;`, a compound assignment, e.g. `x += 1;`, is `x = x + 1;`. + Assign { + local: u32, + value: Expr, + }, + While { + cond: Expr, + body: Vec, + }, + If { + cond: Expr, + then: Vec, + otherwise: Vec, + }, +} + +/// A block's statements, its value if it has one, and where it ends, for +/// errors. +struct Items { + stmts: Vec, + tail: Option, + end: String, +} + +/// An `if` before it's known to be a value or a statement. +struct IfParts { + cond: Expr, + then: Items, + otherwise: Else, + end: String, +} + +enum Else { + None, + If(Box), + Block(Items), +} + +impl IfParts { + /// Whether every branch has a value, so the `if` is a value. + fn has_values(&self) -> bool { + self.then.tail.is_some() + && match &self.otherwise { + Else::None => false, + Else::If(parts) => parts.has_values(), + Else::Block(items) => items.tail.is_some(), + } + } + + fn into_expr(self) -> Result { + let then = items_value(self.then)?; + let otherwise = match self.otherwise { + Else::None => bail!( + "expected `else`, an `if` needs a value for both branches {}", + self.end + ), + Else::If(parts) => parts.into_expr()?, + Else::Block(items) => items_value(items)?, + }; + Ok(Expr::If( + Box::new(self.cond), + Box::new(then), + Box::new(otherwise), + )) + } + + fn into_stmt(self) -> Result { + let then = items_stmts(self.then)?; + let otherwise = match self.otherwise { + Else::None => vec![], + Else::If(parts) => vec![parts.into_stmt()?], + Else::Block(items) => items_stmts(items)?, + }; + Ok(Stmt::If { + cond: self.cond, + then, + otherwise, + }) + } +} + +/// A block as a value, which must end with one. +fn items_value(items: Items) -> Result { + let Some(tail) = items.tail else { + bail!("expected a value at the end of the block {}", items.end); + }; + Ok(match items.stmts.is_empty() { + true => tail, + false => Expr::Block(items.stmts, Box::new(tail)), + }) +} + +/// A block of statements, a loop's or an `if` statement's, without a value. +fn items_stmts(items: Items) -> Result> { + if items.tail.is_some() { + bail!( + "a `while` loop's or an `if` statement's block can't end with a value {}", + items.end + ); + } + Ok(items.stmts) +} + +/// `let name: ty = init;`, binding a local, or nothing for `_`. +struct Let { + local: Option, + ty: Option, + init: Expr, +} + +/// A parsed closure, before its types are checked. +pub struct Closure { + params: u32, + locals: u32, + body: Expr, + /// Whether the expression reads `$init`, the function's arguments. + pub uses_seed: bool, +} + +/// An expression generating the next item of a stream from the previous +/// `params` items, the function's arguments in `seeds`, and the position. +pub struct Generator { + pub params: u32, + /// The number of items each evaluation generates, more than one when the + /// closure returns an array. + pub count: u32, + pub item: Prim, + pub seeds: Vec, + locals: Vec, + body: Expr, +} + +/// Parses a closure, without checking its types. +/// +/// The function's arguments, named in `args` as Rust names them, e.g. +/// `start_value` for `start-value`, are in scope outside the closure's +/// parameters, which shadow them, as do `let` bindings. Reading one reads +/// `$init`. +pub fn parse_closure<'a>(src: &'a str, args: &'a [String]) -> Result { + let mut parser = Parser { + src, + tokens: tokenize(src)?, + pos: 0, + scope: args + .iter() + .enumerate() + .map(|(index, name)| (name.as_str(), Binding::Arg(index as u32))) + .collect(), + params: 0, + param_names: vec![], + locals: 0, + uses_seed: false, + }; + let body = parser.closure()?; + if let Some(token) = parser.tokens.get(parser.pos) { + bail!("unexpected `{}` at {}", token.text(src), token.at(src)); + } + Ok(Closure { + params: parser.params, + locals: parser.locals, + body, + uses_seed: parser.uses_seed, + }) +} + +impl Closure { + /// Checks the closure generates items of type `item`, where `$init` holds + /// arguments of the types in `seeds`, of the parameters named `names`, as + /// Rust names them, e.g. `start_value` for `start-value`. + pub fn check(mut self, item: Prim, seeds: &[Prim], names: &[String]) -> Result { + resolve_seed_names(&mut self.body, names)?; + let mut generator = Generator { + params: self.params, + count: 1, + item, + seeds: seeds.to_vec(), + locals: vec![Prim::Bool; self.locals as usize], + body: Expr::Bool(false), + }; + generator.type_locals(&self.body)?; + let body = self.body; + generator.count = generator.check_result(&body)?; + generator.body = body; + Ok(generator) + } +} + +/// Replaces each `$init.name` with the index of the parameter it names. +fn resolve_seed_names(e: &mut Expr, names: &[String]) -> Result<()> { + match e { + Expr::SeedName(name, at) => { + let Some(index) = names.iter().position(|n| n == name) else { + let names: Vec = names.iter().map(|n| format!("`{n}`")).collect(); + bail!( + "`$init` has no field `{name}` at {at}, the function's parameters are {}", + match names.is_empty() { + true => "none".to_string(), + false => names.join(", "), + } + ); + }; + *e = Expr::Seed(index as u32); + } + Expr::Neg(e) | Expr::WrapNeg(e) | Expr::Not(e) | Expr::Cast(e, _) | Expr::Bytes(e, _) => { + resolve_seed_names(e, names)? + } + Expr::Binary(_, l, r) | Expr::Method(_, l, r) => { + resolve_seed_names(l, names)?; + resolve_seed_names(r, names)?; + } + Expr::If(c, t, e) => { + for e in [c, t, e] { + resolve_seed_names(e, names)?; + } + } + Expr::Block(stmts, e) => { + resolve_stmt_seed_names(stmts, names)?; + resolve_seed_names(e, names)?; + } + Expr::Array(elements) => { + for e in elements { + resolve_seed_names(e, names)?; + } + } + _ => {} + } + Ok(()) +} + +fn resolve_stmt_seed_names(stmts: &mut [Stmt], names: &[String]) -> Result<()> { + for stmt in stmts { + match stmt { + Stmt::Let(l) => resolve_seed_names(&mut l.init, names)?, + Stmt::Assign { value, .. } => resolve_seed_names(value, names)?, + Stmt::While { cond, body } => { + resolve_seed_names(cond, names)?; + resolve_stmt_seed_names(body, names)?; + } + Stmt::If { + cond, + then, + otherwise, + } => { + resolve_seed_names(cond, names)?; + resolve_stmt_seed_names(then, names)?; + resolve_stmt_seed_names(otherwise, names)?; + } + } + } + Ok(()) +} + +#[derive(Clone, Copy, PartialEq)] +enum Kind { + Ident, + Var, + Int(u128), + Float(f64), + Punct(&'static str), +} + +#[derive(Clone, Copy)] +struct Token { + kind: Kind, + start: usize, + end: usize, +} + +impl Token { + fn text<'a>(&self, src: &'a str) -> &'a str { + &src[self.start..self.end] + } + + fn at(&self, src: &str) -> String { + position(src, self.start..self.end) + } +} + +/// Punctuation, longest first so `||` isn't read as two `|`. +const PUNCT: [&str; 59] = [ + "**%=", "<<%=", ">>%=", "**%", "**=", "<<%", ">>%", "<<=", ">>=", "+%=", "-%=", "*%=", "/%=", + "%%=", "**", "&&", "||", "<<", ">>", "<=", ">=", "==", "!=", "+%", "-%", "*%", "/%", "%%", + "+=", "-=", "*=", "/=", "%=", "&=", "^=", "|=", "+", "-", "*", "/", "%", "!", "&", "|", "^", + "<", ">", "=", "(", ")", "{", "}", "[", "]", ",", ";", ":", ".", "_", +]; + +/// Assignment operators, `=` and compound assignments, e.g. `+=`. +const ASSIGN: [(&str, Option); 20] = [ + ("=", None), + ("+=", Some(Op::Add)), + ("-=", Some(Op::Sub)), + ("*=", Some(Op::Mul)), + ("/=", Some(Op::Div)), + ("%=", Some(Op::Rem)), + ("<<=", Some(Op::Shl)), + (">>=", Some(Op::Shr)), + ("&=", Some(Op::BitAnd)), + ("^=", Some(Op::BitXor)), + ("|=", Some(Op::BitOr)), + ("+%=", Some(Op::WrapAdd)), + ("-%=", Some(Op::WrapSub)), + ("*%=", Some(Op::WrapMul)), + ("/%=", Some(Op::WrapDiv)), + ("%%=", Some(Op::WrapRem)), + ("<<%=", Some(Op::WrapShl)), + (">>%=", Some(Op::WrapShr)), + ("**=", Some(Op::Pow)), + ("**%=", Some(Op::WrapPow)), +]; + +fn tokenize(src: &str) -> Result> { + let bytes = src.as_bytes(); + let mut tokens = vec![]; + let mut pos = 0; + while pos < bytes.len() { + let c = bytes[pos]; + let start = pos; + if c.is_ascii_whitespace() { + pos += 1; + continue; + } + let kind = if c.is_ascii_alphabetic() || (c == b'_' && is_ident(bytes.get(pos + 1))) { + while is_ident(bytes.get(pos)) { + pos += 1; + } + Kind::Ident + } else if c == b'$' && is_ident(bytes.get(pos + 1)) { + pos += 1; + while is_ident(bytes.get(pos)) { + pos += 1; + } + Kind::Var + } else if c.is_ascii_digit() { + number(src, &mut pos)? + } else if let Some(p) = PUNCT.into_iter().find(|p| src[pos..].starts_with(p)) { + pos += p.len(); + Kind::Punct(p) + } else { + let c = src[pos..].chars().next().expect("a char"); + bail!( + "unexpected `{c}` at {}", + position(src, pos..pos + c.len_utf8()) + ); + }; + tokens.push(Token { + kind, + start, + end: pos, + }); + } + Ok(tokens) +} + +fn is_ident(c: Option<&u8>) -> bool { + c.is_some_and(|c| c.is_ascii_alphanumeric() || *c == b'_') +} + +/// An integer, in decimal, hex `0x`, octal `0o` or binary `0b`, or a decimal +/// float, digits may be separated by `_`. +fn number(src: &str, pos: &mut usize) -> Result { + let bytes = src.as_bytes(); + let start = *pos; + let digits = |pos: &mut usize, radix: u32| { + while bytes + .get(*pos) + .is_some_and(|c| *c == b'_' || (*c as char).is_digit(radix)) + { + *pos += 1; + } + }; + let radix = match (bytes[*pos], bytes.get(*pos + 1)) { + (b'0', Some(b'x')) => 16, + (b'0', Some(b'o')) => 8, + (b'0', Some(b'b')) => 2, + _ => 10, + }; + let at = || position(src, start..start + 1); + if radix != 10 { + *pos += 2; + digits(pos, radix); + if is_ident(bytes.get(*pos)) { + bail!("literal suffixes aren't supported at {}", at()); + } + let text: String = src[start + 2..*pos].chars().filter(|c| *c != '_').collect(); + return u128::from_str_radix(&text, radix) + .map(Kind::Int) + .map_err(|_| anyhow!("invalid integer at {}", at())); + } + digits(pos, 10); + let mut float = false; + // a `.` followed by a digit, or by nothing that could start a field or + // method, makes a float, e.g. `1.5` or `1.` + if bytes.get(*pos) == Some(&b'.') + && !bytes + .get(*pos + 1) + .is_some_and(|c| *c == b'.' || *c == b'_' || c.is_ascii_alphabetic()) + { + float = true; + *pos += 1; + digits(pos, 10); + } + if matches!(bytes.get(*pos), Some(b'e' | b'E')) { + float = true; + *pos += 1; + if matches!(bytes.get(*pos), Some(b'+' | b'-')) { + *pos += 1; + } + digits(pos, 10); + } + if is_ident(bytes.get(*pos)) { + bail!("literal suffixes aren't supported at {}", at()); + } + let text: String = src[start..*pos].chars().filter(|c| *c != '_').collect(); + match float { + true => text + .parse() + .map(Kind::Float) + .map_err(|_| anyhow!("invalid float at {}", at())), + false => text + .parse() + .map(Kind::Int) + .map_err(|_| anyhow!("invalid integer at {}", at())), + } +} + +/// What a name in scope refers to. +#[derive(Clone, Copy)] +enum Binding { + /// One of the function's arguments, which are never assigned. + Arg(u32), + Param(u32), + /// A `let` binding, and whether it's `let mut`. + Local(u32, bool), +} + +struct Parser<'a> { + src: &'a str, + tokens: Vec, + pos: usize, + /// Names in scope, later names shadowing earlier ones. + scope: Vec<(&'a str, Binding)>, + params: u32, + /// The closure's parameters' names, `None` for `_`, for `$call`'s fields. + param_names: Vec>, + locals: u32, + uses_seed: bool, +} + +impl<'a> Parser<'a> { + fn peek(&self) -> Option { + self.tokens.get(self.pos).map(|t| t.kind) + } + + fn peek_punct(&self, p: &str) -> bool { + matches!(self.peek(), Some(Kind::Punct(q)) if q == p) + } + + fn peek_ident(&self, name: &str) -> bool { + self.tokens + .get(self.pos) + .is_some_and(|t| t.kind == Kind::Ident && t.text(self.src) == name) + } + + fn eat_punct(&mut self, p: &str) -> bool { + let found = self.peek_punct(p); + if found { + self.pos += 1; + } + found + } + + fn expect_punct(&mut self, p: &str) -> Result<()> { + if !self.eat_punct(p) { + bail!("expected `{p}` {}", self.at()); + } + Ok(()) + } + + /// Describes where the next token is, for errors. + fn at(&self) -> String { + match self.tokens.get(self.pos) { + Some(token) => format!( + "at {}, found `{}`", + token.at(self.src), + token.text(self.src) + ), + None => "at the end".to_string(), + } + } + + /// A name to bind, a parameter or a `let`, or `None` for `_`. + fn binding_name(&mut self, what: &str) -> Result> { + let token = self.tokens.get(self.pos).copied(); + match token { + Some(t) if t.kind == Kind::Var => bail!( + "`{}` can't be bound at {}, `$init`, `$call` and `$i` are built in, and can't \ + be shadowed", + t.text(self.src), + t.at(self.src) + ), + Some(t) if t.kind == Kind::Punct("_") => { + self.pos += 1; + Ok(None) + } + Some(t) if t.kind == Kind::Ident && !is_keyword(t.text(self.src)) => { + self.pos += 1; + Ok(Some(t.text(self.src))) + } + _ => bail!("expected {what} {}", self.at()), + } + } + + /// `|a, b| expr`, or `|| expr` without parameters. + fn closure(&mut self) -> Result { + if !self.eat_punct("||") { + self.expect_punct("|")?; + while !self.eat_punct("|") { + let token = self.tokens[self.pos.min(self.tokens.len().saturating_sub(1))]; + let name = self.binding_name("a parameter name")?; + if let Some(name) = name { + // only another parameter conflicts, an argument is shadowed + if self + .scope + .iter() + .any(|(n, b)| *n == name && matches!(b, Binding::Param(_))) + { + bail!("duplicate parameter `{name}` at {}", token.at(self.src)); + } + self.scope.push((name, Binding::Param(self.params))); + } + self.param_names.push(name); + self.params += 1; + if !self.eat_punct(",") && !self.peek_punct("|") { + bail!("expected `,` or `|` {}", self.at()); + } + } + } + self.expr() + } + + fn expr(&mut self) -> Result { + self.binary(0) + } + + /// Binary operators by precedence, lowest first. Comparisons don't chain. + fn binary(&mut self, level: usize) -> Result { + const LEVELS: [&[(&str, Op)]; 9] = [ + &[("||", Op::Or)], + &[("&&", Op::And)], + &[ + ("==", Op::Eq), + ("!=", Op::Ne), + ("<=", Op::Le), + (">=", Op::Ge), + ("<", Op::Lt), + (">", Op::Gt), + ], + &[("|", Op::BitOr)], + &[("^", Op::BitXor)], + &[("&", Op::BitAnd)], + &[ + ("<<", Op::Shl), + (">>", Op::Shr), + ("<<%", Op::WrapShl), + (">>%", Op::WrapShr), + ], + &[ + ("+", Op::Add), + ("-", Op::Sub), + ("+%", Op::WrapAdd), + ("-%", Op::WrapSub), + ], + &[ + ("*", Op::Mul), + ("/", Op::Div), + ("%", Op::Rem), + ("*%", Op::WrapMul), + ("/%", Op::WrapDiv), + ("%%", Op::WrapRem), + ], + ]; + let Some(ops) = LEVELS.get(level) else { + return self.cast(); + }; + let mut lhs = self.binary(level + 1)?; + while let Some((_, op)) = ops.iter().find(|(p, _)| self.peek_punct(p)) { + self.pos += 1; + let rhs = self.binary(level + 1)?; + lhs = Expr::Binary(*op, Box::new(lhs), Box::new(rhs)); + if op.is_comparison() && ops.iter().any(|(p, _)| self.peek_punct(p)) { + bail!("comparison operators can't be chained {}", self.at()); + } + } + Ok(lhs) + } + + /// `e as ty`, binding tighter than binary operators and looser than unary. + fn cast(&mut self) -> Result { + let mut e = self.unary()?; + while self.peek_ident("as") { + self.pos += 1; + let ty = self.type_name()?; + e = Expr::Cast(Box::new(e), ty); + } + Ok(e) + } + + fn type_name(&mut self) -> Result { + let token = self.tokens.get(self.pos).copied(); + match token.and_then(|t| (t.kind == Kind::Ident).then(|| Prim::from_name(t.text(self.src)))) + { + Some(Some(ty)) => { + self.pos += 1; + Ok(ty) + } + _ => bail!("expected a type, e.g. `u8`, `i32` or `f64`, {}", self.at()), + } + } + + fn unary(&mut self) -> Result { + if self.eat_punct("-") { + return Ok(match self.unary()? { + // a negative literal, so the minimum of a signed type is in range + Expr::Int(v) => Expr::Int(-v), + Expr::Float(v) => Expr::Float(-v), + e => Expr::Neg(Box::new(e)), + }); + } + if self.eat_punct("-%") { + return Ok(Expr::WrapNeg(Box::new(self.unary()?))); + } + if self.eat_punct("!") { + return Ok(Expr::Not(Box::new(self.unary()?))); + } + self.power() + } + + /// `base ** exponent` or `base **% exponent`, binding tighter than a unary + /// operator on its left, so `-x ** 2` is `-(x ** 2)`. The exponent is a + /// unary expression, which makes `**` right associative, `a ** b ** c` is + /// `a ** (b ** c)`, and allows `x ** -1`. + fn power(&mut self) -> Result { + let base = self.postfix()?; + for (punct, op) in [("**", Op::Pow), ("**%", Op::WrapPow)] { + if self.eat_punct(punct) { + let exponent = self.unary()?; + return Ok(Expr::Binary(op, Box::new(base), Box::new(exponent))); + } + } + Ok(base) + } + + /// Method calls, `e.method(arg)`. + fn postfix(&mut self) -> Result { + let mut e = self.primary()?; + while self.peek_punct(".") { + let dot = self.tokens[self.pos]; + self.pos += 1; + let token = self.tokens.get(self.pos).copied(); + let Some(name) = token + .filter(|t| t.kind == Kind::Ident) + .map(|t| t.text(self.src)) + else { + bail!("expected a method name {}", self.at()); + }; + let bytes = match name { + "to_le_bytes" => Some(false), + "to_be_bytes" => Some(true), + _ => None, + }; + let method = Method::from_name(name); + if bytes.is_none() && method.is_none() { + bail!( + "unknown method `{name}` at {}, the methods are `wrapping_add`, \ + `wrapping_sub`, `wrapping_mul`, `rotate_left`, `rotate_right`, \ + `to_le_bytes` and `to_be_bytes`", + token.expect("a name").at(self.src) + ); + } + if matches!(e, Expr::Int(_) | Expr::Float(_)) { + bail!( + "can't call `{name}` on a literal at {}, its type is ambiguous", + dot.at(self.src) + ); + } + self.pos += 1; + self.expect_punct("(")?; + e = match (method, bytes) { + (Some(method), _) => { + if self.peek_punct(")") { + bail!("`{name}` takes an argument {}", self.at()); + } + let arg = self.expr()?; + Expr::Method(method, Box::new(e), Box::new(arg)) + } + (None, Some(big)) => { + if !self.peek_punct(")") { + bail!("`{name}` takes no arguments {}", self.at()); + } + Expr::Bytes(Box::new(e), big) + } + (None, None) => unreachable!("checked above"), + }; + self.expect_punct(")")?; + } + Ok(e) + } + + fn primary(&mut self) -> Result { + let Some(token) = self.tokens.get(self.pos).copied() else { + bail!("expected an expression at the end"); + }; + match token.kind { + Kind::Int(v) => { + self.pos += 1; + match i128::try_from(v) { + Ok(v) => Ok(Expr::Int(v)), + Err(_) => bail!("literal {v} is too large at {}", token.at(self.src)), + } + } + Kind::Float(v) => { + self.pos += 1; + Ok(Expr::Float(v)) + } + Kind::Punct("(") => { + self.pos += 1; + let e = self.expr()?; + self.expect_punct(")")?; + Ok(e) + } + Kind::Punct("{") => self.block(), + Kind::Punct("[") => { + self.pos += 1; + let mut elements = vec![]; + while !self.eat_punct("]") { + elements.push(self.expr()?); + if !self.eat_punct(",") && !self.peek_punct("]") { + bail!("expected `,` or `]` {}", self.at()); + } + } + if elements.is_empty() { + bail!( + "an array must have at least one item at {}", + token.at(self.src) + ); + } + Ok(Expr::Array(elements)) + } + Kind::Var => { + self.pos += 1; + match token.text(self.src) { + "$i" => Ok(Expr::Index), + "$init" => { + self.uses_seed = true; + if !self.eat_punct(".") { + bail!( + "expected `.` and a field, `$init` is a tuple of the \ + function's arguments, e.g. `$init.0`, {}", + self.at() + ); + } + let digits = self + .tokens + .get(self.pos) + .is_some_and(|t| t.text(self.src).bytes().all(|c| c.is_ascii_digit())); + let field = self.tokens.get(self.pos).copied(); + match self.peek() { + Some(Kind::Int(index)) if digits && index <= u32::MAX as u128 => { + self.pos += 1; + Ok(Expr::Seed(index as u32)) + } + Some(Kind::Ident) => { + let field = field.expect("a field"); + self.pos += 1; + Ok(Expr::SeedName( + field.text(self.src).to_string(), + field.at(self.src), + )) + } + _ => bail!( + "expected a field of `$init`, an index, e.g. `0`, or a \ + parameter's name, {}", + self.at() + ), + } + } + "$call" => self.call_field(), + name => bail!( + "unknown variable `{name}` at {}, the variables are `$init`, `$call` and \ + `$i`", + token.at(self.src) + ), + } + } + Kind::Ident => match token.text(self.src) { + "true" => { + self.pos += 1; + Ok(Expr::Bool(true)) + } + "false" => { + self.pos += 1; + Ok(Expr::Bool(false)) + } + "if" => { + self.pos += 1; + self.if_parts()?.into_expr() + } + name if is_keyword(name) => bail!("expected an expression {}", self.at()), + name => match self.scope.iter().rev().find(|(n, _)| *n == name) { + Some((_, binding)) => { + self.pos += 1; + Ok(match *binding { + Binding::Arg(index) => { + self.uses_seed = true; + Expr::Seed(index) + } + Binding::Param(index) => Expr::Param(index), + Binding::Local(id, _) => Expr::Local(id), + }) + } + None => bail!("unknown name `{name}` at {}", token.at(self.src)), + }, + }, + _ => bail!("expected an expression {}", self.at()), + } + } + + /// `$call.0` or `$call.name`, after the `$call`, one of the closure's + /// parameters, the arguments of the current item's evaluation, which can't + /// be shadowed. + fn call_field(&mut self) -> Result { + if !self.eat_punct(".") { + bail!( + "expected `.` and a field, `$call` is the closure's parameters, e.g. \ + `$call.0`, {}", + self.at() + ); + } + let Some(field) = self.tokens.get(self.pos).copied() else { + bail!("expected a field of `$call` at the end"); + }; + let text = field.text(self.src); + let index = match field.kind { + Kind::Int(index) if text.bytes().all(|c| c.is_ascii_digit()) => { + if index >= self.params as u128 { + bail!( + "`$call.{index}` doesn't exist at {}, `$call` has {} fields, one for each \ + of the closure's parameters", + field.at(self.src), + self.params + ); + } + index as u32 + } + Kind::Ident => match self.param_names.iter().position(|n| *n == Some(text)) { + Some(index) => index as u32, + None => { + let names: Vec = self + .param_names + .iter() + .flatten() + .map(|n| format!("`{n}`")) + .collect(); + bail!( + "`$call` has no field `{text}` at {}, the closure's parameters are {}", + field.at(self.src), + match names.is_empty() { + true => "unnamed".to_string(), + false => names.join(", "), + } + ); + } + }, + _ => bail!( + "expected a field of `$call`, an index, e.g. `0`, or a parameter's name, {}", + self.at() + ), + }; + self.pos += 1; + Ok(Expr::Param(index)) + } + + /// `if c { … } else { … }`, after the `if`, a value or a statement. + fn if_parts(&mut self) -> Result { + let cond = self.expr()?; + if !self.peek_punct("{") { + bail!("expected `{{` {}", self.at()); + } + let then = self.items()?; + let end = self.at(); + let otherwise = match self.peek_ident("else") { + false => Else::None, + true => { + self.pos += 1; + if self.peek_ident("if") { + self.pos += 1; + Else::If(Box::new(self.if_parts()?)) + } else { + if !self.peek_punct("{") { + bail!("expected `{{` or `if` {}", self.at()); + } + Else::Block(self.items()?) + } + } + }; + Ok(IfParts { + cond, + then, + otherwise, + end, + }) + } + + /// A block as a value, e.g. `{ let a = …; a + 1 }`. + fn block(&mut self) -> Result { + let items = self.items()?; + items_value(items) + } + + /// `{ statements… value }`, the value is optional. The bindings are in + /// scope until the end of the block. + fn items(&mut self) -> Result { + self.expect_punct("{")?; + let scope = self.scope.len(); + let mut stmts = vec![]; + let tail = loop { + if self.peek_punct("}") { + break None; + } + if self.peek_ident("let") { + stmts.push(self.let_stmt()?); + continue; + } + if self.peek_ident("while") { + self.pos += 1; + let cond = self.expr()?; + if !self.peek_punct("{") { + bail!("expected `{{` {}", self.at()); + } + let body = items_stmts(self.items()?)?; + self.eat_punct(";"); + stmts.push(Stmt::While { cond, body }); + continue; + } + if self.peek_ident("if") { + self.pos += 1; + let parts = self.if_parts()?; + // an `if` with values is the block's value, otherwise a statement + if parts.has_values() { + if !self.peek_punct("}") { + bail!( + "expected `}}`, an `if` with values must be the block's value {}", + self.at() + ); + } + break Some(parts.into_expr()?); + } + stmts.push(parts.into_stmt()?); + self.eat_punct(";"); + continue; + } + if let Some(stmt) = self.assignment()? { + stmts.push(stmt); + continue; + } + let e = self.expr()?; + if !self.peek_punct("}") { + bail!("expected `}}` {}", self.at()); + } + break Some(e); + }; + let end = self.at(); + self.expect_punct("}")?; + self.scope.truncate(scope); + Ok(Items { stmts, tail, end }) + } + + /// `let name: ty = init;`, or `let mut …`, the binding is in scope after its + /// initializer. + fn let_stmt(&mut self) -> Result { + self.pos += 1; + let mutable = self.peek_ident("mut"); + if mutable { + self.pos += 1; + } + let name = self.binding_name("a name to bind")?; + let ty = match self.eat_punct(":") { + true => Some(self.type_name()?), + false => None, + }; + self.expect_punct("=")?; + let init = self.expr()?; + self.expect_punct(";")?; + let local = name.map(|name| { + let id = self.locals; + self.locals += 1; + self.scope.push((name, Binding::Local(id, mutable))); + id + }); + Ok(Stmt::Let(Let { local, ty, init })) + } + + /// `x = value;` or a compound assignment, e.g. `x += 1;`, to a `let mut` + /// binding, when the next tokens are a name and an assignment operator. + fn assignment(&mut self) -> Result> { + let Some(token) = self.tokens.get(self.pos).copied() else { + return Ok(None); + }; + // `$i = …;`, `$init.n = …;` or `$call.n = …;`, the built in variables never + // change + if token.kind == Kind::Var { + let field = matches!( + self.tokens.get(self.pos + 1).map(|t| t.kind), + Some(Kind::Punct(".")) + ); + let next = self.tokens.get(self.pos + if field { 3 } else { 1 }); + if let Some(Kind::Punct(punct)) = next.map(|t| t.kind) + && ASSIGN.iter().any(|(p, _)| *p == punct) + { + bail!( + "`{}` can't be assigned at {}, `$init`, `$call` and `$i` are built in, and \ + never change, bind a copy with `let mut`", + token.text(self.src), + token.at(self.src) + ); + } + return Ok(None); + } + let Some(Kind::Punct(punct)) = self.tokens.get(self.pos + 1).map(|t| t.kind) else { + return Ok(None); + }; + let Some((_, op)) = ASSIGN.iter().find(|(p, _)| *p == punct) else { + return Ok(None); + }; + if token.kind != Kind::Ident || is_keyword(token.text(self.src)) { + return Ok(None); + } + let name = token.text(self.src); + let at = token.at(self.src); + let id = match self.scope.iter().rev().find(|(n, _)| *n == name) { + Some((_, Binding::Local(id, true))) => *id, + Some((_, Binding::Local(_, false))) => { + bail!("can't assign to `{name}` at {at}, declare it with `let mut {name}`") + } + Some((_, Binding::Param(_))) => bail!( + "can't assign to the parameter `{name}` at {at}, bind a copy with \ + `let mut {name} = {name};`" + ), + Some((_, Binding::Arg(_))) => bail!( + "can't assign to the argument `{name}` at {at}, arguments never change, bind \ + a copy with `let mut {name} = {name};`" + ), + None => bail!("unknown name `{name}` at {at}"), + }; + self.pos += 2; + let value = self.expr()?; + self.expect_punct(";")?; + let value = match op { + None => value, + Some(op) => Expr::Binary(*op, Box::new(Expr::Local(id)), Box::new(value)), + }; + Ok(Some(Stmt::Assign { local: id, value })) + } +} + +fn is_keyword(name: &str) -> bool { + matches!( + name, + "if" | "else" | "true" | "false" | "let" | "as" | "while" | "mut" + ) +} + +/// The type of an expression, known from the expression itself, or a literal +/// whose type comes from its context. +#[derive(Clone, Copy)] +enum Synth { + Known(Prim), + Int, + Float, +} + +impl Synth { + /// The type of an expression whose context doesn't decide it, literals + /// default to `i32` or `f64`, as in Rust. + fn resolve(self) -> Prim { + match self { + Synth::Known(ty) => ty, + Synth::Int => Prim::S32, + Synth::Float => Prim::F64, + } + } +} + +/// The parameters of the function generating an item, then temporaries for the +/// operands and result of an operation, then for exponentiation's loop, then +/// the `let` bindings. +const STATE: u32 = 0; +const PTR: u32 = 1; +const A32: u32 = 2; +const B32: u32 = 3; +const A64: u32 = 4; +const B64: u32 = 5; +const R64: u32 = 6; +const F32: u32 = 7; +const F64: u32 = 8; +const POW_NEGATIVE: u32 = 9; +const POW_RESULT_32: u32 = 10; +const POW_BASE_32: u32 = 11; +const POW_EXPONENT: u32 = 12; +const POW_RESULT_64: u32 = 13; +const POW_BASE_64: u32 = 14; +const POW_RESULT_F32: u32 = 15; +const POW_BASE_F32: u32 = 16; +const POW_RESULT_F64: u32 = 17; +const POW_BASE_F64: u32 = 18; +const FIRST_LOCAL: u32 = 19; + +impl Generator { + fn seed(&self, field: u32) -> Result { + match self.seeds.get(field as usize) { + Some(ty) => Ok(*ty), + None => bail!( + "`$init.{field}` doesn't exist, `$init` has {} fields, one for each of the \ + function's arguments", + self.seeds.len() + ), + } + } + + /// Assigns the `let` bindings of statements their types, in order. + fn type_stmts(&mut self, stmts: &[Stmt]) -> Result<()> { + for stmt in stmts { + match stmt { + Stmt::Let(l) => { + self.type_locals(&l.init)?; + let ty = match (l.ty, self.synth(&l.init)?) { + (Some(ty), _) => ty, + (None, Synth::Known(ty)) => ty, + // only literals, `let x = 1;` + (None, _) => match l.local { + Some(_) => bail!( + "the type of a `let` bound to a literal must be given, e.g. \ + `let x: u32 = 1;`" + ), + None => Prim::Bool, + }, + }; + if let Some(id) = l.local { + self.locals[id as usize] = ty; + } + } + Stmt::Assign { value, .. } => self.type_locals(value)?, + Stmt::While { cond, body } => { + self.type_locals(cond)?; + self.type_stmts(body)?; + } + Stmt::If { + cond, + then, + otherwise, + } => { + self.type_locals(cond)?; + self.type_stmts(then)?; + self.type_stmts(otherwise)?; + } + } + } + Ok(()) + } + + /// Assigns each `let` binding its type, in the order they're bound. + fn type_locals(&mut self, e: &Expr) -> Result<()> { + match e { + Expr::Block(stmts, e) => { + self.type_stmts(stmts)?; + self.type_locals(e) + } + Expr::Neg(e) + | Expr::WrapNeg(e) + | Expr::Not(e) + | Expr::Cast(e, _) + | Expr::Bytes(e, _) => self.type_locals(e), + Expr::Array(elements) => elements.iter().try_for_each(|e| self.type_locals(e)), + Expr::Binary(_, l, r) | Expr::Method(_, l, r) => { + self.type_locals(l)?; + self.type_locals(r) + } + Expr::If(c, t, e) => { + self.type_locals(c)?; + self.type_locals(t)?; + self.type_locals(e) + } + _ => Ok(()), + } + } + + /// The type of `e` from `e` alone, before considering its context. + fn synth(&self, e: &Expr) -> Result { + Ok(match e { + Expr::Int(_) => Synth::Int, + Expr::Float(_) => Synth::Float, + Expr::Bool(_) => Synth::Known(Prim::Bool), + Expr::Param(_) => Synth::Known(self.item), + Expr::Local(id) => Synth::Known(self.locals[*id as usize]), + Expr::SeedName(..) => unreachable!("resolved before types are checked"), + Expr::Seed(field) => Synth::Known(self.seed(*field)?), + Expr::Index => Synth::Known(Prim::U64), + Expr::Neg(e) | Expr::WrapNeg(e) | Expr::Not(e) => self.synth(e)?, + Expr::Binary(op, l, r) => match op { + _ if op.is_comparison() => Synth::Known(Prim::Bool), + Op::And | Op::Or => Synth::Known(Prim::Bool), + _ if op.is_shift() => self.synth(l)?, + Op::Pow | Op::WrapPow => self.synth(l)?, + _ => join(op.symbol(), self.synth(l)?, self.synth(r)?)?, + }, + Expr::If(_, t, e) => match (self.synth(t)?, self.synth(e)?) { + (Synth::Known(a), Synth::Known(b)) if a != b => bail!( + "expected {} for the `else` branch, found {}", + a.name(), + b.name() + ), + (a, b) => join("if", a, b)?, + }, + Expr::Block(_, e) => self.synth(e)?, + Expr::Cast(_, ty) => Synth::Known(*ty), + Expr::Method(m, r, a) => match m.is_rotate() { + true => self.synth(r)?, + false => join(m.name(), self.synth(r)?, self.synth(a)?)?, + }, + Expr::Array(_) | Expr::Bytes(..) => bail!("{ARRAY_POSITION}"), + }) + } + + /// Checks the closure's result, returning the number of items it gives: + /// the length of an array, or 1. + fn check_result(&self, e: &Expr) -> Result { + let item = self.item; + match e { + Expr::Array(elements) => { + for e in elements { + self.check(e, item)?; + } + Ok(elements.len() as u32) + } + Expr::Bytes(value, big) => { + let name = bytes_name(*big); + if item != Prim::U8 { + bail!( + "`{name}` gives bytes, the stream's items must be u8, not {}", + item.name() + ); + } + let source = self.synth(value)?.resolve(); + if source == Prim::Bool { + bail!("`{name}` isn't supported for bool"); + } + self.check(value, source)?; + Ok(source.size()) + } + Expr::Block(stmts, e) => { + self.check_stmts(stmts)?; + self.check_result(e) + } + Expr::If(c, t, e) => { + self.check_condition(c, "an `if`")?; + let (a, b) = (self.check_result(t)?, self.check_result(e)?); + if a != b { + bail!( + "the `if` branches give {a} and {b} items, they must give the same \ + number" + ); + } + Ok(a) + } + _ => { + if let Synth::Known(ty) = self.synth(e)? + && ty != item + { + bail!( + "expected the closure to return {}, found {}", + item.name(), + ty.name() + ); + } + self.check(e, item)?; + Ok(1) + } + } + } + + fn check_stmts(&self, stmts: &[Stmt]) -> Result<()> { + for stmt in stmts { + match stmt { + Stmt::Let(l) => { + let ty = match l.local { + Some(id) => self.locals[id as usize], + None => l.ty.unwrap_or(self.synth(&l.init)?.resolve()), + }; + self.check(&l.init, ty)?; + } + Stmt::Assign { local, value } => { + let ty = self.locals[*local as usize]; + if let Synth::Known(actual) = self.synth(value)? + && actual != ty + { + bail!( + "expected {} for an assignment, found {}", + ty.name(), + actual.name() + ); + } + self.check(value, ty)?; + } + Stmt::While { cond, body } => { + self.check_condition(cond, "a `while`")?; + self.check_stmts(body)?; + } + Stmt::If { + cond, + then, + otherwise, + } => { + self.check_condition(cond, "an `if`")?; + self.check_stmts(then)?; + self.check_stmts(otherwise)?; + } + } + } + Ok(()) + } + + fn check_condition(&self, c: &Expr, what: &str) -> Result<()> { + if let Synth::Known(actual) = self.synth(c)? + && actual != Prim::Bool + { + bail!( + "expected bool for {what} condition, found {}", + actual.name() + ); + } + self.check(c, Prim::Bool) + } + + /// The type of an exponent, its own type, or for a literal, `u32` for an + /// integer base as Rust's `pow`, and `i32` for a float base as `powi`. + fn exponent_type(&self, r: &Expr, base: Prim) -> Result { + Ok(match self.synth(r)? { + Synth::Known(exponent) => exponent, + _ if base.is_float() => Prim::S32, + _ => Prim::U32, + }) + } + + /// The operands' type for a comparison. + fn comparison_type(&self, l: &Expr, r: &Expr) -> Result { + Ok(join("comparison", self.synth(l)?, self.synth(r)?)?.resolve()) + } + + /// The type of a shift amount, its own type, or the shifted value's type + /// for a literal. + fn amount_type(&self, r: &Expr, ty: Prim) -> Result { + Ok(match self.synth(r)? { + Synth::Known(amount) => amount, + _ => ty, + }) + } + + /// Checks `e` has type `ty`. + fn check(&self, e: &Expr, ty: Prim) -> Result<()> { + let found = |actual: Prim| -> Result<()> { + if actual != ty { + bail!("expected {}, found {}", ty.name(), actual.name()); + } + Ok(()) + }; + match e { + Expr::Int(v) => { + if ty == Prim::Bool { + bail!("expected bool, found integer {v}"); + } + if ty.is_int() { + let (min, max) = ty.range(); + if *v < min || *v > max { + bail!("literal {v} is out of range for {}", ty.name()); + } + } + } + Expr::Float(v) => { + if !ty.is_float() { + bail!("expected {}, found float {v}", ty.name()); + } + if ty == Prim::F32 && !(*v as f32).is_finite() { + bail!("literal {v} is out of range for f32"); + } + } + Expr::Bool(_) => found(Prim::Bool)?, + Expr::Param(_) => found(self.item)?, + Expr::Local(id) => found(self.locals[*id as usize])?, + Expr::SeedName(..) => unreachable!("resolved before types are checked"), + Expr::Seed(field) => found(self.seed(*field)?)?, + Expr::Index => found(Prim::U64)?, + Expr::Neg(e) => { + if !ty.signed() && !ty.is_float() { + bail!("`-` can't negate {}", ty.name()); + } + self.check(e, ty)?; + } + Expr::WrapNeg(e) => { + if !ty.is_int() { + bail!("`-%` isn't supported for {}", ty.name()); + } + self.check(e, ty)?; + } + Expr::Not(e) => { + if ty.is_float() { + bail!("`!` isn't supported for {}", ty.name()); + } + self.check(e, ty)?; + } + Expr::Binary(op, l, r) => { + let symbol = op.symbol(); + match op { + _ if op.is_comparison() => { + found(Prim::Bool)?; + let operands = self.comparison_type(l, r)?; + self.check(l, operands)?; + self.check(r, operands)?; + } + Op::And | Op::Or => { + found(Prim::Bool)?; + for e in [l, r] { + if let Synth::Known(t) = self.synth(e)? + && t != Prim::Bool + { + bail!("`{symbol}` expects bool operands"); + } + self.check(e, Prim::Bool)?; + } + } + Op::Pow | Op::WrapPow => { + if ty == Prim::Bool { + bail!("`{symbol}` expects numeric operands"); + } + if *op == Op::WrapPow && ty.is_float() { + bail!("`{symbol}` isn't supported for {}", ty.name()); + } + self.check(l, ty)?; + let exponent = self.exponent_type(r, ty)?; + if !exponent.is_int() { + bail!( + "the exponent of `{symbol}` must be an integer, found {}", + exponent.name() + ); + } + self.check(r, exponent)?; + } + _ if op.is_shift() => { + if ty == Prim::Bool { + bail!("`{symbol}` expects numeric operands"); + } + if ty.is_float() { + bail!("`{symbol}` isn't supported for {}", ty.name()); + } + self.check(l, ty)?; + let amount = self.amount_type(r, ty)?; + if !amount.is_int() { + bail!("a shift amount must be an integer, found {}", amount.name()); + } + self.check(r, amount)?; + } + _ if op.is_wrapping() => { + if !ty.is_int() { + bail!("`{symbol}` isn't supported for {}", ty.name()); + } + self.check(l, ty)?; + self.check(r, ty)?; + } + Op::BitAnd | Op::BitOr | Op::BitXor => { + if ty.is_float() { + bail!("`{symbol}` isn't supported for {}", ty.name()); + } + self.check(l, ty)?; + self.check(r, ty)?; + } + _ => { + if ty == Prim::Bool { + bail!("`{symbol}` expects numeric operands"); + } + if ty.is_float() && *op == Op::Rem { + bail!("`{symbol}` isn't supported for {}", ty.name()); + } + self.check(l, ty)?; + self.check(r, ty)?; + } + } + } + Expr::If(c, t, e) => { + self.check_condition(c, "an `if`")?; + self.check(t, ty)?; + self.check(e, ty)?; + } + Expr::Block(stmts, e) => { + self.check_stmts(stmts)?; + self.check(e, ty)?; + } + Expr::Array(_) | Expr::Bytes(..) => bail!("{ARRAY_POSITION}"), + Expr::Cast(e, target) => { + found(*target)?; + if *target == Prim::Bool { + bail!("can't cast to bool, compare instead, e.g. `x != 0`"); + } + let source = self.synth(e)?.resolve(); + if source == Prim::Bool && !target.is_int() { + bail!("can't cast bool to {}", target.name()); + } + self.check(e, source)?; + } + Expr::Method(m, r, a) => { + if !ty.is_int() { + bail!("`{}` isn't supported for {}", m.name(), ty.name()); + } + self.check(r, ty)?; + self.check(a, if m.is_rotate() { Prim::U32 } else { ty })?; + } + } + Ok(()) + } + + /// `generate(state, ptr) -> ok` reads the previous items from `ptr`, and + /// `$init` and `$i` from the task's `state`, storing the next item after + /// the previous items, returning 0 when it can't be generated. + pub fn function(&self) -> Function { + use ValType::*; + let mut locals = vec![ + (2, I32), + (3, I64), + (1, F32), + (1, F64), + (3, I32), + (3, I64), + (2, F32), + (2, F64), + ]; + locals.extend(self.locals.iter().map(|ty| (1, ty.val_type()))); + let mut f = Function::new(locals); + let i = &mut f.instructions(); + i.block(BlockType::Empty); + self.emit_result(i, &self.body, 0); + i.i32_const(1).return_().end().i32_const(0).end(); + f + } + + /// Emits the closure's result, storing its items after the previous items. + fn emit_result(&self, i: &mut InstructionSink, e: &Expr, fail: u32) { + let item = self.item; + let offset = self.params * item.size(); + match e { + Expr::Array(elements) => { + for (index, e) in elements.iter().enumerate() { + i.local_get(PTR); + self.emit(i, e, item, fail); + item.store(i, offset + index as u32 * item.size()); + } + } + Expr::Bytes(value, big) => { + let source = self.synth(value).map_or(Prim::U64, Synth::resolve); + let wide = source.size() == 8; + self.emit(i, value, source, fail); + match source { + Prim::F32 => { + i.i32_reinterpret_f32(); + } + Prim::F64 => { + i.i64_reinterpret_f64(); + } + _ => {} + } + let temp = if wide { A64 } else { A32 }; + i.local_set(temp); + let byte = |offset: u32| MemArg { + offset: offset.into(), + align: 0, + memory_index: 0, + }; + if *big { + // most significant byte first + for index in 0..source.size() { + let shift = (source.size() - 1 - index) * 8; + i.local_get(PTR).local_get(temp); + match wide { + true => i + .i64_const(shift as i64) + .i64_shr_u() + .i64_store8(byte(offset + index)), + false => i + .i32_const(shift as i32) + .i32_shr_u() + .i32_store8(byte(offset + index)), + }; + } + } else { + // wasm stores little endian + i.local_get(PTR).local_get(temp); + match source.size() { + 1 => i.i32_store8(byte(offset)), + 2 => i.i32_store16(byte(offset)), + 4 => i.i32_store(byte(offset)), + _ => i.i64_store(byte(offset)), + }; + } + } + Expr::Block(stmts, e) => { + self.emit_stmts(i, stmts, fail); + self.emit_result(i, e, fail); + } + Expr::If(c, t, e) => { + self.emit(i, c, Prim::Bool, fail); + i.if_(BlockType::Empty); + self.emit_result(i, t, fail + 1); + i.else_(); + self.emit_result(i, e, fail + 1); + i.end(); + } + _ => { + i.local_get(PTR); + self.emit(i, e, item, fail); + item.store(i, offset); + } + } + } + + fn emit_stmts(&self, i: &mut InstructionSink, stmts: &[Stmt], fail: u32) { + for stmt in stmts { + match stmt { + Stmt::Let(l) => { + let ty = match l.local { + Some(id) => self.locals[id as usize], + None => l.ty.unwrap_or_else(|| { + self.synth(&l.init).map_or(Prim::S32, Synth::resolve) + }), + }; + self.emit(i, &l.init, ty, fail); + match l.local { + Some(id) => i.local_set(FIRST_LOCAL + id), + None => i.drop(), + }; + } + Stmt::Assign { local, value } => { + self.emit(i, value, self.locals[*local as usize], fail); + i.local_set(FIRST_LOCAL + local); + } + // the loop's block and loop are 2 more blocks to break out of + Stmt::While { cond, body } => { + i.block(BlockType::Empty).loop_(BlockType::Empty); + self.emit(i, cond, Prim::Bool, fail + 2); + i.i32_eqz().br_if(1); + self.emit_stmts(i, body, fail + 2); + i.br(0).end().end(); + } + Stmt::If { + cond, + then, + otherwise, + } => { + self.emit(i, cond, Prim::Bool, fail); + i.if_(BlockType::Empty); + self.emit_stmts(i, then, fail + 1); + if !otherwise.is_empty() { + i.else_(); + self.emit_stmts(i, otherwise, fail + 1); + } + i.end(); + } + } + } + } + + /// Emits `e` as type `ty`, `fail` is the depth of the block to break to + /// when an item can't be generated. + fn emit(&self, i: &mut InstructionSink, e: &Expr, ty: Prim, fail: u32) { + match e { + Expr::Int(v) => match ty.val_type() { + ValType::I64 => { + i.i64_const(*v as i64); + } + ValType::F32 => { + i.f32_const((*v as f32).into()); + } + ValType::F64 => { + i.f64_const((*v as f64).into()); + } + _ => { + i.i32_const(*v as i32); + } + }, + Expr::Float(v) => match ty { + Prim::F32 => { + i.f32_const((*v as f32).into()); + } + _ => { + i.f64_const((*v).into()); + } + }, + Expr::Bool(v) => { + i.i32_const(*v as i32); + } + Expr::Param(index) => { + i.local_get(PTR); + ty.load(i, index * ty.size()); + } + Expr::Local(id) => { + i.local_get(FIRST_LOCAL + id); + } + Expr::SeedName(..) => unreachable!("resolved before types are checked"), + Expr::Seed(field) => { + i.local_get(STATE); + ty.load(i, SEEDS + field * 8); + } + Expr::Index => { + i.local_get(STATE).i64_load(MemArg { + offset: INDEX.into(), + align: 3, + memory_index: 0, + }); + } + Expr::Neg(e) => { + if ty.is_float() { + self.emit(i, e, ty, fail); + match ty { + Prim::F32 => i.f32_neg(), + _ => i.f64_neg(), + }; + } else if ty.narrow() { + i.i64_const(0); + self.emit(i, e, ty, fail); + i.i64_extend_i32_s().i64_sub(); + range(i, ty, fail); + } else { + // only the minimum can't be negated + self.emit(i, e, ty, fail); + i.local_tee(A64) + .i64_const(i64::MIN) + .i64_eq() + .br_if(fail) + .i64_const(0) + .local_get(A64) + .i64_sub(); + } + } + Expr::WrapNeg(e) => { + match ty.wide() { + true => i.i64_const(0), + false => i.i32_const(0), + }; + self.emit(i, e, ty, fail); + wrapping_op(i, Op::WrapSub, ty, fail); + } + Expr::Not(e) => { + self.emit(i, e, ty, fail); + if ty == Prim::Bool { + i.i32_eqz(); + } else if ty.narrow() && !ty.signed() { + i.i32_const(ty.range().1 as i32).i32_xor(); + } else if ty.narrow() { + i.i32_const(-1).i32_xor(); + } else { + i.i64_const(-1).i64_xor(); + } + } + Expr::If(c, t, e) => { + self.emit(i, c, Prim::Bool, fail); + i.if_(BlockType::Result(ty.val_type())); + self.emit(i, t, ty, fail + 1); + i.else_(); + self.emit(i, e, ty, fail + 1); + i.end(); + } + Expr::Block(stmts, e) => { + self.emit_stmts(i, stmts, fail); + self.emit(i, e, ty, fail); + } + Expr::Array(_) | Expr::Bytes(..) => unreachable!("checked as the result"), + Expr::Cast(e, target) => { + let source = self.synth(e).map_or(*target, Synth::resolve); + self.emit(i, e, source, fail); + convert(i, source, *target); + } + Expr::Method(m, r, a) => { + self.emit(i, r, ty, fail); + self.emit(i, a, if m.is_rotate() { Prim::U32 } else { ty }, fail); + method(i, *m, ty); + } + Expr::Binary(Op::And, l, r) => { + self.emit(i, l, Prim::Bool, fail); + i.if_(BlockType::Result(ValType::I32)); + self.emit(i, r, Prim::Bool, fail + 1); + i.else_().i32_const(0).end(); + } + Expr::Binary(Op::Or, l, r) => { + self.emit(i, l, Prim::Bool, fail); + i.if_(BlockType::Result(ValType::I32)).i32_const(1).else_(); + self.emit(i, r, Prim::Bool, fail + 1); + i.end(); + } + Expr::Binary(op, l, r) if op.is_comparison() => { + let operands = self.comparison_type(l, r).unwrap_or(ty); + self.emit(i, l, operands, fail); + self.emit(i, r, operands, fail); + match operands { + Prim::Bool => bool_op(i, *op), + _ => compare(i, *op, operands), + } + } + Expr::Binary(op @ (Op::Pow | Op::WrapPow), l, r) => { + let exponent = self.exponent_type(r, ty).unwrap_or(Prim::U32); + self.emit(i, l, ty, fail); + self.emit(i, r, exponent, fail); + pow(i, *op, ty, exponent, fail); + } + Expr::Binary(op, l, r) if op.is_shift() => { + let amount = self.amount_type(r, ty).unwrap_or(ty); + self.emit(i, l, ty, fail); + self.emit(i, r, amount, fail); + shift(i, *op, ty, amount, fail); + } + Expr::Binary(op, l, r) => { + self.emit(i, l, ty, fail); + self.emit(i, r, ty, fail); + match ty { + _ if op.is_wrapping() => wrapping_op(i, *op, ty, fail), + Prim::Bool => bool_op(i, *op), + _ if ty.is_float() => float_op(i, *op, ty, fail), + _ if ty.narrow() => narrow_op(i, *op, ty, fail), + _ => wide_op(i, *op, ty, fail), + } + } + } + } +} + +const ARRAY_POSITION: &str = + "an array can only be the closure's result, e.g. `|| [a, b]` or `|| x.to_le_bytes()`"; + +fn bytes_name(big: bool) -> &'static str { + match big { + true => "to_be_bytes", + false => "to_le_bytes", + } +} + +/// The joined type of two operands that must have the same type. +fn join(what: &str, a: Synth, b: Synth) -> Result { + Ok(match (a, b) { + (Synth::Known(a), Synth::Known(b)) if a != b => bail!( + "mismatched types for `{what}`, {} and {}", + a.name(), + b.name() + ), + (Synth::Known(a), _) | (_, Synth::Known(a)) => Synth::Known(a), + (Synth::Float, _) | (_, Synth::Float) => Synth::Float, + _ => Synth::Int, + }) +} + +/// Re-extends the low bits of an i32 holding a narrow integer, after an +/// operation that may have set the bits above the type's width. +fn canonical(i: &mut InstructionSink, ty: Prim) { + let unused = 32 - ty.bits() as i32; + if !ty.narrow() || unused == 0 { + return; + } + if ty.signed() { + i.i32_const(unused).i32_shl().i32_const(unused).i32_shr_s(); + } else { + i.i32_const(ty.range().1 as i32).i32_and(); + } +} + +/// Checks the i64 result of a narrow integer operation fits `ty`, wrapping it +/// to an i32. +fn range(i: &mut InstructionSink, ty: Prim, fail: u32) { + let (min, max) = ty.range(); + i.local_tee(R64); + if ty.signed() { + i.i64_const(min as i64) + .i64_lt_s() + .br_if(fail) + .local_get(R64) + .i64_const(max as i64) + .i64_gt_s() + .br_if(fail); + } else { + i.i64_const(max as i64).i64_gt_u().br_if(fail); + } + i.local_get(R64).i32_wrap_i64(); +} + +/// An arithmetic or bitwise operation on integers of at most 32 bits, checked +/// by computing in 64 bits. +fn narrow_op(i: &mut InstructionSink, op: Op, ty: Prim, fail: u32) { + let (min, _) = ty.range(); + i.local_set(B32).local_set(A32); + let extend = |i: &mut InstructionSink, local| { + i.local_get(local); + match ty.signed() { + true => i.i64_extend_i32_s(), + false => i.i64_extend_i32_u(), + }; + }; + match op { + Op::BitAnd | Op::BitOr | Op::BitXor => { + i.local_get(A32).local_get(B32); + match op { + Op::BitAnd => i.i32_and(), + Op::BitOr => i.i32_or(), + _ => i.i32_xor(), + }; + } + _ => { + if matches!(op, Op::Div | Op::Rem) { + i.local_get(B32).i32_eqz().br_if(fail); + if ty.signed() { + // Rust's checked `%` also fails for the minimum and -1 + i.local_get(A32) + .i32_const(min as i32) + .i32_eq() + .local_get(B32) + .i32_const(-1) + .i32_eq() + .i32_and() + .br_if(fail); + } + } + extend(i, A32); + extend(i, B32); + match (op, ty.signed()) { + (Op::Add, _) => i.i64_add(), + (Op::Sub, _) => i.i64_sub(), + (Op::Mul, _) => i.i64_mul(), + (Op::Div, true) => i.i64_div_s(), + (Op::Div, false) => i.i64_div_u(), + (Op::Rem, true) => i.i64_rem_s(), + _ => i.i64_rem_u(), + }; + range(i, ty, fail); + } + } +} + +/// An arithmetic or bitwise operation on 64-bit integers, checked as Rust's +/// checked arithmetic. +fn wide_op(i: &mut InstructionSink, op: Op, ty: Prim, fail: u32) { + let signed = ty.signed(); + i.local_set(B64).local_set(A64); + match op { + Op::Add | Op::Sub => { + i.local_get(A64).local_get(B64); + match op { + Op::Add => i.i64_add(), + _ => i.i64_sub(), + }; + i.local_set(R64); + match (op, signed) { + // the sum wrapped below either operand + (Op::Add, false) => i.local_get(R64).local_get(A64).i64_lt_u(), + (Op::Sub, false) => i.local_get(A64).local_get(B64).i64_lt_u(), + // the operands' signs agree and the result's sign differs + (Op::Add, true) => i + .local_get(A64) + .local_get(R64) + .i64_xor() + .local_get(B64) + .local_get(R64) + .i64_xor() + .i64_and() + .i64_const(0) + .i64_lt_s(), + // the operands' signs differ and the result's sign differs from a + _ => i + .local_get(A64) + .local_get(B64) + .i64_xor() + .local_get(A64) + .local_get(R64) + .i64_xor() + .i64_and() + .i64_const(0) + .i64_lt_s(), + }; + i.br_if(fail).local_get(R64); + } + Op::Mul => { + if signed { + min_by_minus_one(i, B64, A64, fail); + } + i.local_get(A64).local_get(B64).i64_mul().local_set(R64); + // the product divided by a non-zero operand gives back the other + i.local_get(A64) + .i64_const(0) + .i64_ne() + .if_(BlockType::Empty) + .local_get(R64) + .local_get(A64); + match signed { + true => i.i64_div_s(), + false => i.i64_div_u(), + }; + i.local_get(B64) + .i64_ne() + .br_if(fail + 1) + .end() + .local_get(R64); + } + Op::Div | Op::Rem => { + i.local_get(B64).i64_eqz().br_if(fail); + if signed { + min_by_minus_one(i, A64, B64, fail); + } + i.local_get(A64).local_get(B64); + match (op, signed) { + (Op::Div, true) => i.i64_div_s(), + (Op::Div, false) => i.i64_div_u(), + (_, true) => i.i64_rem_s(), + _ => i.i64_rem_u(), + }; + } + _ => { + i.local_get(A64).local_get(B64); + match op { + Op::BitAnd => i.i64_and(), + Op::BitOr => i.i64_or(), + _ => i.i64_xor(), + }; + } + } +} + +/// `a << b` or `a >> b`, where `a` is a `ty` and the amount `b` is an +/// `amount`. The amount must be less than `ty`'s bits, read as unsigned. Bits +/// shifted past the type's width are dropped, as in Rust. +fn shift(i: &mut InstructionSink, op: Op, ty: Prim, amount: Prim, fail: u32) { + let (a, b) = ( + if ty.wide() { A64 } else { A32 }, + if amount.wide() { B64 } else { B32 }, + ); + let wrapping = matches!(op, Op::WrapShl | Op::WrapShr); + i.local_set(b).local_set(a); + if !wrapping { + i.local_get(b); + match amount.wide() { + true => i.i64_const(ty.bits() as i64).i64_ge_u(), + false => i.i32_const(ty.bits() as i32).i32_ge_u(), + }; + i.br_if(fail); + } + i.local_get(a).local_get(b); + match (ty.wide(), amount.wide()) { + (true, false) => { + i.i64_extend_i32_u(); + } + (false, true) => { + i.i32_wrap_i64(); + } + _ => {} + } + // a wrapping shift takes the amount modulo the type's bits + if wrapping { + match ty.wide() { + true => i.i64_const(ty.bits() as i64 - 1).i64_and(), + false => i.i32_const(ty.bits() as i32 - 1).i32_and(), + }; + } + match (op, ty.wide(), ty.signed()) { + (Op::Shl | Op::WrapShl, true, _) => { + i.i64_shl(); + } + (Op::Shl | Op::WrapShl, false, _) => { + i.i32_shl(); + canonical(i, ty); + } + (_, true, true) => { + i.i64_shr_s(); + } + (_, true, false) => { + i.i64_shr_u(); + } + (_, false, true) => { + i.i32_shr_s(); + } + _ => { + i.i32_shr_u(); + } + } +} + +/// `base ** exponent`, by squaring: for each bit of the exponent, lowest +/// first, the result is multiplied by the base when the bit is set, and the +/// base squared while bits are left. The exponent is any integer, counted in +/// 64 bits. Integers multiply as `*`, failing on overflow, or as `*%` for +/// `**%`, and fail for a negative exponent. A float's negative exponent gives +/// the reciprocal, and its result must be finite. +fn pow(i: &mut InstructionSink, op: Op, ty: Prim, exponent: Prim, fail: u32) { + let (result, base) = match ty { + Prim::F32 => (POW_RESULT_F32, POW_BASE_F32), + Prim::F64 => (POW_RESULT_F64, POW_BASE_F64), + _ if ty.wide() => (POW_RESULT_64, POW_BASE_64), + _ => (POW_RESULT_32, POW_BASE_32), + }; + // the exponent, extended to 64 bits + if !exponent.wide() { + match exponent.signed() { + true => i.i64_extend_i32_s(), + false => i.i64_extend_i32_u(), + }; + } + i.local_set(POW_EXPONENT).local_set(base); + i.i32_const(0).local_set(POW_NEGATIVE); + if exponent.signed() { + i.local_get(POW_EXPONENT).i64_const(0).i64_lt_s(); + match ty.is_float() { + // an integer's negative power isn't an integer + false => { + i.br_if(fail); + } + // a float's is the reciprocal, of its magnitude read as unsigned + true => { + i.local_tee(POW_NEGATIVE) + .if_(BlockType::Empty) + .i64_const(0) + .local_get(POW_EXPONENT) + .i64_sub() + .local_set(POW_EXPONENT) + .end(); + } + } + } + match ty { + Prim::F32 => i.f32_const(1.0f32.into()), + Prim::F64 => i.f64_const(1.0f64.into()), + _ if ty.wide() => i.i64_const(1), + _ => i.i32_const(1), + }; + i.local_set(result); + // multiplies the two values on the stack, inside the loop's block, loop + // and an if, 3 blocks deeper than `fail` + let multiply = |i: &mut InstructionSink| match (op, ty) { + (_, Prim::F32) => { + i.f32_mul(); + } + (_, Prim::F64) => { + i.f64_mul(); + } + (Op::WrapPow, _) => wrapping_op(i, Op::WrapMul, ty, fail + 3), + _ if ty.narrow() => narrow_op(i, Op::Mul, ty, fail + 3), + _ => wide_op(i, Op::Mul, ty, fail + 3), + }; + i.block(BlockType::Empty) + .loop_(BlockType::Empty) + .local_get(POW_EXPONENT) + .i64_eqz() + .br_if(1) + .local_get(POW_EXPONENT) + .i64_const(1) + .i64_and() + .i32_wrap_i64() + .if_(BlockType::Empty) + .local_get(result) + .local_get(base); + multiply(i); + i.local_set(result) + .end() + .local_get(POW_EXPONENT) + .i64_const(1) + .i64_shr_u() + .local_tee(POW_EXPONENT) + .i64_const(0) + .i64_ne() + .if_(BlockType::Empty) + .local_get(base) + .local_get(base); + multiply(i); + i.local_set(base).end().br(0).end().end(); + if !ty.is_float() { + i.local_get(result); + return; + } + i.local_get(POW_NEGATIVE) + .if_(BlockType::Result(ty.val_type())); + match ty { + Prim::F32 => i.f32_const(1.0f32.into()).local_get(result).f32_div(), + _ => i.f64_const(1.0f64.into()).local_get(result).f64_div(), + }; + i.else_().local_get(result).end(); + // the result must be finite, NaN isn't less than infinity either + match ty { + Prim::F32 => i + .local_tee(F32) + .f32_abs() + .f32_const(f32::INFINITY.into()) + .f32_lt() + .i32_eqz() + .br_if(fail) + .local_get(F32), + _ => i + .local_tee(F64) + .f64_abs() + .f64_const(f64::INFINITY.into()) + .f64_lt() + .i32_eqz() + .br_if(fail) + .local_get(F64), + }; +} + +/// Arithmetic on floats, the result must be finite. +fn float_op(i: &mut InstructionSink, op: Op, ty: Prim, fail: u32) { + match (ty, op) { + (Prim::F32, Op::Add) => i.f32_add(), + (Prim::F32, Op::Sub) => i.f32_sub(), + (Prim::F32, Op::Mul) => i.f32_mul(), + (Prim::F32, _) => i.f32_div(), + (_, Op::Add) => i.f64_add(), + (_, Op::Sub) => i.f64_sub(), + (_, Op::Mul) => i.f64_mul(), + _ => i.f64_div(), + }; + // NaN isn't less than infinity either + match ty { + Prim::F32 => i + .local_tee(F32) + .f32_abs() + .f32_const(f32::INFINITY.into()) + .f32_lt() + .i32_eqz() + .br_if(fail) + .local_get(F32), + _ => i + .local_tee(F64) + .f64_abs() + .f64_const(f64::INFINITY.into()) + .f64_lt() + .i32_eqz() + .br_if(fail) + .local_get(F64), + }; +} + +/// A method on integers of type `ty`, wrapping arithmetic never fails, and a +/// rotation's `u32` amount is taken modulo `ty`'s bits. +fn method(i: &mut InstructionSink, m: Method, ty: Prim) { + match (m, ty.wide()) { + (Method::WrappingAdd, _) => wrapping_op(i, Op::WrapAdd, ty, 0), + (Method::WrappingSub, _) => wrapping_op(i, Op::WrapSub, ty, 0), + (Method::WrappingMul, _) => wrapping_op(i, Op::WrapMul, ty, 0), + (Method::RotateLeft, true) => { + i.i64_extend_i32_u().i64_rotl(); + } + (Method::RotateRight, true) => { + i.i64_extend_i32_u().i64_rotr(); + } + (_, false) if ty.bits() == 32 => { + match m { + Method::RotateLeft => i.i32_rotl(), + _ => i.i32_rotr(), + }; + } + (_, false) => { + // rotate the type's bits within an i32, then re-extend them + let bits = ty.bits() as i32; + // a left rotation shifts the bits left, then the overflow right + let left = m == Method::RotateLeft; + let shift = |i: &mut InstructionSink, left: bool| { + match left { + true => i.i32_shl(), + false => i.i32_shr_u(), + }; + }; + i.i32_const(bits - 1) + .i32_and() + .local_set(B32) + .i32_const((1 << bits) - 1) + .i32_and() + .local_set(A32) + .local_get(A32) + .local_get(B32); + shift(i, left); + i.local_get(A32) + .i32_const(bits) + .local_get(B32) + .i32_sub() + .i32_const(bits - 1) + .i32_and(); + shift(i, !left); + i.i32_or(); + if ty.signed() { + canonical(i, ty); + } else { + i.i32_const((1 << bits) - 1).i32_and(); + } + } + } +} + +/// Wrapping arithmetic on integers, as Rust's `wrapping_*` methods: the +/// result wraps around the type's range. Division and remainder still fail +/// for a zero divisor. +fn wrapping_op(i: &mut InstructionSink, op: Op, ty: Prim, fail: u32) { + match (op, ty.wide()) { + (Op::WrapAdd, true) => { + i.i64_add(); + } + (Op::WrapSub, true) => { + i.i64_sub(); + } + (Op::WrapMul, true) => { + i.i64_mul(); + } + (Op::WrapAdd, false) => { + i.i32_add(); + canonical(i, ty); + } + (Op::WrapSub, false) => { + i.i32_sub(); + canonical(i, ty); + } + (Op::WrapMul, false) => { + i.i32_mul(); + canonical(i, ty); + } + // narrow integers divide in 64 bits, where the minimum divided by -1 + // doesn't overflow, then wrap + (_, false) => { + i.local_set(B32) + .local_set(A32) + .local_get(B32) + .i32_eqz() + .br_if(fail); + for local in [A32, B32] { + i.local_get(local); + match ty.signed() { + true => i.i64_extend_i32_s(), + false => i.i64_extend_i32_u(), + }; + } + match (op, ty.signed()) { + (Op::WrapDiv, true) => i.i64_div_s(), + (Op::WrapDiv, false) => i.i64_div_u(), + (_, true) => i.i64_rem_s(), + _ => i.i64_rem_u(), + }; + i.i32_wrap_i64(); + canonical(i, ty); + } + (_, true) => { + i.local_set(B64) + .local_set(A64) + .local_get(B64) + .i64_eqz() + .br_if(fail); + match (op, ty.signed()) { + // the minimum divided by -1 wraps to the minimum, negating + (Op::WrapDiv, true) => { + i.local_get(B64) + .i64_const(-1) + .i64_eq() + .if_(BlockType::Result(ValType::I64)) + .i64_const(0) + .local_get(A64) + .i64_sub() + .else_() + .local_get(A64) + .local_get(B64) + .i64_div_s() + .end(); + } + (Op::WrapDiv, false) => { + i.local_get(A64).local_get(B64).i64_div_u(); + } + // wasm's remainder of the minimum and -1 is 0, as Rust's wrapping_rem + (_, true) => { + i.local_get(A64).local_get(B64).i64_rem_s(); + } + _ => { + i.local_get(A64).local_get(B64).i64_rem_u(); + } + } + } + } +} + +/// Converts a `source` value to `target`, as Rust's `as`: integers are +/// truncated, or sign or zero extended, floats are rounded, and floats +/// converted to integers saturate, with NaN becoming zero. +fn convert(i: &mut InstructionSink, source: Prim, target: Prim) { + use Prim::*; + if source == target { + return; + } + match (source.is_float(), target.is_float()) { + // integers, including bool, to integers + (false, false) => match (source.wide(), target.wide()) { + (false, true) => { + match source.signed() { + true => i.i64_extend_i32_s(), + false => i.i64_extend_i32_u(), + }; + } + (true, false) => { + i.i32_wrap_i64(); + canonical(i, target); + } + (false, false) => canonical(i, target), + (true, true) => {} + }, + (false, true) => { + match (source.wide(), source.signed(), target) { + (false, true, F32) => i.f32_convert_i32_s(), + (false, false, F32) => i.f32_convert_i32_u(), + (true, true, F32) => i.f32_convert_i64_s(), + (true, false, F32) => i.f32_convert_i64_u(), + (false, true, _) => i.f64_convert_i32_s(), + (false, false, _) => i.f64_convert_i32_u(), + (true, true, _) => i.f64_convert_i64_s(), + (true, false, _) => i.f64_convert_i64_u(), + }; + } + (true, true) => { + match target { + F32 => i.f32_demote_f64(), + _ => i.f64_promote_f32(), + }; + } + (true, false) => { + match (source, target.wide(), target.signed()) { + (F32, true, true) => i.i64_trunc_sat_f32_s(), + (F32, true, false) => i.i64_trunc_sat_f32_u(), + (F32, false, true) => i.i32_trunc_sat_f32_s(), + (F32, false, false) => i.i32_trunc_sat_f32_u(), + (_, true, true) => i.i64_trunc_sat_f64_s(), + (_, true, false) => i.i64_trunc_sat_f64_u(), + (_, false, true) => i.i32_trunc_sat_f64_s(), + _ => i.i32_trunc_sat_f64_u(), + }; + // saturate to a narrower integer's range + if target.narrow() && target.bits() < 32 { + let (min, max) = target.range(); + let clamp = + |i: &mut InstructionSink, bound: i32, past: fn(&mut InstructionSink)| { + i.local_set(A32) + .i32_const(bound) + .local_get(A32) + .local_get(A32) + .i32_const(bound); + past(i); + i.select(); + }; + match target.signed() { + true => { + clamp(i, max as i32, |i| { + i.i32_gt_s(); + }); + clamp(i, min as i32, |i| { + i.i32_lt_s(); + }); + } + false => clamp(i, max as i32, |i| { + i.i32_gt_u(); + }), + } + } + } + } +} + +/// Fails when `x` is the minimum and `y` is -1, which overflows dividing and +/// multiplying. +fn min_by_minus_one(i: &mut InstructionSink, x: u32, y: u32, fail: u32) { + i.local_get(x) + .i64_const(i64::MIN) + .i64_eq() + .local_get(y) + .i64_const(-1) + .i64_eq() + .i32_and() + .br_if(fail); +} + +/// Operations on bools, as 0 or 1 i32s, `false` is less than `true`. +fn bool_op(i: &mut InstructionSink, op: Op) { + match op { + Op::BitAnd => i.i32_and(), + Op::BitOr => i.i32_or(), + Op::BitXor | Op::Ne => i.i32_ne(), + Op::Eq => i.i32_eq(), + Op::Lt => i.i32_lt_u(), + Op::Le => i.i32_le_u(), + Op::Gt => i.i32_gt_u(), + _ => i.i32_ge_u(), + }; +} + +fn compare(i: &mut InstructionSink, op: Op, ty: Prim) { + let signed = ty.signed(); + match ty.val_type() { + ValType::I64 => match (op, signed) { + (Op::Eq, _) => i.i64_eq(), + (Op::Ne, _) => i.i64_ne(), + (Op::Lt, true) => i.i64_lt_s(), + (Op::Lt, false) => i.i64_lt_u(), + (Op::Le, true) => i.i64_le_s(), + (Op::Le, false) => i.i64_le_u(), + (Op::Gt, true) => i.i64_gt_s(), + (Op::Gt, false) => i.i64_gt_u(), + (_, true) => i.i64_ge_s(), + _ => i.i64_ge_u(), + }, + ValType::F32 => match op { + Op::Eq => i.f32_eq(), + Op::Ne => i.f32_ne(), + Op::Lt => i.f32_lt(), + Op::Le => i.f32_le(), + Op::Gt => i.f32_gt(), + _ => i.f32_ge(), + }, + ValType::F64 => match op { + Op::Eq => i.f64_eq(), + Op::Ne => i.f64_ne(), + Op::Lt => i.f64_lt(), + Op::Le => i.f64_le(), + Op::Gt => i.f64_gt(), + _ => i.f64_ge(), + }, + _ => match (op, signed) { + (Op::Eq, _) => i.i32_eq(), + (Op::Ne, _) => i.i32_ne(), + (Op::Lt, true) => i.i32_lt_s(), + (Op::Lt, false) => i.i32_lt_u(), + (Op::Le, true) => i.i32_le_s(), + (Op::Le, false) => i.i32_le_u(), + (Op::Gt, true) => i.i32_gt_s(), + (Op::Gt, false) => i.i32_gt_u(), + (_, true) => i.i32_ge_s(), + _ => i.i32_ge_u(), + }, + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The tree-sitter grammar's test corpus, see grammars/constants-expression/README.md. + const CORPUS: [&str; 9] = [ + include_str!("../../../grammars/constants-expression/test/corpus/arrays.txt"), + include_str!("../../../grammars/constants-expression/test/corpus/blocks.txt"), + include_str!("../../../grammars/constants-expression/test/corpus/closures.txt"), + include_str!("../../../grammars/constants-expression/test/corpus/conditionals.txt"), + include_str!("../../../grammars/constants-expression/test/corpus/errors.txt"), + include_str!("../../../grammars/constants-expression/test/corpus/literals.txt"), + include_str!("../../../grammars/constants-expression/test/corpus/loops.txt"), + include_str!("../../../grammars/constants-expression/test/corpus/operators.txt"), + include_str!("../../../grammars/constants-expression/test/corpus/variables.txt"), + ]; + + /// Each test of a corpus: its name, whether it's an error, and the input. + fn cases(corpus: &str) -> Vec<(String, bool, String)> { + let lines: Vec<&str> = corpus.lines().collect(); + let header = |line: &str| line.len() >= 3 && line.chars().all(|c| c == '='); + let mut cases = vec![]; + let mut i = 0; + while i < lines.len() { + if !header(lines[i]) { + i += 1; + continue; + } + let name = lines[i + 1].to_string(); + let mut j = i + 2; + let mut error = false; + while !header(lines[j]) { + error |= lines[j].trim() == ":error"; + j += 1; + } + let start = j + 1; + let end = start + + lines[start..] + .iter() + .position(|l| *l == "---") + .expect("---"); + cases.push((name, error, lines[start..end].join("\n").trim().to_string())); + i = end; + } + cases + } + + #[test] + fn it_accepts_the_same_syntax_as_the_tree_sitter_grammar() { + let cases: Vec<_> = CORPUS.iter().flat_map(|c| cases(c)).collect(); + assert!(cases.len() >= 40, "found {} cases", cases.len()); + for (name, error, input) in cases { + let result = parse_closure(&input, &[]); + assert_eq!( + result.is_err(), + error, + "{name}: {input:?} {}", + match &result { + Ok(_) => "parsed".to_string(), + Err(e) => format!("failed: {e}"), + } + ); + } + } +} diff --git a/crates/componentized-constants/src/lib.rs b/crates/componentized-constants/src/lib.rs index fe3e166..e59817d 100644 --- a/crates/componentized-constants/src/lib.rs +++ b/crates/componentized-constants/src/lib.rs @@ -2,21 +2,23 @@ use anyhow::{Context, Result, bail}; use std::path::Path; use wasm_encoder::{ CodeSection, ConstExpr, CustomSection, DataSection, ExportKind, ExportSection, Function, - FunctionSection, Ieee32, Ieee64, Instruction, MemorySection, MemoryType, Module, TypeSection, - ValType, + FunctionSection, Ieee32, Ieee64, Instruction, MemorySection, MemoryType, Module, ValType, }; use wasm_metadata::Producers; use wasm_wave::untyped::UntypedValue; use wit_component::{ComponentEncoder, DecodedWasm, StringEncoding, metadata}; -use wit_parser::abi::{AbiVariant, WasmType}; +use wit_parser::abi::{AbiVariant, WasmSignature, WasmType}; use wit_parser::{ - LiftLowerAbi, ManglingAndAbi, Resolve, WasmExport, WasmExportKind, WorldId, WorldItem, WorldKey, + FunctionKind, LiftLowerAbi, ManglingAndAbi, Resolve, WasmExport, WasmExportKind, WorldId, + WorldItem, WorldKey, }; use values::{Flat, Layout, optional_fields, parser_error}; use wasm_wave::ast::Node; mod docs; +mod expr; +mod streams; pub mod types; mod values; @@ -73,12 +75,29 @@ pub fn decode_world(bytes: &[u8], world: Option<&str>) -> Result<(Resolve, World /// constant value. /// /// The world may only export functions (directly, or within interfaces) that -/// are synchronous and take no parameters. Their values may not reach +/// take no parameters, synchronous or async. Their values may not reach /// resources, handles, futures, streams, error contexts or maps, though the /// result types may include them in branches the values don't take, e.g. /// `none` for an `option>`. Types the exports reference from other /// interfaces become imports of the resulting component. /// +/// A function may return a `stream`, whose value is a list of the stream's +/// items. Only an async function can return a stream with items, which it +/// writes as they're read. Any function can return an empty stream, `[]`. +/// A stream of integers, floats or `bool` may continue with items generated by +/// a closure following an `@expression` tag, from the items before them: +/// +/// ```wit +/// /// @value [1, 1] +/// /// @expression |a, b| a + b +/// fibonacci: async func() -> stream; +/// ``` +/// +/// The stream ends once an item can't be represented, e.g. on overflow. +/// Instead of a `@value`, an async function may take the items the stream +/// starts from as arguments, one for each of the closure's parameters, e.g. +/// `async func(a: u32, b: u32) -> stream`. +/// /// Each function's value is a WAVE expression following a `@value` tag in its /// doc comment: /// @@ -139,13 +158,24 @@ struct Constant<'a> { enum Value<'a> { Override(&'a Node), Doc(UntypedValue<'a>), + /// A stream generated from the function's arguments. + Arguments, +} + +/// What a constant's function returns. +enum Body<'a> { + /// Flat values, or a pointer to the value in memory. + Flat(WasmSignature, Vec), + /// A stream of items in memory. + Stream(streams::Stream<'a>), } impl Constant<'_> { - fn node(&self) -> &Node { + fn node(&self) -> Option<&Node> { match &self.value { - Value::Override(node) => node, - Value::Doc(value) => value.node(), + Value::Override(node) => Some(node), + Value::Doc(value) => Some(value.node()), + Value::Arguments => None, } } } @@ -227,6 +257,49 @@ fn create_module( for (interface, func, node) in funcs { let name = qualified_name(resolve, interface, func); let location = resolve.source_map.render_location(func.span); + // a stream generated from the arguments has no value of its own + if !func.params.is_empty() { + if node.is_some() { + bail!( + "invalid overrides: `{name}` can't be overridden, its stream is \ + generated from its arguments" + ); + } + // an expression reading `$init` may also list the items it starts + // from, otherwise the arguments are the items it starts from. An + // expression that doesn't parse is reported with its own error. + let args = arg_names(func); + let uses_seed = docs::expression(&func.docs).is_some_and(|src| { + expr::parse_closure(src.trim(), &args).map_or(true, |closure| closure.uses_seed) + }); + let constant = match (docs::value(&func.docs), uses_seed) { + (Some(_), false) => bail!( + "invalid `{}` for `{name}` at {location}: its stream starts from its \ + arguments, remove the tag, or read the arguments with `$init`", + docs::TAG + ), + (Some(src), true) => Constant { + interface, + func, + src, + value: Value::Doc( + UntypedValue::parse(src) + .map_err(|e| parser_error(src, e)) + .with_context(|| { + format!("invalid `{}` for `{name}` at {location}", docs::TAG) + })?, + ), + }, + (None, _) => Constant { + interface, + func, + src: "", + value: Value::Arguments, + }, + }; + constants.push(constant); + continue; + } let constant = match (node, docs::value(&func.docs)) { (Some(node), _) => Constant { interface, @@ -256,14 +329,11 @@ fn create_module( } } - // encode the values and generate a function returning each one + // encode the values, the flat values each function returns or the items of + // the stream it returns let mut layout = Layout::new(resolve, DATA_BASE)?; - let mut types = TypeSection::new(); - let mut functions = FunctionSection::new(); - let mut code = CodeSection::new(); - let mut exports = ExportSection::new(); - - for (index, constant) in constants.iter().enumerate() { + let mut bodies = vec![]; + for constant in &constants { let func = constant.func; let ty = func.result.as_ref().expect("checked by check_function"); let sig = resolve.wasm_signature(AbiVariant::GuestExport, func); @@ -272,7 +342,7 @@ fn create_module( let name = qualified_name(resolve, constant.interface, func); match constant.value { Value::Override(_) => format!("invalid override for `{name}`"), - Value::Doc(_) => format!( + Value::Doc(_) | Value::Arguments => format!( "invalid `{}` for `{name}` at {}", docs::TAG, resolve.source_map.render_location(func.span) @@ -281,41 +351,207 @@ fn create_module( }; layout.set_source(constant.src); - let flat = if sig.retptr { - vec![Flat::I32( - layout.store_new(ty, node).with_context(context)? as i32 - )] + // an expression always comes from the WIT, overrides only replace the + // items it starts from + let expression = docs::expression(&func.docs); + let expression_context = || { + format!( + "invalid `{}` for `{}` at {}", + docs::EXPRESSION_TAG, + qualified_name(resolve, constant.interface, func), + resolve.source_map.render_location(func.span) + ) + }; + let stream = streams::stream_type(resolve, ty); + if expression.is_some() && stream.is_none() { + return Err(anyhow::anyhow!( + "only functions returning a stream can generate items" + )) + .with_context(expression_context); + } + + let body = if let Some((stream_ty, item)) = stream { + let closure = match expression { + Some(src) => Some( + expr::parse_closure(src.trim(), &arg_names(func)) + .with_context(expression_context)?, + ), + None => None, + }; + let uses_seed = closure.as_ref().is_some_and(|closure| closure.uses_seed); + let generator = match closure { + Some(closure) => { + let prim = expr::Prim::from_type(resolve, &item) + .context("only streams of integers, floats or bool can generate items") + .with_context(expression_context)?; + let seeds = match uses_seed { + true => seed_types(resolve, func).with_context(expression_context)?, + false => vec![], + }; + // `$init`'s fields by name, as Rust names the parameters + let names = arg_names(func); + Some( + closure + .check(prim, &seeds, &names) + .with_context(expression_context)?, + ) + } + None => None, + }; + if !func.params.is_empty() && !uses_seed { + // a stream starting from the function's arguments + let generator = generator.expect("parameters require an expression"); + let stream = + arguments_stream(resolve, constant.interface, func, stream_ty, generator) + .with_context(expression_context)?; + bodies.push(Body::Stream(stream)); + continue; + } + if uses_seed { + check_arguments(resolve, func).with_context(expression_context)?; + } + // the listed items, or none for a stream generated from `$init` without + // a `@value` + let (ptr, len, stride) = match (node, &generator) { + (Some(node), _) => layout.stream(&item, node).with_context(context)?, + (None, Some(generator)) => (0, 0, generator.item.size()), + (None, None) => unreachable!("only generated streams lack a value"), + }; + let generates = generator.is_some(); + if let Some(node) = node + && (len > 0 || generates) + && !matches!(func.kind, FunctionKind::AsyncFreestanding) + { + return Err(layout.stream_requires_async(node)).with_context(context); + } + if let Some(generator) = &generator + && len < generator.params + { + return match node { + Some(node) => Err(layout.too_few_items(node, generator.params)), + None => Err(anyhow::anyhow!( + "the expression reads the previous {n} items, list at least {n} with \ + `{}`", + docs::TAG, + n = generator.params + )), + } + .with_context(context); + } + Body::Stream(streams::Stream { + interface: constant.interface, + func, + ty: stream_ty, + ptr, + len, + stride, + args: 0, + generator, + }) } else { - let mut flat = vec![]; - layout.flatten(ty, node, &mut flat).with_context(context)?; - flat + let node = node.expect("only streams are generated from arguments"); + if sig.retptr { + let flat = vec![Flat::I32( + layout.store_new(ty, node).with_context(context)? as i32 + )]; + Body::Flat(sig, flat) + } else { + let mut flat = vec![]; + layout.flatten(ty, node, &mut flat).with_context(context)?; + Body::Flat(sig, flat) + } }; + bodies.push(body); + } - let results: Vec = sig.results.iter().map(val_type).collect(); - types.ty().function([], results); - functions.function(index as u32); + // streams import intrinsics, which come first in the function index space, + // and add helper functions following the constants + let streams: Vec<&streams::Stream> = bodies + .iter() + .filter_map(|body| match body { + Body::Stream(stream) => Some(stream), + Body::Flat(..) => None, + }) + .collect(); + let mut types = streams::Types::new(); + let mut imports = streams::Imports::new(); + for body in &bodies { + match body { + Body::Flat(sig, _) => { + let results: Vec = sig.results.iter().map(val_type).collect(); + types.push(&[], &results); + } + // the arguments a stream starts from, returning its readable end or + // an async callback code + Body::Stream(stream) => { + types.push(&stream.params(), &[ValType::I32]); + } + } + } + let plan = streams::Plan::new(resolve, &streams, &mut types, &mut imports); + let first = imports.len(); + let helpers = streams::Helpers::new(&plan, &streams, first + constants.len() as u32); - let mut body = Function::new([]); - for value in flat { - body.instruction(&match value { - Flat::I32(v) => Instruction::I32Const(v), - Flat::I64(v) => Instruction::I64Const(v), - Flat::F32(v) => Instruction::F32Const(Ieee32::new(v)), - Flat::F64(v) => Instruction::F64Const(Ieee64::new(v)), - }); + // generate a function returning each value + let mut functions = FunctionSection::new(); + let mut code = CodeSection::new(); + let mut exports = ExportSection::new(); + let mut stream_index = 0; + for (index, (constant, body)) in constants.iter().zip(&bodies).enumerate() { + let func = constant.func; + functions.function(index as u32); + let mut abi = LiftLowerAbi::Sync; + let mut callback = None; + match body { + Body::Flat(_, flat) => { + let mut body = Function::new([]); + for value in flat { + body.instruction(&match *value { + Flat::I32(v) => Instruction::I32Const(v), + Flat::I64(v) => Instruction::I64Const(v), + Flat::F32(v) => Instruction::F32Const(Ieee32::new(v)), + Flat::F64(v) => Instruction::F64Const(Ieee64::new(v)), + }); + } + body.instruction(&Instruction::End); + code.function(&body); + } + Body::Stream(_) => { + code.function(&helpers.export(&streams, stream_index)); + callback = helpers.callback(stream_index); + if callback.is_some() { + abi = LiftLowerAbi::AsyncCallback; + } + stream_index += 1; + } } - body.instruction(&Instruction::End); - code.function(&body); - let name = resolve.wasm_export_name( - ManglingAndAbi::Legacy(LiftLowerAbi::Sync), - WasmExport::Func { - interface: constant.interface, - func, - kind: WasmExportKind::Normal, - }, + let export = |kind| { + resolve.wasm_export_name( + ManglingAndAbi::Legacy(abi), + WasmExport::Func { + interface: constant.interface, + func, + kind, + }, + ) + }; + exports.export( + &export(WasmExportKind::Normal), + ExportKind::Func, + first + index as u32, ); - exports.export(&name, ExportKind::Func, index as u32); + if let Some(callback) = callback { + exports.export( + &export(WasmExportKind::Callback), + ExportKind::Func, + callback, + ); + } + } + for (ty, body) in helpers.functions(&streams, &mut types) { + functions.function(ty); + code.function(&body); } let mut memories = MemorySection::new(); @@ -345,10 +581,18 @@ fn create_module( let component_type = metadata::encode(resolve, world, StringEncoding::UTF8, Some(&producers), true)?; + let globals = helpers.globals(layout.base() + layout.data().len() as u32); + let mut module = Module::new(); - module.section(&types); + module.section(types.section()); + if imports.len() > 0 { + module.section(imports.section()); + } module.section(&functions); module.section(&memories); + if let Some(globals) = &globals { + module.section(globals); + } module.section(&exports); module.section(&code); module.section(&data); @@ -359,6 +603,96 @@ fn create_module( Ok(module.finish()) } +/// The names of the function's arguments in expressions, as Rust names them, +/// e.g. `start_value` for `start-value`. +fn arg_names(func: &wit_parser::Function) -> Vec { + func.params + .iter() + .map(|p| p.name.replace('-', "_")) + .collect() +} + +/// The types of `$init`, the function's arguments, which must be integers, +/// floats or `bool`. +fn seed_types(resolve: &Resolve, func: &wit_parser::Function) -> Result> { + if func.params.is_empty() { + bail!("`$init` holds the function's arguments, but the function takes none"); + } + func.params + .iter() + .map(|param| { + expr::Prim::from_type(resolve, ¶m.ty).with_context(|| { + format!( + "parameter `{}` can't be read with `$init`, its type must be an integer, \ + float or bool", + param.name + ) + }) + }) + .collect() +} + +/// A function generating a stream from its arguments must be async, and take +/// at most 16 parameters. +fn check_arguments(resolve: &Resolve, func: &wit_parser::Function) -> Result<()> { + if !matches!(func.kind, FunctionKind::AsyncFreestanding) { + bail!( + "a stream generated from arguments must be returned by an async function, e.g. \ + `async func(n: u8) -> stream`" + ); + } + if resolve + .wasm_signature(AbiVariant::GuestExportAsync, func) + .indirect_params + { + bail!("a stream can be generated from at most 16 arguments"); + } + Ok(()) +} + +/// A stream generated from the arguments of `func`, which must be async and +/// take a parameter of the stream's item type for each of the expression's. +fn arguments_stream<'a>( + resolve: &Resolve, + interface: Option<&'a WorldKey>, + func: &'a wit_parser::Function, + ty: wit_parser::TypeId, + generator: expr::Generator, +) -> Result> { + check_arguments(resolve, func)?; + let params = func.params.len() as u32; + if params != generator.params { + let count = |n| match n { + 1 => "1 parameter".to_string(), + n => format!("{n} parameters"), + }; + bail!( + "the expression takes {}, the function takes {}", + count(generator.params), + count(params) + ); + } + for param in &func.params { + if expr::Prim::from_type(resolve, ¶m.ty) != Some(generator.item) { + bail!( + "parameter `{}` must have the stream's item type, `{}`", + param.name, + generator.item.name() + ); + } + } + Ok(streams::Stream { + interface, + func, + ty, + ptr: 0, + len: 0, + stride: generator.item.size(), + args: params, + generator: Some(generator), + }) +} + /// Whether the WAVE source contains nothing but whitespace and comments. fn is_blank(src: &str) -> bool { src.lines() diff --git a/crates/componentized-constants/src/streams.rs b/crates/componentized-constants/src/streams.rs new file mode 100644 index 0000000..473ae14 --- /dev/null +++ b/crates/componentized-constants/src/streams.rs @@ -0,0 +1,939 @@ +//! Exports returning a `stream`, whose value is written as a list of the +//! stream's items. +//! +//! An empty stream is created and closed by a synchronous export, returning the +//! readable end. A stream with items needs a task that outlives the call, so the +//! export must be an `async func`. It's lifted with a callback: the export +//! creates the stream, returns the readable end with `task.return`, then writes +//! the items from the data segment, waiting whenever the reader hasn't taken +//! them all. The task ends once every item is written or the reader drops the +//! stream. +//! +//! A stream may continue past its listed items with items generated by an +//! `@expression`, see [`crate::expr`]. The listed items are written first, then +//! batches of generated items, until an item can't be generated or the reader +//! drops the stream. +//! +//! Each task's state is a record in linear memory, allocated from a free list +//! and kept in the task's context slot: +//! +//! ```text +//! offset 0: the waitable set the task waits on +//! offset 4: the number of listed items left to write +//! ``` +//! +//! A generated stream's record continues: +//! +//! ```text +//! offset 8: the number of items generated into the buffer +//! offset 12: the number of generated items written +//! offset 16: 1 once no more items can be generated +//! offset 24: `$i`, the position of the next item generated, a u64 +//! offset 32: `$init`, each of the function's arguments in 8 bytes +//! then: the buffer, the previous items the expression reads followed by +//! the batch of generated items +//! ``` +//! +//! None of this is emitted unless an export returns a stream, so other +//! components are unchanged. + +use wasm_encoder::{ + BlockType, ConstExpr, EntityType, Function, GlobalSection, GlobalType, ImportSection, + InstructionSink, MemArg, TypeSection, ValType, +}; +use wit_parser::{ + LiftLowerAbi, ManglingAndAbi, Resolve, StreamIntrinsic, Type, TypeDefKind, TypeId, WasmImport, + WorldKey, +}; + +use crate::expr::Generator; +use crate::types::dealias; + +/// `waitable-set.wait`'s callback code, the set is in the upper 28 bits. +const CALLBACK_WAIT: i32 = 2; +const CALLBACK_EXIT: i32 = 0; +/// The event delivered when a write to a stream finishes. +const EVENT_STREAM_WRITE: i32 = 3; +/// Returned by an async `stream.write` that hasn't finished yet. +const BLOCKED: i32 = -1; +/// A finished copy's result is the item count shifted over a 4-bit status, a +/// `COMPLETED` status is zero. +const COPY_STATUS_BITS: i32 = 4; +const COPY_STATUS_MASK: i32 = 0xf; + +const SET: MemArg = MemArg { + offset: 0, + align: 2, + memory_index: 0, +}; +const REMAINING: MemArg = MemArg { + offset: 4, + align: 2, + memory_index: 0, +}; +const FILLED: MemArg = MemArg { + offset: 8, + align: 2, + memory_index: 0, +}; +const WRITTEN: MemArg = MemArg { + offset: 12, + align: 2, + memory_index: 0, +}; +const DONE: MemArg = MemArg { + offset: 16, + align: 2, + memory_index: 0, +}; +/// The position of the next item generated, `$i`. +pub const INDEX: u32 = 24; +/// The function's arguments, `$init`, each in 8 bytes. +pub const SEEDS: u32 = 32; +const INDEX_ARG: MemArg = MemArg { + offset: INDEX as u64, + align: 3, + memory_index: 0, +}; +/// The number of items generated at a time, rounded up to a multiple of the +/// items each evaluation of the expression generates. +const BATCH: u32 = 64; + +/// The stream type and item type when `ty` is a stream with items, following +/// aliases. +pub fn stream_type(resolve: &Resolve, ty: &Type) -> Option<(TypeId, Type)> { + match dealias(resolve, *ty) { + Type::Id(id) => match resolve.types[id].kind { + TypeDefKind::Stream(Some(item)) => Some((id, item)), + _ => None, + }, + _ => None, + } +} + +/// An export returning a stream of `len` items stored from `ptr`, each `stride` +/// bytes, then any items the generator generates from the items before them. +pub struct Stream<'a> { + pub interface: Option<&'a WorldKey>, + pub func: &'a wit_parser::Function, + pub ty: TypeId, + pub ptr: u32, + pub len: u32, + pub stride: u32, + /// The number of arguments the stream starts from, rather than listed + /// items. + pub args: u32, + pub generator: Option, +} + +impl Stream<'_> { + /// Whether the export writes items after returning the stream, rather + /// than returning an empty, closed stream. + pub fn is_async(&self) -> bool { + self.len > 0 || self.generator.is_some() + } + + /// The core type of each of the function's arguments, `$init`, or the + /// items the stream starts from. + pub fn params(&self) -> Vec { + match &self.generator { + Some(generator) if !generator.seeds.is_empty() => { + generator.seeds.iter().map(|ty| ty.val_type()).collect() + } + Some(generator) => vec![generator.item.val_type(); self.args as usize], + None => vec![], + } + } + + /// Where the buffer starts in the task's state record, after the seeds. + fn buffer(&self) -> u32 { + let seeds = self.generator.as_ref().map_or(0, |g| g.seeds.len() as u32); + SEEDS + 8 * seeds + } + + fn end(&self) -> u32 { + self.ptr + self.len * self.stride + } + + /// The size of the task's state record. + fn state_size(&self) -> u32 { + match &self.generator { + Some(generator) => (self.buffer() + + (generator.params + batch(generator)) * self.stride) + .next_multiple_of(8), + None => 8, + } + } +} + +/// The functions an export uses, by index. +struct Intrinsics { + new: u32, + drop_writable: u32, + write: Option, + task_return: Option, +} + +/// Core functions the module imports, the async built-ins and each stream's +/// intrinsics. +pub struct Imports { + section: ImportSection, + /// The module and name of each import, by index. There are few, so they're + /// searched rather than hashed. + names: Vec<(String, String)>, +} + +impl Imports { + pub fn new() -> Self { + Self { + section: ImportSection::new(), + names: vec![], + } + } + + pub fn len(&self) -> u32 { + self.names.len() as u32 + } + + pub fn section(&self) -> &ImportSection { + &self.section + } + + fn import( + &mut self, + types: &mut Types, + (module, name): (String, String), + params: &[ValType], + results: &[ValType], + ) -> u32 { + let import = (module, name); + if let Some(index) = self.names.iter().position(|i| *i == import) { + return index as u32; + } + let ty = types.intern(params, results); + self.section + .import(&import.0, &import.1, EntityType::Function(ty)); + self.names.push(import); + self.names.len() as u32 - 1 + } + + fn root( + &mut self, + types: &mut Types, + name: &str, + params: &[ValType], + results: &[ValType], + ) -> u32 { + self.import(types, ("$root".into(), name.into()), params, results) + } +} + +/// The type section, a function type for each constant followed by types +/// shared by the imports and helper functions. +pub struct Types { + section: TypeSection, + count: u32, + /// Types shared by imports and helpers, searched rather than hashed. + interned: Vec<(Vec, Vec, u32)>, +} + +impl Types { + pub fn new() -> Self { + Self { + section: TypeSection::new(), + count: 0, + interned: vec![], + } + } + + pub fn section(&self) -> &TypeSection { + &self.section + } + + /// Adds a function type, returning its index. + pub fn push(&mut self, params: &[ValType], results: &[ValType]) -> u32 { + self.section + .ty() + .function(params.iter().copied(), results.iter().copied()); + self.count += 1; + self.count - 1 + } + + /// Adds a function type unless an identical type was interned already. + fn intern(&mut self, params: &[ValType], results: &[ValType]) -> u32 { + if let Some((.., index)) = self + .interned + .iter() + .find(|(p, r, _)| p == params && r == results) + { + return *index; + } + let index = self.push(params, results); + self.interned + .push((params.to_vec(), results.to_vec(), index)); + index + } +} + +/// Built-ins shared by every async stream export. +struct Builtins { + waitable_set_new: u32, + waitable_set_drop: u32, + waitable_join: u32, + context_get: u32, + context_set: u32, +} + +/// Imports the functions the streams use, before any other function is +/// defined as imports come first in the function index space. +pub struct Plan { + intrinsics: Vec, + builtins: Option, +} + +impl Plan { + pub fn new( + resolve: &Resolve, + streams: &[&Stream], + types: &mut Types, + imports: &mut Imports, + ) -> Self { + use ValType::{I32, I64}; + let intrinsics = streams + .iter() + .map(|stream| { + let name = |intrinsic, async_| { + resolve.wasm_import_name( + ManglingAndAbi::Legacy(LiftLowerAbi::Sync), + WasmImport::StreamIntrinsic { + interface: stream.interface, + func: stream.func, + ty: Some(stream.ty), + intrinsic, + exported: true, + async_, + }, + ) + }; + let new = imports.import(types, name(StreamIntrinsic::New, false), &[], &[I64]); + let drop_writable = imports.import( + types, + name(StreamIntrinsic::DropWritable, false), + &[I32], + &[], + ); + let (write, task_return) = match stream.is_async() { + false => (None, None), + true => { + let write = imports.import( + types, + name(StreamIntrinsic::Write, true), + &[I32, I32, I32], + &[I32], + ); + let module = match stream.interface { + Some(key) => format!("[export]{}", resolve.name_world_key(key)), + None => "[export]$root".to_string(), + }; + let task_return = imports.import( + types, + (module, format!("[task-return]{}", stream.func.name)), + &[I32], + &[], + ); + (Some(write), Some(task_return)) + } + }; + Intrinsics { + new, + drop_writable, + write, + task_return, + } + }) + .collect(); + let builtins = streams.iter().any(|s| s.is_async()).then(|| Builtins { + waitable_set_new: imports.root(types, "[waitable-set-new]", &[], &[I32]), + waitable_set_drop: imports.root(types, "[waitable-set-drop]", &[I32], &[]), + waitable_join: imports.root(types, "[waitable-join]", &[I32, I32], &[]), + context_get: imports.root(types, "[context-get-0]", &[], &[I32]), + context_set: imports.root(types, "[context-set-0]", &[I32], &[]), + }); + Self { + intrinsics, + builtins, + } + } +} + +/// Emits the helper functions following the constants, `first` is the index +/// of the first helper. +pub struct Helpers<'p> { + plan: &'p Plan, + alloc: u32, + free: u32, + /// The size of every state record, the largest any stream needs. + state_size: u32, + /// The functions of each async stream, by stream. + async_funcs: Vec>, +} + +/// The functions writing an async stream, by index. +#[derive(Clone, Copy)] +struct AsyncFuncs { + pump: u32, + callback: u32, + /// Generates an item, for a stream with a generator. + generate: Option, +} + +impl<'p> Helpers<'p> { + pub fn new(plan: &'p Plan, streams: &[&Stream], first: u32) -> Self { + let mut next = first + 2; + let mut take = || { + next += 1; + next - 1 + }; + let async_funcs = streams + .iter() + .map(|stream| { + stream.is_async().then(|| AsyncFuncs { + pump: take(), + callback: take(), + generate: stream.generator.as_ref().map(|_| take()), + }) + }) + .collect(); + let state_size = streams.iter().map(|s| s.state_size()).max().unwrap_or(8); + Self { + plan, + alloc: first, + free: first + 1, + state_size, + async_funcs, + } + } + + /// The callback to export with the stream's export, when it's async. + pub fn callback(&self, index: usize) -> Option { + self.async_funcs[index].map(|funcs| funcs.callback) + } + + /// The export returning the stream at `index`. + pub fn export(&self, streams: &[&Stream], index: usize) -> Function { + let stream = streams[index]; + let intrinsics = &self.plan.intrinsics[index]; + // locals follow the arguments + let params = stream.params().len() as u32; + let (pair, state, writable) = (params, params + 1, params + 2); + let mut f = Function::new([(1, ValType::I64), (2, ValType::I32)]); + let i = &mut f.instructions(); + // the writable end is in the upper 32 bits, the readable end the lower + i.call(intrinsics.new) + .local_tee(pair) + .i64_const(32) + .i64_shr_u() + .i32_wrap_i64() + .local_set(writable); + match (self.async_funcs[index], &self.plan.builtins) { + (Some(funcs), Some(builtins)) => { + i.call(self.alloc) + .local_tee(state) + .call(builtins.waitable_set_new) + .i32_store(SET) + .local_get(state) + .i32_const(stream.len as i32) + .i32_store(REMAINING); + if let Some(generator) = &stream.generator { + let buffer = stream.buffer(); + for field in [WRITTEN, DONE] { + i.local_get(state).i32_const(0).i32_store(field); + } + // the position of the first item generated, after the items + // the stream starts from + i.local_get(state) + .i64_const((stream.len + stream.args) as i64) + .i64_store(INDEX_ARG); + // the arguments are `$init`, or the first batch, which the + // expression starts from once they're written + for (arg, ty) in generator.seeds.iter().enumerate() { + i.local_get(state).local_get(arg as u32); + ty.store(i, SEEDS + 8 * arg as u32); + } + let generated = buffer + generator.params * stream.stride; + for arg in 0..stream.args { + i.local_get(state).local_get(arg); + generator.item.store(i, generated + arg * stream.stride); + } + i.local_get(state) + .i32_const(stream.args as i32) + .i32_store(FILLED); + // otherwise, the expression starts from the last listed items + let window = generator.params * stream.stride; + if window > 0 && stream.args == 0 { + i.local_get(state) + .i32_const(buffer as i32) + .i32_add() + .i32_const((stream.end() - window) as i32) + .i32_const(window as i32) + .memory_copy(0, 0); + } + } + i.local_get(state) + .call(builtins.context_set) + .local_get(pair) + .i32_wrap_i64() + .call(intrinsics.task_return.expect("async stream")) + .local_get(state) + .local_get(writable) + .call(funcs.pump); + } + // an empty stream is closed before it's returned + _ => { + i.local_get(writable) + .call(intrinsics.drop_writable) + .local_get(pair) + .i32_wrap_i64(); + } + } + i.end(); + f + } + + /// The functions following the constants, in index order: the allocator + /// then each async stream's pump and callback, with their types. + pub fn functions(&self, streams: &[&Stream], types: &mut Types) -> Vec<(u32, Function)> { + use ValType::I32; + let Some(builtins) = &self.plan.builtins else { + return vec![]; + }; + let mut functions = vec![ + (types.intern(&[], &[I32]), self.alloc_fn()), + (types.intern(&[I32], &[]), self.free_fn()), + ]; + for (index, stream) in streams.iter().enumerate() { + let Some(funcs) = self.async_funcs[index] else { + continue; + }; + let intrinsics = &self.plan.intrinsics[index]; + let pump = match (&stream.generator, funcs.generate) { + (Some(generator), Some(generate)) => { + self.generated_pump_fn(stream, generator, generate, intrinsics, builtins) + } + _ => self.pump_fn(stream, intrinsics, builtins), + }; + functions.push((types.intern(&[I32, I32], &[I32]), pump)); + functions.push(( + types.intern(&[I32, I32, I32], &[I32]), + self.callback_fn(stream, funcs, builtins), + )); + if let Some(generator) = &stream.generator { + functions.push((types.intern(&[I32, I32], &[I32]), generator.function())); + } + } + functions + } + + /// The globals the allocator uses, the free list and the top of the heap, + /// which starts at `heap`, past the data. + pub fn globals(&self, heap: u32) -> Option { + self.plan.builtins.as_ref()?; + let mut globals = GlobalSection::new(); + let ty = GlobalType { + val_type: ValType::I32, + mutable: true, + shared: false, + }; + globals.global(ty, &ConstExpr::i32_const(0)); + globals.global(ty, &ConstExpr::i32_const(heap.next_multiple_of(8) as i32)); + Some(globals) + } + + /// Takes a state record from the free list, or from the top of the heap, + /// growing memory when the heap reaches its end. + fn alloc_fn(&self) -> Function { + let (free_list, heap) = (0, 1); + let p = 0; + let mut f = Function::new([(1, ValType::I32)]); + f.instructions() + .global_get(free_list) + .if_(BlockType::Result(ValType::I32)) + .global_get(free_list) + .local_tee(p) + .i32_load(SET) + .global_set(free_list) + .local_get(p) + .else_() + .global_get(heap) + .local_tee(p) + .i32_const(self.state_size as i32) + .i32_add() + .global_set(heap) + .global_get(heap) + .memory_size(0) + .i32_const(16) + .i32_shl() + .i32_gt_u() + .if_(BlockType::Empty) + .i32_const(1) + .memory_grow(0) + .i32_const(-1) + .i32_eq() + .if_(BlockType::Empty) + .unreachable() + .end() + .end() + .local_get(p) + .end() + .end(); + f + } + + /// Returns a state record to the free list. + fn free_fn(&self) -> Function { + let free_list = 0; + let p = 0; + let mut f = Function::new([]); + f.instructions() + .local_get(p) + .global_get(free_list) + .i32_store(SET) + .local_get(p) + .global_set(free_list) + .end(); + f + } + + /// `pump(state, writable) -> callback code` writes the items left, until a + /// write blocks, then waits for it. Once every item is written, or the + /// reader drops the stream, the stream is closed and the task exits. + fn pump_fn(&self, stream: &Stream, intrinsics: &Intrinsics, builtins: &Builtins) -> Function { + let (state, writable, remaining, result) = (0, 1, 2, 3); + let end = stream.end(); + let mut f = Function::new([(2, ValType::I32)]); + let i = &mut f.instructions(); + i.block(BlockType::Empty) + .loop_(BlockType::Empty) + .local_get(state) + .i32_load(REMAINING) + .local_tee(remaining) + .i32_eqz() + .br_if(1) + // write the items left, which end the data + .local_get(writable) + .i32_const(end as i32) + .local_get(remaining) + .i32_const(stream.stride as i32) + .i32_mul() + .i32_sub() + .local_get(remaining) + .call(intrinsics.write.expect("async stream")) + .local_tee(result) + .i32_const(BLOCKED) + .i32_eq() + .if_(BlockType::Empty) + .local_get(writable) + .local_get(state) + .i32_load(SET) + .call(builtins.waitable_join) + .local_get(state) + .i32_load(SET) + .i32_const(4) + .i32_shl() + .i32_const(CALLBACK_WAIT) + .i32_or() + .return_() + .end(); + consume(i, state, result); + i.br(0) + .end() + .end() + // close the stream and free the task's state + .local_get(writable) + .i32_const(0) + .call(builtins.waitable_join) + .local_get(writable) + .call(intrinsics.drop_writable) + .local_get(state) + .i32_load(SET) + .call(builtins.waitable_set_drop) + .local_get(state) + .call(self.free) + .i32_const(CALLBACK_EXIT) + .end(); + f + } + + /// `pump(state, writable) -> callback code` for a stream with a generator: + /// writes the listed items left, then the generated items, generating + /// another batch once the last is written. Waits whenever a write blocks. + /// Once an item can't be generated and the items before it are written, + /// or the reader drops the stream, the stream is closed and the task + /// exits. + fn generated_pump_fn( + &self, + stream: &Stream, + generator: &Generator, + generate: u32, + intrinsics: &Intrinsics, + builtins: &Builtins, + ) -> Function { + let (state, writable, remaining, result, filled, written, n) = (0, 1, 2, 3, 4, 5, 6); + let stride = stream.stride as i32; + let write = intrinsics.write.expect("async stream"); + let buffer = stream.buffer() as i32; + let generated = buffer + (generator.params * stream.stride) as i32; + // waits when the write in `result` blocks + let wait = |i: &mut InstructionSink| { + i.local_get(result) + .i32_const(BLOCKED) + .i32_eq() + .if_(BlockType::Empty) + .local_get(writable) + .local_get(state) + .i32_load(SET) + .call(builtins.waitable_join) + .local_get(state) + .i32_load(SET) + .i32_const(4) + .i32_shl() + .i32_const(CALLBACK_WAIT) + .i32_or() + .return_() + .end(); + }; + let mut f = Function::new([(5, ValType::I32)]); + let i = &mut f.instructions(); + i.block(BlockType::Empty).loop_(BlockType::Empty); + + // the listed items left, which end the data + i.local_get(state) + .i32_load(REMAINING) + .local_tee(remaining) + .if_(BlockType::Empty) + .local_get(writable) + .i32_const(stream.end() as i32) + .local_get(remaining) + .i32_const(stride) + .i32_mul() + .i32_sub() + .local_get(remaining) + .call(write) + .local_set(result); + wait(i); + account(i, state, result); + i.br(1).end(); + + // the generated items left + i.local_get(state) + .i32_load(WRITTEN) + .local_tee(written) + .local_get(state) + .i32_load(FILLED) + .local_tee(filled) + .i32_lt_u() + .if_(BlockType::Empty) + .local_get(writable) + .local_get(state) + .i32_const(generated) + .i32_add() + .local_get(written) + .i32_const(stride) + .i32_mul() + .i32_add() + .local_get(filled) + .local_get(written) + .i32_sub() + .call(write) + .local_set(result); + wait(i); + account(i, state, result); + i.br(1).end(); + + // every item is written + i.local_get(state).i32_load(DONE).br_if(1); + + // the last items of the batch become the items the expression reads + let window = (generator.params * stream.stride) as i32; + if window > 0 { + i.local_get(state) + .i32_const(buffer) + .i32_add() + .local_get(state) + .i32_const(buffer) + .i32_add() + .local_get(filled) + .i32_const(stride) + .i32_mul() + .i32_add() + .i32_const(window) + .memory_copy(0, 0); + } + + // generate a batch, stopping at items that can't be generated, each + // evaluation generating `count` items + let count = generator.count as i32; + i.i32_const(0) + .local_set(n) + .block(BlockType::Empty) + .loop_(BlockType::Empty) + .local_get(n) + .i32_const(batch(generator) as i32) + .i32_eq() + .br_if(1) + .local_get(state) + .local_get(state) + .i32_const(buffer) + .i32_add() + .local_get(n) + .i32_const(stride) + .i32_mul() + .i32_add() + .call(generate) + .i32_eqz() + .if_(BlockType::Empty) + .local_get(state) + .i32_const(1) + .i32_store(DONE) + .br(2) + .end() + .local_get(n) + .i32_const(count) + .i32_add() + .local_set(n) + // the next position, the stream ends after the items at the last + // positions, when the position wraps + .local_get(state) + .local_get(state) + .i64_load(INDEX_ARG) + .i64_const(count as i64) + .i64_add() + .i64_store(INDEX_ARG) + .local_get(state) + .i64_load(INDEX_ARG) + .i64_const(count as i64) + .i64_lt_u() + .if_(BlockType::Empty) + .local_get(state) + .i32_const(1) + .i32_store(DONE) + .br(2) + .end() + .br(0) + .end() + .end() + .local_get(state) + .local_get(n) + .i32_store(FILLED) + .local_get(state) + .i32_const(0) + .i32_store(WRITTEN) + .br(0) + .end() + .end(); + + // close the stream and free the task's state + i.local_get(writable) + .i32_const(0) + .call(builtins.waitable_join) + .local_get(writable) + .call(intrinsics.drop_writable) + .local_get(state) + .i32_load(SET) + .call(builtins.waitable_set_drop) + .local_get(state) + .call(self.free) + .i32_const(CALLBACK_EXIT) + .end(); + f + } + + /// `callback(event, writable, result) -> callback code` accounts for the + /// finished write then pumps the items left. + fn callback_fn(&self, stream: &Stream, funcs: AsyncFuncs, builtins: &Builtins) -> Function { + let (event, writable, result, state) = (0, 1, 2, 3); + let mut f = Function::new([(1, ValType::I32)]); + let i = &mut f.instructions(); + i.local_get(event) + .i32_const(EVENT_STREAM_WRITE) + .i32_ne() + .if_(BlockType::Empty) + .unreachable() + .end() + .call(builtins.context_get) + .local_set(state); + match stream.generator { + Some(_) => account(i, state, result), + None => consume(i, state, result), + } + i.local_get(state) + .local_get(writable) + .call(funcs.pump) + .end(); + f + } +} + +/// The number of items generated at a time, a multiple of the items each +/// evaluation generates. +fn batch(generator: &Generator) -> u32 { + BATCH.next_multiple_of(generator.count) +} + +/// Accounts for a finished write of a stream with a generator: the items it +/// copied were listed items while any are left, otherwise generated items. +/// When the reader dropped the stream, nothing is left to write. +fn account(i: &mut InstructionSink, state: u32, result: u32) { + i.local_get(result) + .i32_const(COPY_STATUS_MASK) + .i32_and() + .if_(BlockType::Empty); + for field in [REMAINING, FILLED, WRITTEN] { + i.local_get(state).i32_const(0).i32_store(field); + } + i.local_get(state) + .i32_const(1) + .i32_store(DONE) + .else_() + .local_get(state) + .i32_load(REMAINING) + .if_(BlockType::Empty) + .local_get(state) + .local_get(state) + .i32_load(REMAINING) + .local_get(result) + .i32_const(COPY_STATUS_BITS) + .i32_shr_u() + .i32_sub() + .i32_store(REMAINING) + .else_() + .local_get(state) + .local_get(state) + .i32_load(WRITTEN) + .local_get(result) + .i32_const(COPY_STATUS_BITS) + .i32_shr_u() + .i32_add() + .i32_store(WRITTEN) + .end() + .end(); +} + +/// Subtracts the items a finished write copied from the items left, or leaves +/// none when the reader dropped the stream. +fn consume(i: &mut InstructionSink, state: u32, result: u32) { + i.local_get(state) + .local_get(state) + .i32_load(REMAINING) + .local_get(result) + .i32_const(COPY_STATUS_BITS) + .i32_shr_u() + .i32_sub() + .i32_const(0) + .local_get(result) + .i32_const(COPY_STATUS_MASK) + .i32_and() + .i32_eqz() + .select() + .i32_store(REMAINING); +} diff --git a/crates/componentized-constants/src/types.rs b/crates/componentized-constants/src/types.rs index cff1126..69c1047 100644 --- a/crates/componentized-constants/src/types.rs +++ b/crates/componentized-constants/src/types.rs @@ -2,22 +2,28 @@ use anyhow::{Result, bail}; use wit_parser::{Function, FunctionKind, Handle, Resolve, Type, TypeDefKind}; /// Ensures a function can be implemented as a constant: it must be a -/// synchronous, freestanding function that takes no parameters and returns a -/// value. +/// freestanding function, synchronous or async, that takes no parameters and +/// returns a value. A function generating a stream with an `@expression` may +/// take the items the stream starts from as parameters. /// /// The result type isn't checked here; any type is allowed so long as the /// function's value doesn't reach a type that values can't be expressed for, /// see [`crate::values`]. pub fn check_function(func: &Function) -> Result<()> { - if !matches!(func.kind, FunctionKind::Freestanding) { + crate::docs::check(&func.docs).map_err(|e| e.context(format!("function `{}`", func.name)))?; + if !matches!( + func.kind, + FunctionKind::Freestanding | FunctionKind::AsyncFreestanding + ) { + bail!("function `{}` must be a freestanding function", func.name); + } + if !func.params.is_empty() && crate::docs::expression(&func.docs).is_none() { bail!( - "function `{}` must be a synchronous freestanding function", + "function `{}` must not accept parameters, unless it generates a stream from \ + them with an `@expression`", func.name ); } - if !func.params.is_empty() { - bail!("function `{}` must not accept parameters", func.name); - } if func.result.is_none() { bail!("function `{}` must return a value", func.name); } diff --git a/crates/componentized-constants/src/values.rs b/crates/componentized-constants/src/values.rs index 4118512..42b59b9 100644 --- a/crates/componentized-constants/src/values.rs +++ b/crates/componentized-constants/src/values.rs @@ -329,7 +329,8 @@ impl<'a> Layout<'a> { /// Values can't be expressed for resources, handles, futures, streams, /// error contexts or maps. Types may still contain them, so long as a /// value doesn't reach them, e.g. `none` for an `option>`, or `[]` - /// for a `list>`. + /// for a `list>`. A function's result may be a stream, see + /// [`crate::streams`]. fn unsupported(&self, ty: &Type, node: &Node) -> Error { // TODO restore map support once WAVE defines a syntax for maps, see the // `TypeDefKind::Map` TODOs in `store` and `flatten` @@ -394,6 +395,32 @@ impl<'a> Layout<'a> { Ok((ptr, bytes.len() as u32)) } + /// Stores the items of a stream, written as a list, returning their + /// address, count and size. + pub fn stream(&mut self, item: &Type, node: &Node) -> Result<(u32, u32, u32)> { + let (ptr, len) = self.list(item, node)?; + Ok((ptr, len, self.size(item) as u32)) + } + + /// Streams with items are written by a task that outlives the call, which + /// only an async function has. + pub fn stream_requires_async(&self, node: &Node) -> Error { + self.error( + node, + "a stream with items must be returned by an async function, e.g. \ + `async func() -> stream`", + ) + } + + /// The expression generating a stream's items reads the items before + /// them, starting from the listed items. + pub fn too_few_items(&self, node: &Node, params: u32) -> Error { + self.error( + node, + &format!("the expression reads the previous {params} items, list at least {params}"), + ) + } + fn list(&mut self, elem: &Type, node: &Node) -> Result<(u32, u32)> { let values: Vec<&Node> = self.parse(node.as_list())?.collect(); let stride = self.size(elem); @@ -594,7 +621,7 @@ pub fn parser_error(src: &str, e: ParserError) -> Error { } /// Formats the start of a span as `line:column` (1-based). -fn position(src: &str, span: Range) -> String { +pub(crate) fn position(src: &str, span: Range) -> String { let before = &src[..span.start.min(src.len())]; let line = before.matches('\n').count() + 1; let column = before.rsplit('\n').next().unwrap_or("").chars().count() + 1; diff --git a/crates/componentized-constants/tests/integration_test.rs b/crates/componentized-constants/tests/integration_test.rs index f39886c..67e4896 100644 --- a/crates/componentized-constants/tests/integration_test.rs +++ b/crates/componentized-constants/tests/integration_test.rs @@ -1,8 +1,8 @@ use anyhow::{Context, Result}; use componentized_constants::value::{Type, Value, WasmValue}; use componentized_constants::{Overrides, create_component}; -use test_harness::{call, call_with}; -use wasmtime::component::{ResourceType, Val}; +use test_harness::{call, call_with, read_streams, read_streams_until, read_streams_with}; +use wasmtime::component::{ComponentType, Lift, ResourceType, Val}; use wit_parser::Resolve; const ALL_WIT: &str = "tests/fixtures/all"; @@ -562,3 +562,1739 @@ fn it_rejects_invalid_values() { ); } } + +const STREAMS_WIT: &str = "package a:b; + interface i { + type bytes = stream; + /// @value [1, 2, 3, 4, 5] + numbers: async func() -> bytes; + /// @value [\"a\", \"bc\", \"\", \"🦀\"] + words: async func() -> stream; + /// @value [{x: 1, y: -1}, {x: -2, y: 2}] + points: async func() -> stream; + /// @value [] + nothing: async func() -> stream; + /// @value [] + closed: func() -> stream; + /// @value 42 + answer: async func() -> u32; + record point { x: s32, y: s32 } + } + world w { + export i; + /// @value [true, false] + export switches: async func() -> stream; + }"; + +#[derive(ComponentType, Lift, Debug, PartialEq, Clone)] +#[component(record)] +struct Point { + x: i32, + y: i32, +} + +#[test] +fn it_returns_streams() -> Result<()> { + let component = build(STREAMS_WIT, None)?; + let iface = "a:b/i"; + // reads taking every item, and reads taking fewer items than are written + for per_read in [100, 2, 1] { + let context = || format!("{per_read} items per read"); + let numbers = read_streams::(&component, (iface, "numbers"), per_read, 3) + .with_context(context)?; + assert_eq!(numbers, vec![vec![1, 2, 3, 4, 5]; 6], "{}", context()); + let words = read_streams::(&component, (iface, "words"), per_read, 2) + .with_context(context)?; + let expected: Vec = ["a", "bc", "", "🦀"].map(String::from).into(); + assert_eq!(words, vec![expected; 4], "{}", context()); + let points = read_streams::(&component, (iface, "points"), per_read, 1) + .with_context(context)?; + let expected = vec![Point { x: 1, y: -1 }, Point { x: -2, y: 2 }]; + assert_eq!(points, vec![expected; 2], "{}", context()); + let switches = read_streams::(&component, ("", "switches"), per_read, 1) + .with_context(context)?; + assert_eq!(switches, vec![vec![true, false]; 2], "{}", context()); + } + for name in ["nothing", "closed"] { + let empty = read_streams::(&component, (iface, name), 1, 2)?; + assert_eq!(empty, vec![Vec::::new(); 4], "{name}"); + } + Ok(()) +} + +#[test] +fn it_stops_writing_when_the_reader_closes_the_stream() -> Result<()> { + let component = build(STREAMS_WIT, None)?; + // the second round of calls shows the closed streams' tasks ended cleanly + let numbers = read_streams_until::(&component, ("a:b/i", "numbers"), 1, 2, 2)?; + assert_eq!(numbers, vec![vec![1, 2]; 4]); + Ok(()) +} + +#[test] +fn it_grows_memory_for_the_state_of_each_stream() -> Result<()> { + // the data starts at 8 and ends just short of the first page, so the state + // of a second stream needs another page + let items = vec!["7"; 65536 - 8 - 8].join(","); + let component = build( + "package a:b; world w { export f: async func() -> stream; }", + Some(&format!("{{f: [{items}]}}")), + )?; + let streams = read_streams::(&component, ("", "f"), 65536, 3)?; + assert_eq!(streams, vec![vec![7; 65536 - 16]; 6]); + Ok(()) +} + +#[test] +fn it_returns_values_from_async_functions() -> Result<()> { + let component = build(STREAMS_WIT, None)?; + let actual = call(&component, &[], &[("a:b/i", "answer")])?; + assert_eq!(actual, vec![Val::U32(42)]); + Ok(()) +} + +#[test] +fn it_overrides_streams() -> Result<()> { + let component = build(STREAMS_WIT, Some("{i: {numbers: [9], closed: []}}"))?; + let numbers = read_streams::(&component, ("a:b/i", "numbers"), 1, 1)?; + assert_eq!(numbers, vec![vec![9]; 2]); + Ok(()) +} + +#[test] +fn it_rejects_streams_with_items_from_sync_functions() { + let err = build_err( + "package a:b; world w { export f: func() -> stream; }", + "{f: [1]}", + ); + assert!( + err.contains("a stream with items must be returned by an async function"), + "{err}" + ); +} + +#[test] +fn it_rejects_invalid_stream_items() { + let wit = "package a:b; world w { export f: async func() -> stream; }"; + for (value, expected) in [ + ("[256]", "expected u8"), + ("1", "invalid value type"), + ("[\"a\"]", "invalid value type"), + ] { + let err = build_err(wit, &format!("{{f: {value}}}")); + assert!( + err.contains(expected), + "{value}: expected {expected:?} in {err:?}" + ); + } + + // a stream without a type has no items to write + let err = build_err( + "package a:b; world w { export f: async func() -> stream; }", + "{f: []}", + ); + assert!(err.contains("are not supported"), "{err}"); +} + +const GENERATORS_WIT: &str = "package a:b; + world w { + /// @value [0] + /// @expression |n| n + 1 + export incrementor: async func() -> stream; + /// @value [false] + /// @expression |value| !value + export flip-flop: async func() -> stream; + /// @value [1, 1] + /// @expression |a, b| a + b + export fibonacci: async func() -> stream; + /// @value [1, 1] + /// @expression |a, b| a + b + export long-fibonacci: async func() -> stream; + /// @value [0] + /// @expression |n| n - 1 + export countdown: async func() -> stream; + /// @value [1] + /// @expression |n| n * 2 + export doubling: async func() -> stream; + /// @value [9223372036854775805] + /// @expression |n| n + 1 + export near-max: async func() -> stream; + /// @value [1.0] + /// @expression |x| x * 10.0 + export powers: async func() -> stream; + /// @value [1] + /// @expression |x| x * 10 + export small-powers: async func() -> stream; + /// @value [3] + /// @expression |n| 12 / (n - 1) + export divide: async func() -> stream; + /// @value [0] + /// @expression |n| if n == 9 { 0 } else { n + 1 } + export digits: async func() -> stream; + /// @value [1] + /// @expression |n| n << 1 + export shift: async func() -> stream; + /// @value [1] + /// @expression |n| n << 1 + export signed-shift: async func() -> stream; + /// @value [] + /// @expression || 7 + export sevens: async func() -> stream; + /// @value [1, 2, 3] + /// @expression |a, b| a + b + export window: async func() -> stream; + /// @value [-5] + /// @expression |n| if n < 0 && n % 2 != 0 { -n } else { n - 3 } + export bounce: async func() -> stream; + }"; + +fn generated( + component: &[u8], + name: &str, + limit: usize, +) -> Result> { + let mut streams = vec![]; + // reads taking every item, and reads taking fewer items than are written + for per_read in [1000, 7, 1] { + let read = read_streams_until::(component, ("", name), per_read, 2, limit) + .with_context(|| format!("{name}, {per_read} items per read"))?; + streams.extend(read); + } + let first = streams.remove(0); + for stream in streams { + assert_eq!(stream, first, "{name}: streams differ"); + } + Ok(first) +} + +/// The sequence from `seed`, until `next` can't produce an item. +fn sequence(seed: &[T], next: impl Fn(&[T]) -> Option) -> Vec { + let mut items = seed.to_vec(); + while let Some(item) = next(&items) { + items.push(item); + } + items +} + +#[test] +fn it_generates_streams_until_an_item_cannot_be_represented() -> Result<()> { + let component = build(GENERATORS_WIT, None)?; + let all = usize::MAX; + assert_eq!( + generated::(&component, "incrementor", all)?, + (0..=255).collect::>() + ); + assert_eq!( + generated::(&component, "fibonacci", all)?, + sequence(&[1u32, 1], |s| s[s.len() - 2].checked_add(s[s.len() - 1])) + ); + assert_eq!( + generated::(&component, "long-fibonacci", all)?, + sequence(&[1u64, 1], |s| s[s.len() - 2].checked_add(s[s.len() - 1])) + ); + assert_eq!( + generated::(&component, "countdown", all)?, + (-128..=0).rev().collect::>() + ); + assert_eq!( + generated::(&component, "doubling", all)?, + (0..64).map(|n| 1u64 << n).collect::>() + ); + assert_eq!( + generated::(&component, "near-max", all)?, + vec![i64::MAX - 2, i64::MAX - 1, i64::MAX] + ); + assert_eq!( + generated::(&component, "powers", all)?, + sequence(&[1.0f64], |s| Some(s[s.len() - 1] * 10.0) + .filter(|x| x.is_finite())) + ); + assert_eq!( + generated::(&component, "small-powers", all)?, + sequence(&[1.0f32], |s| Some(s[s.len() - 1] * 10.0) + .filter(|x| x.is_finite())) + ); + assert_eq!( + generated::(&component, "divide", all)?, + vec![3, 6, 2, 12, 1] + ); + Ok(()) +} + +#[test] +fn it_generates_unbounded_streams() -> Result<()> { + let component = build(GENERATORS_WIT, None)?; + assert_eq!( + generated::(&component, "flip-flop", 5)?, + vec![false, true, false, true, false] + ); + assert_eq!( + generated::(&component, "digits", 12)?, + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1] + ); + // bits shifted past the type's width are dropped, as in Rust + let mut shifts: Vec = (0..32).map(|n| 1 << n).collect(); + shifts.extend([0, 0]); + assert_eq!(generated::(&component, "shift", 34)?, shifts); + assert_eq!( + generated::(&component, "signed-shift", 10)?, + vec![1, 2, 4, 8, 16, 32, 64, -128, 0, 0] + ); + assert_eq!(generated::(&component, "sevens", 3)?, vec![7, 7, 7]); + assert_eq!( + generated::(&component, "window", 7)?, + vec![1, 2, 3, 5, 8, 13, 21] + ); + assert_eq!( + generated::(&component, "bounce", 6)?, + vec![-5, 5, 2, -1, 1, -2] + ); + // longer than a batch of generated items + assert_eq!( + generated::(&component, "flip-flop", 1000)?, + (0..1000).map(|n| n % 2 == 1).collect::>() + ); + Ok(()) +} + +#[test] +fn it_overrides_the_items_generated_streams_start_from() -> Result<()> { + let component = build( + GENERATORS_WIT, + Some("{incrementor: [250], fibonacci: [5, 8, 13]}"), + )?; + assert_eq!( + generated::(&component, "incrementor", usize::MAX)?, + (250..=255).collect::>() + ); + assert_eq!( + generated::(&component, "fibonacci", 5)?, + vec![5, 8, 13, 21, 34] + ); + Ok(()) +} + +#[test] +fn it_rejects_invalid_expressions() { + let wit = |ty: &str, value: &str, expression: &str| { + format!( + "package a:b; world w {{ + /// @value {value} + /// @expression {expression} + export f: async func() -> {ty}; + }}" + ) + }; + for (wit, expected) in [ + (wit("u8", "1", "|n| n"), "only functions returning a stream"), + ( + wit("stream", "[\"a\"]", "|s| s"), + "only streams of integers, floats or bool", + ), + (wit("stream", "[1]", "|a, b| a + b"), "list at least 2"), + (wit("stream", "[1]", "|n| m"), "unknown name `m`"), + (wit("stream", "[1]", "|n| n + true"), "mismatched types"), + ( + wit("stream", "[1]", "|n| n + 256"), + "literal 256 is out of range for u8", + ), + (wit("stream", "[1]", "|n| -n"), "`-` can't negate u8"), + ( + wit("stream", "[1]", "|n| n + 1.5"), + "expected u8, found float", + ), + ( + wit("stream", "[1.0]", "|x| x % 2.0"), + "isn't supported for f64", + ), + ( + wit("stream", "[1]", "|n| n < 2 < 3"), + "can't be chained", + ), + ( + wit("stream", "[1]", "|n| if n > 1 { 1 }"), + "expected `else`", + ), + ( + wit("stream", "[1]", "|n| n > 1"), + "expected the closure to return u8", + ), + ( + wit("stream", "[1]", "|n| n + 1u8"), + "literal suffixes aren't supported", + ), + (wit("stream", "[1]", "n + 1"), "expected `|`"), + ( + wit("stream", "[1]", "|n| n +"), + "expected an expression at the end", + ), + ( + wit("stream", "[1]", "|n, n| n"), + "duplicate parameter `n`", + ), + ] { + let err = format!("{:#}", build(&wit, None).expect_err(&wit)); + assert!(err.contains(expected), "expected {expected:?} in {err:?}"); + } + + // positions are the line and column within the expression + let err = format!( + "{:#}", + build( + "package a:b; world w { + /// @value [1] + /// @expression |n| + /// n + $ + export f: async func() -> stream; + }", + None + ) + .expect_err("position") + ); + assert!(err.contains("unexpected `$` at 2:9"), "{err}"); + + // generated streams are unbounded, so need an async function + let err = format!( + "{:#}", + build( + "package a:b; world w { + /// @value [] + /// @expression || 1 + export f: func() -> stream; + }", + None + ) + .expect_err("sync") + ); + assert!( + err.contains("must be returned by an async function"), + "{err}" + ); +} + +const ARGUMENTS_WIT: &str = "package a:b; + interface i { + /// @expression |a, b| a + b + fibonacci: async func(a: u32, b: u32) -> stream; + } + world w { + export i; + /// @expression |n| n + 1 + export incrementor: async func(n: u8) -> stream; + /// @expression |value| !value + export flip-flop: async func(value: bool) -> stream; + /// @expression |x| x / 2.0 + export halving: async func(x: f64) -> stream; + }"; + +#[test] +fn it_generates_streams_from_arguments() -> Result<()> { + let component = build(ARGUMENTS_WIT, None)?; + let all = usize::MAX; + for per_read in [1000, 7, 1] { + let fibonacci = read_streams_with::<_, u32>( + &component, + ("a:b/i", "fibonacci"), + (1u32, 1u32), + per_read, + 2, + all, + )?; + let expected = sequence(&[1u32, 1], |s| s[s.len() - 2].checked_add(s[s.len() - 1])); + assert_eq!(fibonacci, vec![expected; 4], "{per_read} items per read"); + } + let fibonacci = + read_streams_with::<_, u32>(&component, ("a:b/i", "fibonacci"), (5u32, 8u32), 1, 1, 4)?; + assert_eq!(fibonacci, vec![vec![5, 8, 13, 21]; 2]); + let incrementor = + read_streams_with::<_, u8>(&component, ("", "incrementor"), (250u8,), 3, 1, all)?; + assert_eq!(incrementor, vec![(250..=255).collect::>(); 2]); + let flip_flop = read_streams_with::<_, bool>(&component, ("", "flip-flop"), (true,), 2, 1, 3)?; + assert_eq!(flip_flop, vec![vec![true, false, true]; 2]); + let halving = read_streams_with::<_, f64>(&component, ("", "halving"), (8.0,), 2, 1, 4)?; + assert_eq!(halving, vec![vec![8.0, 4.0, 2.0, 1.0]; 2]); + Ok(()) +} + +#[test] +fn it_rejects_overrides_for_streams_generated_from_arguments() { + for overrides in ["{incrementor: [1]}", "{i: {fibonacci: [1, 1]}}"] { + let err = format!( + "{:#}", + build(ARGUMENTS_WIT, Some(overrides)).expect_err(overrides) + ); + assert!( + err.contains("can't be overridden, its stream is generated from its arguments"), + "{overrides}: {err}" + ); + } +} + +#[test] +fn it_rejects_invalid_streams_generated_from_arguments() { + let wit = + |docs: &str, func: &str| format!("package a:b; world w {{ {docs} export f: {func}; }}"); + for (wit, expected) in [ + ( + wit( + "/// @expression |a, b| a + b\n", + "async func(a: u32) -> stream", + ), + "the expression takes 2 parameters, the function takes 1 parameter", + ), + ( + wit( + "/// @expression |a| a\n", + "async func(a: u32, b: u32) -> stream", + ), + "the expression takes 1 parameter, the function takes 2 parameters", + ), + ( + wit( + "/// @expression |a| a\n", + "async func(a: u8) -> stream", + ), + "parameter `a` must have the stream's item type, `u32`", + ), + ( + wit("/// @expression |a| a\n", "func(a: u32) -> stream"), + "must be returned by an async function", + ), + ( + wit( + "/// @value [1]\n/// @expression |a| a\n", + "async func(a: u32) -> stream", + ), + "its stream starts from its arguments, remove the tag", + ), + ( + wit("/// @value [1]\n", "async func(a: u32) -> stream"), + "must not accept parameters, unless it generates a stream", + ), + ( + wit("/// @expression |a| a\n", "async func(a: u32) -> u32"), + "only functions returning a stream can generate items", + ), + ] { + let err = format!("{:#}", build(&wit, None).expect_err(&wit)); + assert!(err.contains(expected), "expected {expected:?} in {err:?}"); + } +} + +#[test] +fn it_rejects_duplicate_tags() { + for docs in [ + "/// @value 1\n/// @value 2\n", + "/// @value [1]\n/// @expression |n| n + 1\n/// @expression |n| n + 2\n", + ] { + let wit = format!( + "package a:b; interface i {{ {docs} f: async func() -> stream; }} world w {{ export i; }}" + ); + let err = format!("{:#}", build(&wit, None).expect_err(&wit)); + assert!( + err.contains("interface `i`: function `f`: duplicate `@"), + "{docs}: {err}" + ); + } +} + +#[test] +fn it_generates_xorshift_pseudorandom_numbers() -> Result<()> { + // xorshift64, x ^= x << 13; x ^= x >> 7; x ^= x << 17, written out without `let` + let component = build( + "package a:b; world w { + /// @expression |x| ((x ^ x << 13) ^ (x ^ x << 13) >> 7) ^ ((x ^ x << 13) ^ (x ^ x << 13) >> 7) << 17 + export pseudorandom: async func(seed: u64) -> stream; + }", + None, + )?; + let xorshift = |mut x: u64| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x + }; + for seed in [1u64, 42, u64::MAX] { + let mut expected = vec![seed]; + while expected.len() < 1000 { + expected.push(xorshift(*expected.last().unwrap())); + } + let actual = + read_streams_with::<_, u64>(&component, ("", "pseudorandom"), (seed,), 100, 1, 1000)?; + assert_eq!(actual, vec![expected; 2], "seed {seed}"); + } + Ok(()) +} + +const SEEDED_WIT: &str = "package a:b; + world w { + /// SplitMix64, from the seed and the position + /// @expression || { + /// let z = $init.0.wrapping_add($i.wrapping_add(1).wrapping_mul(0x9E3779B97F4A7C15)); + /// let z = (z ^ z >> 30).wrapping_mul(0xBF58476D1CE4E5B9); + /// let z = (z ^ z >> 27).wrapping_mul(0x94D049BB133111EB); + /// z ^ z >> 31 + /// } + export splitmix: async func(seed: u64) -> stream; + /// SplitMix64, with wrapping operators + /// @expression || { + /// let z = $init.0 +% ($i +% 1) *% 0x9E3779B97F4A7C15; + /// let z = (z ^ z >> 30) *% 0xBF58476D1CE4E5B9; + /// let z = (z ^ z >> 27) *% 0x94D049BB133111EB; + /// z ^ z >> 31 + /// } + export splitmix-operators: async func(seed: u64) -> stream; + /// @expression || ($init.0 as u64 * $i) as u8 + export truncated: async func(seed: u32) -> stream; + /// @value [1] + /// @expression |n| n + $init.0 + $init.1 as u32 + export stepped: async func(step: u32, extra: u8) -> stream; + /// @expression || $init.0 && $i % 2 == 0 + export gated: async func(open: bool) -> stream; + }"; + +/// SplitMix64 as Rust, from a seed and a position. +fn splitmix(seed: u64, i: u64) -> u64 { + let z = seed.wrapping_add(i.wrapping_add(1).wrapping_mul(0x9E3779B97F4A7C15)); + let z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + let z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) +} + +#[test] +fn it_generates_streams_from_seed_and_position() -> Result<()> { + let component = build(SEEDED_WIT, None)?; + for seed in [0u64, 42, u64::MAX] { + let actual = + read_streams_with::<_, u64>(&component, ("", "splitmix"), (seed,), 100, 2, 1000)?; + let expected: Vec = (0..1000).map(|i| splitmix(seed, i)).collect(); + assert_eq!(actual, vec![expected.clone(); 4], "seed {seed}"); + let actual = read_streams_with::<_, u64>( + &component, + ("", "splitmix-operators"), + (seed,), + 100, + 1, + 1000, + )?; + assert_eq!(actual, vec![expected; 2], "seed {seed}"); + } + let truncated = read_streams_with::<_, u8>(&component, ("", "truncated"), (7u32,), 9, 1, 300)?; + let expected: Vec = (0..300u64).map(|i| (7 * i) as u8).collect(); + assert_eq!(truncated, vec![expected; 2]); + // the listed items come first, and the arguments aren't items + let stepped = read_streams_with::<_, u32>(&component, ("", "stepped"), (10u32, 5u8), 3, 1, 4)?; + assert_eq!(stepped, vec![vec![1, 16, 31, 46]; 2]); + let gated = read_streams_with::<_, bool>(&component, ("", "gated"), (true,), 3, 1, 4)?; + assert_eq!(gated, vec![vec![true, false, true, false]; 2]); + Ok(()) +} + +#[test] +fn it_reads_the_position() -> Result<()> { + let component = build( + "package a:b; world w { + /// @value [10, 20] + /// @expression |_, b| b + $i + export f: async func() -> stream; + }", + None, + )?; + // the first item generated is at position 2 + assert_eq!( + generated::(&component, "f", 5)?, + vec![10, 20, 22, 25, 29] + ); + Ok(()) +} + +const CASTS_WIT: &str = "package a:b; + world w { + /// @value [] + /// @expression || ($i as i64 - 3) as u8 + export negative-to-unsigned: async func() -> stream; + /// @value [] + /// @expression || ($i as f64 * 100.0 - 150.0) as i8 + export saturating: async func() -> stream; + /// @value [] + /// @expression || ($i as f64 * 1.0e10) as u32 + export saturating-wide: async func() -> stream; + /// @value [] + /// @expression || ($i % 2 == 0) as u8 + export from-bool: async func() -> stream; + /// @value [] + /// @expression || ($i as i16 * 1000) as i8 + export truncating-signed: async func() -> stream; + /// @value [] + /// @expression || $i as f32 / 4.0 + export to-float: async func() -> stream; + /// @value [] + /// @expression || ($i as u8).wrapping_add(250) + export wrapping-add: async func() -> stream; + /// @value [] + /// @expression || ($i as i8).wrapping_mul(100) + export wrapping-mul: async func() -> stream; + /// @value [] + /// @expression || (0 as u16).wrapping_sub($i as u16) + export wrapping-sub: async func() -> stream; + /// @value [] + /// @expression || (0b1000_0101 as u8).rotate_left($i as u32) + export rotate-u8: async func() -> stream; + /// @value [] + /// @expression || (-30000 as i16).rotate_right($i as u32) + export rotate-s16: async func() -> stream; + /// @value [] + /// @expression || { let x: u32 = 0x8000_0001; x.rotate_left($i as u32) } + export rotate-u32: async func() -> stream; + /// @value [] + /// @expression || (-2 as i64).rotate_right($i as u32) + export rotate-s64: async func() -> stream; + /// @value [] + /// @expression || { let a: u8 = 3; let b = a * 2; let a = b + a; { let _ = $i; a } } + export lets: async func() -> stream; + }"; + +#[test] +fn it_casts_and_wraps_as_rust() -> Result<()> { + let c = build(CASTS_WIT, None)?; + let n = 70u64; + assert_eq!( + generated::(&c, "negative-to-unsigned", 5)?, + (0..5).map(|i: i64| (i - 3) as u8).collect::>() + ); + assert_eq!( + generated::(&c, "saturating", 4)?, + (0..4) + .map(|i| (i as f64 * 100.0 - 150.0) as i8) + .collect::>() + ); + assert_eq!( + generated::(&c, "saturating-wide", 3)?, + (0..3) + .map(|i| (i as f64 * 1.0e10) as u32) + .collect::>() + ); + assert_eq!(generated::(&c, "from-bool", 4)?, vec![1, 0, 1, 0]); + assert_eq!( + generated::(&c, "truncating-signed", 30)?, + (0..30).map(|i: i16| (i * 1000) as i8).collect::>() + ); + assert_eq!( + generated::(&c, "to-float", 5)?, + (0..5).map(|i| i as f32 / 4.0).collect::>() + ); + assert_eq!( + generated::(&c, "wrapping-add", n as usize)?, + (0..n) + .map(|i| (i as u8).wrapping_add(250)) + .collect::>() + ); + assert_eq!( + generated::(&c, "wrapping-mul", n as usize)?, + (0..n) + .map(|i| (i as i8).wrapping_mul(100)) + .collect::>() + ); + assert_eq!( + generated::(&c, "wrapping-sub", n as usize)?, + (0..n) + .map(|i| 0u16.wrapping_sub(i as u16)) + .collect::>() + ); + assert_eq!( + generated::(&c, "rotate-u8", n as usize)?, + (0..n) + .map(|i| 0b1000_0101u8.rotate_left(i as u32)) + .collect::>() + ); + assert_eq!( + generated::(&c, "rotate-s16", n as usize)?, + (0..n) + .map(|i| (-30000i16).rotate_right(i as u32)) + .collect::>() + ); + assert_eq!( + generated::(&c, "rotate-u32", n as usize)?, + (0..n) + .map(|i| 0x8000_0001u32.rotate_left(i as u32)) + .collect::>() + ); + assert_eq!( + generated::(&c, "rotate-s64", n as usize)?, + (0..n) + .map(|i| (-2i64).rotate_right(i as u32)) + .collect::>() + ); + assert_eq!(generated::(&c, "lets", 2)?, vec![9, 9]); + Ok(()) +} + +#[test] +fn it_rejects_invalid_seeds_casts_methods_and_lets() { + let wit = |expression: &str, func: &str| { + format!( + "package a:b; world w {{ + /// @value [] + /// @expression {expression} + export f: {func}; + }}" + ) + }; + for (wit, expected) in [ + ( + wit("|| $init.0", "async func() -> stream"), + "`$init` holds the function's arguments, but the function takes none", + ), + ( + wit("|| $init.1", "async func(a: u8) -> stream"), + "`$init.1` doesn't exist, `$init` has 1 fields", + ), + ( + wit("|| $init", "async func(a: u8) -> stream"), + "expected `.` and a field, `$init` is a tuple", + ), + ( + wit("|| $init.0 as u8", "async func(a: string) -> stream"), + "parameter `a` can't be read with `$init`", + ), + ( + wit("|| $x", "async func() -> stream"), + "unknown variable `$x`", + ), + ( + wit("|| $i", "async func() -> stream"), + "expected the closure to return u8, found u64", + ), + ( + wit("|| $init.0 * $i", "async func(seed: u32) -> stream"), + "mismatched types for `*`, u32 and u64", + ), + ( + wit("|| $i as bool", "async func() -> stream"), + "can't cast to bool", + ), + ( + wit("|| 1.wrapping_add(2)", "async func() -> stream"), + "can't call `wrapping_add` on a literal", + ), + ( + wit( + "|| ($i as u8).saturating_add(2)", + "async func() -> stream", + ), + "unknown method `saturating_add`", + ), + ( + wit("|| { let x = 1; x }", "async func() -> stream"), + "the type of a `let` bound to a literal must be given", + ), + ( + wit("|| { let x: u8 = 1 x }", "async func() -> stream"), + "expected `;`", + ), + ( + wit( + "|| ($i as f64).rotate_left(1)", + "async func() -> stream", + ), + "`rotate_left` isn't supported for f64", + ), + ( + wit("|| $i as u9", "async func() -> stream"), + "expected a type", + ), + ( + wit("|| { let as: u8 = 1; as }", "async func() -> stream"), + "expected a name to bind", + ), + ] { + let err = format!("{:#}", build(&wit, None).expect_err(&wit)); + assert!(err.contains(expected), "expected {expected:?} in {err:?}"); + } + + // overrides are rejected, even with listed items + let err = format!( + "{:#}", + build(SEEDED_WIT, Some("{stepped: [5]}")).expect_err("overrides") + ); + assert!(err.contains("can't be overridden"), "{err}"); +} + +/// Wrapping operators on every integer type, against Rust's `wrapping_*` +/// methods: every pair of operands for 8-bit types, and pseudorandom pairs +/// for wider types, both derived from the position. +#[test] +fn it_wraps_with_wrapping_operators_as_rust() -> Result<()> { + const OPS: [(&str, &str); 8] = [ + ("add", "+%"), + ("sub", "-%"), + ("mul", "*%"), + ("div", "/%"), + ("rem", "%%"), + ("shl", "<<%"), + ("shr", ">>%"), + ("neg", "-%"), + ]; + let types = [ + ("u8", "u8", true), + ("s8", "i8", true), + ("u16", "u16", false), + ("s16", "i16", false), + ("u32", "u32", false), + ("s32", "i32", false), + ("u64", "u64", false), + ("s64", "i64", false), + ]; + let mut wit = String::from("package a:b; world w {\n"); + for (wit_ty, rust_ty, exhaustive) in types { + let (a, b) = match exhaustive { + true => ( + format!("($i >> 8) as u8 as {rust_ty}"), + format!("$i as u8 as {rust_ty}"), + ), + false => ( + format!("($i *% 0x9E3779B97F4A7C15 ^ $i >> 7) as {rust_ty}"), + format!("($i *% 0xD1B54A32D192ED03 ^ $i >> 3) as {rust_ty}"), + ), + }; + for (name, op) in OPS { + let value = match name { + "neg" => "-%a".to_string(), + // a zero divisor still ends the stream + "div" | "rem" => format!("a {op} if b == 0 {{ 1 }} else {{ b }}"), + _ => format!("a {op} b"), + }; + wit.push_str(&format!( + "/// @value []\n/// @expression || {{ let a = {a}; let b = {b}; {value} }}\n\ + export {name}-{wit_ty}: async func() -> stream<{wit_ty}>;\n" + )); + } + } + wit.push('}'); + let component = build(&wit, None)?; + + macro_rules! check { + ($wit_ty:literal, $t:ty, $limit:expr, $a:expr, $b:expr) => {{ + let a = |i: u64| -> $t { $a(i) }; + let b = |i: u64| -> $t { $b(i) }; + let nonzero = |i: u64| -> $t { if b(i) == 0 { 1 } else { b(i) } }; + let expected: [( + &str, + fn(u64, &dyn Fn(u64) -> $t, &dyn Fn(u64) -> $t, &dyn Fn(u64) -> $t) -> $t, + ); 8] = [ + ("add", |i, a, b, _| a(i).wrapping_add(b(i))), + ("sub", |i, a, b, _| a(i).wrapping_sub(b(i))), + ("mul", |i, a, b, _| a(i).wrapping_mul(b(i))), + ("div", |i, a, _, n| a(i).wrapping_div(n(i))), + ("rem", |i, a, _, n| a(i).wrapping_rem(n(i))), + ("shl", |i, a, b, _| a(i).wrapping_shl(b(i) as u32)), + ("shr", |i, a, b, _| a(i).wrapping_shr(b(i) as u32)), + ("neg", |i, a, _, _| a(i).wrapping_neg()), + ]; + for (name, f) in expected { + let export = format!("{name}-{}", $wit_ty); + let actual = read_streams_until::<$t>(&component, ("", &export), 4096, 1, $limit)?; + let expected: Vec<$t> = + (0..$limit as u64).map(|i| f(i, &a, &b, &nonzero)).collect(); + assert_eq!(actual, vec![expected; 2], "{export}"); + } + }}; + } + let mix_a = |i: u64| i.wrapping_mul(0x9E3779B97F4A7C15) ^ (i >> 7); + let mix_b = |i: u64| i.wrapping_mul(0xD1B54A32D192ED03) ^ (i >> 3); + check!("u8", u8, 65536, |i: u64| (i >> 8) as u8, |i: u64| i as u8); + check!( + "s8", + i8, + 65536, + |i: u64| (i >> 8) as u8 as i8, + |i: u64| i as u8 as i8 + ); + check!("u16", u16, 5000, |i| mix_a(i) as u16, |i| mix_b(i) as u16); + check!("s16", i16, 5000, |i| mix_a(i) as i16, |i| mix_b(i) as i16); + check!("u32", u32, 5000, |i| mix_a(i) as u32, |i| mix_b(i) as u32); + check!("s32", i32, 5000, |i| mix_a(i) as i32, |i| mix_b(i) as i32); + check!("u64", u64, 5000, mix_a, mix_b); + check!("s64", i64, 5000, |i| mix_a(i) as i64, |i| mix_b(i) as i64); + Ok(()) +} + +#[test] +fn it_wraps_the_minimum_divided_by_minus_one() -> Result<()> { + let component = build( + "package a:b; world w { + /// @value [] + /// @expression || { let a: i64 = -9223372036854775808; a /% -1 } + export div: async func() -> stream; + /// @value [] + /// @expression || { let a: i64 = -9223372036854775808; a %% -1 } + export rem: async func() -> stream; + /// @value [] + /// @expression || { let a: i32 = -2147483648; a /% -1 } + export div32: async func() -> stream; + /// @value [] + /// @expression || $i as u8 /% 0 + export by-zero: async func() -> stream; + }", + None, + )?; + assert_eq!(generated::(&component, "div", 2)?, vec![i64::MIN; 2]); + assert_eq!(generated::(&component, "rem", 2)?, vec![0; 2]); + assert_eq!(generated::(&component, "div32", 2)?, vec![i32::MIN; 2]); + assert_eq!(generated::(&component, "by-zero", 2)?, Vec::::new()); + Ok(()) +} + +#[test] +fn it_rejects_wrapping_operators_on_floats_and_bools() { + for (ty, expression, expected) in [ + ("f64", "|| $i as f64 +% 1.0", "`+%` isn't supported for f64"), + ("bool", "|| true *% false", "`*%` isn't supported for bool"), + ("f32", "|| -%($i as f32)", "`-%` isn't supported for f32"), + ("f64", "|| $i as f64 <<% 1", "`<<%` isn't supported for f64"), + ] { + let wit = format!( + "package a:b; world w {{ /// @value []\n/// @expression {expression}\nexport f: async func() -> stream<{ty}>; }}" + ); + let err = format!("{:#}", build(&wit, None).expect_err(&wit)); + assert!(err.contains(expected), "expected {expected:?} in {err:?}"); + } +} + +const ARRAYS_WIT: &str = "package a:b; + world w { + /// @expression || { + /// let z = $init.0 +% ($i / 8 +% 1) *% 0x9E3779B97F4A7C15; + /// let z = (z ^ z >> 30) *% 0xBF58476D1CE4E5B9; + /// let z = (z ^ z >> 27) *% 0x94D049BB133111EB; + /// (z ^ z >> 31).to_le_bytes() + /// } + export pseudorandom-bytes: async func(seed: u64) -> stream; + /// @value [] + /// @expression || ($i as u32 *% 0x01020304).to_be_bytes() + export big-endian: async func() -> stream; + /// @value [] + /// @expression || ($i as f64 / 8.0).to_le_bytes() + export float-bytes: async func() -> stream; + /// @value [1, 1] + /// @expression |a, b| [a + b, a + b + b] + export fibonacci-pairs: async func() -> stream; + /// @value [] + /// @expression || [$i, $i + 1, $i + 2] + export positions: async func() -> stream; + /// @value [] + /// @expression || if $i / 2 % 2 == 0 { [1, 2] } else { [3, 4] } + export alternating: async func() -> stream; + }"; + +#[test] +fn it_generates_several_items_from_an_array() -> Result<()> { + let component = build(ARRAYS_WIT, None)?; + for seed in [0u64, 42] { + let expected: Vec = (0..1000) + .flat_map(|n| splitmix(seed, n).to_le_bytes()) + .collect(); + for per_read in [4096, 3] { + let actual = read_streams_with::<_, u8>( + &component, + ("", "pseudorandom-bytes"), + (seed,), + per_read, + 2, + 8000, + )?; + assert_eq!( + actual, + vec![expected.clone(); 4], + "seed {seed}, {per_read} per read" + ); + } + } + let expected: Vec = (0..250u64) + .flat_map(|g| ((g * 4) as u32).wrapping_mul(0x01020304).to_be_bytes()) + .collect(); + assert_eq!(generated::(&component, "big-endian", 1000)?, expected); + let expected: Vec = (0..125u64) + .flat_map(|g| ((g * 8) as f64 / 8.0).to_le_bytes()) + .collect(); + assert_eq!(generated::(&component, "float-bytes", 1000)?, expected); + // a pair is generated whole, or not at all + let mut expected = vec![1u32, 1]; + loop { + let (a, b) = (expected[expected.len() - 2], expected[expected.len() - 1]); + match ( + a.checked_add(b), + a.checked_add(b).and_then(|c| c.checked_add(b)), + ) { + (Some(c), Some(d)) => expected.extend([c, d]), + _ => break, + } + } + assert_eq!( + generated::(&component, "fibonacci-pairs", usize::MAX)?, + expected + ); + // `$i` is the position of each group's first item + assert_eq!( + generated::(&component, "positions", 1000)?, + (0..1000).collect::>() + ); + assert_eq!( + generated::(&component, "alternating", 8)?, + vec![1, 2, 3, 4, 1, 2, 3, 4] + ); + Ok(()) +} + +#[test] +fn it_rejects_invalid_arrays() { + let wit = |ty: &str, expression: &str| { + format!( + "package a:b; world w {{ /// @value []\n/// @expression {expression}\nexport f: async func() -> stream<{ty}>; }}" + ) + }; + for (wit, expected) in [ + ( + wit("u32", "|| $i.to_le_bytes()"), + "`to_le_bytes` gives bytes, the stream's items must be u8, not u32", + ), + ( + wit("u8", "|| 1 + [2]"), + "an array can only be the closure's result", + ), + ( + wit("u8", "|| { let a = [1, 2]; a }"), + "an array can only be the closure's result", + ), + ( + wit("u8", "|| if $i == 0 { [1, 2] } else { [3] }"), + "the `if` branches give 2 and 1 items", + ), + (wit("u8", "|| []"), "an array must have at least one item"), + ( + wit("u8", "|| $i.to_le_bytes(1)"), + "`to_le_bytes` takes no arguments", + ), + ( + wit("u8", "|| ($i as u8).wrapping_add()"), + "`wrapping_add` takes an argument", + ), + ( + wit("u8", "|| ($i == 0).to_be_bytes()"), + "`to_be_bytes` isn't supported for bool", + ), + ( + wit("u8", "|| [1, 256]"), + "literal 256 is out of range for u8", + ), + ] { + let err = format!("{:#}", build(&wit, None).expect_err(&wit)); + assert!(err.contains(expected), "expected {expected:?} in {err:?}"); + } +} + +const LOOPS_WIT: &str = "package a:b; + world w { + /// @value [2] + /// @expression |p| { + /// let mut n = p + 1; + /// let mut d: u64 = 2; + /// while d * d <= n { + /// if n % d == 0 { + /// n += 1; + /// d = 2; + /// } else { + /// d += 1; + /// } + /// } + /// n + /// } + export primes: async func() -> stream; + /// @value [] + /// @expression || { + /// let mut n = $i + 1; + /// let mut steps: u64 = 0; + /// while n != 1 { + /// if n % 2 == 0 { n /= 2; } else { n = 3 * n + 1; } + /// steps += 1; + /// } + /// steps + /// } + export collatz: async func() -> stream; + /// @value [] + /// @expression || { + /// let mut f: u64 = 1; + /// let mut k: u64 = 1; + /// while k <= $i { f *= k; k += 1; } + /// f + /// } + export factorials: async func() -> stream; + /// @value [] + /// @expression || { + /// let mut x = $i; + /// x *%= 0x9E3779B97F4A7C15; + /// x ^= x >> 29; + /// x +%= 7; + /// x <<%= 67; + /// x + /// } + export hash: async func() -> stream; + /// @value [] + /// @expression || { + /// let n = $i + 1; + /// let mut sum: u64 = 0; + /// let mut d: u64 = 1; + /// while d <= n { + /// let mut m = n; + /// while m >= d { m -= d; } + /// if m == 0 { sum += d; } + /// d += 1; + /// } + /// sum + /// } + export divisor-sums: async func() -> stream; + /// @value [] + /// @expression || { + /// let mut sign: i8 = 0; + /// let x = $i as i8 - 2; + /// if x < 0 { sign = -1; } else if x > 0 { sign = 1; } + /// sign + /// } + export signs: async func() -> stream; + }"; + +#[test] +fn it_loops() -> Result<()> { + let component = build(LOOPS_WIT, None)?; + let is_prime = |n: u64| { + n >= 2 + && (2..) + .take_while(|d| d * d <= n) + .all(|d| !n.is_multiple_of(d)) + }; + let primes: Vec = (2..).filter(|n| is_prime(*n)).take(2000).collect(); + assert_eq!(generated::(&component, "primes", 2000)?, primes); + let collatz = |mut n: u64| { + let mut steps = 0; + while n != 1 { + n = if n.is_multiple_of(2) { + n / 2 + } else { + 3 * n + 1 + }; + steps += 1; + } + steps + }; + assert_eq!( + generated::(&component, "collatz", 500)?, + (1..=500).map(collatz).collect::>() + ); + // overflow inside a loop ends the stream, after 20! + let factorials: Vec = (0..=20u64).map(|n| (1..=n).product()).collect(); + assert_eq!( + generated::(&component, "factorials", usize::MAX)?, + factorials + ); + let hash = |i: u64| { + let mut x = i.wrapping_mul(0x9E3779B97F4A7C15); + x ^= x >> 29; + x.wrapping_add(7).wrapping_shl(67) + }; + assert_eq!( + generated::(&component, "hash", 300)?, + (0..300).map(hash).collect::>() + ); + let divisor_sum = |n: u64| (1..=n).filter(|d| n.is_multiple_of(*d)).sum::(); + assert_eq!( + generated::(&component, "divisor-sums", 60)?, + (1..=60).map(divisor_sum).collect::>() + ); + assert_eq!( + generated::(&component, "signs", 5)?, + vec![-1, -1, 0, 1, 1] + ); + Ok(()) +} + +#[test] +fn it_rejects_invalid_loops_and_assignments() { + let wit = |expression: &str| { + format!( + "package a:b; world w {{ /// @value [1]\n/// @expression {expression}\nexport f: async func() -> stream; }}" + ) + }; + for (expression, expected) in [ + ( + "|n| { let x = n; x = 2; x }", + "can't assign to `x` at 1:18, declare it with `let mut x`", + ), + ("|n| { n = 2; n }", "can't assign to the parameter `n`"), + ( + "|n| { let mut x = n; x = true; x }", + "expected u8 for an assignment, found bool", + ), + ( + "|n| { let mut x = n; x += 256; x }", + "literal 256 is out of range for u8", + ), + ( + "|n| { while n { } n }", + "expected bool for a `while` condition, found u8", + ), + ( + "|n| { while n > 1 { n } n }", + "a `while` loop's or an `if` statement's block can't end with a value", + ), + ( + "|n| { if n > 1 { n } n }", + "a `while` loop's or an `if` statement's block can't end with a value", + ), + ( + "|n| { let mut x = n; while x > 1 { x -= 1; } }", + "expected a value at the end of the block", + ), + ( + "|n| { if n > 1 { 1 } else { 2 } + 1 }", + "an `if` with values must be the block's value", + ), + ("|n| { y = 1; n }", "unknown name `y`"), + ("|n| { let mut = n; n }", "expected a name to bind"), + ( + "|n| { let x = { let mut y = n; y += 1; }; x }", + "expected a value at the end of the block", + ), + ] { + let wit = wit(expression); + let err = format!("{:#}", build(&wit, None).expect_err(expression)); + assert!( + err.contains(expected), + "{expression}: expected {expected:?} in {err:?}" + ); + } +} + +const POW_WIT: &str = "package a:b; + world w { + /// @value [] + /// @expression || (3 as u64) ** ($i as u32) + export powers-u64: async func() -> stream; + /// @value [] + /// @expression || (-3 as i64) ** ($i as u32) + export powers-s64: async func() -> stream; + /// @value [] + /// @expression || (2 as u8) ** ($i as u32) + export powers-u8: async func() -> stream; + /// @value [] + /// @expression || (-2 as i8) ** ($i as u32) + export powers-s8: async func() -> stream; + /// @value [] + /// @expression || (7 as u32) ** ($i as u32) + export powers-u32: async func() -> stream; + /// @value [] + /// @expression || (-5 as i16) ** ($i as u32) + export powers-s16: async func() -> stream; + /// @value [] + /// @expression || (1.5 as f64) ** ($i as i32 - 40) + export powi-f64: async func() -> stream; + /// @value [] + /// @expression || (10 as f32) ** ($i as i32) + export powi-f32: async func() -> stream; + /// @value [] + /// @expression || { let x = $i as i32; -x ** 2 } + export negated: async func() -> stream; + /// @value [] + /// @expression || (2 as u32) ** 3 ** ($i as u32) + export right-associative: async func() -> stream; + /// @value [] + /// @expression || { let mut x = ($i + 2) as u32; x **= 2; x **%= 3; x } + export compound: async func() -> stream; + /// @expression || $init.0 * ($init.1 ** $i) + export geometric: async func(n: u64, r: u64) -> stream; + /// @value [] + /// @expression || (2 as i64) ** ($i as i64 - 1) + export negative-exponent: async func() -> stream; + /// @value [] + /// @expression || (2 as f64) ** ($i as i64 - 3) + export float-wide-exponent: async func() -> stream; + /// @value [] + /// @expression || (1 as u8) ** ($i * 0x1_0000_0001) + export huge-exponent: async func() -> stream; + }"; + +#[test] +fn it_raises_to_powers_as_rust() -> Result<()> { + let c = build(POW_WIT, None)?; + // checked powers end the stream at the first that overflows + fn powers(base: T, pow: impl Fn(T, u32) -> Option) -> Vec { + (0..).map_while(|e| pow(base, e)).collect() + } + let all = usize::MAX; + assert_eq!( + generated::(&c, "powers-u64", all)?, + powers(3u64, u64::checked_pow) + ); + assert_eq!( + generated::(&c, "powers-s64", all)?, + powers(-3i64, i64::checked_pow) + ); + assert_eq!( + generated::(&c, "powers-u8", all)?, + powers(2u8, u8::checked_pow) + ); + assert_eq!( + generated::(&c, "powers-s8", all)?, + powers(-2i8, i8::checked_pow) + ); + assert_eq!( + generated::(&c, "powers-u32", all)?, + powers(7u32, u32::checked_pow) + ); + assert_eq!( + generated::(&c, "powers-s16", all)?, + powers(-5i16, i16::checked_pow) + ); + + // floats, as `powi`, within rounding, the stream ends when a power isn't finite + let close = |actual: &[f64], expected: &[f64]| { + assert_eq!(actual.len(), expected.len()); + for (a, e) in actual.iter().zip(expected) { + assert!((a - e).abs() <= e.abs() * 1e-12, "{a} != {e}"); + } + }; + let actual = generated::(&c, "powi-f64", 100)?; + let expected: Vec = (0..100).map(|i| 1.5f64.powi(i - 40)).collect(); + close(&actual, &expected); + let actual: Vec = generated::(&c, "powi-f32", all)? + .into_iter() + .map(f64::from) + .collect(); + let expected: Vec = (0..) + .map(|i| 10f32.powi(i)) + .take_while(|x| x.is_finite()) + .map(f64::from) + .collect(); + close(&actual, &expected); + + // `**` binds tighter than a unary operator on its left, and associates right + assert_eq!( + generated::(&c, "negated", 5)?, + (0..5).map(|x: i32| -(x * x)).collect::>() + ); + assert_eq!( + generated::(&c, "right-associative", 3)?, + vec![2, 8, 512] + ); + // the exponent may be any integer + let geometric = + read_streams_with::<_, u64>(&c, ("", "geometric"), (5u64, 3u64), 64, 1, usize::MAX)?; + let expected: Vec = (0..) + .map_while(|e| 3u64.checked_pow(e).and_then(|p| 5u64.checked_mul(p))) + .collect(); + assert_eq!(geometric, vec![expected; 2]); + // an integer's negative power ends the stream + assert_eq!( + generated::(&c, "negative-exponent", 3)?, + Vec::::new() + ); + assert_eq!( + generated::(&c, "float-wide-exponent", 6)?, + vec![0.125, 0.25, 0.5, 1.0, 2.0, 4.0] + ); + // exponents past u32 + assert_eq!(generated::(&c, "huge-exponent", 3)?, vec![1, 1, 1]); + assert_eq!( + generated::(&c, "compound", 50)?, + (0..50u32) + .map(|i| ((i + 2) * (i + 2)).wrapping_pow(3)) + .collect::>() + ); + Ok(()) +} + +/// `**%` on every integer type, against Rust's `wrapping_pow`: every base and +/// exponent below 256 for 8-bit types, pseudorandom pairs for wider types. +#[test] +fn it_wraps_powers_as_rust() -> Result<()> { + let types = [ + ("u8", "u8", true), + ("s8", "i8", true), + ("u16", "u16", false), + ("s32", "i32", false), + ("u64", "u64", false), + ("s64", "i64", false), + ]; + let mut wit = String::from("package a:b; world w {\n"); + for (wit_ty, rust_ty, exhaustive) in types { + let base = match exhaustive { + true => format!("($i >> 8) as u8 as {rust_ty}"), + false => format!("($i *% 0x9E3779B97F4A7C15 ^ $i >> 7) as {rust_ty}"), + }; + wit.push_str(&format!( + "/// @value []\n/// @expression || ({base}) **% ($i as u8 as u32)\n\ + export pow-{wit_ty}: async func() -> stream<{wit_ty}>;\n" + )); + } + wit.push('}'); + let component = build(&wit, None)?; + let mix = |i: u64| i.wrapping_mul(0x9E3779B97F4A7C15) ^ (i >> 7); + macro_rules! check { + ($wit_ty:literal, $t:ty, $limit:expr, $base:expr) => {{ + let export = format!("pow-{}", $wit_ty); + let actual = read_streams_until::<$t>(&component, ("", &export), 4096, 1, $limit)?; + let expected: Vec<$t> = (0..$limit as u64) + .map(|i| ($base(i) as $t).wrapping_pow(i as u8 as u32)) + .collect(); + assert_eq!(actual, vec![expected; 2], "{export}"); + }}; + } + check!("u8", u8, 65536, |i: u64| (i >> 8) as u8); + check!("s8", i8, 65536, |i: u64| (i >> 8) as u8 as i8); + check!("u16", u16, 5000, mix); + check!("s32", i32, 5000, mix); + check!("u64", u64, 5000, mix); + check!("s64", i64, 5000, mix); + Ok(()) +} + +#[test] +fn it_rejects_invalid_powers() { + let wit = |ty: &str, expression: &str| { + format!( + "package a:b; world w {{ /// @value []\n/// @expression {expression}\nexport f: async func() -> stream<{ty}>; }}" + ) + }; + for (wit, expected) in [ + ( + wit("u64", "|| $i ** ($i as f64)"), + "the exponent of `**` must be an integer, found f64", + ), + ( + wit("f64", "|| ($i as f64) ** ($i == 0)"), + "the exponent of `**` must be an integer, found bool", + ), + (wit("u64", "|| $i ** 1.5"), "expected u32, found float 1.5"), + ( + wit("f64", "|| ($i as f64) **% 2"), + "`**%` isn't supported for f64", + ), + (wit("bool", "|| true ** 2"), "`**` expects numeric operands"), + (wit("u64", "|| $i as u64 ** 2"), "unexpected `**`"), + (wit("u64", "|| $i **"), "expected an expression at the end"), + ] { + let err = format!("{:#}", build(&wit, None).expect_err(&wit)); + assert!(err.contains(expected), "expected {expected:?} in {err:?}"); + } +} + +#[test] +fn it_reads_seed_fields_by_name() -> Result<()> { + let component = build( + "package a:b; world w { + /// @expression || $init.n * ($init.r ** $i) + export geometric: async func(n: u64, r: u64) -> stream; + /// @expression || $init.start_value + $init.1 * $i + export arithmetic: async func(start-value: u64, step: u64) -> stream; + /// @expression || { + /// let mut total = $init.base; + /// let mut k: u64 = 0; + /// while k < $i { total += $init.step; k += 1; } + /// total + /// } + export looped: async func(base: u64, step: u64) -> stream; + }", + None, + )?; + let geometric = + read_streams_with::<_, u64>(&component, ("", "geometric"), (5u64, 3u64), 64, 1, 4)?; + assert_eq!(geometric, vec![vec![5, 15, 45, 135]; 2]); + let arithmetic = + read_streams_with::<_, u64>(&component, ("", "arithmetic"), (10u64, 4u64), 64, 1, 4)?; + assert_eq!(arithmetic, vec![vec![10, 14, 18, 22]; 2]); + let looped = read_streams_with::<_, u64>(&component, ("", "looped"), (1u64, 2u64), 64, 1, 4)?; + assert_eq!(looped, vec![vec![1, 3, 5, 7]; 2]); + + let err = format!( + "{:#}", + build( + "package a:b; world w { + /// @expression || $init.m * $i + export f: async func(n: u64, start-value: u64) -> stream; + }", + None, + ) + .expect_err("unknown field") + ); + assert!( + err.contains( + "`$init` has no field `m` at 1:10, the function's parameters are `n`, `start_value`" + ), + "{err}" + ); + Ok(()) +} + +#[test] +fn it_reads_arguments_by_name() -> Result<()> { + let component = build( + "package a:b; world w { + /// @expression || n * (r ** $i) + export geometric: async func(n: u64, r: u64) -> stream; + /// @expression || { let n = n + 1; n * $i } + export shadowed: async func(n: u64) -> stream; + /// @expression || start_value + $i + export kebab: async func(start-value: u64) -> stream; + /// @value [0] + /// @expression |v| v + step + export stepping: async func(step: u64) -> stream; + /// @expression |n| n + 1 + export items: async func(n: u64) -> stream; + }", + None, + )?; + let read = |name: &str, args: (u64,)| { + read_streams_with::<_, u64>(&component, ("", name), args, 64, 1, 4) + }; + let geometric = + read_streams_with::<_, u64>(&component, ("", "geometric"), (5u64, 3u64), 64, 1, 4)?; + assert_eq!(geometric, vec![vec![5, 15, 45, 135]; 2]); + // a `let` shadows an argument, which is never changed + assert_eq!(read("shadowed", (2,))?, vec![vec![0, 3, 6, 9]; 2]); + assert_eq!(read("kebab", (10,))?, vec![vec![10, 11, 12, 13]; 2]); + // read by name, the arguments aren't items, the stream starts from `@value` + assert_eq!(read("stepping", (5,))?, vec![vec![0, 5, 10, 15]; 2]); + // a closure parameter shadows an argument, the arguments are the first items + assert_eq!(read("items", (7,))?, vec![vec![7, 8, 9, 10]; 2]); + + let err = format!( + "{:#}", + build( + "package a:b; world w { + /// @expression || { n = 1; n } + export f: async func(n: u64) -> stream; + }", + None, + ) + .expect_err("assigned argument") + ); + assert!( + err.contains("can't assign to the argument `n` at 1:6, arguments never change"), + "{err}" + ); + Ok(()) +} + +#[test] +fn it_rejects_shadowing_or_assigning_builtin_variables() { + for (expression, expected) in [ + ("|n| { let $i = 1; n }", "`$i` can't be bound at 1:11"), + ( + "|n| { let mut $init = 1; n }", + "`$init` can't be bound at 1:15", + ), + ("|$i| $i", "`$i` can't be bound at 1:2"), + ("|n| { $i = 1; n }", "`$i` can't be assigned at 1:7"), + ("|n| { $i += 1; n }", "`$i` can't be assigned at 1:7"), + ("|n| { $init.0 = 1; n }", "`$init` can't be assigned at 1:7"), + ( + "|n| { $init.x *%= 2; n }", + "`$init` can't be assigned at 1:7", + ), + ] { + let wit = format!( + "package a:b; world w {{ /// @value [1]\n/// @expression {expression}\nexport f: async func(x: u64) -> stream; }}" + ); + let err = format!("{:#}", build(&wit, None).expect_err(expression)); + assert!( + err.contains(expected), + "{expression}: expected {expected:?} in {err:?}" + ); + } +} + +#[test] +fn it_never_changes_seed_or_position() -> Result<()> { + let component = build( + "package a:b; world w { + /// @expression || { + /// let mut copy = $i; + /// copy += 100; + /// let mut seed = $init.0; + /// seed *= 3; + /// let mut n = n; + /// n += 1; + /// [$i, $init.0, $init.n, n, copy + seed] + /// } + export copies: async func(n: u64) -> stream; + }", + None, + )?; + // each evaluation sees the arguments unchanged, and its own position, however + // its copies changed, here and in earlier evaluations + let items = read_streams_with::<_, u64>(&component, ("", "copies"), (7u64,), 64, 1, 15)?; + let expected: Vec = (0..3u64) + .flat_map(|g| { + let i = g * 5; + [i, 7, 7, 8, i + 100 + 21] + }) + .collect(); + assert_eq!(items, vec![expected; 2]); + Ok(()) +} + +#[test] +fn it_reads_the_closures_parameters_from_call() -> Result<()> { + let component = build( + "package a:b; world w { + /// @value [1, 1] + /// @expression |a, b| { let a: u64 = 100; $call.a + b + a - 100 } + export shadowed: async func() -> stream; + /// @value [1, 1] + /// @expression |_, b| $call.0 + b + export unnamed: async func() -> stream; + /// @value [1, 1] + /// @expression |a, b| a + $call.b + export named: async func() -> stream; + }", + None, + )?; + let fibonacci = vec![1u64, 1, 2, 3, 5, 8, 13, 21]; + for name in ["shadowed", "unnamed", "named"] { + assert_eq!(generated::(&component, name, 8)?, fibonacci, "{name}"); + } + + for (expression, expected) in [ + ( + "|a| $call.1", + "`$call.1` doesn't exist at 1:11, `$call` has 1 fields", + ), + ( + "|a, _| $call.b", + "`$call` has no field `b` at 1:14, the closure's parameters are `a`", + ), + ( + "|| $call.0", + "`$call.0` doesn't exist at 1:10, `$call` has 0 fields", + ), + ( + "|a| $call", + "expected `.` and a field, `$call` is the closure's parameters", + ), + ("|a| { let $call = 1; a }", "`$call` can't be bound at 1:11"), + ("|a| { $call.a = 1; a }", "`$call` can't be assigned at 1:7"), + ( + "|a| $args.0", + "unknown variable `$args` at 1:5, the variables are `$init`, `$call` and `$i`", + ), + ] { + let wit = format!( + "package a:b; world w {{ /// @value [1]\n/// @expression {expression}\nexport f: async func() -> stream; }}" + ); + let err = format!("{:#}", build(&wit, None).expect_err(expression)); + assert!( + err.contains(expected), + "{expression}: expected {expected:?} in {err:?}" + ); + } + Ok(()) +} diff --git a/crates/test-harness/Cargo.toml b/crates/test-harness/Cargo.toml index f253683..67ff7c3 100644 --- a/crates/test-harness/Cargo.toml +++ b/crates/test-harness/Cargo.toml @@ -7,4 +7,5 @@ publish = false [dependencies] anyhow = { workspace = true } +futures = { workspace = true } wasmtime = { workspace = true } diff --git a/crates/test-harness/src/lib.rs b/crates/test-harness/src/lib.rs index be794cf..0b3914a 100644 --- a/crates/test-harness/src/lib.rs +++ b/crates/test-harness/src/lib.rs @@ -2,15 +2,21 @@ //! `target/components/` by make. use anyhow::{Context, Result, bail}; +use futures::channel::oneshot; use std::{ collections::HashSet, path::{Path, PathBuf}, + pin::Pin, process::Command, - sync::Mutex, + sync::{Arc, Mutex}, + task::{Context as TaskContext, Poll}, }; use wasmtime::{ - Config, Engine, Store, - component::{Component, Linker, Val}, + Config, Engine, Store, StoreContextMut, + component::{ + Component, ComponentNamedList, Func, Instance, Lift, Linker, Lower, Source, StreamConsumer, + StreamReader, StreamResult, Val, + }, }; /// Calls each named export (`interface` may be empty for world-level @@ -35,6 +41,130 @@ pub fn call_with( link: impl FnOnce(&mut Linker<()>) -> Result<()>, calls: &[(&str, &str)], ) -> Result> { + let (mut store, instance) = instantiate(bytes, link)?; + let mut results = vec![]; + for (interface, name) in calls { + let func = export(&mut store, &instance, interface, name); + let mut result = [Val::Bool(false)]; + func.call(&mut store, &[], &mut result)?; + results.push(result[0].clone()); + } + Ok(results) +} + +/// Calls the named export, which returns a `stream`, and reads the stream +/// until the writer closes it. Each read takes at most `per_read` items, so a +/// writer sees reads that take only some of the items it offers. +/// +/// The export is called `calls` times at once, then as many times again once +/// those streams are closed, returning the items of each stream in call order. +pub fn read_streams( + bytes: &[u8], + export: (&str, &str), + per_read: usize, + calls: usize, +) -> Result>> { + read_streams_until(bytes, export, per_read, calls, usize::MAX) +} + +/// Like [`read_streams`], closing each stream once `limit` items are read. +pub fn read_streams_until( + bytes: &[u8], + export: (&str, &str), + per_read: usize, + calls: usize, + limit: usize, +) -> Result>> { + read_streams_with(bytes, export, (), per_read, calls, limit) +} + +/// Like [`read_streams_until`], calling the export with `params`. +pub fn read_streams_with( + bytes: &[u8], + (interface, name): (&str, &str), + params: P, + per_read: usize, + calls: usize, + limit: usize, +) -> Result>> +where + P: ComponentNamedList + Lower + Clone + Send + Sync + 'static, + T: Lift + Send + Sync + 'static, +{ + let (mut store, instance) = instantiate(bytes, |_| Ok(()))?; + let func = export(&mut store, &instance, interface, name); + let func = func.typed::,)>(&store)?; + let streams = + futures::executor::block_on(store.run_concurrent(async |accessor| -> Result<_> { + let mut streams = vec![]; + for _ in 0..2 { + let mut round = vec![]; + for _ in 0..calls { + let (reader,) = func.call_concurrent(accessor, params.clone()).await?; + let items = Arc::new(Mutex::new(vec![])); + let (closed, on_close) = oneshot::channel(); + let collect = Collect { + items: items.clone(), + per_read, + limit, + closed: Some(closed), + }; + accessor.with(|mut store| reader.pipe(&mut store, collect))?; + round.push((items, on_close)); + } + for (items, on_close) in round { + on_close.await?; + streams.push(std::mem::take(&mut *items.lock().unwrap())); + } + } + Ok(streams) + }))??; + Ok(streams) +} + +/// Collects a stream's items, signalling once the stream is closed, which +/// drops the consumer. The consumer closes the stream once `limit` items are +/// read. +struct Collect { + items: Arc>>, + per_read: usize, + limit: usize, + closed: Option>, +} + +impl StreamConsumer for Collect { + type Item = T; + + fn poll_consume( + self: Pin<&mut Self>, + _cx: &mut TaskContext<'_>, + mut store: StoreContextMut, + mut source: Source<'_, T>, + _finish: bool, + ) -> Poll> { + let mut items = self.items.lock().unwrap(); + let mut buffer = Vec::with_capacity(self.per_read.min(self.limit - items.len())); + source.read(&mut store, &mut buffer)?; + items.extend(buffer); + Poll::Ready(Ok(match items.len() < self.limit { + true => StreamResult::Completed, + false => StreamResult::Dropped, + })) + } +} + +impl Drop for Collect { + fn drop(&mut self) { + if let Some(closed) = self.closed.take() { + let _ = closed.send(()); + } + } +} + +fn instantiate( + bytes: &[u8], + link: impl FnOnce(&mut Linker<()>) -> Result<()>, +) -> Result<(Store<()>, Instance)> { let mut config = Config::new(); config.wasm_component_model_fixed_length_lists(true); // types that values can't reach may still appear in result types @@ -46,27 +176,24 @@ pub fn call_with( link(&mut linker)?; let mut store = Store::new(&engine, ()); let component = Component::from_binary(&engine, bytes)?; - let instance = linker.instantiate(&mut store, &component)?; + let instance = futures::executor::block_on(linker.instantiate_async(&mut store, &component))?; + Ok((store, instance)) +} - let mut results = vec![]; - for (interface, name) in calls { - let parent = match *interface { - "" => None, - interface => Some( - instance - .get_export_index(&mut store, None, interface) - .expect("interface export"), - ), - }; - let index = instance - .get_export_index(&mut store, parent.as_ref(), name) - .unwrap_or_else(|| panic!("missing export {name}")); - let func = instance.get_func(&mut store, index).expect("func export"); - let mut result = [Val::Bool(false)]; - func.call(&mut store, &[], &mut result)?; - results.push(result[0].clone()); - } - Ok(results) +/// The named export, `interface` may be empty for world-level functions. +fn export(store: &mut Store<()>, instance: &Instance, interface: &str, name: &str) -> Func { + let parent = match interface { + "" => None, + interface => Some( + instance + .get_export_index(&mut *store, None, interface) + .expect("interface export"), + ), + }; + let index = instance + .get_export_index(&mut *store, parent.as_ref(), name) + .unwrap_or_else(|| panic!("missing export {name}")); + instance.get_func(&mut *store, index).expect("func export") } /// Root directory of the workspace. diff --git a/docs/expressions.md b/docs/expressions.md new file mode 100644 index 0000000..c6e72ef --- /dev/null +++ b/docs/expressions.md @@ -0,0 +1,120 @@ +# Expressions + +An `@expression` generates the items of a stream returned by an exported function. It's a closure written in a subset of Rust, with a few additions, returning the stream's next item each time it's evaluated. + +This guide summarizes how to declare generated streams and what expressions can do. [grammars/constants-expression](../grammars/constants-expression/README.md) specifies expressions in full: their grammar, typing, evaluation and every diagnostic, with a tree-sitter grammar for editors and language servers. + +- [Generating a stream](#generating-a-stream) +- [Built-in variables](#built-in-variables) +- [Syntax](#syntax) +- [Types](#types) +- [When a stream ends](#when-a-stream-ends) +- [Several items at once](#several-items-at-once) + +## Generating a stream + +A function with an `@expression` must be `async` and return a `stream` of integers, floats or `bool`. Its items come from one of three places. + +**Listed items.** The closure's parameters are the stream's previous items, oldest first. The stream starts with the items `@value` lists, at least one for each parameter, and the closure continues from the last of them. Overrides replace the listed items, but not the expression. + +```wit +/// @value [1, 1] +/// @expression |a, b| a + b +fibonacci: async func() -> stream; +``` + +**Arguments as items.** The function takes the items the stream starts from as arguments, one for each of the closure's parameters, of the stream's item type. `fibonacci(1, 1)` streams `1, 1, 2, 3, 5, …`. + +```wit +/// @expression |a, b| a + b +fibonacci: async func(a: u64, b: u64) -> stream; +``` + +**Arguments as inputs.** The expression reads the function's arguments by name, or through `$init`. They're inputs, not items, so they can have any integer, float or `bool` type. The stream starts from an optional `@value`, or from nothing. + +```wit +/// @expression || n * r ** $i +geometric: async func(n: u64, r: u64) -> stream; +``` + +A function with arguments can't be overridden. + +## Built-in variables + +| Variable | Holds | Example | +| --- | --- | --- | +| `$init` | the arguments of the call that started the stream, the same for every item, by index or by name | `$init.0`, `$init.n` | +| `$call` | the closure's parameters, the arguments of the current evaluation, by index or by name | `$call.0`, `$call.a` | +| `$i` | the position in the stream of the item being generated, a `u64` | `$i` | + +Arguments can also be read directly by name, e.g. `n`, and closure parameters by their names. Those names can be shadowed by a `let`, or an argument by a closure parameter, but `$init`, `$call` and `$i` can't be. None of them ever change. Names with `-` in WIT use `_`: an argument `start-value` is `start_value`. + +## Syntax + +Expressions follow Rust's syntax and precedence, unless noted: + +- **Literals**: integers, in decimal, hex, octal or binary, floats, and `true` and `false`. Literals have no suffixes, they take their type from their context. +- **Arithmetic**: `+`, `-`, `*`, `/` and `%` (not for floats), and `**` for powers, as in Python, e.g. `r ** $i`, binding tighter than `*`, and associating right. +- **Wrapping arithmetic**, as in Zig: `+%`, `-%`, `*%`, `/%`, `%%`, `**%`, `<<%`, `>>%` and unary `-%` wrap around the type's range instead of ending the stream. +- **Bitwise and logical**: `&`, `|`, `^`, `<<`, `>>`, `!`, `&&` and `||`. +- **Comparisons**: `==`, `!=`, `<`, `<=`, `>` and `>=`, which don't chain. +- **Casts**: `as`, e.g. `$i as u8`, which truncate and saturate as in Rust. +- **Methods**: `wrapping_add`, `wrapping_sub`, `wrapping_mul`, `rotate_left` and `rotate_right`, and `to_le_bytes` and `to_be_bytes`. +- **Blocks**: `{ let a = …; a + 1 }`. `let mut` bindings can be assigned, `x = …;`, or with a compound assignment, `x += …;`, for any binary operator. +- **Control flow**: `if c { a } else { b }`, as a value or a statement, and `while` loops: + + ```wit + /// @value [2] + /// @expression |v| { + /// let mut n = v + 1; + /// let mut d: u64 = 2; + /// while d * d <= n { + /// if n % d == 0 { + /// n += 1; + /// d = 2; + /// } else { + /// d += 1; + /// } + /// } + /// n + /// } + prime: async func() -> stream; + ``` + +There's no `match`, `loop`, `for`, `break`, `return`, indexing, tuples, strings or comments. + +## Types + +The closure's parameters and its result have the stream's item type, comparisons are `bool`, and `$i` is a `u64`. Operands must have the same type: there are no implicit conversions, so mixed types need `as`, e.g. `n as u64 * $i` for a `u32` `n`. A `let` bound to a literal needs its type, e.g. `let d: u64 = 2;`. + +## When a stream ends + +A stream ends when its next item can't be represented, rather than overflowing, as Rust's checked arithmetic would fail: on overflow, division or remainder by zero, shifting by at least the type's bits, a negative integer power, or a float result that isn't finite. So the `fibonacci` above ends at the largest Fibonacci number a `u64` holds. + +A stream that never fails, e.g. `|value| !value`, is unbounded, writing items until the reader closes it. The wrapping operators never fail, except when dividing by zero, so a pseudorandom generator built from them never ends: + +```wit +/// @expression || { +/// let z = seed +% ($i +% 1) *% 0x9E3779B97F4A7C15; +/// let z = (z ^ z >> 30) *% 0xBF58476D1CE4E5B9; +/// let z = (z ^ z >> 27) *% 0x94D049BB133111EB; +/// z ^ z >> 31 +/// } +seeded-random: async func(seed: u64) -> stream; +``` + +A `while` loop whose condition never becomes `false` never ends, and nor does the stream's task, so its reader waits forever. + +## Several items at once + +The closure can return an array, whose elements are each an item: `[a, b]`, or `x.to_le_bytes()` and `x.to_be_bytes()` for a stream of `u8`. `$i` is then the position of the array's first item. `seeded-random-bytes` in the `random` interface writes each 64-bit number as 8 bytes, computing it once for every 8: + +```wit +/// @expression || { +/// let z = seed +% ($i / 8 +% 1) *% 0x9E3779B97F4A7C15; +/// let z = (z ^ z >> 30) *% 0xBF58476D1CE4E5B9; +/// let z = (z ^ z >> 27) *% 0x94D049BB133111EB; +/// (z ^ z >> 31).to_le_bytes() +/// } +seeded-random-bytes: async func(seed: u64) -> stream; +``` diff --git a/grammars/constants-expression/.gitignore b/grammars/constants-expression/.gitignore new file mode 100644 index 0000000..4d03459 --- /dev/null +++ b/grammars/constants-expression/.gitignore @@ -0,0 +1,6 @@ +# generated by `tree-sitter generate`, and built by `tree-sitter test` +/src/ +*.dylib +*.so +*.dll +*.wasm diff --git a/grammars/constants-expression/README.md b/grammars/constants-expression/README.md new file mode 100644 index 0000000..364ddcd --- /dev/null +++ b/grammars/constants-expression/README.md @@ -0,0 +1,458 @@ +# Expressions + +An expression generates the items of a stream returned by an exported function, continuing from the items before them. Expressions are a subset of Rust's closures, e.g. `|a, b| a + b`. + +This document specifies expressions fully, for tools such as language servers: where they appear, their lexical and syntactic grammar, names, types, evaluation, the rules for the functions declaring them, every diagnostic, and what can be completed at each position. A [tree-sitter grammar](#tree-sitter-grammar) implements the syntax. + +For a shorter guide to writing expressions, see [docs/expressions.md](../../docs/expressions.md). + +The reference implementation is [`crates/componentized-constants/src/expr.rs`](../../crates/componentized-constants/src/expr.rs). The grammar's test corpus is also run against it, so the two accept the same syntax. + +## Contents + +- [Expressions](#expressions) + - [Where expressions appear](#where-expressions-appear) + - [Lexical grammar](#lexical-grammar) + - [Syntax](#syntax) + - [Names](#names) + - [Types](#types) + - [Evaluation](#evaluation) + - [Declarations](#declarations) + - [Diagnostics](#diagnostics) + - [Lexical](#lexical) + - [Syntax and names](#syntax-and-names) + - [Types](#types-1) + - [Declarations](#declarations-1) + - [Completions](#completions) + - [tree-sitter grammar](#tree-sitter-grammar) + +## Where expressions appear + +An expression follows an `@expression` tag in the doc comment of a function exported by the world, either directly or in an exported interface. + +```wit +/// The Fibonacci sequence. +/// +/// @value [1, 1] +/// @expression |a, b| a + b +fibonacci: async func() -> stream; +``` + +A doc comment's contents are its `///` lines with the `///` removed. Within the contents: + +- A tag starts a line, after any whitespace, and is followed by whitespace or the end of the line. The tags are `@value` and `@expression`, so `@expressions` and `see @expression` aren't tags. +- A tag's text runs from the end of the tag to the start of the next line starting with a tag, or to the end of the contents. Text may span lines. +- The expression is the `@expression` tag's text with leading and trailing whitespace removed. +- A doc comment may have at most one of each tag, a second `@value` or `@expression` is an error. + +The tags may appear in either order, after any other documentation. + +Positions in diagnostics are `line:column` within the expression, both counted from 1 in characters, where line 1 starts at the expression's first character. + +## Lexical grammar + +The expression is a sequence of tokens separated by optional whitespace. There are no comments. + +| Token | Pattern | Notes | +| ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| whitespace | `[ \t\n\r\f]+` | ASCII only, ignored between tokens | +| identifier | `[A-Za-z][A-Za-z0-9_]*` or `_[A-Za-z0-9_]+` | ASCII only, except the keywords | +| keyword | `if`, `else`, `true`, `false`, `let`, `mut`, `while`, `as` | reserved, never identifiers | +| variable | `$init`, `$call`, `$i` | `$` followed by an identifier, other names are errors | +| `_` | `_` | alone, it ignores a parameter or a `let` value | +| integer | `[0-9][0-9_]*`, `0x_*[0-9a-fA-F][0-9a-fA-F_]*`, `0o_*[0-7][0-7_]*`, `0b_*[01][01_]*` | decimal, hex, octal or binary | +| float | `[0-9][0-9_]*\.([0-9][0-9_]*)?([eE][+-]?_*[0-9][0-9_]*)?` or `[0-9][0-9_]*[eE][+-]?_*[0-9][0-9_]*` | decimal only, e.g. `1.5`, `1.`, `1e3`, `2.5E-3` | +| punctuation | `**%=` `<<%=` `>>%=` `**%` `**=` `<<%` `>>%` `<<=` `>>=` `+%=` `-%=` `*%=` `/%=` `%%=` `**` `\|\|` `&&` `<<` `>>` `<=` `>=` `==` `!=` `+%` `-%` `*%` `/%` `%%` `+=` `-=` `*=` `/=` `%=` `&=` `^=` `\|=` `+` `-` `*` `/` `%` `!` `&` `\|` `^` `<` `>` `=` `(` `)` `{` `}` `[` `]` `,` `;` `:` `.` | the longest match is taken, so `\|\|`, `+%` and `+=` are one token, and `+ %` is two | + +Rules beyond the patterns: + +- `_` separates digits and is otherwise ignored, so `1_000` is `1000`. A number can't start with `_`, which would be an identifier. +- A number can't be followed directly by an identifier character: literal suffixes such as `1u8` or `1.5f32` aren't supported. Use a cast, e.g. `1 as u8`, or a typed `let`. +- A `.` after decimal digits makes a float only when it isn't followed by another `.`, a `_` or a letter. So `1.` is a float, and in `$init.0.wrapping_add(1)` the `0` is an integer followed by `.`. +- Type names in casts and `let` annotations are identifiers: `bool`, `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64`, `f32` and `f64` as in Rust, and `s8`, `s16`, `s32` and `s64` as in WIT. +- Any other character, including non-ASCII letters, is an error. + +## Syntax + +In EBNF, where `{ x }` repeats `x` zero or more times and `[ x ]` makes it optional: + +```ebnf +closure = ( "||" | "|" [ parameters ] "|" ) expression ; +parameters = parameter { "," parameter } [ "," ] ; +parameter = identifier | "_" ; + +expression = or ; +or = and { "||" and } ; +and = comparison { "&&" comparison } ; +comparison = bit_or [ ( "==" | "!=" | "<" | "<=" | ">" | ">=" ) bit_or ] ; +bit_or = bit_xor { "|" bit_xor } ; +bit_xor = bit_and { "^" bit_and } ; +bit_and = shift { "&" shift } ; +shift = additive { ( "<<" | ">>" | "<<%" | ">>%" ) additive } ; +additive = multiplicative { ( "+" | "-" | "+%" | "-%" ) multiplicative } ; +multiplicative = cast { ( "*" | "/" | "%" | "*%" | "/%" | "%%" ) cast } ; +cast = unary { "as" type } ; +unary = ( "-" | "-%" | "!" ) unary | power ; +power = postfix [ ( "**" | "**%" ) unary ] ; +postfix = primary { "." method "(" [ expression ] ")" } ; +primary = integer | float | "true" | "false" | identifier + | ( "$init" | "$call" ) "." ( integer | identifier ) | "$i" + | "(" expression ")" | if | block | array ; +array = "[" expression { "," expression } [ "," ] "]" ; +if = "if" expression block [ "else" ( block | if ) ] ; +block = "{" { statement } [ expression ] "}" ; +statement = let | assignment | while | if [ ";" ] ; +let = "let" [ "mut" ] ( identifier | "_" ) [ ":" type ] "=" expression ";" ; +assignment = identifier ( "=" | "+=" | "-=" | "*=" | "/=" | "%=" | "<<=" | ">>=" | "&=" | "^=" | "|=" + | "+%=" | "-%=" | "*%=" | "/%=" | "%%=" | "<<%=" | ">>%=" | "**=" | "**%=" ) expression ";" ; +while = "while" expression block [ ";" ] ; +method = "wrapping_add" | "wrapping_sub" | "wrapping_mul" + | "rotate_left" | "rotate_right" | "to_le_bytes" | "to_be_bytes" ; +type = "bool" | "u8" | "u16" | "u32" | "u64" | "i8" | "i16" | "i32" | "i64" + | "s8" | "s16" | "s32" | "s64" | "f32" | "f64" ; +``` + +The whole expression must be one `closure`, with nothing after it. The field after `$init.` or `$call.` is a decimal integer without `_`, or a name. `to_le_bytes` and `to_be_bytes` take no argument, the other methods take one. + +Operators by precedence, highest first, as in Rust: + +| Precedence | Operators | Associativity | +| ---------- | --------------------------- | ----------------------------- | +| 1 | method calls `.method(…)` | left | +| 2 | `**` `**%` | right, and tighter than a unary operator on its left, so `-x ** 2` is `-(x ** 2)` | +| 3 | unary `-` `-%` `!` | prefix | +| 4 | `as` | left | +| 5 | `*` `/` `%` `*%` `/%` `%%` | left | +| 6 | `+` `-` `+%` `-%` | left | +| 7 | `<<` `>>` `<<%` `>>%` | left | +| 8 | `&` | left | +| 9 | `^` | left | +| 10 | `\|` | left | +| 11 | `==` `!=` `<` `<=` `>` `>=` | none, `a < b < c` is an error | +| 12 | `&&` | left | +| 13 | `\|\|` | left | + +So `-x.wrapping_add(1)` negates the call, `-x as u8` casts the negation, and `x as u64 * 2` multiplies the cast. `**`'s base is a method call or tighter, and its exponent a unary expression, so `2 ** 3 ** 2` is `2 ** 9`, `x ** -1` needs no parentheses, and a cast needs them, `(x as u64) ** 2`. + +`**` raises to a power, as in Python, `**%` with wrapping. The wrapping operators, ending in `%` as in Zig, have the precedence of the operators they wrap, so `a +% b *% c` is `a +% (b *% c)`. + +`if` and blocks are expressions, so they may be operands, e.g. `1 + if c { a } else { b }`. A block is statements followed by an optional value: `let` bindings, assignments, `while` loops and `if` statements. Where a block is a value, as an operand, a `let` value, an `if` expression's branch or the closure's body, it must end with a value, and an `if` must have an `else`, with a value for each branch. A `while` loop's body and an `if` statement's branches are statements only, without a value, and an `if` statement may leave out `else`. An `if` starting a statement is a statement, unless it's the last thing in its block and every branch has a value, when it's the block's value, as in Rust. `else if` chains conditions. The grammar accepts both shapes of blocks and `if`s everywhere, these rules are checked after parsing, see [Syntax and names](#syntax-and-names). + +`|` both delimits the parameters and is the bitwise or operator. Only the first two `|` (or the first `||`) delimit the parameters, so `|a| a | 1` is valid. `| |` with whitespace is also a closure without parameters. + +Not supported: `match`, `loop`, `for`, `break`, `continue`, `return`, statements other than `let`, assignment, `while` and `if`, assignment to anything but a `let mut` binding, other methods, method calls on literals, fields other than `$init`'s, type annotations on parameters such as `|a: u32|`, array repeats such as `[0; 4]`, indexing, ranges, tuples, strings, chars and comments. + +## Names + +An identifier in the body is a `let` binding, a closure parameter, or one of the function's arguments, the innermost first. + +- **Parameters.** With `N` parameters, the next item is computed from the previous `N` items of the stream, oldest first: generating item `i`, the first parameter is item `i - N` and the last is item `i - 1`. A parameter is an identifier or `_`, which takes a position without a name and may appear more than once. Parameter names must be unique. +- **`let` bindings.** A binding is in scope from the end of its `let` to the end of its block. A later binding of the same name shadows an earlier binding or a parameter, as in Rust, so `let x = x ^ x << 13;` reads the previous `x`. `let _ = …;` evaluates its value without binding it. A `let mut` binding can be assigned, `x = value;`, or with a compound assignment, `x op= value;`, which is `x = x op value;`, for each binary arithmetic, bitwise, shift or wrapping operator. Other bindings and the parameters can't be assigned. +- **`$init`** holds the function's arguments, read by field: by index, `$init.0` is the first argument, `$init.1` the second, and so on, or by the parameter's name, as Rust names it, with `-` replaced by `_`, e.g. `$init.start_value` for a parameter `start-value`. It can only be used in a function with parameters, see [Declarations](#declarations). +- **`$call`** holds the closure's parameters, the arguments of the current evaluation, read by field: by index, `$call.0` is the first parameter, or by its name, `$call.a`. Unlike the parameter's name, `$call` can't be shadowed, so `$call.a` reads the parameter even after `let a = …;`, and `$call.0` reads a parameter ignored with `_`. It can be used in any expression, with as many fields as the closure has parameters. +- **Arguments.** The function's arguments are in scope by name, outside the closure's parameters, named as `$init`'s fields are, so `n` is `$init.n`. Reading one reads `$init`, so the arguments are inputs, not items, see [Declarations](#declarations). A closure parameter or `let` binding of the same name shadows an argument, so in `|a, b| a + b` for `func(a: u32, b: u32)`, `a` and `b` are the previous items. An argument is never assigned, even when its name is shadowed by `let mut`. +- **`$i`** is the position in the stream of the item being generated, counting from 0 and including the items the stream starts from. After `@value [1, 1]`, the first item generated has `$i` 2. When the closure returns an array, `$i` is the position of its first item, so it advances by the array's length: 0, 8, 16, … for `to_le_bytes` of a `u64`. + +`$init`, `$call` and `$i` are built in: they can't be shadowed, by a parameter or `let`, nor assigned, and nothing else changes them. Operators and methods give new values, and a copy, `let mut x = $i;`, can change without changing `$i`. `$init` and the arguments are the same for every item, `$call` is the parameters of the current evaluation, and `$i` is the same throughout an evaluation, advancing only between items. + +`if`, `else`, `true`, `false`, `let`, `mut`, `while` and `as` are keywords, so they can't be bound. + +## Types + +Every expression has a type: `bool`, `u8`, `s8`, `u16`, `s16`, `u32`, `s32`, `u64`, `s64`, `f32` or `f64`. Types are named in messages by their WIT names, so Rust's `i32` is `s32`. + +The stream's item type `T` must be one of these, after following type aliases. The closure's body has type `T`, and so does each parameter. `$init.k` has the type of the function's `k`-th parameter, and `$i` is a `u64`. + +Literals take the type their context needs. An integer literal can be any integer or float type, a float literal any float type. Where the context doesn't decide, e.g. both operands of a comparison are literals, or a literal is cast, an integer literal is an `s32` and a float literal an `f64`, as in Rust. So `0x8000_0001 as u32` is out of range for `s32`: write `let x: u32 = 0x8000_0001;` instead. A `let` bound to only literals must give its type. + +| Expression | Operands | Type | Rules | +| -------------------------------------------------------- | ----------------------------------------------- | ------------------------ | ------------------------------------------------------------------ | +| integer literal | | an integer or float type | in range for an integer type | +| float literal | | a float type | finite as an `f32` | +| `true`, `false` | | `bool` | | +| parameter, `let` binding, `$init.k`, `$call.k`, `$i` | | its type | | +| `-e` | `e: X` | `X` | `X` must be a signed integer or a float | +| `-%e` | `e: X` | `X` | `X` must be an integer | +| `!e` | `e: X` | `X` | logical not for `bool`, bitwise not for an integer, not for floats | +| `a * b`, `a / b`, `a + b`, `a - b` | `X`, `X` | `X` | `X` must be an integer or float | +| `a % b` | `X`, `X` | `X` | `X` must be an integer | +| `a +% b`, `a -% b`, `a *% b`, `a /% b`, `a %% b` | `X`, `X` | `X` | `X` must be an integer | +| `a ** b`, `a **% b` | `a: X`, `b`: any integer | `X` | `X` must be an integer or float, `**%` an integer, a literal exponent is a `u32`, or an `s32` for a float base | +| `a << b`, `a >> b`, `a <<% b`, `a >>% b` | `a: X`, `b`: any integer | `X` | `X` must be an integer, a literal amount takes `X` | +| `a & b`, `a ^ b`, `a \| b` | `X`, `X` | `X` | `X` must be an integer or `bool`, `bool` without short-circuiting | +| `a == b`, `a != b`, `a < b`, `a <= b`, `a > b`, `a >= b` | `X`, `X` | `bool` | `false` is less than `true` | +| `a && b`, `a \|\| b` | `bool`, `bool` | `bool` | short-circuiting | +| `e as Y` | `e`: a number, or `bool` when `Y` is an integer | `Y` | `Y` can't be `bool`, see [Evaluation](#evaluation) | +| `a.wrapping_add(b)`, `wrapping_sub`, `wrapping_mul` | `X`, `X` | `X` | `X` must be an integer | +| `a.rotate_left(b)`, `rotate_right` | `a: X`, `b: u32` | `X` | `X` must be an integer | +| `if c { a } else { b }` | `c: bool`, `a` and `b` the same type | the type of `a` | | +| `{ let x: Y = e; …; v }` | `e: Y`, or `e`'s type without `: Y` | the type of `v` | | +| `x = e;`, `x op= e;` | `e`, or `x op e`, the type of `x` | | a statement, `x` a `let mut` binding | +| `while c { … }` | `c: bool` | | a statement | +| `if c { … } else { … }` | `c: bool` | | a statement, when its branches have no values | +| `( e )` | | the type of `e` | | + +The closure may instead return an array, whose elements are each an item, in order, so one evaluation generates several items. Arrays can only be the closure's result: the body, a block's value or an `if`'s branches in that position, where both branches must give the same number of items. Elsewhere, an array is an error. + +| Result | Items | Rules | +| ----------------- | ------------------------------------ | ------------------------------------------------------------------- | +| `[a, b, …]` | each element, at least one | each element must be the item type `T` | +| `x.to_le_bytes()` | `x`'s bytes, least significant first | `T` must be `u8`, `x` any integer or float, as Rust's `to_le_bytes` | +| `x.to_be_bytes()` | `x`'s bytes, most significant first | as `to_le_bytes` | + +Operands of the same type must have exactly the same type, there are no implicit conversions: `$init.0 * $i` with a `u32` seed is an error, `$init.0 as u64 * $i` isn't. + +A `-` applied directly to a literal, even through parentheses, makes a negative literal, so `-128` is in range for `s8`. For an unsigned type, a negative literal is out of range. + +## Evaluation + +A stream with an expression writes the items it starts from, see [Declarations](#declarations), then items generated from them, one at a time, until an item can't be generated. When the reader closes the stream, nothing more is generated. A stream whose expression never fails is unbounded. Each call returns a new stream, from the same items and arguments, so the same call always generates the same items. + +Operands, `let` values, statements and array elements are evaluated in order, left to right. An array's items are generated together: if any element fails, the stream ends before the array's first item, so the items written are always whole arrays. `&&`, `||` and `if` evaluate only what they need, so a failure in an operand or branch that isn't evaluated doesn't end the stream. A `let` value that fails ends the stream, even when the binding isn't used. + +A `while` loop evaluates its condition, then its body, until the condition is `false`. A failure inside a loop ends the stream like any other, which ends most runaway loops: a counter that only grows overflows. A loop whose condition never becomes `false` without failing, e.g. `while true { }`, never ends, and nor does the stream's task, as in Rust, so the reader waits forever for the next item. + +An item can't be generated, and the stream ends without it, when evaluating the expression would fail Rust's checked arithmetic, or a float result isn't finite: + +| Operation | Integers | Floats | +| ------------------------------------------------------------------------- | ------------------------------------------------------- | --------------------------------------------------------- | +| `a + b`, `a - b`, `a * b` | the result is out of the type's range | the result is infinite or NaN | +| `a ** b` | the result is out of the type's range, or the exponent is negative | the result is infinite or NaN, a negative exponent gives the reciprocal | +| `a / b` | `b` is zero, or `a` is the signed minimum and `b` is -1 | the result is infinite or NaN, including when `b` is zero | +| `a % b` | `b` is zero, or `a` is the signed minimum and `b` is -1 | not supported | +| `-a` | `a` is the signed minimum | never fails | +| `a << b`, `a >> b` | `b`, read as unsigned, is at least the type's bits | not supported | +| `a /% b`, `a %% b` | `b` is zero | not supported | +| `!`, `&`, `^`, `\|`, comparisons, `as`, methods, other wrapping operators | never fail | never fail, comparisons follow IEEE 754 | + +The stream also ends after the item at position `u64::MAX`, as `$i` can't go further, which never happens in practice. + +Shifts only check the amount: bits shifted past the type's width are dropped, as in Rust, so `1u32 << 31 << 1` is `0`. `>>` is arithmetic for signed types and logical for unsigned types. + +The wrapping operators wrap around the type's range instead of failing, as Rust's `wrapping_*` methods: + +| Operator | As Rust's | Result | +| ---------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------ | +| `a +% b`, `a -% b`, `a *% b` | `wrapping_add`, `wrapping_sub`, `wrapping_mul` | the result modulo 2 to the type's bits | +| `a /% b`, `a %% b` | `wrapping_div`, `wrapping_rem` | as `/` and `%`, except the signed minimum and -1 give the minimum and 0 | +| `a <<% b`, `a >>% b` | `wrapping_shl`, `wrapping_shr` | a shift by `b` modulo the type's bits, e.g. `x <<% 33` shifts a `u32` by 1 | +| `a **% b` | `wrapping_pow` | the result modulo 2 to the type's bits | +| `-%a` | `wrapping_neg` | 0 minus `a`, wrapping, so it's allowed for unsigned types, e.g. `-%(1 as u8)` is 255 | + +The methods `wrapping_add`, `wrapping_sub` and `wrapping_mul` are the same as `+%`, `-%` and `*%`. `rotate_left` and `rotate_right` rotate the type's bits by the amount modulo the type's bits. + +Casts with `as` never fail, following Rust: + +| From | To | Result | +| ------- | ------- | ------------------------------------------------------------------------------------------------------- | +| integer | integer | truncated to the target's bits, or extended with the source's sign or zeros, e.g. `-3 as u8` is 253 | +| `bool` | integer | 0 or 1 | +| integer | float | the nearest float | +| float | float | rounded to `f32`, or exact to `f64` | +| float | integer | rounded toward zero, saturating at the target's range, with NaN becoming 0, e.g. `-150.0 as i8` is -128 | + +## Declarations + +A function with an `@expression` must return a `stream`, possibly through type aliases, where `T` is one of the types above. The function must be `async`, since its stream is written after the call returns, and may be unbounded. It takes one of three forms. + +**Listed items.** The function takes no parameters. Its `@value` tag lists the items the stream starts from, which must number at least as many as the closure's parameters. The closure starts from the last of them. The `@value` is required, though it may be `[]` when the closure takes no parameters. An override replaces the listed items, but not the expression. `$init` can't be used. + +```wit +/// @value [1, 1] +/// @expression |a, b| a + b +fibonacci: async func() -> stream; +``` + +**Arguments as items.** The expression doesn't read `$init`, or an argument by name. The function takes the items the stream starts from as parameters, one for each of the closure's, by position, each of type `T`. Names may differ from the closure's. The arguments are the stream's first items. The function has no `@value`. + +```wit +/// @expression |a, b| a + b +fibonacci: async func(a: u32, b: u32) -> stream; +``` + +**Arguments as `$init`.** The expression reads `$init`, or an argument by name. The function's parameters may have any of the types above, and are only read through `$init`, they aren't items. The stream starts from the items listed by an optional `@value`, which must number at least as many as the closure's parameters, and are none without a `@value`. + +```wit +/// @expression || { +/// let z = $init.0.wrapping_add($i.wrapping_add(1).wrapping_mul(0x9E3779B97F4A7C15)); +/// let z = (z ^ z >> 30).wrapping_mul(0xBF58476D1CE4E5B9); +/// let z = (z ^ z >> 27).wrapping_mul(0x94D049BB133111EB); +/// z ^ z >> 31 +/// } +pseudorandom: async func(seed: u64) -> stream; +``` + +A function with parameters can't be overridden, and takes at most 16 parameters. Without an `@expression`, an exported function can't take parameters. + +## Diagnostics + +Diagnostics found while creating a component are reported for the function, e.g. ``invalid `@expression` for `fibonacci` at world.wit:4:5: unknown name `c` at 1:12``. Diagnostics with a position report it as `at line:column` within the expression, see [Where expressions appear](#where-expressions-appear), or `at the end` when the expression ends early. Type diagnostics have no position, so a language server should place them on the expression they describe. + +In the messages below, `T`, `X` and `Y` are types by their WIT names, e.g. `u32` or `s8`, `op` is an operator, `n` and `k` are numbers, and `at` is a position, followed by the token found there. + +### Lexical + +| Message | Cause | +| -------------------------------------- | ------------------------------------------------------------------------ | +| ``unexpected `c` at`` | a character that starts no token, including a `$` not followed by a name | +| `invalid integer at` | a prefix without digits, e.g. `0x`, or an integer too large for 128 bits | +| `invalid float at` | an exponent without digits, e.g. `1e` | +| `literal suffixes aren't supported at` | a number followed by an identifier character, e.g. `1u8` | + +### Syntax and names + +| Message | Cause | +| ------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | +| ``expected `\|` at`` | the expression doesn't start with a closure's parameters | +| `expected a parameter name at` | a parameter that isn't an identifier or `_`, including a keyword | +| ``expected `,` or `\|` at`` | parameters not separated by `,` or ended by `\|` | +| ``duplicate parameter `name` at`` | a parameter name used twice | +| `expected an expression at` | a token that can't start an expression, or the end, where an operand is expected | +| ``expected `)` at``, ``expected `}` at``, ``expected `(` at`` | an unclosed parenthesis or block, or a method without its argument | +| ``expected `{` at``, ``expected `{` or `if` at`` | an `if` condition or `else` without a block | +| ``expected `else`, an `if` needs a value for both branches at`` | an `if` as a value without `else` | +| `expected a value at the end of the block at` | a block as a value ending with a statement, e.g. a `let` | +| ``a `while` loop's or an `if` statement's block can't end with a value at`` | a value at the end of a loop's body or an `if` statement's branch | +| ``expected `}`, an `if` with values must be the block's value at`` | an `if` with values starting a statement that continues, e.g. `if c { 1 } else { 2 } + 1` | +| ``can't assign to `x` at …, declare it with `let mut x` `` | an assignment to a `let` binding without `mut` | +| ``can't assign to the parameter `x` at …, bind a copy with `let mut x = x;` `` | an assignment to a parameter | +| ``can't assign to the argument `x` at …, arguments never change, bind a copy with `let mut x = x;` `` | an assignment to one of the function's arguments | +| ``` `$i` can't be bound at …, `$init`, `$call` and `$i` are built in, and can't be shadowed ``` | `$init`, `$call` or `$i` as a parameter or `let` name | +| ``` `$i` can't be assigned at …, `$init`, `$call` and `$i` are built in, and never change, bind a copy with `let mut` ``` | an assignment to `$i`, or a field of `$init` or `$call` | +| `comparison operators can't be chained at` | a comparison as the operand of another, e.g. `a < b < c` | +| `expected a name to bind at` | a `let` without an identifier or `_`, including a keyword | +| ``expected `=` at``, ``expected `;` at`` | a `let` without a value, or a `let` or assignment not ended by `;` | +| ``expected a type, e.g. `u8`, `i32` or `f64`, at`` | an `as` or `let` annotation without a type name | +| `expected a method name at` | a `.` not followed by a name, e.g. `n.0` | +| ``unknown method `name` at …, the methods are …`` | a method other than the seven supported | +| ``` `name` takes an argument at ```, ``` `name` takes no arguments at ``` | a method called with the wrong number of arguments | +| `an array must have at least one item at` | `[]` | +| ``expected `,` or `]` at`` | array elements not separated by `,` or ended by `]` | +| ``can't call `name` on a literal at …, its type is ambiguous`` | a method call on a literal, e.g. `1.wrapping_add(2)` | +| ``expected `.` and a field, `$init` is a tuple of the function's arguments, e.g. `$init.0`, at`` | `$init` without a field | +| ``expected a field of `$init`, an index, e.g. `0`, or a parameter's name, at`` | a `$init.` not followed by a decimal integer or a name | +| ``expected `.` and a field, `$call` is the closure's parameters, e.g. `$call.0`, at`` | `$call` without a field | +| ``expected a field of `$call`, an index, e.g. `0`, or a parameter's name, at`` | a `$call.` not followed by a decimal integer or a name | +| ``` `$call.k` doesn't exist at …, `$call` has n fields, one for each of the closure's parameters ``` | a field past the closure's parameters | +| ``` `$call` has no field `name` at …, the closure's parameters are … ``` | a field by a name that isn't one of the closure's parameters | +| ``unknown variable `$name` at …, the variables are `$init`, `$call` and `$i` `` | any other `$` name | +| ``unknown name `name` at`` | an identifier that isn't a parameter, a `let` binding in scope, or an argument | +| `literal n is too large at` | an integer literal too large for any supported type | +| ``unexpected `token` at`` | tokens after the closure's body | + +### Types + +| Message | Cause | +| ----------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| `expected the closure to return T, found X` | the body's type isn't the item type | +| `expected X, found Y` | a parameter, binding, `$init.k`, `$i` or cast of type `Y` where `X` is needed | +| `expected bool, found integer n` | an integer literal where a `bool` is needed | +| `literal n is out of range for X` | an integer literal outside `X`'s range, or a float literal infinite as `f32` | +| `expected X, found float n` | a float literal where an integer or `bool` is needed | +| ``mismatched types for `op`, X and Y`` | operands of different types, e.g. `n + true`, also for methods, `if` branches and comparisons | +| ``expected X for the `else` branch, found Y`` | `if` branches of different types | +| ``expected bool for an `if` condition, found X``, ``expected bool for a `while` condition, found X`` | a condition that isn't a `bool` | +| `expected X for an assignment, found Y` | an assigned value of another type than the binding | +| ``` `-` can't negate X ``` | `-` on an unsigned type | +| ``` `!` isn't supported for X ``` | `!` on a float | +| ``` `-%` isn't supported for X ``` | `-%` on a float or `bool` | +| ``` `op` expects bool operands ``` | `&&` or `\|\|` on operands that aren't `bool` | +| ``` `op` expects numeric operands ``` | arithmetic or shifts on `bool` operands | +| ``` `op` isn't supported for X ``` | `%`, `<<`, `>>`, `&`, `^` or `\|` on floats, or a wrapping operator on a float or `bool` | +| ``the exponent of `**` must be an integer, found X`` | a float or `bool` exponent | +| `a shift amount must be an integer, found X` | a shift by a float or `bool` | +| ``` `method` isn't supported for X ``` | a method on a float or `bool` | +| ``can't cast to bool, compare instead, e.g. `x != 0` `` | `as bool` | +| `can't cast bool to X` | `as` from `bool` to a float | +| ``the type of a `let` bound to a literal must be given, e.g. `let x: u32 = 1;` `` | a `let` without a type, whose value is only literals | +| ``an array can only be the closure's result, e.g. `\|\| [a, b]` or `\|\| x.to_le_bytes()` `` | an array, or bytes, as an operand or `let` value | +| ``the `if` branches give n and m items, they must give the same number`` | `if` branches returning arrays of different lengths, or an array and a single item | +| ``` `to_le_bytes` gives bytes, the stream's items must be u8, not T ``` | bytes in a stream of another type | +| ``` `to_le_bytes` isn't supported for bool ``` | bytes of a `bool` | +| ``` `$init.k` doesn't exist, `$init` has n fields, one for each of the function's arguments ``` | a field past the function's parameters | +| ``` `$init` has no field `name` at …, the function's parameters are … ``` | a field by a name that isn't one of the function's parameters | + +### Declarations + +| Message | Cause | +| ---------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| ``function `f`: duplicate `@expression` tag, a doc comment may have one of each tag`` | a second `@expression`, or `@value`, tag in the doc comment | +| `only functions returning a stream can generate items` | an `@expression` on a function not returning a stream | +| `only streams of integers, floats or bool can generate items` | a stream of another item type | +| ``a stream with items must be returned by an async function, e.g. `async func() -> stream` `` | a listed-items function that isn't `async` | +| ``a stream generated from arguments must be returned by an async function, e.g. `async func(n: u8) -> stream` `` | a function with parameters that isn't `async` | +| `the expression reads the previous n items, list at least n` | fewer listed items than the closure's parameters | +| ``the expression reads the previous n items, list at least n with `@value` `` | a `$init` function without a `@value`, whose closure has parameters | +| ``missing value for `f` at …: add a `@value` tag to its doc comment, or an override`` | a listed-items function without a `@value` | +| `the expression takes n parameters, the function takes m parameters` | an arguments-as-items function whose parameter count differs from the closure's | +| ``parameter `p` must have the stream's item type, `T` `` | an argument as an item that isn't a `T` | +| ``` `$init` holds the function's arguments, but the function takes none ``` | `$init` in a function without parameters | +| ``parameter `p` can't be read with `$init`, its type must be an integer, float or bool`` | a `$init` function with a parameter of another type | +| `a stream can be generated from at most 16 arguments` | more than 16 parameters | +| ``invalid `@value` for `f` at …: its stream starts from its arguments, remove the tag, or read the arguments with `$init` `` | an arguments-as-items function with a `@value` | +| ``invalid overrides: `f` can't be overridden, its stream is generated from its arguments`` | an override for a function with parameters | +| ``function `f` must not accept parameters, unless it generates a stream from them with an `@expression` `` | parameters without an `@expression` | + +## Completions + +What can be completed depends on the position and, for operands, on the type the context expects: + +| Position | Expected type | +| ------------------------------------------------------------------------------------------------ | ---------------------------------------------- | +| the closure's body | `T` | +| an `if` condition | `bool` | +| an `if` branch, a block's value | the type the `if` or block is expected to have | +| a `let` value | its annotated type, otherwise any | +| an operand of `&&`, `\|\|` | `bool` | +| an operand of arithmetic, wrapping operators, `&`, `^`, `\|`, `-`, `-%`, `!`, or a shifted value | the expected type, matching the other operand | +| a shift amount | any integer | +| an operand of a comparison, or the value of a cast | any, matching the other operand | +| a `wrapping_*` method's receiver and argument | the expected type | +| a rotation's receiver, its amount | the expected type, `u32` | +| inside `( )` | the type the parentheses are expected to have | + +Completions by position: + +- **A doc comment line**: the tags `@value` and `@expression` that the comment doesn't have yet. +- **The start of an expression**: a closure with the parameters it needs. For an arguments-as-items function, its parameter names, e.g. `|a, b| `. Otherwise any names, with at most as many parameters as listed items, or `|| ` for a `$init` function without a `@value`. +- **A parameter**: for an arguments-as-items function, the function's parameter name at the same position, or `_`. +- **An operand**: by expected type, the parameters, `let` bindings and arguments in scope of that type, `$init.k` for each argument of that type, `$call.k` for each parameter when that's the item type, `$i` for `u64`, a literal as the type allows, `(`, `{`, `if`, `-` for a signed integer or float, `-%` for an integer, and `!` for an integer or `bool`. Any value followed by `as` reaches a numeric type, so values of other numeric types are completions too, with `as` and the type. An expected `bool` can also start a comparison, so operands of any type are completions too, as can `true` and `false`. +- **After `$init`**: `.` and the fields, by name and by index, `0` up to the number of arguments, with their types. +- **After `$call`**: `.` and the fields, by name and by index, `0` up to the number of the closure's parameters, of the item type. +- **After a complete operand of type `X`**: `.` and the methods for an integer, `as` and the types it can be cast to, and the binary operators valid for `X` and the expected type, e.g. after a `u32`, `+ - * / % << >> & ^ |`, the wrapping operators `+% -% *% /% %% <<% >>%`, `**` and `**%`, and the comparisons, plus `&&` and `||` after a `bool`. Then `)` inside parentheses, or `}` inside a block. After an `if` condition, `{`. After an `if`'s first block, `else`. After `else`, `{` or `if`. +- **The start of a block, or after a statement**: `let`, `while`, `if`, the `let mut` bindings in scope followed by the assignment operators valid for their type, and the operands for the block's value, where it has one. +- **The closure's result**: also `[`, and after a value, `.to_le_bytes()` and `.to_be_bytes()` when the items are `u8`. +- **After `let`**: `mut`, or a name. After a name, `:` and the types, or `=`. After an assignment's or `let`'s value, `;`. After a `while` condition, `{`. +- Don't offer a comparison operator after a comparison, since they don't chain. + +Literal hints can show the expected type's range, e.g. `0..=255` for `u8`, as literals outside it are errors. + +## tree-sitter grammar + +[`grammars/constants-expression`](../../grammars/constants-expression) is a [tree-sitter](https://tree-sitter.github.io) grammar for the [syntax](#syntax) of a single expression, the text of an `@expression` tag. It parses a whole closure, recovering from errors with `ERROR` and `MISSING` nodes, so editors can report syntax errors and find the node at the cursor. Names, types and declarations are checked separately, as described above. + +| Node | Fields | Children | +| ---------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -------------------------------------- | +| `closure` | `parameters: parameters` (absent for `\|\|`), `body` | | +| `parameters` | | `parameter`s | +| `parameter` | | an `identifier`, or the `_` token | +| `binary_expression` | `left`, `operator` (the operator token), `right` | | +| `unary_expression` | `operator` (`-` or `!`), `operand` | | +| `cast_expression` | `value`, `type: primitive_type` | | +| `method_call` | `receiver`, `method: identifier`, `argument` (absent without one) | | +| `array_expression` | | the elements | +| `builtin_field` | `variable` (the `$init` or `$call` token), `field: field_index`, or `field: identifier` for a name | | +| `index` | | | +| `if_expression` | `condition`, `consequence: block`, `alternative: block` or `if_expression` (absent without `else`) | | +| `block` | | statements, `let_declaration`, `assignment_statement`, `while_statement` or `if_expression`, then an optional expression | +| `let_declaration` | `pattern: identifier` (absent for `_`), `type: primitive_type` (optional), `value` | `mutable` for `let mut` | +| `assignment_statement` | `left: identifier`, `operator`, `right` | | +| `while_statement` | `condition`, `body: block` | | +| `parenthesized_expression` | | an expression | +| `identifier`, `integer`, `float`, `boolean`, `primitive_type`, `field_index` | | | + +A negative literal parses as a `unary_expression` of `-` and the literal. A block's final `if` parses the same whether it's a statement or the block's value. + +[`queries/highlights.scm`](./queries/highlights.scm) highlights expressions. To highlight them in WIT, a WIT grammar can inject the `constants_expression` language into the text following an `@expression` tag in a doc comment, extracted as described in [Where expressions appear](#where-expressions-appear). + +`make test-grammar` generates the parser with the tree-sitter CLI pinned in [`tools/Cargo.toml`](../../tools/Cargo.toml), using its built-in JavaScript runtime, then tests it against [`test/corpus`](./test/corpus). The corpus is also parsed by the reference implementation, which must accept every test and reject every `:error` test. Add cases to the corpus when changing the syntax, so the grammar and the implementation stay in step. diff --git a/grammars/constants-expression/grammar.js b/grammars/constants-expression/grammar.js new file mode 100644 index 0000000..d57167c --- /dev/null +++ b/grammars/constants-expression/grammar.js @@ -0,0 +1,234 @@ +/** + * @file Expressions generating the items of a stream, a subset of Rust's closures + * @license Apache-2.0 + * + * Mirrors the parser in crates/componentized-constants/src/expr.rs, see + * README.md. Each precedence level is its own rule, lowest first, + * which makes comparisons non-associative as in Rust: `a < b < c` is an error. + */ + +/// +// @ts-check + +/** + * Binary operators by precedence, lowest first, each level's operands are the + * next level's. A level is a hidden rule choosing between the next level and a + * binary expression, a hidden rule of its own, aliased to `binary_expression`. + */ +const LEVELS = [ + ['or', ['||']], + ['and', ['&&']], + // comparisons don't chain, both operands are the next level + ['comparison', ['==', '!=', '<', '<=', '>', '>='], {nonAssociative: true}], + ['bit_or', ['|']], + ['bit_xor', ['^']], + ['bit_and', ['&']], + // `%` suffixed operators wrap around the type's range, as in Zig + ['shift', ['<<', '>>', '<<%', '>>%']], + ['additive', ['+', '-', '+%', '-%']], + ['multiplicative', ['*', '/', '%', '*%', '/%', '%%']], +]; + +/** @param {string[]} operators */ +const operator = operators => operators.length === 1 ? operators[0] : choice(...operators); + +/** The rules for each level, `_` and `__expression`. */ +const levelRules = Object.fromEntries(LEVELS.flatMap(([name, operators, options], index) => { + const level = `_${name}`; + const expression = `_${name}_expression`; + const next = index + 1 < LEVELS.length ? `_${LEVELS[index + 1][0]}` : '_cast'; + return [ + [level, $ => choice($[next], alias($[expression], $.binary_expression))], + [expression, $ => seq( + field('left', options?.nonAssociative ? $[next] : $[level]), + field('operator', operator(operators)), + field('right', $[next]), + )], + ]; +})); + +module.exports = grammar({ + name: 'constants_expression', + + // ASCII whitespace + extras: _ => [/[ \t\n\r\f]/], + + word: $ => $.identifier, + + reserved: { + global: _ => ['if', 'else', 'true', 'false', 'let', 'as', 'while', 'mut'], + }, + + rules: { + // `|a, b| a + b`, or `|| 7` without parameters + closure: $ => seq( + choice( + '||', + seq('|', optional(field('parameters', $.parameters)), '|'), + ), + field('body', $._expression), + ), + + parameters: $ => seq( + $.parameter, + repeat(seq(',', $.parameter)), + optional(','), + ), + + // `_` ignores a parameter + parameter: $ => choice($.identifier, '_'), + + _expression: $ => $._or, + + ...levelRules, + + // `x as u8`, binding tighter than binary operators and looser than unary + _cast: $ => choice($._unary, $.cast_expression), + + cast_expression: $ => seq( + field('value', $._cast), + 'as', + field('type', $.primitive_type), + ), + + _unary: $ => choice($.unary_expression, $._power), + + // `a ** b`, binding tighter than a unary operator on its left, and right + // associative, as the exponent is a unary expression + _power: $ => choice($._postfix, alias($._power_expression, $.binary_expression)), + + _power_expression: $ => seq( + field('left', $._postfix), + field('operator', choice('**', '**%')), + field('right', $._unary), + ), + + unary_expression: $ => seq( + field('operator', choice('-', '-%', '!')), + field('operand', $._unary), + ), + + _postfix: $ => choice($._primary, $.method_call), + + // `x.wrapping_mul(y)` or `x.to_le_bytes()`, the methods and their + // arguments are checked by name + method_call: $ => seq( + field('receiver', $._postfix), + '.', + field('method', $.identifier), + '(', + optional(field('argument', $._expression)), + ')', + ), + + _primary: $ => choice( + $.integer, + $.float, + $.boolean, + $.identifier, + $.builtin_field, + $.index, + $.parenthesized_expression, + $.if_expression, + $.block, + $.array_expression, + ), + + // `[a, b, c]`, only valid as the closure's result + array_expression: $ => seq( + '[', + $._expression, + repeat(seq(',', $._expression)), + optional(','), + ']', + ), + + // `$init.0` or `$init.name`, a field of the function's arguments, or `$call.0` + // or `$call.name`, of the closure's parameters, by index or by name + builtin_field: $ => seq( + field('variable', choice('$init', '$call')), + '.', + field('field', choice($.field_index, $.identifier)), + ), + + field_index: _ => /[0-9]+/, + + // `$i`, the position of the item being generated + index: _ => '$i', + + primitive_type: _ => choice( + 'bool', 'u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', + 's8', 's16', 's32', 's64', 'f32', 'f64', + ), + + parenthesized_expression: $ => seq('(', $._expression, ')'), + + // `if c { a } else { b }`, both branches are required + if_expression: $ => seq( + 'if', + field('condition', $._expression), + field('consequence', $.block), + optional(seq('else', field('alternative', choice($.block, $.if_expression)))), + ), + + mutable: _ => 'mut', + + // `{ let a = x; let b: u32 = y; a + b }`, statements then an optional value + block: $ => seq('{', repeat($._statement), optional($._expression), '}'), + + _statement: $ => choice( + $.let_declaration, + $.assignment_statement, + $.while_statement, + // an `if` starting a statement is the whole statement, as in Rust, a final `if` + // with values is the block's value, which the tree doesn't distinguish + prec(1, seq($.if_expression, optional(';'))), + ), + + // `x = y;` or a compound assignment, e.g. `x += 1;` + assignment_statement: $ => seq( + field('left', $.identifier), + field('operator', choice( + '=', '+=', '-=', '*=', '/=', '%=', '<<=', '>>=', '&=', '^=', '|=', + '+%=', '-%=', '*%=', '/%=', '%%=', '<<%=', '>>%=', '**=', '**%=', + )), + field('right', $._expression), + ';', + ), + + while_statement: $ => seq( + 'while', + field('condition', $._expression), + field('body', $.block), + optional(';'), + ), + + let_declaration: $ => seq( + 'let', + optional($.mutable), + field('pattern', choice($.identifier, '_')), + optional(seq(':', field('type', $.primitive_type))), + '=', + field('value', $._expression), + ';', + ), + + boolean: _ => choice('true', 'false'), + + // decimal, hex, octal or binary, digits may be separated by `_`, without a suffix + integer: _ => token(choice( + /[0-9][0-9_]*/, + /0x_*[0-9a-fA-F][0-9a-fA-F_]*/, + /0o_*[0-7][0-7_]*/, + /0b_*[01][01_]*/, + )), + + // `1.5`, `1.`, `1e3` or `1.5e-3`, without a suffix + float: _ => token(choice( + /[0-9][0-9_]*\.([0-9][0-9_]*)?([eE][+-]?_*[0-9][0-9_]*)?/, + /[0-9][0-9_]*[eE][+-]?_*[0-9][0-9_]*/, + )), + + identifier: _ => /[A-Za-z][A-Za-z0-9_]*|_[A-Za-z0-9_]+/, + }, +}); diff --git a/grammars/constants-expression/queries/highlights.scm b/grammars/constants-expression/queries/highlights.scm new file mode 100644 index 0000000..867f396 --- /dev/null +++ b/grammars/constants-expression/queries/highlights.scm @@ -0,0 +1,30 @@ +; Highlights for expressions generating the items of a stream + +(parameter (identifier) @variable.parameter) +(let_declaration pattern: (identifier) @variable) +(method_call method: (identifier) @function.method.call) +["$init" "$call" (index)] @variable.builtin +(field_index) @property +(builtin_field field: (identifier) @variable.parameter) +(primitive_type) @type.builtin +(parameter "_" @variable.parameter.builtin) +(identifier) @variable + +(integer) @number +(float) @number.float +(boolean) @boolean + +["if" "else"] @keyword.conditional +["let" (mutable)] @keyword +"while" @keyword.repeat +"as" @keyword.operator + +(binary_expression operator: _ @operator) +(assignment_statement operator: _ @operator) +(assignment_statement left: (identifier) @variable) +(unary_expression operator: _ @operator) + +["|" "||"] @punctuation.bracket +["(" ")" "{" "}" "[" "]"] @punctuation.bracket +["," ";" "." ":"] @punctuation.delimiter +"=" @operator diff --git a/grammars/constants-expression/test/corpus/arrays.txt b/grammars/constants-expression/test/corpus/arrays.txt new file mode 100644 index 0000000..d593615 --- /dev/null +++ b/grammars/constants-expression/test/corpus/arrays.txt @@ -0,0 +1,108 @@ +================================================== +Array literal +================================================== + +|a, b| [a + b, a + b + b] + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (array_expression + (binary_expression + left: (identifier) + right: (identifier)) + (binary_expression + left: (binary_expression + left: (identifier) + right: (identifier)) + right: (identifier)))) + +================================================== +Trailing comma +================================================== + +|| [$i, $i + 1,] + +--- + +(closure + body: (array_expression + (index) + (binary_expression + left: (index) + right: (integer)))) + +================================================== +Bytes +================================================== + +|| $i.to_le_bytes() + +--- + +(closure + body: (method_call + receiver: (index) + method: (identifier))) + +================================================== +Big endian bytes +================================================== + +|| ($i as u32).to_be_bytes() + +--- + +(closure + body: (method_call + receiver: (parenthesized_expression + (cast_expression + value: (index) + type: (primitive_type))) + method: (identifier))) + +================================================== +Arrays in if branches +================================================== + +|| if $i == 0 { [1, 2] } else { [3, 4] } + +--- + +(closure + body: (if_expression + condition: (binary_expression + left: (index) + right: (integer)) + consequence: (block + (array_expression + (integer) + (integer))) + alternative: (block + (array_expression + (integer) + (integer))))) + +================================================== +Array as a block's value +================================================== + +|| { let z = $i *% 3; z.to_le_bytes() } + +--- + +(closure + body: (block + (let_declaration + pattern: (identifier) + value: (binary_expression + left: (index) + right: (integer))) + (method_call + receiver: (identifier) + method: (identifier)))) diff --git a/grammars/constants-expression/test/corpus/blocks.txt b/grammars/constants-expression/test/corpus/blocks.txt new file mode 100644 index 0000000..a308d6c --- /dev/null +++ b/grammars/constants-expression/test/corpus/blocks.txt @@ -0,0 +1,169 @@ +================================================== +Block as the body +================================================== + +|n| { n } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (identifier))) + +================================================== +Let bindings +================================================== + +|| { let a = $i; let b: u64 = a * 2; a + b } + +--- + +(closure + body: (block + (let_declaration + pattern: (identifier) + value: (index)) + (let_declaration + pattern: (identifier) + type: (primitive_type) + value: (binary_expression + left: (identifier) + right: (integer))) + (binary_expression + left: (identifier) + right: (identifier)))) + +================================================== +Shadowing +================================================== + +|x| { let x = x ^ x << 13; let x = x ^ x >> 7; x ^ x << 17 } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + pattern: (identifier) + value: (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer)))) + (let_declaration + pattern: (identifier) + value: (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer)))) + (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer))))) + +================================================== +Ignored binding +================================================== + +|n| { let _ = n * 2; n } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + value: (binary_expression + left: (identifier) + right: (integer))) + (identifier))) + +================================================== +Nested blocks +================================================== + +|n| { let a = { let b = n; b + 1 }; a } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + pattern: (identifier) + value: (block + (let_declaration + pattern: (identifier) + value: (identifier)) + (binary_expression + left: (identifier) + right: (integer)))) + (identifier))) + +================================================== +SplitMix64 +================================================== + +|| { + let z = $init.0.wrapping_add($i.wrapping_add(1).wrapping_mul(0x9E3779B97F4A7C15)); + let z = (z ^ z >> 30).wrapping_mul(0xBF58476D1CE4E5B9); + let z = (z ^ z >> 27).wrapping_mul(0x94D049BB133111EB); + z ^ z >> 31 +} + +--- + +(closure + body: (block + (let_declaration + pattern: (identifier) + value: (method_call + receiver: (builtin_field + field: (field_index)) + method: (identifier) + argument: (method_call + receiver: (method_call + receiver: (index) + method: (identifier) + argument: (integer)) + method: (identifier) + argument: (integer)))) + (let_declaration + pattern: (identifier) + value: (method_call + receiver: (parenthesized_expression + (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer)))) + method: (identifier) + argument: (integer))) + (let_declaration + pattern: (identifier) + value: (method_call + receiver: (parenthesized_expression + (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer)))) + method: (identifier) + argument: (integer))) + (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer))))) diff --git a/grammars/constants-expression/test/corpus/closures.txt b/grammars/constants-expression/test/corpus/closures.txt new file mode 100644 index 0000000..84c771d --- /dev/null +++ b/grammars/constants-expression/test/corpus/closures.txt @@ -0,0 +1,123 @@ +================================================== +One parameter +================================================== + +|n| n + 1 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (integer))) + +================================================== +Two parameters +================================================== + +|a, b| a + b + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (identifier))) + +================================================== +Trailing comma +================================================== + +|a, b,| a + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (identifier)) + +================================================== +No parameters +================================================== + +|| 7 + +--- + +(closure + body: (integer)) + +================================================== +No parameters with a space +================================================== + +| | 7 + +--- + +(closure + body: (integer)) + +================================================== +Ignored parameter +================================================== + +|_, b| b + +--- + +(closure + parameters: (parameters + (parameter) + (parameter + (identifier))) + body: (identifier)) + +================================================== +Identifiers with underscores +================================================== + +|_prev, next_1| next_1 + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (identifier)) + +================================================== +Whitespace and newlines +================================================== + +|a, + b| + a + + b + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (identifier))) diff --git a/grammars/constants-expression/test/corpus/conditionals.txt b/grammars/constants-expression/test/corpus/conditionals.txt new file mode 100644 index 0000000..c0603d9 --- /dev/null +++ b/grammars/constants-expression/test/corpus/conditionals.txt @@ -0,0 +1,103 @@ +================================================== +If else +================================================== + +|n| if n == 9 { 0 } else { n + 1 } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (if_expression + condition: (binary_expression + left: (identifier) + right: (integer)) + consequence: (block + (integer)) + alternative: (block + (binary_expression + left: (identifier) + right: (integer))))) + +================================================== +Else if +================================================== + +|n| if n < 0 { -1 } else if n > 0 { 1 } else { 0 } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (if_expression + condition: (binary_expression + left: (identifier) + right: (integer)) + consequence: (block + (unary_expression + operand: (integer))) + alternative: (if_expression + condition: (binary_expression + left: (identifier) + right: (integer)) + consequence: (block + (integer)) + alternative: (block + (integer))))) + +================================================== +If as an operand +================================================== + +|n| 1 + if n > 1 { n } else { 1 } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (integer) + right: (if_expression + condition: (binary_expression + left: (identifier) + right: (integer)) + consequence: (block + (identifier)) + alternative: (block + (integer))))) + +================================================== +Nested conditions +================================================== + +|a, b| if (a > 0) && !(b > 0) { a } else { b } + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (if_expression + condition: (binary_expression + left: (parenthesized_expression + (binary_expression + left: (identifier) + right: (integer))) + right: (unary_expression + operand: (parenthesized_expression + (binary_expression + left: (identifier) + right: (integer))))) + consequence: (block + (identifier)) + alternative: (block + (identifier)))) diff --git a/grammars/constants-expression/test/corpus/errors.txt b/grammars/constants-expression/test/corpus/errors.txt new file mode 100644 index 0000000..5e3bb8b --- /dev/null +++ b/grammars/constants-expression/test/corpus/errors.txt @@ -0,0 +1,450 @@ +================================================== +Missing closure +:error +================================================== + +n + 1 + +--- + + + +================================================== +Chained comparison +:error +================================================== + +|a, b, c| a < b < c + +--- + + + +================================================== +Keyword parameter +:error +================================================== + +|if| 1 + +--- + + + +================================================== +Literal suffix +:error +================================================== + +|n| n + 1u8 + +--- + + + +================================================== +Missing operand +:error +================================================== + +|n| n + + +--- + + + +================================================== +Unknown character +:error +================================================== + +|n| n $ 1 + +--- + + + +================================================== +Field access +:error +================================================== + +|n| n.abs + +--- + + + +================================================== +Unbalanced parentheses +:error +================================================== + +|n| (n + 1 + +--- + + + +================================================== +Empty prefix +:error +================================================== + +|| 0x + +--- + + + +================================================== +Missing body +:error +================================================== + +|n| + +--- + + + +================================================== +Assignment +:error +================================================== + +|n| n = 1 + +--- + + + +================================================== +Seed without a field +:error +================================================== + +|| $init + +--- + + + +================================================== +Unknown variable +:error +================================================== + +|| $x + +--- + + + +================================================== +Method call on a literal +:error +================================================== + +|| 1.wrapping_add(2) + +--- + + + +================================================== +Field of a parameter +:error +================================================== + +|n| n.0 + +--- + + + +================================================== +Let without a semicolon +:error +================================================== + +|n| { let a = n a } + +--- + + + +================================================== +Let without a value +:error +================================================== + +|n| { let a; a } + +--- + + + +================================================== +Let as an expression +:error +================================================== + +|n| let a = n + +--- + + + +================================================== +Cast to an unknown type +:error +================================================== + +|n| n as u9 + +--- + + + +================================================== +Keyword binding +:error +================================================== + +|n| { let as = n; as } + +--- + + + +================================================== +Seed field in hex +:error +================================================== + +|| $init.0x1 + +--- + + + +================================================== +Wrapping operator without an operand +:error +================================================== + +|a| a +% + +--- + + + +================================================== +Remainder as a prefix +:error +================================================== + +|a| %%a + +--- + + + +================================================== +Wrapping operator with a space +:error +================================================== + +|a, b| a + % b + +--- + + + +================================================== +Empty array +:error +================================================== + +|| [] + +--- + + + +================================================== +Unclosed array +:error +================================================== + +|| [1, 2 + +--- + + + +================================================== +Array without commas +:error +================================================== + +|| [1 2] + +--- + + + +================================================== +Method call on a literal's bytes +:error +================================================== + +|| 1.to_le_bytes() + +--- + + + +================================================== +Assignment as the closure's body +:error +================================================== + +|n| n += 1 + +--- + + + +================================================== +Assignment without a semicolon +:error +================================================== + +|n| { let mut x = n; x = 1 x } + +--- + + + +================================================== +While without a block +:error +================================================== + +|n| { while n > 1; n } + +--- + + + +================================================== +Let without a name +:error +================================================== + +|n| { let mut = n; n } + +--- + + + +================================================== +Power without an exponent +:error +================================================== + +|a| a ** + +--- + + + +================================================== +Power of a cast without parentheses +:error +================================================== + +|a| a as u64 ** 2 + +--- + + + +================================================== +Seed field that's neither an index nor a name +:error +================================================== + +|| $init.(0) + +--- + + + +================================================== +Binding a built in variable +:error +================================================== + +|n| { let $i = 1; n } + +--- + + + +================================================== +Built in variable as a parameter +:error +================================================== + +|$init| 1 + +--- + + + +================================================== +Assigning a built in variable +:error +================================================== + +|n| { $i = 1; n } + +--- + + + +================================================== +Assigning a seed field +:error +================================================== + +|n| { $init.0 += 1; n } + +--- + + + +================================================== +Call without a field +:error +================================================== + +|a| $call + +--- + + + +================================================== +Binding call +:error +================================================== + +|a| { let $call = a; a } + +--- + + diff --git a/grammars/constants-expression/test/corpus/literals.txt b/grammars/constants-expression/test/corpus/literals.txt new file mode 100644 index 0000000..28fcf55 --- /dev/null +++ b/grammars/constants-expression/test/corpus/literals.txt @@ -0,0 +1,73 @@ +================================================== +Decimal integers +================================================== + +|| 1_000_000 + +--- + +(closure + body: (integer)) + +================================================== +Hex, octal and binary integers +================================================== + +|| 0xff + 0o17 + 0b1010 + 0x_FF_ff + +--- + +(closure + body: (binary_expression + left: (binary_expression + left: (binary_expression + left: (integer) + right: (integer)) + right: (integer)) + right: (integer))) + +================================================== +Floats +================================================== + +|| 1.5 + 1. + 1e3 + 1.5e-3 + 2E+10 + 1_000.000_1 + +--- + +(closure + body: (binary_expression + left: (binary_expression + left: (binary_expression + left: (binary_expression + left: (binary_expression + left: (float) + right: (float)) + right: (float)) + right: (float)) + right: (float)) + right: (float))) + +================================================== +Booleans +================================================== + +|| true != false + +--- + +(closure + body: (binary_expression + left: (boolean) + right: (boolean))) + +================================================== +Negative literals +================================================== + +|| -128 + +--- + +(closure + body: (unary_expression + operand: (integer))) diff --git a/grammars/constants-expression/test/corpus/loops.txt b/grammars/constants-expression/test/corpus/loops.txt new file mode 100644 index 0000000..652dc5f --- /dev/null +++ b/grammars/constants-expression/test/corpus/loops.txt @@ -0,0 +1,302 @@ +================================================== +Primes +================================================== + +|p| { + let mut n = p + 1; + let mut d: u64 = 2; + while d * d <= n { + if n % d == 0 { + n += 1; + d = 2; + } else { + d += 1; + } + } + n +} + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + (mutable) + pattern: (identifier) + value: (binary_expression + left: (identifier) + right: (integer))) + (let_declaration + (mutable) + pattern: (identifier) + type: (primitive_type) + value: (integer)) + (while_statement + condition: (binary_expression + left: (binary_expression + left: (identifier) + right: (identifier)) + right: (identifier)) + body: (block + (if_expression + condition: (binary_expression + left: (binary_expression + left: (identifier) + right: (identifier)) + right: (integer)) + consequence: (block + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer))) + alternative: (block + (assignment_statement + left: (identifier) + right: (integer)))))) + (identifier))) + +================================================== +Compound assignments +================================================== + +|n| { let mut x = n; x -= 1; x *= 2; x /= 3; x %= 4; x <<= 1; x >>= 1; x &= 7; x ^= 1; x |= 8; x } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + (mutable) + pattern: (identifier) + value: (identifier)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (identifier))) + +================================================== +Wrapping compound assignments +================================================== + +|n| { let mut x = n; x +%= 1; x -%= 2; x *%= 3; x /%= 4; x %%= 5; x <<%= 6; x >>%= 7; x } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + (mutable) + pattern: (identifier) + value: (identifier)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (identifier))) + +================================================== +If statement without else +================================================== + +|n| { let mut x = n; if x > 1 { x = 1; } x } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + (mutable) + pattern: (identifier) + value: (identifier)) + (if_expression + condition: (binary_expression + left: (identifier) + right: (integer)) + consequence: (block + (assignment_statement + left: (identifier) + right: (integer)))) + (identifier))) + +================================================== +Else if statements +================================================== + +|n| { let mut x = n; if x < 1 { x = 0; } else if x > 9 { x = 9; } else { x += 1; } x } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + (mutable) + pattern: (identifier) + value: (identifier)) + (if_expression + condition: (binary_expression + left: (identifier) + right: (integer)) + consequence: (block + (assignment_statement + left: (identifier) + right: (integer))) + alternative: (if_expression + condition: (binary_expression + left: (identifier) + right: (integer)) + consequence: (block + (assignment_statement + left: (identifier) + right: (integer))) + alternative: (block + (assignment_statement + left: (identifier) + right: (integer))))) + (identifier))) + +================================================== +Nested loops +================================================== + +|n| { let mut s = n; while s > 1 { let mut t = s; while t > 0 { t -= 1; } s -= 1; }; s } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + (mutable) + pattern: (identifier) + value: (identifier)) + (while_statement + condition: (binary_expression + left: (identifier) + right: (integer)) + body: (block + (let_declaration + (mutable) + pattern: (identifier) + value: (identifier)) + (while_statement + condition: (binary_expression + left: (identifier) + right: (integer)) + body: (block + (assignment_statement + left: (identifier) + right: (integer)))) + (assignment_statement + left: (identifier) + right: (integer)))) + (identifier))) + +================================================== +Final if is the value +================================================== + +|n| { let x = n; if x > 1 { x } else { 1 } } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + pattern: (identifier) + value: (identifier)) + (if_expression + condition: (binary_expression + left: (identifier) + right: (integer)) + consequence: (block + (identifier)) + alternative: (block + (integer))))) + +================================================== +Power assignments +================================================== + +|n| { let mut x = n; x **= 2; x **%= 3; x } + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (block + (let_declaration + (mutable) + pattern: (identifier) + value: (identifier)) + (assignment_statement + left: (identifier) + right: (integer)) + (assignment_statement + left: (identifier) + right: (integer)) + (identifier))) diff --git a/grammars/constants-expression/test/corpus/operators.txt b/grammars/constants-expression/test/corpus/operators.txt new file mode 100644 index 0000000..fe2d082 --- /dev/null +++ b/grammars/constants-expression/test/corpus/operators.txt @@ -0,0 +1,445 @@ +================================================== +Multiplication binds tighter than addition +================================================== + +|a, b, c| a + b * c + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (identifier)))) + +================================================== +Left associativity +================================================== + +|a, b, c| a - b - c + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (binary_expression + left: (identifier) + right: (identifier)) + right: (identifier))) + +================================================== +Shifts below arithmetic +================================================== + +|a, b| a << b + 1 + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer)))) + +================================================== +Bitwise precedence +================================================== + +|a, b, c| a | b ^ c & a + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (identifier))))) + +================================================== +Comparison below bitwise +================================================== + +|a, b| a & 1 == b | 2 + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (binary_expression + left: (identifier) + right: (integer)) + right: (binary_expression + left: (identifier) + right: (integer)))) + +================================================== +Logical operators +================================================== + +|a, b| a > 1 && b < 2 || a == b + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (binary_expression + left: (binary_expression + left: (identifier) + right: (integer)) + right: (binary_expression + left: (identifier) + right: (integer))) + right: (binary_expression + left: (identifier) + right: (identifier)))) + +================================================== +Unary operators +================================================== + +|a, b| -a * !b + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (unary_expression + operand: (identifier)) + right: (unary_expression + operand: (identifier)))) + +================================================== +Nested unary operators +================================================== + +|a| - -a + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (unary_expression + operand: (unary_expression + operand: (identifier)))) + +================================================== +Parentheses +================================================== + +|a, b, c| (a + b) * c + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (parenthesized_expression + (binary_expression + left: (identifier) + right: (identifier))) + right: (identifier))) + +================================================== +Remainder and division +================================================== + +|a| a % 7 / 2 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (binary_expression + left: (identifier) + right: (integer)) + right: (integer))) + +================================================== +Wrapping operators +================================================== + +|a, b| a +% b *% 3 -% a /% b %% 7 + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer))) + right: (binary_expression + left: (binary_expression + left: (identifier) + right: (identifier)) + right: (integer)))) + +================================================== +Wrapping shifts +================================================== + +|a, b| a <<% b >>% 1 + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (binary_expression + left: (identifier) + right: (identifier)) + right: (integer))) + +================================================== +Wrapping negation +================================================== + +|a| -%a +% -a + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (unary_expression + operand: (identifier)) + right: (unary_expression + operand: (identifier)))) + +================================================== +Wrapping and checked operators share a level +================================================== + +|a, b| a + b +% a - b -% a + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (binary_expression + left: (binary_expression + left: (binary_expression + left: (identifier) + right: (identifier)) + right: (identifier)) + right: (identifier)) + right: (identifier))) + +================================================== +Wrapping operators without spaces +================================================== + +|a, b| a+%b*%2 + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (binary_expression + left: (identifier) + right: (integer)))) + +================================================== +Power +================================================== + +|a, b| a ** b + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (identifier))) + +================================================== +Power binds tighter than multiplication +================================================== + +|a, b| 2 * a ** b + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter + (identifier))) + body: (binary_expression + left: (integer) + right: (binary_expression + left: (identifier) + right: (identifier)))) + +================================================== +Power binds tighter than a unary operator on its left +================================================== + +|a| -a ** 2 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (unary_expression + operand: (binary_expression + left: (identifier) + right: (integer)))) + +================================================== +Power is right associative +================================================== + +|a| a ** 2 ** 3 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (binary_expression + left: (integer) + right: (integer)))) + +================================================== +Negative exponent +================================================== + +|a| a ** -1 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (unary_expression + operand: (integer)))) + +================================================== +Wrapping power +================================================== + +|a| a **% 3 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (identifier) + right: (integer))) + +================================================== +Power of a method call +================================================== + +|a| a.wrapping_add(1) ** 2 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (method_call + receiver: (identifier) + method: (identifier) + argument: (integer)) + right: (integer))) diff --git a/grammars/constants-expression/test/corpus/variables.txt b/grammars/constants-expression/test/corpus/variables.txt new file mode 100644 index 0000000..3e0b7e1 --- /dev/null +++ b/grammars/constants-expression/test/corpus/variables.txt @@ -0,0 +1,209 @@ +================================================== +Seed fields +================================================== + +|| $init.0 + $init.12 + +--- + +(closure + body: (binary_expression + left: (builtin_field + field: (field_index)) + right: (builtin_field + field: (field_index)))) + +================================================== +Position +================================================== + +|| $i + +--- + +(closure + body: (index)) + +================================================== +Casts +================================================== + +|n| n as u64 * 2 as u64 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (binary_expression + left: (cast_expression + value: (identifier) + type: (primitive_type)) + right: (cast_expression + value: (integer) + type: (primitive_type)))) + +================================================== +Chained casts +================================================== + +|| $i as i32 as u8 + +--- + +(closure + body: (cast_expression + value: (cast_expression + value: (index) + type: (primitive_type)) + type: (primitive_type))) + +================================================== +Cast binds looser than unary +================================================== + +|n| -n as u8 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (cast_expression + value: (unary_expression + operand: (identifier)) + type: (primitive_type))) + +================================================== +Method calls +================================================== + +|n| n.wrapping_mul(3).rotate_left(5) + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (method_call + receiver: (method_call + receiver: (identifier) + method: (identifier) + argument: (integer)) + method: (identifier) + argument: (integer))) + +================================================== +Method call on a seed field +================================================== + +|| $init.0.wrapping_add($i) + +--- + +(closure + body: (method_call + receiver: (builtin_field + field: (field_index)) + method: (identifier) + argument: (index))) + +================================================== +Unary binds looser than a method call +================================================== + +|n| -n.wrapping_add(1) + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (unary_expression + operand: (method_call + receiver: (identifier) + method: (identifier) + argument: (integer)))) + +================================================== +WIT and Rust type names +================================================== + +|n| n as s8 as i8 as f64 + +--- + +(closure + parameters: (parameters + (parameter + (identifier))) + body: (cast_expression + value: (cast_expression + value: (cast_expression + value: (identifier) + type: (primitive_type)) + type: (primitive_type)) + type: (primitive_type))) + +================================================== +Seed fields by name +================================================== + +|| $init.n * ($init.r ** $i) + +--- + +(closure + body: (binary_expression + left: (builtin_field + field: (identifier)) + right: (parenthesized_expression + (binary_expression + left: (builtin_field + field: (identifier)) + right: (index))))) + +================================================== +Seed fields by name and index +================================================== + +|| $init.start_value + $init.1 + +--- + +(closure + body: (binary_expression + left: (builtin_field + field: (identifier)) + right: (builtin_field + field: (field_index)))) + +================================================== +Call fields +================================================== + +|a, _| { let a: u64 = 0; $call.a + $call.1 + a } + +--- + +(closure + parameters: (parameters + (parameter + (identifier)) + (parameter)) + body: (block + (let_declaration + pattern: (identifier) + type: (primitive_type) + value: (integer)) + (binary_expression + left: (binary_expression + left: (builtin_field + field: (identifier)) + right: (builtin_field + field: (field_index))) + right: (identifier)))) diff --git a/grammars/constants-expression/tree-sitter.json b/grammars/constants-expression/tree-sitter.json new file mode 100644 index 0000000..8e04e1c --- /dev/null +++ b/grammars/constants-expression/tree-sitter.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json", + "grammars": [ + { + "name": "constants_expression", + "camelcase": "ConstantsExpression", + "title": "Constants expression", + "scope": "source.constants-expression", + "file-types": [], + "injection-regex": "^constants[_-]expression$", + "highlights": "queries/highlights.scm" + } + ], + "metadata": { + "version": "0.1.0", + "license": "Apache-2.0", + "description": "Expressions generating the items of a stream, for componentized constants", + "links": { + "repository": "https://github.com/componentized/constants" + } + } +} diff --git a/src/main.rs b/src/main.rs index dce53ec..feba180 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,10 +18,16 @@ fn main() { #[command()] /// Create a wasm component whose exported functions return constant values. /// -/// Every function exported by the world must be synchronous, accept no -/// parameters and return a value. Values can't reach maps, handles, futures, -/// streams or error contexts, though result types may include them in -/// branches the value doesn't take, e.g. `none` for an `option>`. +/// Every function exported by the world must accept no parameters and return a +/// value. Values can't reach maps, handles, futures, streams or error +/// contexts, though result types may include them in branches the value +/// doesn't take, e.g. `none` for an `option>`. A function may return a +/// stream, written as a list of its items, which must be an async function +/// unless the stream is empty. A stream may continue with items generated by +/// a Rust closure following an `@expression` tag, e.g. `|a, b| a + b`, from +/// the items before them, ending once an item can't be represented. Instead of +/// a `@value`, the function may take the items the stream starts from as +/// arguments, e.g. `async func(a: u32, b: u32) -> stream`. /// /// Each function's value is a WAVE expression following a `@value` tag in its /// doc comment, e.g. `/// @value 42`. diff --git a/tools/Cargo.toml b/tools/Cargo.toml index 0fb6214..d12cae3 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -14,6 +14,7 @@ path = "lib.rs" [dependencies] static-config = "=0.2.0" +tree-sitter-cli = "=0.27.0" wac-cli = "=0.12.0" wasm-opt = "=0.116.1" wasm-tools = "=1.261.0" diff --git a/wit/atomics.wit b/wit/atomics.wit new file mode 100644 index 0000000..737a825 --- /dev/null +++ b/wit/atomics.wit @@ -0,0 +1,9 @@ +interface atomics { + /// Incrementing integers. Stream closes when it reaches max int. + /// @expression |v| v + 1 + incrementer: async func(seed: u64) -> stream; + + /// Decrementing integers. Stream closes when it reaches zero. + /// @expression |v| v - 1 + decrementer: async func(seed: u64) -> stream; +} diff --git a/wit/factory.wit b/wit/factory.wit index 885072b..1b2d8b5 100644 --- a/wit/factory.wit +++ b/wit/factory.wit @@ -13,12 +13,16 @@ interface factory { /// Creates a component implementing `world` from `wit`. /// - /// Every function the world exports must be synchronous, take no - /// parameters and return a value. Values can't reach maps, handles, - /// futures, streams or error contexts, though result types may include - /// them in branches the value doesn't take, e.g. `none` for an - /// `option>`. Each function's value is a WAVE expression following - /// a `@value` tag in its doc comment. + /// Every function the world exports must take no parameters and return a + /// value. Values can't reach maps, handles, futures, streams or error + /// contexts, though result types may include them in branches the value + /// doesn't take, e.g. `none` for an `option>`. A function may + /// return a stream, written as a list of its items, which must be an + /// async function unless the stream is empty. Each function's value is a + /// WAVE expression following a `@value` tag in its doc comment. A stream + /// may continue with items generated by a Rust closure following an + /// `@expression` tag, e.g. `|a, b| a + b`, from the items before them, + /// starting from either its `@value` or the function's arguments. /// /// - `world`: the world to implement, required when the WIT package /// defines more than one world. diff --git a/wit/math.wit b/wit/math.wit index f3089f8..e7dc527 100644 --- a/wit/math.wit +++ b/wit/math.wit @@ -1,5 +1,6 @@ -/// Common mathematical constants. +/// Common mathematical constants and sequences. interface math { + /// Euler's number (e) /// @value 2.718281828459045 e: func() -> f64; @@ -83,4 +84,59 @@ interface math { /// The full circle constant (τ) /// @value 6.283185307179586 tau: func() -> f64; + + /// Odd numbers. Stream closes when it reaches max int. + /// @value [1] + /// @expression |v| v + 2 + odd: async func() -> stream; + + /// Even numbers. Stream closes when it reaches max int. + /// @value [0] + /// @expression |v| v + 2 + even: async func() -> stream; + + /// Arithmetic sequence. Stream closes when it reaches max int. + /// @expression || n + d * $i + arithmetic: async func(n: u64, d: u64) -> stream; + + /// Geometric sequence. Stream closes when it reaches max int. + /// @expression || n * r ** $i + geometric: async func(n: u64, r: u64) -> stream; + + /// Fibonacci sequence. Stream closes when it reaches max int. + /// @value [1, 1] + /// @expression |a, b| a + b + fibonacci: async func() -> stream; + + /// Factorial sequence. Stream closes when it reaches max int. + /// @value [1, 1] + /// @expression |v| v * $i + factorial: async func() -> stream; + + /// Harmonic sequence. Stream closes when it reaches min-positive. + /// @expression || n / ($i + 1) as f64 + harmonic: async func(n: f64) -> stream; + + /// Figurate sequence. Stream closes when it reaches max int. + /// @expression || $i ** n + figurate: async func(n: u64) -> stream; + + /// Prime numbers, each found by trial division from the previous one. + /// Stream closes when it reaches max int. + /// @value [2] + /// @expression |v| { + /// let mut n = v + 1; + /// let mut d: u64 = 2; + /// while d * d <= n { + /// if n % d == 0 { + /// n += 1; + /// d = 2; + /// } else { + /// d += 1; + /// } + /// } + /// n + /// } + prime: async func() -> stream; + } diff --git a/wit/random.wit b/wit/random.wit new file mode 100644 index 0000000..bfd5555 --- /dev/null +++ b/wit/random.wit @@ -0,0 +1,27 @@ +/// Pseudorandom number generators. +interface random { + + /// Pseudorandom numbers from SplitMix64, each mixed from the seed and its + /// position in the stream. The same seed always gives the same numbers. + /// Not cryptographically secure. The stream is unbounded. + /// + /// @expression || { + /// let z = seed +% ($i +% 1) *% 0x9E3779B97F4A7C15; + /// let z = (z ^ z >> 30) *% 0xBF58476D1CE4E5B9; + /// let z = (z ^ z >> 27) *% 0x94D049BB133111EB; + /// z ^ z >> 31 + /// } + seeded-random: async func(seed: u64) -> stream; + + /// Pseudorandom bytes, each 8 bytes one of `seeded-random`'s numbers for the + /// same seed, little endian. Not cryptographically secure. The stream is + /// unbounded. + /// + /// @expression || { + /// let z = seed +% ($i / 8 +% 1) *% 0x9E3779B97F4A7C15; + /// let z = (z ^ z >> 30) *% 0xBF58476D1CE4E5B9; + /// let z = (z ^ z >> 27) *% 0x94D049BB133111EB; + /// (z ^ z >> 31).to_le_bytes() + /// } + seeded-random-bytes: async func(seed: u64) -> stream; +}