From e7dadeda4dd7969f77ad4d33875557fbd12538e9 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 13 Oct 2026 21:40:16 +0200 Subject: [PATCH 1/6] A TCP stream is one region and one connection to netstack (stage 3 of the netstack data-path track) A received byte was copied three times between the card's buffer and the application: into [tcp]'s heap ring, into a kernel pipe under the global pipe lock, and out of it; a sent byte four times. How a stream ended was the order its two pipes closed in, and a stream's client leaving was learned only through those pipes. Stage 3 of issues/netstack-moves-every-byte-through-two-kernel-pipes.md replaces both pipes with a region the client makes and lends, which is [tcp]'s two buffers, and the connection it was opened on, which carries the request and its answer and then only doorbells and the hang-up. Under the orchestrator's rulings on #905: an accepted stream is a region too (R1), what a passive child held copied into it at its offsets, so std's make_socket_fd and SYS_CONNECTION_JOIN go; mio and socket2 reach a stream through std (R2), which keeps the one table of the process's ends; toyos-transport and toyos-untrusted join std's graph as published SDK crates (R3). - toyos-transport: stream.rs gains the ring back, and session.rs is the duplex session, the client's read, write, poll and registration sequences written once over a lock and doorbells the adapter supplies, and the server's publish, release, finish, fail and ring back. Three loom models drive them, each red under the controls the track names. - [tcp]: a ring's storage is its own or lent; a connection is lent the region's rings, what it held moving there, and the user's read and queue are `consume` and `queue`, with no copy into [tcp]. - The node: a stream is a connection, its lent rings and the server end; a hostile word is a Violation that resets that stream alone; the client's hang-up is the close, and the ownerless clock goes. - netstack: a region is judged by `SharedMemory::lent` before it is mapped; a stream is one watch, its connection. - toyos::net::stream: the region's declaration, the process's ends in a fixed table behind futex locks (the lock libc had moves to toyos::sync), connect, accept, read, write, shutdown, nodelay, register and poll. - std: TcpStream is a handle entered there; FromRawFd, IntoRawFd and the OwnedFd conversions take only the process's own ends; File::from_raw_fd and every spawn path refuse a stream's handle; the rust/ fork routes OwnedFd's drop and try_clone_to_owned through toyos::net. std::os::toyos::net gives mio the bell and the registration. - libc: sockets over toyos::net::stream, and poll asks the rings first. - mio and socket2 forks: through std, and neither names toyos any more. - kernel: SYS_CONNECTION_JOIN and ConnectionEnd::joined go. issues/netstack-spends-two-poller-slots-per-piped-connection.md closes: a stream is one watch and WATCHES_PER_PLACE is gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- Cargo.lock | 15 +- build/ci.rs | 41 + build/sdkversion.rs | 2 + build/sysroot.rs | 21 +- build/toolchain.rs | 4 +- ...ves-every-byte-through-two-kernel-pipes.md | 162 +-- ...s-two-poller-slots-per-piped-connection.md | 32 - kernel/src/object/mod.rs | 7 - kernel/src/object/pipe.rs | 10 - kernel/src/object/service.rs | 12 +- kernel/src/syscall/dispatch.rs | 5 +- kernel/src/syscall/ipc.rs | 17 +- lib/blockring/transport/Cargo.toml | 31 +- lib/blockring/transport/src/lib.rs | 7 +- lib/blockring/transport/src/session.rs | 815 +++++++++++++ lib/blockring/transport/src/stream.rs | 59 +- lib/blockring/transport/tests/loom.rs | 483 ++++++++ lib/untrusted/Cargo.toml | 11 +- rust | 2 +- sdk/abi/src/syscall.rs | 14 - sdk/libc/Cargo.lock | 12 + sdk/libc/src/errcode.rs | 2 + sdk/libc/src/posix_io.rs | 102 +- sdk/libc/src/pthread.rs | 65 +- sdk/libc/src/socket.rs | 150 +-- sdk/std/os/io.rs | 55 +- sdk/std/os/mod.rs | 3 + sdk/std/os/net.rs | 38 + sdk/std/sys/net/connection.rs | 234 ++-- sdk/std/sys/process.rs | 25 + sdk/toyos/Cargo.toml | 3 +- sdk/toyos/src/fs.rs | 6 + sdk/toyos/src/ipc.rs | 4 + sdk/toyos/src/launch.rs | 2 +- sdk/toyos/src/lib.rs | 7 +- sdk/toyos/src/net.rs | 241 +--- sdk/toyos/src/net/stream.rs | 772 +++++++++++++ sdk/toyos/src/poller.rs | 2 +- sdk/toyos/src/relocate/mod.rs | 6 +- sdk/toyos/src/shm.rs | 6 + sdk/toyos/src/sync.rs | 74 ++ sdk/toyos/src/volatile.rs | 14 +- system/netstack/Cargo.toml | 1 + system/netstack/node/Cargo.toml | 3 + system/netstack/node/src/lease/tcp.rs | 24 +- system/netstack/node/src/lib.rs | 8 +- system/netstack/node/src/listeners.rs | 38 +- system/netstack/node/src/places.rs | 23 +- system/netstack/node/src/streams.rs | 550 +++------ system/netstack/node/tests/common/client.rs | 233 ++++ system/netstack/node/tests/common/mod.rs | 2 + system/netstack/node/tests/listeners.rs | 295 ++--- system/netstack/node/tests/streams.rs | 1004 +++++------------ system/netstack/shard/src/lib.rs | 20 +- system/netstack/shard/tcp/src/conn.rs | 52 +- system/netstack/shard/tcp/src/lib.rs | 8 +- system/netstack/shard/tcp/src/ring.rs | 154 ++- system/netstack/shard/tcp/src/rx.rs | 29 +- system/netstack/shard/tcp/src/stack.rs | 72 +- system/netstack/shard/tcp/tests/lent.rs | 146 +++ system/netstack/shard/tcp/tests/take.rs | 130 --- system/netstack/src/client.rs | 6 +- system/netstack/src/lent.rs | 113 ++ system/netstack/src/main.rs | 50 +- system/netstack/src/pipes.rs | 33 +- system/netstack/src/serve.rs | 370 +++--- tests/guest/rust-tests/Cargo.lock | 12 + .../rust-tests/src/bin/abuse_pipe_owner.rs | 9 +- .../rust-tests/src/bin/stream_handles.rs | 255 +++++ 69 files changed, 4712 insertions(+), 2501 deletions(-) delete mode 100644 issues/netstack-spends-two-poller-slots-per-piped-connection.md create mode 100644 lib/blockring/transport/src/session.rs create mode 100644 sdk/std/os/net.rs create mode 100644 sdk/toyos/src/net/stream.rs create mode 100644 sdk/toyos/src/sync.rs create mode 100644 system/netstack/node/tests/common/client.rs create mode 100644 system/netstack/shard/tcp/tests/lent.rs delete mode 100644 system/netstack/shard/tcp/tests/take.rs create mode 100644 system/netstack/src/lent.rs create mode 100644 tests/guest/rust-tests/src/bin/stream_handles.rs diff --git a/Cargo.lock b/Cargo.lock index a66297b3989..536f4a910e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2031,10 +2031,9 @@ dependencies = [ [[package]] name = "mio" version = "1.2.0" -source = "git+https://github.com/ToyOSOrg/mio?branch=toyos#d1edcaaddb7d4b651fb996e034d1b5e5ec9aef25" +source = "git+https://github.com/ToyOSOrg/mio?branch=toyos#836d14f1cca5d3597135216430d296aaa712300e" dependencies = [ "libc", - "toyos", "toyos-abi", "wasi", "windows-sys 0.61.2", @@ -2114,6 +2113,7 @@ dependencies = [ "toyos-net-wire", "toyos-pci-claim", "toyos-tco", + "toyos-transport", "toyos-virtio", ] @@ -3643,11 +3643,9 @@ dependencies = [ [[package]] name = "socket2" version = "0.6.3" -source = "git+https://github.com/ToyOSOrg/socket2?branch=toyos#1d9025096919dbc95ab647f65b99e2b8dff4664d" +source = "git+https://github.com/ToyOSOrg/socket2?branch=toyos#12abd8f732277436da2718073cec90f9d2d67f66" dependencies = [ "libc", - "toyos", - "toyos-abi", "windows-sys 0.61.2", ] @@ -4047,6 +4045,7 @@ version = "0.18.0" dependencies = [ "rustc-std-workspace-core", "toyos-abi", + "toyos-transport", ] [[package]] @@ -4323,6 +4322,7 @@ dependencies = [ "etherparse", "mio", "socket2", + "toyos", "toyos-dhcp", "toyos-dns", "toyos-mdns", @@ -4331,6 +4331,7 @@ dependencies = [ "toyos-net-tcp", "toyos-net-udp", "toyos-net-wire", + "toyos-transport", ] [[package]] @@ -4497,6 +4498,7 @@ name = "toyos-transport" version = "0.1.0" dependencies = [ "loom", + "rustc-std-workspace-core", "toyos-untrusted", ] @@ -4507,6 +4509,9 @@ version = "0.1.0" [[package]] name = "toyos-untrusted" version = "0.1.0" +dependencies = [ + "rustc-std-workspace-core", +] [[package]] name = "toyos-update" diff --git a/build/ci.rs b/build/ci.rs index eb37ecb4f7a..4ceab3acfa7 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -446,6 +446,19 @@ const fn red( Control { krate, feature, test, must_red: true, verdicts } } +/// Every case of the session model a registration reds without its arm, its +/// ring, or both. +const REGISTERED: &[Verdict] = &[ + Fails("a_source_registered_after_bytes_arrived_gets_its_first_event"), + Fails("a_source_registered_as_bytes_arrive_gets_its_first_event"), + Fails("a_source_registered_after_the_end_gets_its_first_event"), + Fails("a_source_registered_as_the_end_arrives_gets_its_first_event"), + Fails("a_source_registered_with_room_gets_its_first_event"), + Fails("a_source_registered_as_room_is_released_gets_its_first_event"), + Fails("a_source_registered_over_a_full_ring_after_a_reset_gets_its_first_event"), + Fails("a_source_registered_over_a_full_ring_as_a_reset_arrives_gets_its_first_event"), +]; + /// Every negative control a model crate declares; `build/build.rs`'s /// `every_model_control_is_run` holds this against the manifests. pub(crate) const CONTROLS: &[Control] = &[ @@ -667,6 +680,34 @@ pub(crate) const CONTROLS: &[Control] = &[ red(TRANSPORT, "want-relaxed", Some("loom"), &[ Fails("a_consumer_that_looked_again_is_rung_for_bytes_it_missed"), Fails("a_producer_that_looked_again_is_rung_for_room_it_missed"), + Fails("two_sleepers_each_wake_when_room_is_released_first"), + Fails("two_sleepers_each_wake_when_data_is_published_first"), + Fails("a_poll_beside_a_blocking_read_is_answered_for_the_bytes_it_left"), + Fails("a_poll_beside_a_blocking_read_is_answered_for_the_end_it_left"), + Fails("a_poll_beside_a_blocking_write_is_answered_for_the_room_it_left"), + Fails("a_source_registered_as_bytes_arrive_gets_its_first_event"), + Fails("a_source_registered_as_the_end_arrives_gets_its_first_event"), + Fails("a_source_registered_as_room_is_released_gets_its_first_event"), + Fails("a_source_registered_over_a_full_ring_as_a_reset_arrives_gets_its_first_event"), + ]), + // A session's lost wakes: a reader and a writer on one doorbell, a take + // not passed on, a registration that neither arms nor rings, and a reset + // a look that arms does not see. + red(TRANSPORT, "one-doorbell", Some("loom"), &[ + Fails("two_sleepers_each_wake_when_room_is_released_first"), + Fails("two_sleepers_each_wake_when_data_is_published_first"), + ]), + red(TRANSPORT, "no-pass-on", Some("loom"), &[ + Fails("a_poll_beside_a_blocking_read_is_answered_for_the_bytes_it_left"), + Fails("a_poll_beside_a_blocking_read_is_answered_for_the_end_it_left"), + Fails("a_poll_beside_a_blocking_write_is_answered_for_the_room_it_left"), + ]), + red(TRANSPORT, "register-no-arm", Some("loom"), REGISTERED), + red(TRANSPORT, "register-no-ring", Some("loom"), REGISTERED), + red(TRANSPORT, "register-neither", Some("loom"), REGISTERED), + red(TRANSPORT, "head-only", Some("loom"), &[ + Fails("a_source_registered_over_a_full_ring_after_a_reset_gets_its_first_event"), + Fails("a_source_registered_over_a_full_ring_as_a_reset_arrives_gets_its_first_event"), ]), red(TRANSPORT, "end-keeps-inflight", None, &[Fails( "inflight::tests::an_end_answers_every_tag_once_and_a_late_completion_nothing", diff --git a/build/sdkversion.rs b/build/sdkversion.rs index 827edbbd429..3e7fd470a67 100644 --- a/build/sdkversion.rs +++ b/build/sdkversion.rs @@ -22,6 +22,8 @@ pub struct Crate { /// The published crates, in the order a publisher must take them. pub const PUBLISHED: &[Crate] = &[ Crate { name: "toyos-abi", dir: "sdk/abi" }, + Crate { name: "toyos-untrusted", dir: "lib/untrusted" }, + Crate { name: "toyos-transport", dir: "lib/blockring/transport" }, Crate { name: "toyos-keymap", dir: "sdk/keymap" }, Crate { name: "toyos-font", dir: "sdk/font" }, Crate { name: "toyos", dir: "sdk/toyos" }, diff --git a/build/sysroot.rs b/build/sysroot.rs index 566b93e1611..8339492a196 100644 --- a/build/sysroot.rs +++ b/build/sysroot.rs @@ -65,12 +65,14 @@ use crate::toolchain::{self, host_triple, GuestTarget, Owner, Role, GUEST_TARGET use whole_toolchain::{whole, Whole}; /// The per-worktree sources that end up inside a sysroot: std links the -/// `toyos-abi` and `toyos` [`std_patch`] hands it and compiles its ToyOS -/// backend from `sdk/std`, and +/// `toyos-abi`, `toyos`, `toyos-transport` and `toyos-untrusted` [`std_patch`] +/// hands it and compiles its ToyOS backend from `sdk/std`, and /// `libtoyos_c.a` is `sdk/libc` with `toyos-elf` and `toyos-osrelease`. -pub const SYSROOT_SOURCES: [&str; 7] = [ +pub const SYSROOT_SOURCES: [&str; 9] = [ "sdk/abi/src", "sdk/toyos/src", + "lib/blockring/transport/src", + "lib/untrusted/src", "sdk/std", "lib/elf/src", "sdk/osrelease/src", @@ -80,9 +82,11 @@ pub const SYSROOT_SOURCES: [&str; 7] = [ /// Their manifests, and the lockfile and cargo configuration libc is built /// under: the features, versions and flags of the same build. -pub(crate) const SYSROOT_MANIFESTS: [&str; 7] = [ +pub(crate) const SYSROOT_MANIFESTS: [&str; 9] = [ "sdk/abi/Cargo.toml", "sdk/toyos/Cargo.toml", + "lib/blockring/transport/Cargo.toml", + "lib/untrusted/Cargo.toml", "lib/elf/Cargo.toml", "sdk/osrelease/Cargo.toml", "sdk/libc/Cargo.toml", @@ -93,7 +97,8 @@ pub(crate) const SYSROOT_MANIFESTS: [&str; 7] = [ /// Of [`SYSROOT_MANIFESTS`], the crates std names by version, which std's /// lockfile resolves with the fork's own through [`std_patch`]: what of a /// worktree can move a freestanding target's dependency versions. -const STD_MANIFESTS: [&str; 2] = ["sdk/abi/Cargo.toml", "sdk/toyos/Cargo.toml"]; +const STD_MANIFESTS: [&str; 4] = + ["sdk/abi/Cargo.toml", "sdk/toyos/Cargo.toml", "lib/blockring/transport/Cargo.toml", "lib/untrusted/Cargo.toml"]; /// One of the crates std names by version, in a worktree. pub(crate) struct StdCrate { @@ -1274,7 +1279,7 @@ pub(crate) mod tests { /// The manifests [`std_crates`] reads, in a worktree at `root` that has /// no other source: what a bootstrap run is handed std's patch from. pub(crate) fn std_manifests(root: &Path) { - for (manifest, name) in STD_MANIFESTS.into_iter().zip(["toyos-abi", "toyos"]) { + for (manifest, name) in STD_MANIFESTS.into_iter().zip(["toyos-abi", "toyos", "toyos-transport", "toyos-untrusted"]) { write(&root.join(manifest), &format!("[package]\nname = \"{name}\"\n")); } } @@ -1315,6 +1320,10 @@ pub(crate) mod tests { format!("patch.crates-io.toyos-abi.path=\"{at}/sdk/abi\""), "--config".to_string(), format!("patch.crates-io.toyos.path=\"{at}/sdk/toyos\""), + "--config".to_string(), + format!("patch.crates-io.toyos-transport.path=\"{at}/lib/blockring/transport\""), + "--config".to_string(), + format!("patch.crates-io.toyos-untrusted.path=\"{at}/lib/untrusted\""), ], ); for name in ["work tree", "work\"tree", "work\\tree", "work\ttree"] { diff --git a/build/toolchain.rs b/build/toolchain.rs index 15817dbf139..6be72d48390 100644 --- a/build/toolchain.rs +++ b/build/toolchain.rs @@ -825,8 +825,10 @@ mod tests { let registry = |name: &str, version: &str| { format!("/home/runner/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/{name}-{version}/src/lib.rs") }; + let rest = format!("{} {}", ours("lib/blockring/transport"), ours("lib/untrusted")); let built_from = |abi: &str, toyos: &str| { - fs::write(built.join("dist/libstd.d"), format!("{}/dist/libstd.rlib: {abi} {toyos}\n", built.display())).unwrap(); + fs::write(built.join("dist/libstd.d"), format!("{}/dist/libstd.rlib: {abi} {toyos} {rest}\n", built.display())) + .unwrap(); std::panic::catch_unwind(|| assert_std_built_from(&root, &built)) .err() .map(|refused| *refused.downcast::().expect("a formatted refusal")) diff --git a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md index 8714da2b721..d26692a9e21 100644 --- a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md +++ b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md @@ -54,43 +54,54 @@ The design and its roast are PR #863's body. - Stage 2, done (PR #878): a byte ring with untrusted cursors and want words in `toyos-transport` (`lib/blockring/transport`), host-tested and modelled in loom. -- Stage 3: a TCP stream is one region and one connection: netstack's receive - and send buffers are the region's two rings, beside [tcp]'s heap storage, - which stays for stage 4's passive children, and the connection carries the - connect and its answer, then only doorbells and the hang-up. A stream - belongs to the process that connected or accepted it: `toyos::net` refuses - by name a handle it holds no end for. netstack rings a reader on the - connection and a writer waiting for send room on a pipe of the stream's - own, so neither drains a wake rung for the other. Within a direction no - thread holds the end across its wait, and a thread that took the doorbell - while another waits on it passes the take on through netstack. A - registration arms the want of each direction it names and rings netstack if - that direction is already ready, and netstack rings back. A fatal `end`, - any but a FIN, makes the send direction ready to every look, as POSIX's - `select` counts a descriptor ready for writing when a write "would not - block, whether or not the function would transfer data successfully", and a - `write` reads it before it copies, answering its error and copying nothing, - as RFC 9293 flushes every segment queue at a reset or a user timeout - (§3.10.7.4, §3.10.8) and answers a SEND that finds no connection an error - (§3.10.2). std, libc and the mio and socket2 forks read and write the rings - through `toyos::net`. std gains `FromRawFd`, `IntoRawFd` and the `OwnedFd` - conversions for `TcpStream`, which take only the process's own ends, and - the `rust/` fork's ToyOS arms of `OwnedFd`'s drop and - `BorrowedFd::try_clone_to_owned` let go of and enter a stream's handle - through `toyos::net`. Today a stream's handle reads - and writes the stream through a `File` or a child's stdio; over the region - it would read doorbells, so `File::from_raw_fd` of a stream's handle panics - with `toyos::net`'s refusal, and every path std offers to move a raw - handle into another process asks `toyos::net` of each handle before - anything moves and answers its refusal: a spawn's stdio slots, - `CommandExt::inherit_handle` into any slot, `CommandExt::endow` and - `Command::provide`, each with no child started and the handle still the - caller's, so no refused move leaves an entry that outlives its handle's - number. `toyos::ipc`'s `send_handles` is refused by its type: it takes - `OwnedHandle`s, and a stream's handle becomes one only through - `OwnedHandle::from_raw`, whose `# Safety` stage 3 extends to exclude a - handle `toyos::net` answers for. The bare `dup`, `SYS_HANDLE_SEND` and - `SYS_SPAWN` syscalls stay unchecked. A regression, +- Stage 3 (PR #905): a TCP stream is one region and one connection, a + connected one and an accepted one alike, as the orchestrator ruled on #905 + (R1): netstack's receive and send buffers are the region's two rings, and + [tcp]'s heap storage stays for a passive child before its accept, at most + `limits::RECEIVE_BUFFER_INITIAL` a child and `limits::LISTEN_READY` + children a listener. An accept takes one of the wakes netstack writes on the + listener's pipe for each child that waits before it makes a region, so a + nonblocking accept with none answers `WouldBlock` having made nothing; it + lends the region, netstack copies what the child held into the receive ring + at its offsets, text past a hole included and the FIN [tcp]'s state, before + it answers, and the answer carries the peer's address and the `nodelay` the + child inherited from its listener. `SYS_CONNECTION_JOIN` goes. The + connection carries the connect or the accept and its answer, then only + doorbells and the hang-up. A stream belongs to the process that connected + or accepted it: `toyos::net` refuses by name a handle it holds no end for. + netstack rings a reader on the connection and a writer waiting for send room + on a pipe of the stream's own, so neither drains a wake rung for the other. + Within a direction no thread holds the end across its wait, and a thread + that took the doorbell while another waits on it passes the take on through + netstack. A registration arms the want of each direction it names and rings + netstack if that direction is already ready, and netstack rings back. A + fatal `end`, any but a FIN, makes the send direction ready to every look, as + POSIX's `select` counts a descriptor ready for writing when a write "would + not block, whether or not the function would transfer data successfully", + and a `write` reads it before it copies, answering its error and copying + nothing, as RFC 9293 flushes every segment queue at a reset or a user + timeout (§3.10.7.4, §3.10.8) and answers a SEND that finds no connection an + error (§3.10.2). std and libc read and write the rings through + `toyos::net`, whose table of the process's ends is the instance std links + (R2): the mio and socket2 forks reach a stream through std, and + `std::os::toyos::net` gives them what std has no word for, the bell to watch + and the registration. `toyos-transport` and `toyos-untrusted`, whose code + the client end is, are published SDK crates in std's graph (R3). std gains + `FromRawFd`, `IntoRawFd` and the `OwnedFd` conversions for `TcpStream`, + which take only the process's own ends, and the `rust/` fork's ToyOS arms of + `OwnedFd`'s drop and `BorrowedFd::try_clone_to_owned` let go of and enter a + stream's handle through `toyos::net`. A stream's handle reads doorbells, so + `File::from_raw_fd` of one panics with `toyos::net`'s refusal, and every + path std offers to move a raw handle into another process asks + `toyos::net` of each handle before anything moves and answers its refusal: + a spawn's stdio slots, `CommandExt::inherit_handle` into any slot, + `CommandExt::endow` and `Command::provide`, each with no child started and + the handle still the caller's, so no refused move leaves an entry that + outlives its handle's number. `toyos::ipc`'s `send_handles` is refused by + its type: it takes `OwnedHandle`s, and a stream's handle becomes one only + through `OwnedHandle::from_raw`, whose `# Safety` excludes a handle + `toyos::net` answers for. The bare `dup`, `SYS_HANDLE_SEND` and `SYS_SPAWN` + syscalls stay unchecked. A regression, `issues/a-stream-cannot-move-between-processes.md`. Exit: three loom models that drive the client end and netstack's ringing code as stage 3 lands them, each red under a control that reverts its decision. In one, a @@ -106,39 +117,39 @@ The design and its roast are PR #863's body. send ring is full, made ready before the registration or racing it, gets its first event; it is red without the registration's arm and ring, red with either of the two removed, red under stage 2's `want-relaxed`, and red - with the fatal `end` where a look reads only the send ring's head. A test: - `toyos::net`, handed a stream handle the process holds no end for, refuses - it by name and writes nothing on its connection, and a stream's own handle, - given out by `into_raw_fd` or `From for OwnedFd` or made by - `try_clone_to_owned`, and taken back, shares the one end and its cursors, - red when giving the handle out lets its entry go. A test: - `File::from_raw_fd` of a stream's handle panics with that refusal, red when - it takes the handle silently. A test per moving path, each handed a - stream's handle: a spawn's stdio slot, `inherit_handle` into a slot past - the stdio slots, `endow` and `provide` each answer the refusal and start no - child, and the stream then reads and writes through that handle; each is - red when its path takes the handle silently, and `endow`'s and `provide`'s - are red when the handle moves and its entry stays. A test: a `write` with - room after a reset answers `ConnectionReset` and copies nothing, red when - `write` copies before it reads the `end`. And a T14 boot of round 3's - `internet_download` on stage 3's head reads, in every run, - `pipe_copy_bytes` under 0.01 per byte downloaded, its C0, hold and spin - per MB posted beside the scout's rows. No tree that lands has that row or - that counter: the boot carries the scout's instrument, `git diff - 777c96b2b 2f9e090b8` (`wt/toyos-netring-scout` over its base), onto stage - 3's head, less the netstack stamps on the pipe writes stage 3 deletes. -- Stage 4: a listener and a datagram socket the same way, and the pipe ABI's - messages, `SYS_CONNECTION_JOIN` and `toyos::net`'s pipe handles go. A - passive child's bytes wait in [tcp]'s heap storage, at most - `limits::RECEIVE_BUFFER_INITIAL` a child and `limits::LISTEN_READY` children - a listener, and its accept copies them into the region before it answers. An - accept takes one of the wakes netstack writes on the listener's connection - for each child that waits before it makes a region, so a nonblocking accept - with none answers `WouldBlock` having made nothing, and the accept's answer - carries the peer's address and the `nodelay` the child inherited from its - listener. netstack copies each datagram record's `len`, `port` and `addr` - out of the region once and judges the copy: a `len` past the bytes the - record holds ends that socket alone. `toyos::net`'s `send_to` refuses + with the fatal `end` where a look reads only the send ring's head. Over the + region, `system/netstack/node/tests/listeners.rs`'s + `a_listener_answers_a_syn_and_wakes_its_owner_when_the_handshake_ends` + passes, and so does a case in which a child holds, before its accept, text + in order, text past a hole and its FIN behind that, the hole is filled after + the accept, and the region reads all of it in order, then its `end`. That + case is red under each of three controls that drop at the accept one thing + the child held: its in-order bytes, which also reds the first, its + out-of-order range, its FIN. A test: `toyos::net`, handed a stream handle + the process holds no end for, refuses it by name and writes nothing on its + connection, and a stream's own handle, given out by `into_raw_fd` or + `From for OwnedFd` or made by `try_clone_to_owned`, and taken + back, shares the one end and its cursors, red when giving the handle out + lets its entry go. A test: `File::from_raw_fd` of a stream's handle panics + with that refusal, red when it takes the handle silently. A test per moving + path, each handed a stream's handle: a spawn's stdio slot, `inherit_handle` + into a slot past the stdio slots, `endow` and `provide` each answer the + refusal and start no child, and the stream then reads and writes through + that handle; each is red when its path takes the handle silently, and + `endow`'s and `provide`'s are red when the handle moves and its entry stays. + A test: a `write` with room after a reset answers `ConnectionReset` and + copies nothing, red when `write` copies before it reads the `end`. And a T14 + boot of round 3's `internet_download` on stage 3's head reads, in every run, + `pipe_copy_bytes` under 0.01 per byte downloaded, its C0, hold and spin per + MB posted beside the scout's rows. No tree that lands has that row or that + counter: the boot carries the scout's instrument, `git diff 777c96b2b + 2f9e090b8` (`wt/toyos-netring-scout` over its base), onto stage 3's head, + less the netstack stamps on the pipe writes stage 3 deletes. +- Stage 4: a listener's connection carries its wakes in place of its pipe, a + datagram socket is a region as a stream is, and the pipe ABI's messages and + `toyos::net`'s pipe handles go. netstack copies each datagram record's + `len`, `port` and `addr` out of the region once and judges the copy: a + `len` past the bytes the record holds ends that socket alone. `toyos::net`'s `send_to` refuses what a sender can judge and writes no record: a payload past `toyos_abi::net::UDP_MAX_PAYLOAD`, as POSIX's `sendto` fails `[EMSGSIZE]`; port 0; an unspecified address. That bound is the one @@ -160,16 +171,7 @@ The design and its roast are PR #863's body. bell of the socket's own that netstack makes at the bind and alone writes, its want `tx_room`, as POSIX's `sendto` "shall block until space is available or a timeout occurs"; nonblocking, which std's - `UdpSocket::set_nonblocking` then sets, it answers `WouldBlock`. Exit: - over the region, - `system/netstack/node/tests/listeners.rs`'s - `a_listener_answers_a_syn_and_wakes_its_owner_when_the_handshake_ends` - passes, and so does a case in which a child holds, before its accept, text - in order, text past a hole and its FIN behind that, the hole is filled after - the accept, and the region reads all of it in order, then its `end`. That - case is red under each of three controls that drop at the accept one thing - the child held: its in-order bytes, which also reds the first, its - out-of-order range, its FIN. And a datagram record whose `len` is past the + `UdpSocket::set_nonblocking` then sets, it answers `WouldBlock`. Exit: a datagram record whose `len` is past the bytes it holds is refused while netstack serves on, red when netstack takes that `len` as written. A case per refusal: an oversized payload, port 0 and an unspecified address each answer their error at `send_to` with no diff --git a/issues/netstack-spends-two-poller-slots-per-piped-connection.md b/issues/netstack-spends-two-poller-slots-per-piped-connection.md deleted file mode 100644 index 83c9f2178f8..00000000000 --- a/issues/netstack-spends-two-poller-slots-per-piped-connection.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-26 ---- - -# netstack spends two poller slots per piped connection - -A stream registers both its pipes on every pass, for as long as the node holds -them: its send pipe `READABLE` while the stack has room for its bytes and -`OTHER_END_GONE` until the kernel has answered that, and its receive pipe -`OTHER_END_GONE` and, while the stack holds bytes the pipe refused, `WRITABLE` -(`system/netstack/src/serve.rs`, `Sockets::watch`, from the node's -`Node::watches`). `MAX_PLACES` (`system/netstack/src/main.rs`) divides the -batch one poller can carry (`Poller::MAX_HANDLES`, less the fixed -registrations, the pending connections and the lookups' clients) by -`serve::WATCHES_PER_PLACE = 2`, so the ceiling on the node's places is half -what one registration each would give. - -The memory budget (an eighth of memory at what a listener can be made to hold, -`PLACE_BYTES`) binds first only on a machine whose eighth holds fewer places -than that ceiling. - -**Exit condition**: one registration per connection, and -`WATCHES_PER_PLACE` deleted. One `OP_WATCH` on a joined `Connection` -carries `READABLE | WRITABLE` for both its pipes, and is refused -`OTHER_END_GONE`: `ops::pipe_end_watch` (`kernel/src/object/ops.rs`) answers a -pipe end alone, and a connection has two other ends for the one bit. So the -exit also needs the kernel to say of a connection which of its directions has -no holder left. - -**Owner**: whoever holds `issues/toyos-has-its-own-network-stack.md`. diff --git a/kernel/src/object/mod.rs b/kernel/src/object/mod.rs index bb532bc8f4f..0fc48e14a1b 100644 --- a/kernel/src/object/mod.rs +++ b/kernel/src/object/mod.rs @@ -61,13 +61,6 @@ impl Held { } } -impl Held { - /// A second reference to what is held, or `None` after the last handle released it. - pub(crate) fn get(&self) -> Option { - self.0.lock().clone() - } -} - /// A kernel object's identity: diagnostic and internal-key only, never an authority — no syscall turns a koid into access. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] pub struct Koid(NonZeroU64); diff --git a/kernel/src/object/pipe.rs b/kernel/src/object/pipe.rs index 8290608b35a..307331ae10c 100644 --- a/kernel/src/object/pipe.rs +++ b/kernel/src/object/pipe.rs @@ -31,12 +31,6 @@ pub struct PipeWriteEnd { } impl PipeReadEnd { - /// A second counted reference to the same pipe, for `SYS_CONNECTION_JOIN` - /// to build another object over it. - pub fn reference(&self) -> PipeReader { - self.reference.get().expect("a live handle names this end") - } - pub fn new(reader: PipeReader) -> Arc { Arc::new(Self { core: Self::new_core(), @@ -62,10 +56,6 @@ impl PipeReadEnd { } impl PipeWriteEnd { - pub fn reference(&self) -> PipeWriter { - self.reference.get().expect("a live handle names this end") - } - pub fn new(writer: PipeWriter) -> Arc { Arc::new(Self { core: Self::new_core(), diff --git a/kernel/src/object/service.rs b/kernel/src/object/service.rs index 739ee6aa2b3..6bb9317b3d0 100644 --- a/kernel/src/object/service.rs +++ b/kernel/src/object/service.rs @@ -21,11 +21,6 @@ impl HandleQueue { Arc::new(Self(Lock::new(Some(VecDeque::new())))) } - /// No reader and never one: what a joined connection's queues are. - fn dead() -> Arc { - Arc::new(Self(Lock::new(None))) - } - /// A refusal hands the batch back: dropped here, it would be capabilities destroyed silently. fn push(&self, batch: Vec) -> Result<(), (Vec, SyscallError)> { let mut guard = self.0.lock(); @@ -69,7 +64,7 @@ pub struct ConnectionEnd { inbox: Arc, outbox: Arc, reference: Held<(PipeReader, PipeWriter)>, - /// The server's end of a port's connection; `None` on a client's end and a joined one. + /// The server's end of a port's connection; `None` on a client's end. stamp: Option, } @@ -111,11 +106,6 @@ impl ConnectionEnd { }) } - /// Two pipe ends that were never a port's, so both handle queues are dead. - pub fn joined(rx: PipeReader, tx: PipeWriter) -> Arc { - Self::new(rx, tx, HandleQueue::dead(), HandleQueue::dead()) - } - pub fn stamp(&self) -> Option<&Stamp> { self.stamp.as_ref() } diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index d5286ccfe98..898ead18650 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -36,7 +36,7 @@ use super::io::{ sys_random, sys_read, sys_read_nonblock, sys_write, sys_write_nonblock, }; use super::ipc::{ - sys_accept, sys_connection_join, sys_handle_recv, sys_handle_send, sys_inbox_setup, + sys_accept, sys_handle_recv, sys_handle_send, sys_inbox_setup, sys_inbox_submit, sys_namespace_build, sys_namespace_open, sys_pipe, sys_pipe_map, sys_port_badge, sys_port_create, sys_port_mint, sys_shm_create, sys_shm_map, }; @@ -393,9 +393,6 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> copy_len as u64 } } - SYS_CONNECTION_JOIN => { - sys_connection_join(RawHandle(a1 as u32), RawHandle(a2 as u32)) - } SYS_PIPE_MAP => sys_pipe_map(RawHandle(a1 as u32)), // What a claimed PCI function's driver is built out of. Each is // gated by the claim handle alone: nothing here takes a bus/device/ diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index b16b2760c70..a9470793b69 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -84,21 +84,6 @@ pub(super) fn sys_pipe_map(h: RawHandle) -> u64 { } } -/// Join a pipe read end and a pipe write end into one duplex connection; the caller keeps both handles and gains no more than it already held. -pub(super) fn sys_connection_join(rx_h: RawHandle, tx_h: RawHandle) -> u64 { - let ends = process::with_process_data(|data| { - let rx = data.handles.get::(rx_h, Rights::READ)?; - let tx = data.handles.get::(tx_h, Rights::WRITE)?; - Ok::<_, crate::object::HandleError>((rx.reference(), tx.reference())) - }); - let (rx, tx) = match ends { - Ok(ends) => ends, - Err(e) => return e.refuse(), - }; - let object = KObjectRef::Connection(crate::object::service::ConnectionEnd::joined(rx, tx)); - process::with_process_data(|data| handle_result(ops::install(&mut data.handles, object))) -} - /// Make a port and install both ends; needs no right and grants none, since a port with no clients is not authority. pub(super) fn sys_port_create() -> u64 { let (acceptor, connector) = port::create(); @@ -327,7 +312,7 @@ pub(super) fn sys_port_mint(h: RawHandle, badge: &[u8]) -> u64 { /// The badge a connection accepted from `acceptor_h`'s port was stamped with, written to `out`; answers its length. /// -/// **The port check is the whole boundary**: any process can make a port and mint any bytes on it, so a stamp is answered only to the acceptor of the port it was made on. A connection with no stamp (a client's end, a joined one) is `InvalidArgument`, one from another port `PermissionDenied`, and one made through an unbadged connector `NotFound`, never an empty badge. +/// **The port check is the whole boundary**: any process can make a port and mint any bytes on it, so a stamp is answered only to the acceptor of the port it was made on. A connection with no stamp (a client's end) is `InvalidArgument`, one from another port `PermissionDenied`, and one made through an unbadged connector `NotFound`, never an empty badge. pub(super) fn sys_port_badge( acceptor_h: RawHandle, conn_h: RawHandle, diff --git a/lib/blockring/transport/Cargo.toml b/lib/blockring/transport/Cargo.toml index 60ea174eac3..5f218ba0d09 100644 --- a/lib/blockring/transport/Cargo.toml +++ b/lib/blockring/transport/Cargo.toml @@ -4,17 +4,20 @@ # in. Nothing here maps, copies or blocks. # # Its defects are orders, lost wakes and hostile peers, so `tests/loom.rs` -# holds the publication edge, the wake handshake and a hostile peer. +# holds the publication edge, the wake handshake, a hostile peer and a +# session's three waits. [package] name = "toyos-transport" -version = "0.1.0" +version = "0.1.0" # never bumped; crates.io's is assigned on publish (build/sdkversion.rs) edition = "2021" -description = "The transport every client/server session runs over: entry rings, byte streams and their wakes, an arena of runs and a tag table, decided over words an adapter hands in." +description = "The transport every client/server session runs over: entry rings, byte streams, sessions and their wakes, an arena of runs and a tag table, decided over words an adapter hands in." license = "MIT OR Apache-2.0" -publish = false +repository = "https://github.com/ToyOSOrg/ToyOS" [features] +# std links it through `toyos`, whose stream end is a session's client. +rustc-dep-of-std = ["core", "toyos-untrusted/rustc-dep-of-std"] # Declared, never enabled by any build that ships: each reverts one decision so # the model that refuses it is shown able to red at all. `build/ci.rs`'s # `CONTROLS` runs each and demands the named test's own FAILED line. @@ -36,8 +39,28 @@ want-relaxed = [] # `Inflight::end` answers nothing and keeps every tag in flight, so a session's # requests outlive it unanswered and a late completion answers one. end-keeps-inflight = [] +# A session's server rings a writer waiting for room on the connection, the +# doorbell a reader waits on, so either can take the other's wake. +one-doorbell = [] +# A thread that took a direction's doorbell leaves it with another waiter +# counted and passes nothing on, so that waiter sleeps over what the take +# announced. +no-pass-on = [] +# A registration looks without arming, so a direction that becomes ready after +# its look rings nobody. +register-no-arm = [] +# A registration arms and looks but rings nobody for a direction it found +# ready, so its first event never comes. +register-no-ring = [] +# Both of the two above. +register-neither = ["register-no-arm", "register-no-ring"] +# A look that arms the send direction, and the server's ring back of it, read +# only the room and not a failure, so a writer over a full ring sleeps through +# a reset. +head-only = [] [dependencies] +core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } toyos-untrusted = { path = "../../untrusted" } [dev-dependencies] diff --git a/lib/blockring/transport/src/lib.rs b/lib/blockring/transport/src/lib.rs index 940dfff917a..eb3f6779c43 100644 --- a/lib/blockring/transport/src/lib.rs +++ b/lib/blockring/transport/src/lib.rs @@ -3,9 +3,9 @@ //! **A session is one connection and, at most, one region.** The connection //! carries the hello, handles, doorbells and the hang-up; the region carries //! [`Producer`]/[`Consumer`] queues of fixed-size entries and an arena of -//! [`Run`]s, or byte [`stream`]s and the wants that say when to ring. A -//! protocol says what the entries and bytes mean; this crate says only what is -//! safe. +//! [`Run`]s, or byte [`stream`]s and the wants that say when to ring, two of +//! which with their `end`s are a duplex [`session`]. A protocol says what the +//! entries and bytes mean; this crate says only what is safe. //! //! **Nothing here holds the region.** An adapter hands every call the `N` //! [`Word`]s an end was made over, and copies bytes itself, through a run's @@ -30,6 +30,7 @@ mod arena; mod inflight; mod queue; +pub mod session; pub mod stream; use core::sync::atomic::{AtomicU32, Ordering}; diff --git a/lib/blockring/transport/src/session.rs b/lib/blockring/transport/src/session.rs new file mode 100644 index 00000000000..99a64e0bac6 --- /dev/null +++ b/lib/blockring/transport/src/session.rs @@ -0,0 +1,815 @@ +//! A duplex byte stream between a client and a server: two [`stream`] rings +//! and two `end` words among `N` words, and two doorbells the server rings. +//! +//! **Nothing here is any protocol's.** The receive ring carries the server's +//! bytes to the client and the send ring the client's to the server. The +//! server's `end` word ends the receive ring and the client's the send ring: +//! [`OPEN`] while the ring runs, [`ORDERLY`] once its producer's last byte is +//! published, and any other value a failure the protocol names, which also +//! makes the client's send direction ready to every look, so a write answers it +//! rather than waiting. An `end` is stored after the last tail with `Release` +//! and loaded before the tail with `Acquire`, so a reader that finds it finds +//! every byte before it. +//! +//! **The client waits on one doorbell a direction** ([`doorbell`]), the one the +//! server rings for that direction's want, so a reader and a writer never take +//! each other's wake. The server waits on one, which every client ring writes. +//! +//! **The client's sequences are written once, here.** Within a direction the +//! half is held to drain, arm, look and copy, and never across a wait. A waiter +//! counts itself in before it drains and out when it returns. A thread that +//! took the doorbell, by a wait or a drain, and leaves the half with another +//! waiter counted arms again, looks, and rings the server if the direction is +//! ready; the server answers by ringing back each ready direction whose want is +//! set ([`Server::rung`]). So a wake one thread took is never one another +//! sleeps over. `WouldBlock` is answered only after the drain, the arm and the +//! look that follows it. A registration ([`register`]) arms each direction it +//! names and rings the server if one is ready, and drains and counts nothing. +//! +//! The adapter supplies the [`Word`]s, the [`Lock`] each half lives behind and +//! the [`Bells`]; it copies bytes through the [`Span`]s it is handed, and blocks +//! only inside [`Bells::wait`]. + +use core::sync::atomic::Ordering; + +use crate::stream::{Consumer, Place, Producer, WAKE}; +use crate::{word, Span, Violation, Word}; + +/// An `end` while its ring runs. +pub const OPEN: u32 = 0; +/// An `end` after its producer's last byte. +pub const ORDERLY: u32 = 1; + +/// A failure: any `end` but [`OPEN`] and [`ORDERLY`]. +const fn fatal(end: u32) -> bool { + end != OPEN && end != ORDERLY +} + +/// A direction, as the client names it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Way { + /// The server's bytes to the client. + Receive, + /// The client's bytes to the server. + Send, +} + +/// A pipe the server writes and the client waits on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Doorbell { + /// The connection the session was opened on. + Connection, + /// A pipe of the session's own. + Bell, +} + +/// The doorbell the server rings for a client waiting on `way`. +pub const fn doorbell(way: Way) -> Doorbell { + match way { + Way::Receive => Doorbell::Connection, + Way::Send if cfg!(feature = "one-doorbell") => Doorbell::Connection, + Way::Send => Doorbell::Bell, + } +} + +/// Which of the client's doorbells the server is to ring. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[must_use] +pub struct Rings { + pub connection: bool, + pub bell: bool, +} + +impl Rings { + fn of(way: Way, rung: bool) -> Self { + match doorbell(way) { + Doorbell::Connection => Self { connection: rung, bell: false }, + Doorbell::Bell => Self { connection: false, bell: rung }, + } + } + + /// Both sets of rings. + pub fn and(self, other: Self) -> Self { + Self { connection: self.connection || other.connection, bell: self.bell || other.bell } + } +} + +/// Where a session's two rings and two `end`s are among `N` words. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Layout { + rx: Place, + tx: Place, + rx_end: usize, + tx_end: usize, +} + +impl Layout { + /// The receive ring at `rx`, the send ring at `tx`, and the two `end`s. + /// A layout whose words are not ten distinct ones below `N` does not + /// compile where it is a constant. + pub const fn new(rx: Place, tx: Place) -> Self { + let [a, b, c, d] = rx.words(); + let [e, f, g, h] = tx.words(); + assert!(RX_END < N && TX_END < N, "an end past the words"); + assert!(distinct(&[a, b, c, d, e, f, g, h, RX_END, TX_END]), "a session's ten words are ten"); + Self { rx, tx, rx_end: RX_END, tx_end: TX_END } + } + + /// The receive ring's `end` as the server last stored it. + pub fn received_end(&self, page: &[W; N]) -> u32 { + word(page, self.rx_end).load(Ordering::Acquire) + } +} + +/// Whether no two of `words` are one. +const fn distinct(words: &[usize]) -> bool { + match words.split_first() { + None => true, + Some((first, rest)) => !holds(rest, *first) && distinct(rest), + } +} + +const fn holds(words: &[usize], word: usize) -> bool { + match words.split_first() { + None => false, + Some((first, rest)) => *first == word || holds(rest, word), + } +} + +/// What a client call answers instead of bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refused { + /// Nonblocking, and the direction is not ready. + WouldBlock, + /// The deadline passed first. + TimedOut, + /// The server hung up without an `end`. + Gone, + /// The receive ring's `end` names this failure. + Failed(u32), + /// The client shut its sending half down. + Shut, + /// The server wrote what no server of the protocol writes. + Violation(Violation), +} + +impl From for Refused { + fn from(violation: Violation) -> Self { + Self::Violation(violation) + } +} + +/// What a drain of a doorbell took. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Took { + Nothing, + /// One ring or more. + Rung, + /// The server holds the doorbell no more. + Gone, +} + +/// How a wait on a doorbell ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Waited { + /// A ring, taken. + Rung, + TimedOut, + /// The server holds the doorbell no more. + Gone, +} + +/// Whether a call waits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Mode { + Nonblocking, + /// Until the direction is ready or, given one, a deadline. + Blocking(Option), +} + +/// The client's doorbells and its ring of the server. +pub trait Bells { + /// A deadline. + type Until: Copy; + /// Takes every ring the doorbell holds without waiting. + fn drain(&self, bell: Doorbell) -> Took; + /// Waits for a ring and takes it. + fn wait(&self, bell: Doorbell, until: Option) -> Waited; + /// Rings the server. + fn ring(&self); +} + +/// The lock a half lives behind: `f` runs holding it. +pub trait Lock { + fn with(&self, f: impl FnOnce(&mut T) -> R) -> R; +} + +mod sealed { + pub trait Sealed {} +} + +/// One of the client's two halves. +pub trait Half: sealed::Sealed { + /// What a look that finds the direction ready saw. + type Seen: Copy; + #[doc(hidden)] + const WAY: Way; + #[doc(hidden)] + fn waiters(&mut self) -> &mut u32; + #[doc(hidden)] + fn gone(&mut self) -> &mut bool; + #[doc(hidden)] + fn look(&mut self, page: &[W; N]) -> Result, Violation>; + /// Its want armed, then a look. + #[doc(hidden)] + fn arm(&mut self, page: &[W; N]) -> Result, Violation>; +} + +/// The client's receiving half. +#[derive(Debug)] +pub struct Receiving { + ring: Consumer, + end: usize, + waiters: u32, + gone: bool, +} + +/// The client's sending half. +#[derive(Debug)] +pub struct Sending { + ring: Producer, + /// The server's `end`, which a failure makes ready. + received: usize, + end: usize, + waiters: u32, + gone: bool, + shut: bool, +} + +/// What a ready receiving half has: bytes, or none and an `end`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Readable { + ready: u32, + end: u32, +} + +/// What a ready sending half has. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Writable { + Room(u32), + Failed(u32), + Shut, +} + +impl sealed::Sealed for Receiving {} +impl sealed::Sealed for Sending {} + +impl Receiving { + /// Bytes, or an `end` behind the last of them: the `end` is loaded first. + fn seen(&mut self, page: &[W; N], armed: Option) -> Result, Violation> { + let end = word(page, self.end).load(Ordering::Acquire); + let ready = match armed { + Some(ready) if end == OPEN => ready, + _ => self.ring.ready(page)?, + }; + Ok((ready > 0 || end != OPEN).then_some(Readable { ready, end })) + } +} + +impl Half for Receiving { + type Seen = Readable; + const WAY: Way = Way::Receive; + + fn waiters(&mut self) -> &mut u32 { + &mut self.waiters + } + + fn gone(&mut self) -> &mut bool { + &mut self.gone + } + + fn look(&mut self, page: &[W; N]) -> Result, Violation> { + self.seen(page, None) + } + + /// The tail the arm read is believed only while no `end` is stored: one + /// stored after it may follow bytes it did not see. + fn arm(&mut self, page: &[W; N]) -> Result, Violation> { + let ready = self.ring.arm(page)?; + self.seen(page, Some(ready)) + } +} + +impl Sending { + fn seen(&self, page: &[W; N], room: u32, failure: bool) -> Option { + if self.shut { + return Some(Writable::Shut); + } + let end = if failure { word(page, self.received).load(Ordering::Acquire) } else { OPEN }; + if fatal(end) { + return Some(Writable::Failed(end)); + } + (room > 0).then_some(Writable::Room(room)) + } +} + +impl Half for Sending { + type Seen = Writable; + const WAY: Way = Way::Send; + + fn waiters(&mut self) -> &mut u32 { + &mut self.waiters + } + + fn gone(&mut self) -> &mut bool { + &mut self.gone + } + + /// The failure first, so a write after one copies nothing. + fn look(&mut self, page: &[W; N]) -> Result, Violation> { + if let Some(seen @ (Writable::Shut | Writable::Failed(_))) = self.seen(page, 0, true) { + return Ok(Some(seen)); + } + let room = self.ring.room(page)?; + Ok(self.seen(page, room, true)) + } + + fn arm(&mut self, page: &[W; N]) -> Result, Violation> { + let room = self.ring.arm(page)?; + Ok(self.seen(page, room, !cfg!(feature = "head-only"))) + } +} + +/// The client's two halves, their cursors stored 0. +pub fn client( + page: &[W; N], + layout: Layout, +) -> (Receiving, Sending) { + let receiving = Receiving { ring: Consumer::new(page, layout.rx), end: layout.rx_end, waiters: 0, gone: false }; + let sending = Sending { + ring: Producer::new(page, layout.tx), + received: layout.rx_end, + end: layout.tx_end, + waiters: 0, + gone: false, + shut: false, + }; + (receiving, sending) +} + +/// Counts a waiter out and, where it took the doorbell and another waits, +/// arms and looks again: `true` to ring the server for a direction found ready. +fn leave, W: Word>(half: &mut H, page: &[W; N], counted: bool, took: bool) -> bool { + if counted { + let waiters = half.waiters(); + *waiters = waiters.saturating_sub(1); + } + let pass_on = !cfg!(feature = "no-pass-on") && took && *half.waiters() > 0; + pass_on && matches!(half.arm(page), Ok(Some(_))) +} + +/// One look of a waiter under its half: the first look, then, counted in, +/// the drain, the arm and the look after it. +fn look_counted, W: Word, B: Bells>( + half: &mut H, + page: &[W; N], + bells: &B, + counted: &mut bool, + took: &mut bool, +) -> Result, Violation> { + if let Some(seen) = half.look(page)? { + return Ok(Some(seen)); + } + if !*counted { + let waiters = half.waiters(); + *waiters = waiters.saturating_add(1); + *counted = true; + } + match bells.drain(doorbell(H::WAY)) { + Took::Rung => *took = true, + Took::Gone => *half.gone() = true, + Took::Nothing => {} + } + half.arm(page) +} + +/// A call that waits for its direction: `act` runs under the half once it is +/// ready, and answers the call and whether the server is to be rung. +fn waiting( + page: &[W; N], + lock: &L, + bells: &B, + mode: Mode, + mut act: impl FnMut(&mut H, H::Seen) -> (Result, bool), +) -> Result +where + H: Half, + W: Word, + L: Lock, + B: Bells, +{ + let (mut took, mut counted, mut expired, mut hung_up) = (false, false, false, false); + loop { + let (answer, ring) = lock.with(|half| { + if hung_up { + *half.gone() = true; + } + let mut ring = false; + let answer = match look_counted(half, page, bells, &mut counted, &mut took) { + Err(violation) => Some(Err(Refused::Violation(violation))), + Ok(Some(seen)) => { + let (answer, rung) = act(half, seen); + ring = rung; + Some(answer) + } + Ok(None) if *half.gone() => Some(Err(Refused::Gone)), + Ok(None) => match mode { + Mode::Nonblocking => Some(Err(Refused::WouldBlock)), + Mode::Blocking(_) if expired => Some(Err(Refused::TimedOut)), + Mode::Blocking(_) => None, + }, + }; + if answer.is_some() { + ring |= leave(half, page, counted, took); + } + (answer, ring) + }); + if ring { + bells.ring(); + } + if let Some(answer) = answer { + return answer; + } + let Mode::Blocking(until) = mode else { return Err(Refused::WouldBlock) }; + match bells.wait(doorbell(H::WAY), until) { + Waited::Rung => took = true, + Waited::TimedOut => expired = true, + Waited::Gone => hung_up = true, + } + } +} + +/// Reads at most `max` bytes, each span copied out by `copy` at its offset +/// into the caller's buffer: the count, at least one, or 0 for an orderly end +/// with every byte read. A read of no bytes answers 0 at once. +pub fn read( + page: &[W; N], + lock: &L, + bells: &B, + mode: Mode, + max: u32, + mut copy: impl FnMut(Span, usize), +) -> Result +where + W: Word, + L: Lock>, + B: Bells, +{ + if max == 0 { + return Ok(0); + } + waiting(page, lock, bells, mode, |half: &mut Receiving, seen: Readable| { + if seen.ready == 0 { + return (if seen.end == ORDERLY { Ok(0) } else { Err(Refused::Failed(seen.end)) }, false); + } + let n = seen.ready.min(max); + let mut at = 0usize; + for span in half.ring.span(0, n).into_iter().flatten() { + copy(span, at); + at = at.saturating_add(span.len); + } + if !half.ring.advance(n) { + return (Err(Refused::Violation(Violation::TailPastDepth)), false); + } + (Ok(n), half.ring.release(page)) + }) +} + +/// Writes at most `len` bytes, each span copied in by `copy` from its offset +/// into the caller's buffer: the count, at least one. A failure the server +/// stored is answered, and nothing copied, whatever room there is. A write of +/// no bytes answers 0 at once. +pub fn write( + page: &[W; N], + lock: &L, + bells: &B, + mode: Mode, + len: u32, + mut copy: impl FnMut(Span, usize), +) -> Result +where + W: Word, + L: Lock>, + B: Bells, +{ + if len == 0 { + return Ok(0); + } + waiting(page, lock, bells, mode, |half: &mut Sending, seen: Writable| match seen { + Writable::Shut => (Err(Refused::Shut), false), + Writable::Failed(end) => (Err(Refused::Failed(end)), false), + Writable::Room(room) => { + let n = room.min(len); + let mut at = 0usize; + for span in half.ring.span(0, n).into_iter().flatten() { + copy(span, at); + at = at.saturating_add(span.len); + } + if !half.ring.advance(n) { + return (Err(Refused::Violation(Violation::HeadPastTail)), false); + } + (Ok(n), half.ring.publish(page)) + } + }) +} + +/// A poll of one direction that found it not ready: counted in until +/// [`unwatch`]. +#[derive(Debug)] +#[must_use] +pub struct Watching { + took: bool, +} + +/// What a poll's look found. +#[derive(Debug)] +pub enum Polled { + Ready, + /// Not ready: the caller waits on the direction's [`doorbell`], then calls + /// [`unwatch`]. + Watching(Watching), +} + +/// A poll of the direction `H` is: ready at once, or counted in, drained, armed +/// and looked at again. +pub fn watch(page: &[W; N], lock: &L, bells: &B) -> Result +where + W: Word, + L: Lock, + B: Bells, + H: Half, +{ + let (polled, ring) = lock.with(|half| { + let (mut counted, mut took) = (false, false); + let seen = look_counted(half, page, bells, &mut counted, &mut took); + let polled = match seen { + Ok(None) if !*half.gone() => return (Ok(Polled::Watching(Watching { took })), false), + Ok(_) => Ok(Polled::Ready), + Err(violation) => Err(violation), + }; + (polled, leave(half, page, counted, took)) + }); + if ring { + bells.ring(); + } + polled +} + +/// The end of a poll [`watch`] counted in, once its wait is over: whether the +/// direction is ready now. +pub fn unwatch(page: &[W; N], lock: &L, bells: &B, watching: Watching) -> Result +where + W: Word, + L: Lock, + B: Bells, + H: Half, +{ + let (ready, ring) = lock.with(|half| { + let ring = leave(half, page, true, watching.took); + let gone = *half.gone(); + (half.look(page).map(|seen| seen.is_some() || gone), ring) + }); + if ring { + bells.ring(); + } + ready +} + +/// What a registration found of one half. +fn registered, W: Word>(half: &mut H, page: &[W; N]) -> Result { + let seen = if cfg!(feature = "register-no-arm") { half.look(page)? } else { half.arm(page)? }; + Ok(seen.is_some()) +} + +/// A source registered for the directions whose halves are given: each one's +/// want armed and looked at, and the server rung if one is ready, so the first +/// event comes whether the direction was ready before or not. +pub fn register( + page: &[W; N], + receive: Option<&RL>, + send: Option<&SL>, + bells: &B, +) -> Result<(), Violation> +where + W: Word, + RL: Lock>, + SL: Lock>, + B: Bells, +{ + let mut ready = false; + if let Some(lock) = receive { + ready |= lock.with(|half| registered(half, page))?; + } + if let Some(lock) = send { + ready |= lock.with(|half| registered(half, page))?; + } + if ready && !cfg!(feature = "register-no-ring") { + bells.ring(); + } + Ok(()) +} + +/// The client sends no more: its `end` stored after its last tail and the +/// server rung, once. A write after it is refused [`Refused::Shut`]. +pub fn shutdown(page: &[W; N], lock: &L, bells: &B) +where + W: Word, + L: Lock>, + B: Bells, +{ + let first = lock.with(|half| { + let first = !half.shut; + if first { + half.shut = true; + word(page, half.end).store(ORDERLY, Ordering::Release); + } + first + }); + if first { + bells.ring(); + } +} + +/// The server's end: the receive ring's producer and the send ring's consumer. +#[derive(Debug)] +pub struct Server { + layout: Layout, + rx: Producer, + tx: Consumer, + /// The receive ring's `end` as stored. + ended: u32, +} + +impl Server { + /// The server's end, its cursors stored 0. + pub fn new(page: &[W; N], layout: Layout) -> Self { + Self { layout, rx: Producer::new(page, layout.rx), tx: Consumer::new(page, layout.tx), ended: OPEN } + } + + /// Bytes published that the client has not released. + pub fn held(&mut self, page: &[W; N]) -> Result { + Ok(RX.saturating_sub(self.rx.room(page)?)) + } + + /// Publishes `n` more bytes, already in the ring behind the tail. + /// [`Violation::HeadPastTail`] where the client's head left no room for + /// them. + pub fn publish(&mut self, page: &[W; N], n: u32) -> Result { + self.rx.room(page)?; + if !self.rx.advance(n) { + return Err(Violation::HeadPastTail); + } + Ok(Rings::of(Way::Receive, self.rx.publish(page))) + } + + /// Bytes the client published past what this end released. + pub fn sent(&mut self, page: &[W; N]) -> Result { + self.tx.ready(page) + } + + /// Gives `n` bytes of the send ring back to the client. + /// [`Violation::TailPastDepth`] where its tail no longer covers them. + pub fn release(&mut self, page: &[W; N], n: u32) -> Result { + self.tx.ready(page)?; + if !self.tx.advance(n) { + return Err(Violation::TailPastDepth); + } + Ok(Rings::of(Way::Send, self.tx.release(page))) + } + + /// Whether the client shut its sending half down: loaded before + /// [`Self::sent`], it says the bytes that call counts are the last. + pub fn shut(&self, page: &[W; N]) -> bool { + word(page, self.layout.tx_end).load(Ordering::Acquire) != OPEN + } + + /// Asks to be rung once the client releases: the client's room after the + /// arm, from which the caller reads what it released. + pub fn want_room(&mut self, page: &[W; N]) -> Result { + self.rx.arm(page) + } + + /// Asks to be rung once the client publishes: what it published after the + /// arm. + pub fn want_data(&mut self, page: &[W; N]) -> Result { + self.tx.arm(page) + } + + /// The receive ring's orderly end, after its last tail, once. + pub fn finish(&mut self, page: &[W; N]) -> Rings { + if self.ended != OPEN { + return Rings::default(); + } + self.ended = ORDERLY; + word(page, self.layout.rx_end).store(ORDERLY, Ordering::Release); + let [_, _, data, _] = self.layout.rx.words(); + Rings::of(Way::Receive, word(page, data).swap(0, WAKE) != 0) + } + + /// The session's failure `code`, past [`ORDERLY`]: stored after the last + /// tail, then both client wants exchanged for 0, each that was set rung, + /// so a client asleep either way wakes and reads why. + pub fn fail(&mut self, page: &[W; N], code: u32) -> Rings { + self.ended = code; + word(page, self.layout.rx_end).store(code, Ordering::Release); + let [_, _, data, _] = self.layout.rx.words(); + let [_, _, _, room] = self.layout.tx.words(); + let receive = Rings::of(Way::Receive, word(page, data).swap(0, WAKE) != 0); + receive.and(Rings::of(Way::Send, word(page, room).swap(0, WAKE) != 0)) + } + + /// The client rang: each of its wants whose direction is ready exchanged + /// for 0, and the doorbells of those that were set. + pub fn rung(&mut self, page: &[W; N]) -> Result { + let receive = self.rx.ring_back(page, self.ended != OPEN)?; + let send = self.tx.ring_back(page, fatal(self.ended) && !cfg!(feature = "head-only"))?; + Ok(Rings::of(Way::Receive, receive).and(Rings::of(Way::Send, send))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::cell::{Cell, RefCell}; + use core::sync::atomic::AtomicU32; + + const N: usize = 10; + const C: u32 = 4; + const LAYOUT: Layout = Layout::new::<8, 9>( + Place::new::<0, 1, 2, 3, 0>(), + Place::new::<4, 5, 6, 7, { C as usize }>(), + ); + + fn page() -> [AtomicU32; N] { + core::array::from_fn(|_| AtomicU32::new(0)) + } + + /// Doorbells that are never rung and never waited on: a test of one + /// thread's decisions. + #[derive(Default)] + struct Quiet { + rang: Cell, + } + + impl Bells for Quiet { + type Until = (); + fn drain(&self, _: Doorbell) -> Took { + Took::Nothing + } + fn wait(&self, _: Doorbell, _: Option<()>) -> Waited { + unreachable!("a nonblocking call waited") + } + fn ring(&self) { + self.rang.set(self.rang.get() + 1); + } + } + + struct Held(RefCell); + + impl Lock for Held { + fn with(&self, f: impl FnOnce(&mut T) -> R) -> R { + f(&mut self.0.borrow_mut()) + } + } + + /// RFC 9293 flushes every queue at a reset (§3.10.7.4) and answers a SEND + /// after it an error (§3.10.2): a write over a send ring with room, after + /// the server stored a failure, answers the failure and copies nothing. + #[test] + fn a_write_with_room_after_a_reset_answers_it_and_copies_nothing() { + const RESET: u32 = 2; + let page = page(); + let (_, sending) = client(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT); + let _ = server.fail(&page, RESET); + let (sending, bells) = (Held(RefCell::new(sending)), Quiet::default()); + let mut copied = 0; + let wrote = write(&page, &sending, &bells, Mode::Nonblocking, 3, |span, _| copied += span.len); + assert_eq!(wrote, Err(Refused::Failed(RESET))); + assert_eq!(copied, 0, "a byte was copied after the reset"); + assert_eq!(server.sent(&page), Ok(0), "a byte was published after the reset"); + } + + /// The `end` behind the last byte: every byte is read, then the end. + #[test] + fn a_read_takes_every_byte_before_the_end() { + let page = page(); + let (receiving, _) = client(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT); + let _ = server.publish(&page, 3); + let _ = server.finish(&page); + let (receiving, bells) = (Held(RefCell::new(receiving)), Quiet::default()); + let read = |max| read(&page, &receiving, &bells, Mode::Nonblocking, max, |_, _| {}); + assert_eq!((read(2), read(2), read(2)), (Ok(2), Ok(1), Ok(0))); + } +} diff --git a/lib/blockring/transport/src/stream.rs b/lib/blockring/transport/src/stream.rs index ead60173214..6ea71516689 100644 --- a/lib/blockring/transport/src/stream.rs +++ b/lib/blockring/transport/src/stream.rs @@ -35,9 +35,9 @@ use crate::{clamp, word, Span, Untrusted, Violation, Word, PUBLISH}; /// seen by the look that follows. const ARM: Ordering = Ordering::Acquire; #[cfg(not(feature = "want-relaxed"))] -const WAKE: Ordering = Ordering::Release; +pub(crate) const WAKE: Ordering = Ordering::Release; #[cfg(feature = "want-relaxed")] -const WAKE: Ordering = Ordering::Relaxed; +pub(crate) const WAKE: Ordering = Ordering::Relaxed; /// The head's edge: a producer that loads the head copies into a byte only /// after the consumer's copy out of it, which the consumer's store carries. const GIVE_BACK: Ordering = if cfg!(feature = "head-relaxed") { Ordering::Relaxed } else { Ordering::Release }; @@ -93,6 +93,11 @@ impl Place { Self { head: HEAD, tail: TAIL, data: DATA, room: ROOM, base: BASE } } + /// Its four words: head, tail and the two wants. + pub(crate) const fn words(&self) -> [usize; 4] { + [self.head, self.tail, self.data, self.room] + } + /// `at` moved `n` on; both are below the lap. fn forward(at: u32, n: u32) -> u32 { let to = at.wrapping_add(n); @@ -187,6 +192,18 @@ impl Producer { word(page, self.place.room).swap(1, ARM); self.room(page) } + + /// The consumer asked to be rung back: `true`, and its want exchanged for + /// 0, if it is set and the consumer has bytes past its head or the ring + /// `ended`. With neither, nothing is exchanged and the want waits for the + /// next publish. + pub fn ring_back(&self, page: &[W; N], ended: bool) -> Result { + let head = Untrusted::new(word(page, self.place.head).load(TAKE_BACK)); + let published = self.published; + let held = + Place::::reach(head, |head| Place::::distance(head, published), Violation::HeadPastTail)?; + Ok((held > 0 || ended) && word(page, self.place.data).swap(0, WAKE) != 0) + } } /// The end of a ring that reads bytes; like [`Producer`], it holds cursors and @@ -253,6 +270,18 @@ impl Consumer { word(page, self.place.data).swap(1, ARM); self.ready(page) } + + /// The producer asked to be rung back: `true`, and its want exchanged for + /// 0, if it is set and the producer has room past what this end released, + /// or `fatal`. With neither, nothing is exchanged and the want waits for + /// the next release. + pub fn ring_back(&self, page: &[W; N], fatal: bool) -> Result { + let tail = Untrusted::new(word(page, self.place.tail).load(Ordering::Acquire)); + let released = self.released; + let unreleased = + Place::::reach(tail, |tail| Place::::distance(released, tail), Violation::TailPastDepth)?; + Ok((unreleased < C || fatal) && word(page, self.place.room).swap(0, WAKE) != 0) + } } #[cfg(test)] @@ -538,6 +567,32 @@ mod tests { } } + /// A ring back exchanges a want only of a peer whose direction is ready, + /// so the want of one that is not stays for the move that makes it ready. + #[test] + fn a_ring_back_answers_only_a_peer_that_armed_and_is_ready() { + const C: u32 = 2; + let page = page(); + let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); + assert_eq!(rx.arm(&page), Ok(0)); + assert_eq!(tx.ring_back(&page, false), Ok(false), "nothing to read: the want stays"); + assert_eq!(tx.ring_back(&page, true), Ok(true), "the ring ended"); + assert_eq!(tx.ring_back(&page, true), Ok(false), "the want was exchanged"); + assert!(tx.advance(2)); + assert!(!tx.publish(&page), "nobody armed"); + assert_eq!(rx.arm(&page), Ok(2)); + assert_eq!(tx.ring_back(&page, false), Ok(true), "bytes past the head"); + assert_eq!(tx.arm(&page), Ok(0), "a full ring"); + assert_eq!(rx.ring_back(&page, false), Ok(false), "no room: the want stays"); + assert_eq!(rx.ring_back(&page, true), Ok(true), "a fatal end"); + assert_eq!(tx.arm(&page), Ok(0)); + assert!(rx.advance(1)); + assert_eq!(rx.ring_back(&page, false), Ok(false), "this end released nothing yet"); + assert!(rx.release(&page), "the release rings it"); + assert_eq!(tx.arm(&page), Ok(1)); + assert_eq!(rx.ring_back(&page, false), Ok(true), "room past what was released"); + } + /// An end rings exactly when the peer armed since the last ring, and only /// when it moved its cursor. #[test] diff --git a/lib/blockring/transport/tests/loom.rs b/lib/blockring/transport/tests/loom.rs index 8403b9b9a6f..4fca91b59e7 100644 --- a/lib/blockring/transport/tests/loom.rs +++ b/lib/blockring/transport/tests/loom.rs @@ -6,6 +6,10 @@ //! - **A hostile peer** leaves the honest end with entries, nothing, or a //! named violation, and never more entries than the ring holds. `no-clamp` //! believes the peer. +//! - **A session's three waits**, each over `session`'s own sequences: a +//! reader and a writer asleep at once, a poll beside a blocking call of one +//! direction, and a source registered while its direction is ready. Each +//! case names the controls that red it. //! //! cargo test -p toyos-transport --features --test loom @@ -274,3 +278,482 @@ fn a_producer_that_looked_again_is_rung_for_room_it_missed() { assert_eq!(consumer.join().expect("the consumer thread"), 0x5A); }); } + +// ---- a session: the client's sequences and the server's rings ---- +// +// The models below drive `session`'s own code: the client's `read`, `write`, +// `watch`, `unwatch` and `register` and the server's `publish`, `release`, +// `finish`, `fail` and `rung`. A doorbell is a kernel pipe as the client waits +// on it: a ring writes a byte and posts its watch, a blocking wait and a drain +// take every byte there, and a watch is answered at once if the pipe holds a +// byte when it is registered, and otherwise by whether it holds one when its +// submitter looks after a post, armed again if it does not +// (`kernel/src/inbox/polls.rs`, `deliver`). loom fails a run that ends with a +// thread waiting. Shared by `Rc`, as the stream models above are. + +use loom::sync::atomic::AtomicBool; +use loom::sync::{Condvar, Mutex}; +use toyos_transport::session::{ + self, Bells, Doorbell, Layout, Lock, Mode, Polled, Receiving, Refused, Rings, Sending, Server, Took, Waited, Way, +}; + +/// Every session model explores its interleavings with at most this many +/// preemptions unless `LOOM_MAX_PREEMPTIONS` names another bound, as the +/// kernel's own models bound theirs. +const PREEMPTIONS: usize = 5; + +fn bounded(model: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + if builder.preemption_bound.is_none() { + builder.preemption_bound = Some(PREEMPTIONS); + } + builder.check(model); +} + +const BYTES: u32 = 2; +const SESSION_WORDS: usize = 10; +const SESSION: Layout = Layout::new::<8, 9>( + stream::Place::new::<0, 1, 2, 3, 0>(), + stream::Place::new::<4, 5, 6, 7, { BYTES as usize }>(), +); +/// A failure the server stores in the receive ring's `end`. +const RESET: u32 = 2; + +type Rx = Receiving; +type Tx = Sending; + +/// A one-byte doorbell pipe. +struct Pipe { + level: Mutex, + posted: Condvar, +} + +struct Level { + bytes: u32, + posts: u32, +} + +impl Pipe { + fn new() -> Self { + Self { level: Mutex::new(Level { bytes: 0, posts: 0 }), posted: Condvar::new() } + } + + /// `SYS_WRITE` of one byte: the watch's post wakes every waiter. + fn ring(&self) { + let mut level = self.level.lock().unwrap(); + level.bytes += 1; + level.posts += 1; + self.posted.notify_all(); + } + + /// A nonblocking `SYS_READ` of every byte there: whether there was one. + fn drain(&self) -> bool { + let mut level = self.level.lock().unwrap(); + std::mem::replace(&mut level.bytes, 0) != 0 + } + + /// A blocking `SYS_READ` and the drain after it. + fn wait(&self) { + let mut level = self.level.lock().unwrap(); + while level.bytes == 0 { + level = self.posted.wait(level).unwrap(); + } + level.bytes = 0; + } + + /// `OP_WATCH` for reading, registered and waited on. + fn watch(&self) { + let mut seen = { + let level = self.level.lock().unwrap(); + if level.bytes != 0 { + return; + } + level.posts + }; + loop { + { + let mut level = self.level.lock().unwrap(); + while level.posts == seen { + level = self.posted.wait(level).unwrap(); + } + } + let level = self.level.lock().unwrap(); + if level.bytes != 0 { + return; + } + seen = level.posts; + } + } +} + +/// A half behind loom's mutex. +struct Held(Mutex); + +impl Lock for Held { + fn with(&self, f: impl FnOnce(&mut T) -> R) -> R { + f(&mut self.0.lock().unwrap()) + } +} + +/// One session: its words and bytes, the client's two halves, the client's two +/// doorbells and the server's one. +struct Session { + page: [Shared; SESSION_WORDS], + bytes: [Cell; 2 * BYTES as usize], + rx: Held, + tx: Held, + connection: Pipe, + bell: Pipe, + server: Pipe, + /// The client is done: the server's loop ends at its next ring. + quit: AtomicBool, +} + +/// The client's doorbells, which never hang up and take no deadline. +impl Bells for Session { + type Until = (); + + fn drain(&self, bell: Doorbell) -> Took { + if self.pipe(bell).drain() { Took::Rung } else { Took::Nothing } + } + + fn wait(&self, bell: Doorbell, _: Option<()>) -> Waited { + self.pipe(bell).wait(); + Waited::Rung + } + + fn ring(&self) { + self.server.ring(); + } +} + +impl Session { + fn new() -> (Rc, Server) { + let page: [Shared; SESSION_WORDS] = core::array::from_fn(|_| Shared(AtomicU32::new(0))); + let (rx, tx) = session::client(&page, SESSION); + let server = Server::new(&page, SESSION); + let s = Session { + page, + bytes: core::array::from_fn(|_| Cell::new(0)), + rx: Held(Mutex::new(rx)), + tx: Held(Mutex::new(tx)), + connection: Pipe::new(), + bell: Pipe::new(), + server: Pipe::new(), + quit: AtomicBool::new(false), + }; + (Rc::new(s), server) + } + + fn pipe(&self, bell: Doorbell) -> &Pipe { + match bell { + Doorbell::Connection => &self.connection, + Doorbell::Bell => &self.bell, + } + } + + /// The server's rings. + fn rings(&self, rings: Rings) { + if rings.connection { + self.connection.ring(); + } + if rings.bell { + self.bell.ring(); + } + } + + /// The client reads one byte, waiting or not. + fn read(&self, mode: Mode<()>) -> Result { + session::read(&self.page, &self.rx, self, mode, 1, |span, _| { + for at in span.offset..span.offset + span.len { + self.bytes[at].get(); + } + }) + } + + /// The client writes one byte, waiting or not. + fn write(&self, mode: Mode<()>) -> Result { + session::write(&self.page, &self.tx, self, mode, 1, |span, _| { + for at in span.offset..span.offset + span.len { + self.bytes[at].set(0x5A); + } + }) + } + + /// The client fills its send ring, before the server's thread runs. + fn fill(&self) { + for _ in 0..BYTES { + assert_eq!(self.write(Mode::Nonblocking), Ok(1)); + } + } + + /// The server publishes `n` bytes into an empty receive ring, which it + /// copies into first. + fn publish(&self, server: &mut Server, n: u32) { + for at in 0..n as usize { + self.bytes[at].set(0xA5); + } + self.rings(server.publish(&self.page, n).expect("the client keeps the protocol")); + } + + /// The server takes back `n` bytes of the send ring. + fn release(&self, server: &mut Server, n: u32) { + assert_eq!(server.sent(&self.page), Ok(BYTES), "the client filled its send ring"); + self.rings(server.release(&self.page, n).expect("the client keeps the protocol")); + } + + /// The server's loop: each ring of the client's answered by + /// [`Server::rung`], until the client is done. + fn serve(&self, mut server: Server) { + loop { + self.server.wait(); + self.rings(server.rung(&self.page).expect("the client keeps the protocol")); + if self.quit.load(Ordering::Acquire) { + return; + } + } + } + + /// The client is done, and says so to the server's loop. + fn done(&self) { + self.quit.store(true, Ordering::Release); + self.server.ring(); + } +} + +/// Which the server's thread does first. +#[derive(Clone, Copy)] +enum First { + Room, + Data, +} + +/// A reader asleep over an empty receive ring and a writer asleep over a full +/// send ring, while the server releases room and publishes data in either +/// order: each wakes, as each waits on a doorbell of its own. +/// `one-doorbell` lets the reader take the writer's wake, or the writer the +/// reader's. +fn two_sleepers(first: First) { + bounded(move || { + let (s, mut server) = Session::new(); + s.fill(); + let net = { + let s = Rc::clone(&s); + loom::thread::spawn(move || match first { + First::Room => { + s.release(&mut server, 1); + s.publish(&mut server, 1); + } + First::Data => { + s.publish(&mut server, 1); + s.release(&mut server, 1); + } + }) + }; + let reader = { + let s = Rc::clone(&s); + loom::thread::spawn(move || assert_eq!(s.read(Mode::Blocking(None)), Ok(1))) + }; + assert_eq!(s.write(Mode::Blocking(None)), Ok(1)); + reader.join().unwrap(); + net.join().unwrap(); + }); +} + +#[test] +fn two_sleepers_each_wake_when_room_is_released_first() { + two_sleepers(First::Room); +} + +#[test] +fn two_sleepers_each_wake_when_data_is_published_first() { + two_sleepers(First::Data); +} + +/// What the server's thread leaves ready, once, for a blocking call to take +/// part of. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Left { + /// Two bytes published; a blocking read takes one. + Bytes, + /// The receive ring's orderly end; a blocking read answers it. + End, + /// Two bytes of a full send ring released; a blocking write fills one. + Room, +} + +/// A blocking call and a `poll` of one direction, the server making the +/// direction ready once and then answering each ring: the poll is answered +/// with the direction ready, whichever of them took the doorbell. +/// `no-pass-on` lets the blocking call take the wake the poll sleeps over. +fn same_direction(left: Left) { + bounded(move || { + let (s, mut server) = Session::new(); + if left == Left::Room { + s.fill(); + } + let net = { + let s = Rc::clone(&s); + loom::thread::spawn(move || { + match left { + Left::Bytes => s.publish(&mut server, 2), + Left::End => s.rings(server.finish(&s.page)), + Left::Room => s.release(&mut server, 2), + } + s.serve(server); + }) + }; + let blocking = { + let s = Rc::clone(&s); + loom::thread::spawn(move || { + let answer = if left == Left::Room { s.write(Mode::Blocking(None)) } else { s.read(Mode::Blocking(None)) }; + let owed = if left == Left::End { 0 } else { 1 }; + assert_eq!(answer, Ok(owed)); + }) + }; + loop { + let polled = if left == Left::Room { + session::watch::<_, _, _, Tx, SESSION_WORDS>(&s.page, &s.tx, &*s) + } else { + session::watch::<_, _, _, Rx, SESSION_WORDS>(&s.page, &s.rx, &*s) + }; + let Polled::Watching(watching) = polled.expect("the server keeps the protocol") else { break }; + let way = if left == Left::Room { Way::Send } else { Way::Receive }; + s.pipe(session::doorbell(way)).watch(); + let ready = if left == Left::Room { + session::unwatch::<_, _, _, Tx, SESSION_WORDS>(&s.page, &s.tx, &*s, watching) + } else { + session::unwatch::<_, _, _, Rx, SESSION_WORDS>(&s.page, &s.rx, &*s, watching) + }; + if ready.expect("the server keeps the protocol") { + break; + } + } + blocking.join().unwrap(); + s.done(); + net.join().unwrap(); + }); +} + +#[test] +fn a_poll_beside_a_blocking_read_is_answered_for_the_bytes_it_left() { + same_direction(Left::Bytes); +} + +#[test] +fn a_poll_beside_a_blocking_read_is_answered_for_the_end_it_left() { + same_direction(Left::End); +} + +#[test] +fn a_poll_beside_a_blocking_write_is_answered_for_the_room_it_left() { + same_direction(Left::Room); +} + +/// What makes the registered direction ready. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Ready { + /// A byte in the receive ring. + Bytes, + /// The receive ring's orderly end. + End, + /// Room in the send ring: empty at the start where it is ready before the + /// registration, released from full where it races it. + Room, + /// A reset stored while the send ring is full. + Reset, +} + +/// A source registered while its direction is ready, or racing the server +/// making it so, gets its first event: its doorbell is watched, as mio's +/// select watches it, and each event leads to the nonblocking operation, +/// until that finds the direction ready. `register-no-arm`, +/// `register-no-ring` and `register-neither` take the registration's arm or +/// ring or both away; `want-relaxed` lets a racing server's clear go unseen; +/// `head-only` makes the reset no readiness to the arm's look or the ring +/// back. +fn registered(ready: Ready, racing: bool) { + bounded(move || { + let (s, mut server) = Session::new(); + if ready == Ready::Reset || (ready == Ready::Room && racing) { + s.fill(); + } + let make_ready = move |s: &Session, server: &mut Server| match ready { + Ready::Bytes => s.publish(server, 1), + Ready::End => s.rings(server.finish(&s.page)), + Ready::Room => s.release(server, 1), + Ready::Reset => s.rings(server.fail(&s.page, RESET)), + }; + if !racing && ready != Ready::Room { + make_ready(&s, &mut server); + } + let net = { + let s = Rc::clone(&s); + loom::thread::spawn(move || { + if racing { + make_ready(&s, &mut server); + } + s.serve(server); + }) + }; + let send = matches!(ready, Ready::Room | Ready::Reset); + let registered = if send { + session::register(&s.page, None::<&Held>, Some(&s.tx), &*s) + } else { + session::register(&s.page, Some(&s.rx), None::<&Held>, &*s) + }; + registered.expect("the server keeps the protocol"); + let way = if send { Way::Send } else { Way::Receive }; + loop { + s.pipe(session::doorbell(way)).watch(); + let answer = if send { s.write(Mode::Nonblocking) } else { s.read(Mode::Nonblocking) }; + match answer { + Err(Refused::WouldBlock) => {} + Ok(_) | Err(Refused::Failed(RESET)) => break, + Err(other) => panic!("the operation answered {other:?}"), + } + } + s.done(); + net.join().unwrap(); + }); +} + +#[test] +fn a_source_registered_after_bytes_arrived_gets_its_first_event() { + registered(Ready::Bytes, false); +} + +#[test] +fn a_source_registered_as_bytes_arrive_gets_its_first_event() { + registered(Ready::Bytes, true); +} + +#[test] +fn a_source_registered_after_the_end_gets_its_first_event() { + registered(Ready::End, false); +} + +#[test] +fn a_source_registered_as_the_end_arrives_gets_its_first_event() { + registered(Ready::End, true); +} + +#[test] +fn a_source_registered_with_room_gets_its_first_event() { + registered(Ready::Room, false); +} + +#[test] +fn a_source_registered_as_room_is_released_gets_its_first_event() { + registered(Ready::Room, true); +} + +#[test] +fn a_source_registered_over_a_full_ring_after_a_reset_gets_its_first_event() { + registered(Ready::Reset, false); +} + +#[test] +fn a_source_registered_over_a_full_ring_as_a_reset_arrives_gets_its_first_event() { + registered(Ready::Reset, true); +} diff --git a/lib/untrusted/Cargo.toml b/lib/untrusted/Cargo.toml index 954395819e7..2540b3d507e 100644 --- a/lib/untrusted/Cargo.toml +++ b/lib/untrusted/Cargo.toml @@ -12,7 +12,14 @@ [package] name = "toyos-untrusted" description = "A number that crossed a trust boundary, and the four ways it gets out." -version = "0.1.0" +version = "0.1.0" # never bumped; crates.io's is assigned on publish (build/sdkversion.rs) edition = "2021" license = "MIT OR Apache-2.0" -publish = false +repository = "https://github.com/ToyOSOrg/ToyOS" + +[features] +# std links it through `toyos-transport`. +rustc-dep-of-std = ["core"] + +[dependencies] +core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } diff --git a/rust b/rust index c1b88133ec7..73f63e2441b 160000 --- a/rust +++ b/rust @@ -1 +1 @@ -Subproject commit c1b88133ec79fc0902d4031d0c58a00b488842b9 +Subproject commit 73f63e2441b93b979229144b58033954ffb1d40d diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index d2e9c5a3512..e3c8ced6357 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -108,9 +108,6 @@ pub const SYS_GET_ENV: u64 = 73; /// [`dup2`]. pub const SYS_HANDLE_DUP_AT: u64 = 74; pub const SYS_CLOCK_EPOCH: u64 = 75; -/// Join a pipe read end and a pipe write end into one duplex `Connection`. -/// See [`connection_join`]. -pub const SYS_CONNECTION_JOIN: u64 = 76; pub const SYS_PIPE_MAP: u64 = 77; pub const SYS_SYMLINK: u64 = 81; pub const SYS_READLINK: u64 = 82; @@ -1966,17 +1963,6 @@ pub fn accept(acceptor: RawHandle) -> Result { check(syscall(SYS_ACCEPT, acceptor.0 as u64, 0, 0, 0)).map(|v| RawHandle(v as u32)) } -/// Join a pipe read end and a pipe write end into one duplex connection. -/// -/// The two ends stay open and this takes references of its own, so the caller -/// closes what it handed in. The result carries no handle-transfer queue: a -/// connection made this way has no peer holding the other half, so -/// [`handle_send`] on one answers [`SyscallError::Gone`]. -pub fn connection_join(rx: RawHandle, tx: RawHandle) -> Result { - check(syscall(SYS_CONNECTION_JOIN, rx.0 as u64, tx.0 as u64, 0, 0)) - .map(|v| RawHandle(v as u32)) -} - /// Handles one [`handle_send`] may carry. /// /// Policy on the primitive: a caller asking for one more is refused by name diff --git a/sdk/libc/Cargo.lock b/sdk/libc/Cargo.lock index 2c169500537..2d766086466 100644 --- a/sdk/libc/Cargo.lock +++ b/sdk/libc/Cargo.lock @@ -30,6 +30,7 @@ name = "toyos" version = "0.18.0" dependencies = [ "toyos-abi", + "toyos-transport", ] [[package]] @@ -55,6 +56,17 @@ dependencies = [ name = "toyos-osrelease" version = "0.1.0" +[[package]] +name = "toyos-transport" +version = "0.1.0" +dependencies = [ + "toyos-untrusted", +] + +[[package]] +name = "toyos-untrusted" +version = "0.1.0" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/sdk/libc/src/errcode.rs b/sdk/libc/src/errcode.rs index 85ee8b9ac72..f6da38e0f3e 100644 --- a/sdk/libc/src/errcode.rs +++ b/sdk/libc/src/errcode.rs @@ -33,10 +33,12 @@ pub(crate) const ENOPROTOOPT: i32 = 92; pub(crate) const EOPNOTSUPP: i32 = 95; pub(crate) const EAFNOSUPPORT: i32 = 97; pub(crate) const EADDRINUSE: i32 = 98; +pub(crate) const ECONNABORTED: i32 = 103; pub(crate) const ECONNRESET: i32 = 104; pub(crate) const ENOTCONN: i32 = 107; pub(crate) const ETIMEDOUT: i32 = 110; pub(crate) const ECONNREFUSED: i32 = 111; +pub(crate) const EHOSTUNREACH: i32 = 113; /// The errno a kernel refusal is answered with. pub(crate) fn of(e: SyscallError) -> i32 { diff --git a/sdk/libc/src/posix_io.rs b/sdk/libc/src/posix_io.rs index a8099f20913..4ba37337a43 100644 --- a/sdk/libc/src/posix_io.rs +++ b/sdk/libc/src/posix_io.rs @@ -598,8 +598,19 @@ pub struct pollfd { pub revents: i16, } +/// A stream socket is asked its rings first, so a level-triggered `poll` of +/// one that is ready answers at once; a direction that is not is counted in, +/// drained, armed and looked at again (`toyos::net::stream::watch`), and its +/// doorbell watched for reading: the connection for `POLLIN`, the bell for +/// `POLLOUT`. Every other descriptor, a listener's wake pipe and a datagram +/// socket's receive pipe for a socket's, is watched as it is. #[no_mangle] pub unsafe extern "C" fn poll(fds: *mut pollfd, nfds: u32, timeout: i32) -> i32 { + use toyos::net::stream::{self, Poll, Way}; + + const POLLIN: i16 = 1; + const POLLOUT: i16 = 4; + if nfds == 0 { if timeout > 0 { syscall::nanosleep(timeout as u64 * 1_000_000); @@ -611,33 +622,76 @@ pub unsafe extern "C" fn poll(fds: *mut pollfd, nfds: u32, timeout: i32) -> i32 // rather than by quietly handing back a smaller ring. A C caller asking to // watch more is not a bug in this library, so it gets POSIX's own answer // for an nfds it cannot serve. - if nfds > toyos::poller::Poller::MAX_HANDLES { + let n = nfds as usize; + let entries: alloc::vec::Vec<(i32, i16)> = (0..n).map(|i| ((*fds.add(i)).fd, (*fds.add(i)).events)).collect(); + let ways = |events: i16| [(POLLIN, Way::Receive), (POLLOUT, Way::Send)].into_iter().filter(move |(flag, _)| events & flag != 0); + let doorbells: usize = entries.iter().filter(|(fd, _)| crate::socket::stream_of(*fd).is_some()).map(|(_, events)| ways(*events).count()).sum(); + if nfds as usize + doorbells > toyos::poller::Poller::MAX_HANDLES as usize { crate::errno::set(EINVAL); return -1; } - let timeout_ns = if timeout < 0 { None } else { Some(timeout as u64 * 1_000_000) }; - - let n = nfds as usize; - let entries: alloc::vec::Vec<(i32, i16)> = (0..n).map(|i| ((*fds.add(i)).fd, (*fds.add(i)).events)).collect(); - let poller = toyos::poller::Poller::new(n as u32); - for (entry, fd, interest) in crate::pollreq::watches(&entries) { - poller.watch_raw(toyos_abi::RawHandle(fd as u32), interest, entry as u64); - } - - let mut ready_set = alloc::vec![false; n]; - poller.wait(1, timeout_ns.unwrap_or(u64::MAX), |token| { - if (token as usize) < n { ready_set[token as usize] = true; } - }); - let mut ready = 0i32; - for i in 0..n { - let answered = ready_set[crate::pollreq::watch_of(&entries, i)]; - let pfd = &mut *fds.add(i); - pfd.revents = 0; - if answered { - pfd.revents = pfd.events; - ready += 1; + let deadline = (timeout >= 0).then(|| toyos_abi::clock::nanos_since_boot().saturating_add(timeout as u64 * 1_000_000)); + // The descriptors that are no stream, each as the kernel handle it is watched by. + let others: alloc::vec::Vec<(i32, i16)> = entries + .iter() + .map(|&(fd, events)| match crate::socket::stream_of(fd) { + Some(_) => (-1, 0), + None => (crate::socket::watched_of(fd).map_or(fd, |handle| handle.0 as i32), events), + }) + .collect(); + loop { + let mut revents = alloc::vec![0i16; n]; + let mut watching = alloc::vec::Vec::new(); + for (i, &(fd, events)) in entries.iter().enumerate() { + let Some(handle) = crate::socket::stream_of(fd) else { continue }; + for (flag, way) in ways(events) { + match stream::watch(handle, way) { + Ok(Poll::Ready) | Err(_) => revents[i] |= flag, + Ok(Poll::Watching(w)) => watching.push((i, flag, stream::doorbell(handle, way), w)), + } + } + } + let ready_now = revents.iter().any(|&r| r != 0); + let left = match deadline { + _ if ready_now => 0, + None => u64::MAX, + Some(at) => at.saturating_sub(toyos_abi::clock::nanos_since_boot()), + }; + let poller = toyos::poller::Poller::new((n + watching.len()) as u32); + for (entry, fd, interest) in crate::pollreq::watches(&others) { + if fd >= 0 { + poller.watch_raw(toyos_abi::RawHandle(fd as u32), interest, entry as u64); + } + } + for (k, (_, _, doorbell, _)) in watching.iter().enumerate() { + if let Ok(doorbell) = doorbell { + poller.watch_raw(*doorbell, toyos::poller::READABLE, (n + k) as u64); + } + } + let mut answered = alloc::vec![false; n + watching.len()]; + poller.wait(if ready_now { 0 } else { 1 }, left, |token| { + if let Some(slot) = answered.get_mut(token as usize) { + *slot = true; + } + }); + for (i, flag, _, w) in watching { + if stream::unwatch(w) { + revents[i] |= flag; + } + } + for i in 0..n { + if others[i].0 >= 0 && answered[crate::pollreq::watch_of(&others, i)] { + revents[i] = entries[i].1; + } + } + let ready = revents.iter().filter(|&&r| r != 0).count(); + let expired = deadline.is_some_and(|at| toyos_abi::clock::nanos_since_boot() >= at); + if ready > 0 || expired { + for (i, r) in revents.into_iter().enumerate() { + (*fds.add(i)).revents = r; + } + return ready as i32; } } - ready -} \ No newline at end of file +} diff --git a/sdk/libc/src/pthread.rs b/sdk/libc/src/pthread.rs index 96800863dbd..cff7e422251 100644 --- a/sdk/libc/src/pthread.rs +++ b/sdk/libc/src/pthread.rs @@ -18,7 +18,7 @@ use alloc::alloc::{alloc as heap_alloc, dealloc as heap_dealloc}; use alloc::boxed::Box; use alloc::vec::Vec; use core::alloc::Layout; -use core::cell::{Cell, UnsafeCell}; +use core::cell::Cell; use core::ptr; use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; @@ -71,64 +71,7 @@ static MASK: Cell = Cell::new(0); /// Blocks of detached threads that have ended, for the next creator to reap. static REAPED: Lock> = Lock::new(Vec::new()); -/// A futex lock around library state: 0 free, 1 held, 2 held with waiters. -pub(crate) struct Lock { - state: AtomicU32, - value: UnsafeCell, -} - -unsafe impl Sync for Lock {} - -pub(crate) struct Held<'a, T>(&'a Lock); - -impl Lock { - pub(crate) const fn new(value: T) -> Self { - Self { state: AtomicU32::new(0), value: UnsafeCell::new(value) } - } - - pub(crate) fn lock(&self) -> Held<'_, T> { - futex_lock(&self.state); - Held(self) - } -} - -impl core::ops::Deref for Held<'_, T> { - type Target = T; - fn deref(&self) -> &T { - // SAFETY: the lock is held. - unsafe { &*self.0.value.get() } - } -} - -impl core::ops::DerefMut for Held<'_, T> { - fn deref_mut(&mut self) -> &mut T { - // SAFETY: the lock is held. - unsafe { &mut *self.0.value.get() } - } -} - -impl Drop for Held<'_, T> { - fn drop(&mut self) { - futex_unlock(&self.0.state); - } -} - -fn futex_lock(state: &AtomicU32) { - if state.compare_exchange(0, 1, Ordering::Acquire, Ordering::Relaxed).is_ok() { - return; - } - while state.swap(2, Ordering::Acquire) != 0 { - // SAFETY: `state` is a live, aligned u32. - unsafe { syscall::futex_wait(state.as_ptr(), 2, None) }; - } -} - -fn futex_unlock(state: &AtomicU32) { - if state.swap(0, Ordering::Release) == 2 { - // SAFETY: `state` is a live, aligned u32. - unsafe { syscall::futex_wake(state.as_ptr(), 1) }; - } -} +pub(crate) use toyos::sync::Lock; fn stack_layout(size: usize) -> Layout { Layout::from_size_align(size, STACK_ALIGN).expect("a thread stack's size is a multiple of its alignment") @@ -344,7 +287,7 @@ pub unsafe extern "C" fn pthread_mutex_lock(mutex: *mut PthreadMutexT) -> i32 { m.count.fetch_add(1, Ordering::Relaxed); return 0; } - futex_lock(&m.state); + toyos::sync::lock_word(&m.state); m.owner.store(me, Ordering::Relaxed); m.count.store(1, Ordering::Relaxed); 0 @@ -378,7 +321,7 @@ pub unsafe extern "C" fn pthread_mutex_unlock(mutex: *mut PthreadMutexT) -> i32 } } m.owner.store(0, Ordering::Relaxed); - futex_unlock(&m.state); + toyos::sync::unlock_word(&m.state); 0 } diff --git a/sdk/libc/src/socket.rs b/sdk/libc/src/socket.rs index ff1c86131c1..366bf355a3b 100644 --- a/sdk/libc/src/socket.rs +++ b/sdk/libc/src/socket.rs @@ -1,17 +1,19 @@ -// BSD sockets — TCP and UDP use pipe-backed data transfer via netstack. +// BSD sockets over netstack: a TCP stream is `toyos::net::stream`'s region +// and connection, and a listener's wakes and a datagram socket's bytes travel +// in pipes. use alloc::alloc::{alloc as heap_alloc, dealloc as heap_dealloc}; use alloc::vec::Vec; use core::ptr; use toyos_abi::RawHandle; use toyos_abi::syscall; -use toyos::net::{NetError, TcpOptions, TcpSocketId, UdpSocketId, OPT_BROADCAST, OPT_NODELAY}; +use toyos::net::stream::{self, Refused, Wait}; +use toyos::net::{NetError, TcpOptions, TcpSocketId, UdpSocketId, OPT_BROADCAST}; use crate::errcode::{ - EACCES, EADDRINUSE, EAFNOSUPPORT, EAGAIN, EBADF, ECONNREFUSED, ECONNRESET, EFAULT, EINVAL, EIO, ENOMEM, ENOPROTOOPT, - ENOSPC, ENOTCONN, EOPNOTSUPP, EPIPE, ETIMEDOUT, + EACCES, EADDRINUSE, EAFNOSUPPORT, EAGAIN, EBADF, ECONNABORTED, ECONNREFUSED, ECONNRESET, EFAULT, EHOSTUNREACH, EINVAL, EIO, + ENOMEM, ENOPROTOOPT, ENOSPC, ENOTCONN, EOPNOTSUPP, EPIPE, ETIMEDOUT, }; -use toyos::net::{read_end, write_refused, StreamRefusal}; use crate::inaddr::{self, SockaddrIn, AF_INET}; use crate::dgramreq; use crate::sockopt::{self, Kept}; @@ -59,7 +61,10 @@ struct SocketEntry { local_port: u16, remote_addr: [u8; 4], remote_port: u16, - // Pipe fds for data path (0 = not set) + /// A stream's connection, which `toyos::net::stream` holds its end by + /// (0 = none). + stream: i32, + // A datagram socket's pipe fds (0 = not set) rx_fd: i32, // read end of rx pipe (netstack→client) tx_fd: i32, // write end of tx pipe (client→netstack) notify_fd: i32, // read end of listener notify pipe @@ -68,12 +73,9 @@ struct SocketEntry { // netstack does not hold yet keeps each for the call that makes it there: // a listener's `nodelay` goes in its `bind`'s request, `broadcast` is // handed over after a datagram socket's bind, and a stream's `nodelay` - // after its `connect`. + // into its region before its connect. nodelay: bool, broadcast: bool, - /// `shutdown` was asked for this half of a stream. - read_shut: bool, - write_shut: bool, } const MAX_SOCKETS: usize = 128; @@ -195,13 +197,12 @@ pub unsafe extern "C" fn socket(domain: i32, sock_type: i32, _protocol: i32) -> local_port: 0, remote_addr: [0; 4], remote_port: 0, + stream: 0, rx_fd: 0, tx_fd: 0, notify_fd: 0, nodelay: false, broadcast: false, - read_shut: false, - write_shut: false, }; let fd = alloc_socket(entry); if fd < 0 { @@ -227,25 +228,16 @@ pub unsafe extern "C" fn connect(fd: i32, addr: *const Sockaddr, addrlen: Sockle match entry.kind { SocketKind::Tcp => { - let conn = match toyos::net::tcp_connect(ip, port, 30000) { + // Nagle's algorithm off from the first segment where it was set before. + let connected = match stream::connect(ip, port, 30000, entry.nodelay) { Ok(c) => c, Err(e) => { set_errno(net_err_to_errno(e)); return -1; } }; - if entry.nodelay { - if let Err(e) = toyos::net::tcp_set_option(conn.socket_id, OPT_NODELAY, 1) { - // `conn`'s pipe ends close where it drops. - let _ = toyos::net::tcp_close(conn.socket_id); - set_errno(net_err_to_errno(e)); - return -1; - } - } - entry.netstack_id = conn.socket_id.0; - entry.local_port = conn.local_port; + entry.local_port = connected.local_port; entry.remote_addr = ip; entry.remote_port = port; entry.connected = true; - entry.rx_fd = conn.rx.into_raw().0 as i32; - entry.tx_fd = conn.tx.into_raw().0 as i32; + entry.stream = connected.handle.0 as i32; 0 } SocketKind::Udp => { @@ -321,7 +313,7 @@ pub unsafe extern "C" fn accept( let mut notify_byte = [0u8; 1]; let _ = syscall::read(RawHandle(notify_fd as u32), &mut notify_byte); - let accepted = match toyos::net::tcp_accept(listener_id) { + let accepted = match stream::accept(listener_id) { Ok(a) => a, Err(e) => { set_errno(net_err_to_errno(e)); return -1; } }; @@ -332,25 +324,23 @@ pub unsafe extern "C" fn accept( let new_entry = SocketEntry { kind: SocketKind::Tcp, - netstack_id: accepted.socket_id.0, + netstack_id: 0, connected: true, bound: false, local_port: accepted.local_port, remote_addr: accepted.remote_addr, remote_port: accepted.remote_port, - rx_fd: accepted.rx.into_raw().0 as i32, - tx_fd: accepted.tx.into_raw().0 as i32, + stream: accepted.handle.0 as i32, + rx_fd: 0, + tx_fd: 0, notify_fd: 0, - nodelay: accepted.options.nodelay(), + nodelay: accepted.nodelay, broadcast: false, - read_shut: false, - write_shut: false, }; let new_fd = alloc_socket(new_entry); if new_fd < 0 { - syscall::close(RawHandle(new_entry.rx_fd as u32)); - syscall::close(RawHandle(new_entry.tx_fd as u32)); - let _ = toyos::net::tcp_close(accepted.socket_id); + // The client's leaving: netstack closes the connection. + stream::let_go(accepted.handle); set_errno(ENOMEM); } new_fd @@ -372,14 +362,10 @@ pub unsafe extern "C" fn send(fd: i32, buf: *const u8, len: usize, _flags: i32) match entry.kind { SocketKind::Tcp => { - if entry.write_shut { - set_errno(EPIPE); - return -1; - } let data = core::slice::from_raw_parts(buf, len); - match syscall::write(RawHandle(entry.tx_fd as u32), data) { + match stream::write(RawHandle(entry.stream as u32), data, Wait::Blocking(None)) { Ok(n) => n as isize, - Err(e) => { set_errno(stream_errno(write_refused(e))); -1 } + Err(refused) => { set_errno(stream_errno(refused)); -1 } } } SocketKind::Udp => { @@ -410,17 +396,10 @@ pub unsafe extern "C" fn recv(fd: i32, buf: *mut u8, len: usize, _flags: i32) -> match entry.kind { SocketKind::Tcp => { - if entry.read_shut { - return 0; - } let data = core::slice::from_raw_parts_mut(buf, len); - let read = syscall::read(RawHandle(entry.rx_fd as u32), data).map_err(|_| StreamRefusal::Other).and_then(|n| match n { - 0 => read_end(syscall::write_nonblock(RawHandle(entry.tx_fd as u32), &[])).map(|()| 0), - n => Ok(n), - }); - match read { + match stream::read(RawHandle(entry.stream as u32), data, Wait::Blocking(None)) { Ok(n) => n as isize, - Err(refusal) => { set_errno(stream_errno(refusal)); -1 } + Err(refused) => { set_errno(stream_errno(refused)); -1 } } } SocketKind::Udp => { @@ -533,12 +512,21 @@ pub unsafe extern "C" fn shutdown(fd: i32, how: i32) -> i32 { }; if let SocketKind::Tcp = entry.kind { - if let Err(e) = toyos::net::tcp_shutdown(TcpSocketId(entry.netstack_id), how as u32) { - set_errno(net_err_to_errno(e)); + if entry.stream == 0 { + set_errno(ENOTCONN); return -1; } - entry.read_shut |= matches!(how, SHUT_RD | SHUT_RDWR); - entry.write_shut |= matches!(how, SHUT_WR | SHUT_RDWR); + let handle = RawHandle(entry.stream as u32); + if !matches!(how, SHUT_RD | SHUT_WR | SHUT_RDWR) { + set_errno(EINVAL); + return -1; + } + if matches!(how, SHUT_RD | SHUT_RDWR) { + let _ = stream::shutdown_read(handle); + } + if matches!(how, SHUT_WR | SHUT_RDWR) { + let _ = stream::shutdown_write(handle); + } } 0 } @@ -547,14 +535,33 @@ const SHUT_RD: i32 = 0; const SHUT_WR: i32 = 1; const SHUT_RDWR: i32 = 2; -fn stream_errno(refusal: StreamRefusal) -> i32 { - match refusal { - StreamRefusal::Reset => ECONNRESET, - StreamRefusal::Again => EAGAIN, - StreamRefusal::Other => EIO, +/// A stream's refusal in POSIX's word: `send`'s and `recv`'s. +fn stream_errno(refused: Refused) -> i32 { + match refused { + Refused::NotThisProcess(_) => EBADF, + Refused::WouldBlock => EAGAIN, + Refused::TimedOut | Refused::ConnectionTimedOut => ETIMEDOUT, + Refused::Reset => ECONNRESET, + Refused::Unreachable => EHOSTUNREACH, + Refused::Aborted => ECONNABORTED, + Refused::Shut => EPIPE, } } +/// The handle of the stream a socket descriptor is, for `poll`. +pub(crate) fn stream_of(fd: i32) -> Option { + let entry = sock_from_fd(fd)?.as_ref()?; + (matches!(entry.kind, SocketKind::Tcp) && entry.stream != 0).then_some(RawHandle(entry.stream as u32)) +} + +/// The kernel handle a socket descriptor that is no stream is watched by, +/// for `poll`: a listener's wake pipe, a datagram socket's receive pipe. +pub(crate) fn watched_of(fd: i32) -> Option { + let entry = sock_from_fd(fd)?.as_ref()?; + let handle = if entry.notify_fd != 0 { entry.notify_fd } else { entry.rx_fd }; + (handle != 0).then_some(RawHandle(handle as u32)) +} + // close (for socket fds) @@ -569,6 +576,8 @@ pub unsafe extern "C" fn close_socket(fd: i32) -> bool { None => return false, }; + // A stream's leaving is its connection's hang-up. + if entry.stream != 0 { stream::let_go(RawHandle(entry.stream as u32)); } // Close pipe fds if entry.rx_fd != 0 { syscall::close(RawHandle(entry.rx_fd as u32)); } if entry.tx_fd != 0 { syscall::close(RawHandle(entry.tx_fd as u32)); } @@ -624,17 +633,18 @@ pub unsafe extern "C" fn setsockopt( }; match option { Kept::NoDelay => { - // netstack holds a listener from `bind` and a connection from - // `connect` or `accept`, and each has its own request. - let id = TcpSocketId(entry.netstack_id); - let asked = match (entry.connected, entry.bound) { - (true, _) => toyos::net::tcp_set_option(id, OPT_NODELAY, on as u32), - (false, true) => toyos::net::tcp_listener_set_option(id, OPT_NODELAY, on as u32), - (false, false) => Ok(()), - }; - if let Err(e) = asked { - set_errno(net_err_to_errno(e)); - return -1; + // A stream's is a word of its region, a listener's a request. + if entry.stream != 0 { + if let Err(refused) = stream::set_nodelay(RawHandle(entry.stream as u32), on) { + set_errno(stream_errno(refused)); + return -1; + } + } else if entry.bound { + let id = TcpSocketId(entry.netstack_id); + if let Err(e) = toyos::net::tcp_listener_set_option(id, toyos::net::OPT_NODELAY, on as u32) { + set_errno(net_err_to_errno(e)); + return -1; + } } entry.nodelay = on; } diff --git a/sdk/std/os/io.rs b/sdk/std/os/io.rs index 3c740ec02ef..b2589594f96 100644 --- a/sdk/std/os/io.rs +++ b/sdk/std/os/io.rs @@ -3,7 +3,7 @@ pub use fd::*; use crate::os::fd; -use crate::sys::{AsInner, FromInner}; +use crate::sys::{AsInner, FromInner, IntoInner}; #[stable(feature = "toyos_ext", since = "1.0.0")] impl AsRawFd for crate::fs::File { @@ -13,11 +13,17 @@ impl AsRawFd for crate::fs::File { } } +/// A stream's handle is read and written through its region, never as a +/// file: taken as one, it panics with `toyos::net`'s refusal. #[stable(feature = "toyos_ext", since = "1.0.0")] impl FromRawFd for crate::fs::File { #[inline] unsafe fn from_raw_fd(fd: RawFd) -> crate::fs::File { - crate::fs::File::from_inner(crate::sys::fs::File::from_fd(toyos_abi::RawHandle(fd as u32))) + let handle = toyos_abi::RawHandle(fd as u32); + if toyos::net::stream::is_stream(handle) { + panic!("{}", toyos::net::stream::Misuse::AsFile(handle)); + } + crate::fs::File::from_inner(crate::sys::fs::File::from_fd(handle)) } } @@ -37,6 +43,51 @@ impl AsRawFd for crate::net::TcpStream { } } +/// A stream's handle taken back: one this process holds the stream's end for +/// shares it, and any other panics with `toyos::net`'s refusal, a stream +/// belonging to the process that connected or accepted it. +#[stable(feature = "toyos_ext", since = "1.0.0")] +impl FromRawFd for crate::net::TcpStream { + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> crate::net::TcpStream { + if let Err(misuse) = toyos::net::stream::adopt(toyos_abi::RawHandle(fd as u32)) { + panic!("{misuse}"); + } + // SAFETY: the caller's, as `FromRawFd` states it. + let owned = unsafe { fd::OwnedFd::from_raw_fd(fd) }; + crate::net::TcpStream::from_inner(crate::sys::net::TcpStream::from_fd(owned)) + } +} + +/// The handle given out keeps reaching the stream's end, for whoever takes it +/// back. +#[stable(feature = "toyos_ext", since = "1.0.0")] +impl IntoRawFd for crate::net::TcpStream { + #[inline] + fn into_raw_fd(self) -> RawFd { + self.into_inner().into_fd().into_raw_fd() + } +} + +#[stable(feature = "toyos_ext", since = "1.0.0")] +impl From for crate::net::TcpStream { + #[inline] + fn from(owned: fd::OwnedFd) -> Self { + if let Err(misuse) = toyos::net::stream::adopt(toyos_abi::RawHandle(owned.as_raw_fd() as u32)) { + panic!("{misuse}"); + } + crate::net::TcpStream::from_inner(crate::sys::net::TcpStream::from_fd(owned)) + } +} + +#[stable(feature = "toyos_ext", since = "1.0.0")] +impl From for fd::OwnedFd { + #[inline] + fn from(stream: crate::net::TcpStream) -> Self { + stream.into_inner().into_fd() + } +} + #[stable(feature = "toyos_ext", since = "1.0.0")] impl AsRawFd for crate::net::TcpListener { #[inline] diff --git a/sdk/std/os/mod.rs b/sdk/std/os/mod.rs index e3a448655fc..5f33503d2b4 100644 --- a/sdk/std/os/mod.rs +++ b/sdk/std/os/mod.rs @@ -9,5 +9,8 @@ pub mod fs; #[stable(feature = "toyos_ext", since = "1.0.0")] pub mod io; +#[stable(feature = "toyos_ext", since = "1.0.0")] +pub mod net; + #[stable(feature = "toyos_ext", since = "1.0.0")] pub mod process; diff --git a/sdk/std/os/net.rs b/sdk/std/os/net.rs new file mode 100644 index 00000000000..f56df24cd29 --- /dev/null +++ b/sdk/std/os/net.rs @@ -0,0 +1,38 @@ +//! What std has no word for of a ToyOS stream: the doorbell a poller watches +//! for its room, and a registration that arms its wants, so a readiness +//! loop's first event comes. +//! +//! A stream's readiness is its region's, and the kernel's readiness of its +//! handles only a wake: its own handle is readable when netstack rang it for +//! bytes or their end, and its bell when netstack rang it for room or a +//! failure. Each says "look", so a source is read and written until +//! `WouldBlock`, which arms the want the next ring answers. + +use crate::io; +use crate::os::fd::RawFd; + +/// ToyOS's extensions of [`crate::net::TcpStream`]. +#[stable(feature = "toyos_ext", since = "1.0.0")] +pub trait TcpStreamExt { + /// The stream's bell, watched for reading where the stream is watched for + /// writing. + #[stable(feature = "toyos_ext", since = "1.0.0")] + fn bell(&self) -> io::Result; + + /// Registers the stream for its readable and writable readiness: each + /// named way's want armed and looked at, and netstack rung where one is + /// ready, which it answers by ringing back. + #[stable(feature = "toyos_ext", since = "1.0.0")] + fn register(&self, readable: bool, writable: bool) -> io::Result<()>; +} + +#[stable(feature = "toyos_ext", since = "1.0.0")] +impl TcpStreamExt for crate::net::TcpStream { + fn bell(&self) -> io::Result { + crate::sys::AsInner::as_inner(self).bell() + } + + fn register(&self, readable: bool, writable: bool) -> io::Result<()> { + crate::sys::AsInner::as_inner(self).register(readable, writable) + } +} diff --git a/sdk/std/sys/net/connection.rs b/sdk/std/sys/net/connection.rs index e0e07ab1d9d..426dfec8ec0 100644 --- a/sdk/std/sys/net/connection.rs +++ b/sdk/std/sys/net/connection.rs @@ -1,6 +1,5 @@ -use toyos::AsHandle; -use toyos::net::{NetError, StreamRefusal, TcpSocketId, UdpSocketId}; -use toyos::poller::{Poller, READABLE, WRITABLE}; +use toyos::net::stream::{self, Refused, Wait}; +use toyos::net::{NetError, TcpSocketId, UdpSocketId}; use toyos_abi::RawHandle; use toyos_abi::syscall::{self, SyscallError}; @@ -11,7 +10,7 @@ use crate::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, OwnedFd}; use crate::sync::Arc; use crate::sync::atomic::Ordering::Relaxed; use crate::sync::atomic::{AtomicBool, AtomicU32}; -use crate::time::{Duration, Instant}; +use crate::time::Duration; // --- Helpers --- @@ -54,60 +53,39 @@ fn syscall_err(e: SyscallError) -> io::Error { /// How long `TcpStream::connect` waits for each address to answer. const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); -/// Wait until `handle` is ready for `flags` or `left` passes. -fn wait_ready(handle: RawHandle, flags: u32, left: Duration) { - let poller = Poller::new(1); - poller.watch_raw(handle, flags, 0); - poller.wait(1, left.as_nanos().min(u64::MAX as u128) as u64, |_| {}); -} - -/// Run `op` until it does not answer `WouldBlock`, waiting for `flags` on -/// `handle` in between; `None` once `timeout_ms` has passed. -fn with_timeout( - handle: RawHandle, - flags: u32, - timeout_ms: u32, - mut op: impl FnMut() -> Result, -) -> Option> { - let deadline = Instant::now() + Duration::from_millis(timeout_ms as u64); - loop { - match op() { - Err(SyscallError::WouldBlock) => {} - done => return Some(done), - } - let left = deadline.saturating_duration_since(Instant::now()); - if left.is_zero() { - return None; - } - wait_ready(handle, flags, left); - } -} - const TIMED_OUT: io::Error = io::const_error!(io::ErrorKind::TimedOut, "timed out"); -/// The connection was reset, or ended by netstack: its send pipe has no reader. -const RESET: io::Error = io::const_error!(io::ErrorKind::ConnectionReset, "connection reset"); +const SHUT_DOWN: io::Error = + io::const_error!(io::ErrorKind::BrokenPipe, "the stream was shut down for writing"); -/// What std answers for a stream's refusal. -fn stream_refused(refusal: StreamRefusal) -> io::Error { +/// What std answers for a stream's refusal. netstack gone is +/// `ConnectionAborted`: neither the peer's reset nor its FIN. +fn stream_refused(refusal: Refused) -> io::Error { match refusal { - StreamRefusal::Reset => RESET, - StreamRefusal::Again => io::ErrorKind::WouldBlock.into(), - StreamRefusal::Other => io::Error::new(io::ErrorKind::Other, "syscall error"), + Refused::NotThisProcess(handle) => { + io::Error::new(io::ErrorKind::InvalidInput, stream::Misuse::NotThisProcess(handle).to_string()) + } + Refused::WouldBlock => io::ErrorKind::WouldBlock.into(), + Refused::TimedOut => TIMED_OUT, + Refused::Reset => io::const_error!(io::ErrorKind::ConnectionReset, "connection reset"), + Refused::ConnectionTimedOut => io::const_error!(io::ErrorKind::TimedOut, "the connection timed out"), + Refused::Unreachable => io::const_error!(io::ErrorKind::HostUnreachable, "the peer is unreachable"), + Refused::Aborted => io::const_error!(io::ErrorKind::ConnectionAborted, "netstack ended the stream"), + Refused::Shut => SHUT_DOWN, } } -const SHUT_DOWN: io::Error = - io::const_error!(io::ErrorKind::BrokenPipe, "the stream was shut down for writing"); - -/// Join rx/tx pipes into a single duplex kernel handle. -fn make_socket_fd(rx: toyos::Pipe, tx: toyos::Pipe) -> io::Result { - let socket_fd = - syscall::connection_join(rx.as_handle(), tx.as_handle()).map_err(syscall_err)?; - // The join takes references of its own; these two are consumed here. - drop(rx); - drop(tx); - Ok(unsafe { OwnedFd::from_raw_fd(socket_fd.0 as i32) }) +/// How a call waits: not at all on a nonblocking stream, and otherwise until +/// its direction is ready or `timeout_ms`, 0 for no timeout, passes. +fn wait(nonblocking: bool, timeout_ms: u32) -> Wait { + match (nonblocking, timeout_ms) { + (true, _) => Wait::Nonblocking, + (false, 0) => Wait::Blocking(None), + (false, ms) => { + let now = toyos_abi::clock::nanos_since_boot(); + Wait::Blocking(Some(now.saturating_add(u64::from(ms) * 1_000_000))) + } + } } // --- Shared socket ownership (prevents double-close on duplicate) --- @@ -128,46 +106,29 @@ impl Drop for NetstackSocket { // --- TcpStream --- +/// A stream's handle, which `toyos::net::stream` holds its one end by: every +/// handle of it in this process shares the region, its cursors and its +/// waiters. The handle's drop lets the end go (`OwnedFd`'s ToyOS arm). pub struct TcpStream { fd: OwnedFd, - socket: Arc, - /// `shutdown` was asked for this half, on this stream or a duplicate. - read_shut: Arc, - write_shut: Arc, - peer: SocketAddr, - local_port: u16, read_timeout_ms: AtomicU32, write_timeout_ms: AtomicU32, - /// What netstack holds for the connection every duplicate names, locked - /// across the request that changes it so the two cannot come to differ. - nodelay: Arc>, nonblocking: AtomicBool, } impl TcpStream { - /// A stream on a connection netstack holds with `nodelay`: off for one a - /// connect made, and what its accept answered for one a listener was - /// handed. - fn new(fd: OwnedFd, id: TcpSocketId, peer: SocketAddr, local_port: u16, nodelay: bool) -> TcpStream { + /// A stream on `fd`, a handle `toyos::net::stream` holds an end for. + pub(crate) fn from_fd(fd: OwnedFd) -> TcpStream { TcpStream { fd, - socket: Arc::new(NetstackSocket::Tcp(id)), - read_shut: Arc::new(AtomicBool::new(false)), - write_shut: Arc::new(AtomicBool::new(false)), - peer, - local_port, read_timeout_ms: AtomicU32::new(0), write_timeout_ms: AtomicU32::new(0), - nodelay: Arc::new(crate::sync::Mutex::new(nodelay)), nonblocking: AtomicBool::new(false), } } - fn socket_id(&self) -> TcpSocketId { - match *self.socket { - NetstackSocket::Tcp(id) => id, - _ => unreachable!(), - } + pub(crate) fn into_fd(self) -> OwnedFd { + self.fd } pub fn connect(addr: A) -> io::Result { @@ -176,13 +137,12 @@ impl TcpStream { pub fn connect_timeout(addr: &SocketAddr, timeout: Duration) -> io::Result { let (ip, port) = addr_to_v4(addr)?; - let conn = toyos::net::tcp_connect(ip, port, duration_to_ms(Some(timeout))) - .map_err(net_err_to_io)?; - let fd = make_socket_fd(conn.rx, conn.tx)?; - Ok(TcpStream::new(fd, conn.socket_id, *addr, conn.local_port, false)) + let connected = stream::connect(ip, port, duration_to_ms(Some(timeout)), false).map_err(net_err_to_io)?; + // SAFETY: the connect made this handle for this stream and nothing else answers for it. + Ok(TcpStream::from_fd(unsafe { OwnedFd::from_raw_fd(connected.handle.0 as i32) })) } - fn raw_handle(&self) -> RawHandle { + fn handle(&self) -> RawHandle { RawHandle(self.fd.as_raw_fd() as u32) } @@ -211,29 +171,8 @@ impl TcpStream { } pub fn read(&self, buf: &mut [u8]) -> io::Result { - if buf.is_empty() || self.read_shut.load(Relaxed) { - return Ok(0); - } - let handle = self.raw_handle(); - let read = if self.nonblocking.load(Relaxed) { - syscall::read_nonblock(handle, buf) - } else { - match self.read_timeout_ms.load(Relaxed) { - 0 => syscall::read(handle, buf), - ms => with_timeout(handle, READABLE, ms, || syscall::read_nonblock(handle, buf)) - .ok_or(TIMED_OUT)?, - } - }; - match read.map_err(syscall_err)? { - 0 => self.ended().map(|()| 0), - n => Ok(n), - } - } - - /// Whether the receive pipe's end was the peer's FIN or not - /// (`toyos::net::read_end`). - fn ended(&self) -> io::Result<()> { - toyos::net::read_end(syscall::write_nonblock(self.raw_handle(), &[])).map_err(stream_refused) + let wait = wait(self.nonblocking.load(Relaxed), self.read_timeout_ms.load(Relaxed)); + stream::read(self.handle(), buf, wait).map_err(stream_refused) } pub fn read_buf(&self, mut cursor: BorrowedCursor<'_, u8>) -> io::Result<()> { @@ -251,23 +190,8 @@ impl TcpStream { } pub fn write(&self, buf: &[u8]) -> io::Result { - if buf.is_empty() { - return Ok(0); - } - if self.write_shut.load(Relaxed) { - return Err(SHUT_DOWN); - } - let handle = self.raw_handle(); - let written = if self.nonblocking.load(Relaxed) { - syscall::write_nonblock(handle, buf) - } else { - match self.write_timeout_ms.load(Relaxed) { - 0 => syscall::write(handle, buf), - ms => with_timeout(handle, WRITABLE, ms, || syscall::write_nonblock(handle, buf)) - .ok_or(TIMED_OUT)?, - } - }; - written.map_err(|e| stream_refused(toyos::net::write_refused(e))) + let wait = wait(self.nonblocking.load(Relaxed), self.write_timeout_ms.load(Relaxed)); + stream::write(self.handle(), buf, wait).map_err(stream_refused) } pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result { @@ -279,43 +203,35 @@ impl TcpStream { } pub fn peer_addr(&self) -> io::Result { - Ok(self.peer) + let ((addr, port), _) = stream::endpoints(self.handle()).map_err(stream_refused)?; + Ok(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::from(addr), port))) } pub fn socket_addr(&self) -> io::Result { - Ok(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(10, 0, 2, 15), self.local_port))) + let (_, local_port) = stream::endpoints(self.handle()).map_err(stream_refused)?; + Ok(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(10, 0, 2, 15), local_port))) } pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { - let how_val = match how { - Shutdown::Read => 0u32, - Shutdown::Write => 1, - Shutdown::Both => 2, - }; - toyos::net::tcp_shutdown(self.socket_id(), how_val).map_err(net_err_to_io)?; if matches!(how, Shutdown::Read | Shutdown::Both) { - self.read_shut.store(true, Relaxed); + stream::shutdown_read(self.handle()).map_err(stream_refused)?; } if matches!(how, Shutdown::Write | Shutdown::Both) { - self.write_shut.store(true, Relaxed); + stream::shutdown_write(self.handle()).map_err(stream_refused)?; } Ok(()) } pub fn duplicate(&self) -> io::Result { - let new_fd = syscall::dup(self.raw_handle()).map_err(syscall_err)?; - Ok(TcpStream { - fd: unsafe { OwnedFd::from_raw_fd(new_fd.0 as i32) }, - socket: Arc::clone(&self.socket), - read_shut: Arc::clone(&self.read_shut), - write_shut: Arc::clone(&self.write_shut), - peer: self.peer, - local_port: self.local_port, + let copy = stream::duplicate(self.handle()).map_err(syscall_err)?; + let duplicate = TcpStream { + // SAFETY: `duplicate` made this handle of the same stream, entered against its end. + fd: unsafe { OwnedFd::from_raw_fd(copy.0 as i32) }, read_timeout_ms: AtomicU32::new(self.read_timeout_ms.load(Relaxed)), write_timeout_ms: AtomicU32::new(self.write_timeout_ms.load(Relaxed)), - nodelay: Arc::clone(&self.nodelay), nonblocking: AtomicBool::new(self.nonblocking.load(Relaxed)), - }) + }; + Ok(duplicate) } pub fn set_linger(&self, _linger: Option) -> io::Result<()> { @@ -327,15 +243,11 @@ impl TcpStream { } pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { - let mut held = self.nodelay.lock().unwrap(); - toyos::net::tcp_set_option(self.socket_id(), toyos::net::OPT_NODELAY, nodelay as u32) - .map_err(net_err_to_io)?; - *held = nodelay; - Ok(()) + stream::set_nodelay(self.handle(), nodelay).map_err(stream_refused) } pub fn nodelay(&self) -> io::Result { - Ok(*self.nodelay.lock().unwrap()) + stream::nodelay(self.handle()).map_err(stream_refused) } pub fn set_keepalive(&self, _keepalive: bool) -> io::Result<()> { @@ -354,8 +266,10 @@ impl TcpStream { Ok(64) } + /// The failure that ended the stream: the receive ring's `end`. pub fn take_error(&self) -> io::Result> { - Ok(None) + let failure = stream::failure(self.handle()).map_err(stream_refused)?; + Ok(failure.map(stream_refused)) } pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> { @@ -370,14 +284,23 @@ impl TcpStream { pub fn as_raw_fd(&self) -> i32 { self.fd.as_raw_fd() } -} -// No Drop impl — Arc handles close on last drop. -// OwnedFd drop closes the pipe-backed socket fd. + /// The stream's bell: the handle a watch for room is made on, for reading. + pub fn bell(&self) -> io::Result { + let bell = stream::doorbell(self.handle(), stream::Way::Send).map_err(stream_refused)?; + Ok(bell.0 as i32) + } + + /// Registered for readiness: each named way's want armed, and netstack rung + /// where one is ready, so the first event comes. + pub fn register(&self, readable: bool, writable: bool) -> io::Result<()> { + stream::register(self.handle(), readable, writable).map_err(stream_refused) + } +} impl fmt::Debug for TcpStream { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TcpStream(fd={}, peer={})", self.fd.as_raw_fd(), self.peer) + write!(f, "TcpStream(fd={})", self.fd.as_raw_fd()) } } @@ -426,15 +349,14 @@ impl TcpListener { syscall::read(notify_fd, &mut byte).map_err(syscall_err)?; } - let accepted = toyos::net::tcp_accept(self.socket_id()).map_err(net_err_to_io)?; - let fd = make_socket_fd(accepted.rx, accepted.tx)?; - + let accepted = stream::accept(self.socket_id()).map_err(net_err_to_io)?; let peer = SocketAddr::V4(SocketAddrV4::new( Ipv4Addr::from(accepted.remote_addr), accepted.remote_port, )); - let nodelay = accepted.options.nodelay(); - Ok((TcpStream::new(fd, accepted.socket_id, peer, accepted.local_port, nodelay), peer)) + // SAFETY: the accept made this handle for this stream and nothing else answers for it. + let fd = unsafe { OwnedFd::from_raw_fd(accepted.handle.0 as i32) }; + Ok((TcpStream::from_fd(fd), peer)) } pub fn duplicate(&self) -> io::Result { diff --git a/sdk/std/sys/process.rs b/sdk/std/sys/process.rs index 378ac8cfe80..f9258973f0c 100644 --- a/sdk/std/sys/process.rs +++ b/sdk/std/sys/process.rs @@ -284,6 +284,7 @@ impl Command { default: Stdio, _needs_stdin: bool, ) -> io::Result<(Process, StdioPipes)> { + self.refuse_streams(&default)?; let route = launch_route::route(launch_route::Asked { supervisor: matches!(self.parent, Parent::Supervisor), endowed: !self.endowments.is_empty(), @@ -444,6 +445,30 @@ impl Command { )) } + /// Refuses, before anything moves, a spawn that would put a stream's handle + /// in another process: in a child's slot, by `Stdio::inherit()`, a `File` + /// or `inherit_handle`, or endowed or provided. A stream belongs to the + /// process that connected or accepted it, and `toyos::net` answers for its + /// handles. + fn refuse_streams(&self, default: &Stdio) -> io::Result<()> { + let stdio = [&self.stdin, &self.stdout, &self.stderr]; + let slots = stdio.into_iter().zip(0u32..).filter_map(|(stdio, slot)| match stdio.as_ref().unwrap_or(default) { + Stdio::Inherit => Some(slot), + Stdio::ParentStdout => Some(1), + Stdio::ParentStderr => Some(2), + Stdio::InheritFile(file) => Some(file.raw_fd() as u32), + Stdio::Null | Stdio::MakePipe | Stdio::MakeTtyPipe | Stdio::InheritPipe(_) => None, + }); + let moved = self.extra_slots.iter().map(|&[_, parent]| parent); + let given = self.endowments.iter().chain(&self.provided).map(|&(_, handle)| handle); + for handle in slots.chain(moved).chain(given).map(toyos_abi::RawHandle) { + if toyos::net::stream::is_stream(handle) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, toyos::net::stream::Misuse::Moved(handle).to_string())); + } + } + Ok(()) + } + /// The environment a direct spawn carries: the caller's, except `HOME`. /// /// **A direct child's `HOME` is one its caller named or one the supervisor answered diff --git a/sdk/toyos/Cargo.toml b/sdk/toyos/Cargo.toml index 3a2906f9ff7..57e4ad42683 100644 --- a/sdk/toyos/Cargo.toml +++ b/sdk/toyos/Cargo.toml @@ -7,7 +7,7 @@ description = "The ToyOS userland SDK: typed handles, IPC, ports, namespaces, su repository = "https://github.com/ToyOSOrg/ToyOS" [features] -rustc-dep-of-std = ["core", "toyos-abi/rustc-dep-of-std"] +rustc-dep-of-std = ["core", "toyos-abi/rustc-dep-of-std", "toyos-transport/rustc-dep-of-std"] # The negative control for the log ring's publication. It makes the store that # publishes a record `Relaxed`, and `kernel-loom`'s `log_ring` model must red: # @@ -29,3 +29,4 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(target_os, values("toyos") [dependencies] core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } toyos-abi = { path = "../abi", default-features = false } +toyos-transport = { path = "../../lib/blockring/transport" } diff --git a/sdk/toyos/src/fs.rs b/sdk/toyos/src/fs.rs index 23028af0585..5eb0cab0102 100644 --- a/sdk/toyos/src/fs.rs +++ b/sdk/toyos/src/fs.rs @@ -129,6 +129,12 @@ ipc_payload! { } } +impl Default for Request { + fn default() -> Self { + Self::new() + } +} + impl Request { pub const fn new() -> Self { Self { fid: 0, offset: 0, len: 0, len2: 0, flags: 0 } diff --git a/sdk/toyos/src/ipc.rs b/sdk/toyos/src/ipc.rs index 01832e09886..53a60d31378 100644 --- a/sdk/toyos/src/ipc.rs +++ b/sdk/toyos/src/ipc.rs @@ -110,6 +110,10 @@ impl IpcHeader { self.len } + pub fn is_empty(&self) -> bool { + self.len == 0 + } + fn to_wire(self) -> [u8; Self::WIRE_SIZE] { let mut bytes = [0u8; Self::WIRE_SIZE]; bytes[..4].copy_from_slice(&self.msg_type.to_ne_bytes()); diff --git a/sdk/toyos/src/launch.rs b/sdk/toyos/src/launch.rs index 86d740daafc..d614941831c 100644 --- a/sdk/toyos/src/launch.rs +++ b/sdk/toyos/src/launch.rs @@ -299,7 +299,7 @@ impl<'a> Request<'a> { /// The child slots the first handles of the batch are for, in order. pub fn slot_numbers(&self) -> impl Iterator + '_ { - self.slots.chunks_exact(4).map(|c| u32::from_le_bytes([c[0], c[1], c[2], c[3]])) + self.slots.as_chunks::<4>().0.iter().map(|c| u32::from_le_bytes(*c)) } pub fn slot_count(&self) -> usize { diff --git a/sdk/toyos/src/lib.rs b/sdk/toyos/src/lib.rs index 1a72dfcdb64..8b4e04da531 100644 --- a/sdk/toyos/src/lib.rs +++ b/sdk/toyos/src/lib.rs @@ -31,6 +31,7 @@ pub mod relocate; pub mod surface; pub mod shm; pub mod syscap; +pub mod sync; pub mod system; pub mod volatile; @@ -80,7 +81,10 @@ impl OwnedHandle { /// /// # Safety /// `raw` must be a live handle in this process's table that nothing else - /// answers for: this closes it when it drops. + /// answers for: this closes it when it drops. A handle + /// [`net::stream`](crate::net::stream) answers for, a stream's, is never + /// one: it reaches its end only through that module, and a stream does not + /// move between processes. pub unsafe fn from_raw(raw: RawHandle) -> Self { Self(raw) } @@ -91,6 +95,7 @@ impl OwnedHandle { /// it. pub(crate) fn into_raw(self) -> RawHandle { let raw = self.0; + #[allow(clippy::disallowed_methods, reason = "the handle is the caller's from here, so this drop must not close it")] core::mem::forget(self); raw } diff --git a/sdk/toyos/src/net.rs b/sdk/toyos/src/net.rs index 0912cd6be90..13c70a4f994 100644 --- a/sdk/toyos/src/net.rs +++ b/sdk/toyos/src/net.rs @@ -1,47 +1,50 @@ //! ToyOS userland networking library. //! //! Owns the netstack IPC protocol and provides client functions for TCP, UDP, and DNS. -//! All networking in ToyOS goes through the `netstack` daemon via message passing -//! and kernel pipes. +//! All networking in ToyOS goes through the `netstack` daemon: a TCP stream is +//! a region and a connection ([`stream`]), and a listener and a datagram socket +//! carry their bytes and wakes in kernel pipes. + +pub mod stream; use crate::ipc::{IpcError, IpcHeader, IpcPayload}; use crate::ipc_payload; -use crate::{Connection, OwnedHandle, Pipe}; +use crate::{Connection, OwnedHandle, Pipe, RawHandle}; #[repr(u32)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MsgType { TcpClose = 4, - TcpShutdown = 7, UdpBind = 8, UdpSendTo = 9, UdpRecvFrom = 10, UdpClose = 11, DnsLookup = 12, - TcpSetOption = 13, TcpListenerSetOption = 14, UdpSetOption = 15, - TcpConnectPiped = 20, TcpBindPiped = 21, - TcpAcceptPiped = 22, + /// A stream's region with [`TcpConnectRequest`], answered with its bell + /// ([`stream::connect`]). + TcpConnect = 23, + /// A stream's region with [`TcpAcceptRequest`], answered with its bell + /// ([`stream::accept`]). + TcpAccept = 24, } impl MsgType { pub fn from_u32(v: u32) -> Option { match v { 4 => Some(Self::TcpClose), - 7 => Some(Self::TcpShutdown), 8 => Some(Self::UdpBind), 9 => Some(Self::UdpSendTo), 10 => Some(Self::UdpRecvFrom), 11 => Some(Self::UdpClose), 12 => Some(Self::DnsLookup), - 13 => Some(Self::TcpSetOption), 14 => Some(Self::TcpListenerSetOption), 15 => Some(Self::UdpSetOption), - 20 => Some(Self::TcpConnectPiped), 21 => Some(Self::TcpBindPiped), - 22 => Some(Self::TcpAcceptPiped), + 23 => Some(Self::TcpConnect), + 24 => Some(Self::TcpAccept), _ => None, } } @@ -82,11 +85,11 @@ pub const ERR_RESOURCE_EXHAUSTED: u32 = 7; pub const ERR_PERMISSION_DENIED: u32 = 8; pub const ERR_OTHER: u32 = 255; -/// Nagle's algorithm off. On a stream, [`tcp_set_option`]. On a listener, -/// its bind's [`TcpOptions`] and then [`tcp_listener_set_option`]: for every -/// connection whose SYN arrives from then on, and one that began before keeps -/// what it has. A connection says what it began with in its accept's answer -/// ([`TcpOptions`]); one a connect made begins with the algorithm on. +/// Nagle's algorithm off, on a listener: its bind's [`TcpOptions`] and then +/// [`tcp_listener_set_option`], for every connection whose SYN arrives from +/// then on, and one that began before keeps what it has. A connection says +/// what it began with in its accept's answer; a stream's own is a word of its +/// region ([`stream::NODELAY`]). pub const OPT_NODELAY: u32 = 1; /// A datagram socket may send to a broadcast address: [`udp_set_option`]. pub const OPT_BROADCAST: u32 = 2; @@ -153,15 +156,16 @@ pub const DATA_FROM_CLIENT: usize = 1; pub const NOTIFY_HANDLES: usize = 1; ipc_payload! { - pub struct TcpConnectPipedRequest { + /// [`MsgType::TcpConnect`], with the stream's region. + pub struct TcpConnectRequest { pub addr: [u8; 4], pub port: u16, pub _pad: u16, pub timeout_ms: u32, } + /// The answer to [`MsgType::TcpConnect`], with the stream's bell. pub struct TcpConnectResponse { - pub socket_id: u32, pub local_port: u16, pub _pad: u16, } @@ -184,29 +188,26 @@ ipc_payload! { pub _pad: u16, } - pub struct TcpShutdownRequest { - pub socket_id: u32, - pub how: u32, + /// [`MsgType::TcpAccept`], with the stream's region. + pub struct TcpAcceptRequest { + pub listener: u32, } - pub struct TcpAcceptPipedRequest { - pub socket_id: u32, - } - - /// The options a listener or a stream holds, a word each: zero is off and - /// anything else on. + /// The options a listener holds, a word each: zero is off and anything + /// else on. #[derive(Debug, PartialEq, Eq)] pub struct TcpOptions { nodelay: u32, } - pub struct TcpAcceptPipedResponse { - pub socket_id: u32, + /// The answer to [`MsgType::TcpAccept`], with the stream's bell: the peer, + /// and whether Nagle's algorithm is off for the stream as its listener + /// had it, which netstack has written into the region's header too. + pub struct TcpAcceptResponse { pub remote_addr: [u8; 4], pub remote_port: u16, pub local_port: u16, - /// What the connection holds as it is handed over. - pub options: TcpOptions, + pub nodelay: u32, } pub struct UdpBindRequest { @@ -239,9 +240,8 @@ ipc_payload! { pub len: u16, } - /// [`MsgType::TcpSetOption`], [`MsgType::TcpListenerSetOption`] and - /// [`MsgType::UdpSetOption`]: the request's type says which kind of socket - /// `socket_id` names. + /// [`MsgType::TcpListenerSetOption`] and [`MsgType::UdpSetOption`]: the + /// request's type says which kind of socket `socket_id` names. pub struct SocketOptionRequest { pub socket_id: u32, pub option: u32, @@ -270,29 +270,12 @@ impl TcpOptions { // Return types -pub struct TcpConnection { - pub rx: Pipe, - pub tx: Pipe, - pub socket_id: TcpSocketId, - pub local_port: u16, -} - pub struct TcpBound { pub notify: Pipe, pub socket_id: TcpSocketId, pub bound_port: u16, } -pub struct TcpAccepted { - pub rx: Pipe, - pub tx: Pipe, - pub socket_id: TcpSocketId, - pub remote_addr: [u8; 4], - pub remote_port: u16, - pub local_port: u16, - pub options: TcpOptions, -} - pub struct UdpBound { pub socket_id: UdpSocketId, pub bound_port: u16, @@ -391,46 +374,6 @@ fn hangup(e: IpcError) -> NetError { } } -/// What a call on a stream answers instead of bytes, for libc and std alike. -/// -/// A stream is two of netstack's pipes, and the receive pipe says only that it -/// ended: netstack lets the send pipe go first when the connection failed, and -/// keeps it after an orderly end for as long as its writer holds it, so what a -/// zero-byte write into it answers after the end is the end's kind — the -/// peer's FIN (RFC 9293 §3.6) or a reset (§3.5.3). -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum StreamRefusal { - /// The connection failed: `ECONNRESET`, `ConnectionReset`. - Reset, - /// The pipe has no room or nothing to read yet. - Again, - /// The handle is not the pipe end the stream was made with. - Other, -} - -/// A read of 0 from a stream's receive pipe, given what a zero-byte write into -/// its send pipe then answered: `Ok` is the peer's FIN, after its last byte. -pub fn read_end(probe: Result) -> Result<(), StreamRefusal> { - use toyos_abi::syscall::SyscallError; - match probe { - Ok(_) | Err(SyscallError::WouldBlock) => Ok(()), - Err(SyscallError::Gone) => Err(StreamRefusal::Reset), - Err(_) => Err(StreamRefusal::Other), - } -} - -/// A write a stream's send pipe refused. One after a shutdown of the sending -/// half is refused before it reaches the pipe, which netstack reads no more -/// but keeps. -pub fn write_refused(refused: toyos_abi::syscall::SyscallError) -> StreamRefusal { - use toyos_abi::syscall::SyscallError; - match refused { - SyscallError::Gone => StreamRefusal::Reset, - SyscallError::WouldBlock => StreamRefusal::Again, - _ => StreamRefusal::Other, - } -} - pub struct PendingResponse(NetstackConn); impl PendingResponse { @@ -458,9 +401,18 @@ impl PendingResponse { self.conn().recv_bytes(&header, buf).map_err(hangup) } + /// The answer, the one handle it hands over, and the connection, which + /// is then the caller's. + pub fn answer_with_handle(self) -> Result<(Resp, RawHandle, Connection), NetError> { + let header = self.recv_checked_header()?; + let answer = self.conn().recv_payload(&header).map_err(hangup)?; + let [handle] = self.conn().recv_handles_exact::<1>().ok_or(NetError::Protocol(header.msg_type))?; + Ok((answer, handle, (self.0).0)) + } + pub fn status(self) -> Result<(), NetError> { let header = self.recv_checked_header()?; - if header.len() > 0 { + if !header.is_empty() { let mut skip = [0u8; 128]; let _ = self.conn().recv_bytes(&header, &mut skip); } @@ -491,27 +443,7 @@ impl DataPath { } } -// TCP client functions - -pub fn tcp_connect( - addr: [u8; 4], - port: u16, - timeout_ms: u32, -) -> Result { - let netstack = NetstackConn::connect()?; - let (rx, tx, handles) = DataPath::create()?.split(); - - let resp: TcpConnectResponse = netstack - .request_with_handles(handles, MsgType::TcpConnectPiped, &TcpConnectPipedRequest { - addr, - port, - _pad: 0, - timeout_ms, - })? - .response()?; - - Ok(TcpConnection { rx, tx, socket_id: TcpSocketId(resp.socket_id), local_port: resp.local_port }) -} +// TCP client functions: a listener's. A stream's are [`stream`]'s. /// A listener that holds no option. pub fn tcp_bind(addr: [u8; 4], port: u16) -> Result { @@ -534,45 +466,13 @@ pub fn tcp_bind_with(addr: [u8; 4], port: u16, options: TcpOptions) -> Result Result { - let netstack = NetstackConn::connect()?; - let (rx, tx, handles) = DataPath::create()?.split(); - - let resp: TcpAcceptPipedResponse = netstack - .request_with_handles(handles, MsgType::TcpAcceptPiped, &TcpAcceptPipedRequest { - socket_id: socket_id.0, - })? - .response()?; - - Ok(TcpAccepted { - rx, - tx, - socket_id: TcpSocketId(resp.socket_id), - remote_addr: resp.remote_addr, - remote_port: resp.remote_port, - local_port: resp.local_port, - options: resp.options, - }) -} - -pub fn tcp_shutdown(socket_id: TcpSocketId, how: u32) -> Result<(), NetError> { - NetstackConn::connect()? - .request(MsgType::TcpShutdown, &TcpShutdownRequest { socket_id: socket_id.0, how })? - .status() -} - +/// The listener's owner lets go of it. pub fn tcp_close(socket_id: TcpSocketId) -> Result<(), NetError> { NetstackConn::connect()? .request(MsgType::TcpClose, &SocketCloseRequest { socket_id: socket_id.0 })? .status() } -pub fn tcp_set_option(socket_id: TcpSocketId, option: u32, value: u32) -> Result<(), NetError> { - NetstackConn::connect()? - .request(MsgType::TcpSetOption, &SocketOptionRequest { socket_id: socket_id.0, option, value })? - .status() -} - pub fn tcp_listener_set_option(socket_id: TcpSocketId, option: u32, value: u32) -> Result<(), NetError> { NetstackConn::connect()? .request(MsgType::TcpListenerSetOption, &SocketOptionRequest { socket_id: socket_id.0, option, value })? @@ -665,29 +565,6 @@ mod tests { use super::*; use toyos_abi::syscall::SyscallError; - /// Every refusal but the two a pipe answers on purpose. - const OTHERS: [SyscallError; 4] = - [SyscallError::InvalidArgument, SyscallError::PermissionDenied, SyscallError::BadAddress, SyscallError::Io]; - - #[test] - fn a_read_of_zero_is_the_peer_s_fin_only_while_the_send_pipe_is_read() { - assert_eq!(read_end(Ok(0)), Ok(())); - assert_eq!(read_end(Err(SyscallError::WouldBlock)), Ok(()), "a full send pipe still has its reader"); - assert_eq!(read_end(Err(SyscallError::Gone)), Err(StreamRefusal::Reset)); - for other in OTHERS { - assert_eq!(read_end(Err(other)), Err(StreamRefusal::Other), "{other:?}"); - } - } - - #[test] - fn a_write_the_send_pipe_refuses_for_its_reader_is_a_reset() { - assert_eq!(write_refused(SyscallError::Gone), StreamRefusal::Reset); - assert_eq!(write_refused(SyscallError::WouldBlock), StreamRefusal::Again); - for other in OTHERS { - assert_eq!(write_refused(other), StreamRefusal::Other, "{other:?}"); - } - } - /// `SYS_HANDLE_SEND` into a connection whose server end has gone. The first /// refusal a `tcp_bind` can meet, because a request that hands netstack pipe /// ends moves the handles before it writes the frame. @@ -782,25 +659,19 @@ mod tests { )); } - /// The accept's answer as a client decodes it: the options last, after - /// the ports, any non-zero word an option that is on, and an answer - /// without them refused. + /// The accept's answer as a client decodes it: the peer, the port, and + /// `nodelay` last, any non-zero word on, and an answer without it refused. #[test] - fn an_accepts_answer_ends_in_its_connections_options() { - let mut wire = [0x44, 0x33, 0x22, 0x11, 192, 0, 2, 7, 0x39, 0x30, 0x16, 0, 0, 0, 0, 0]; - let answer: TcpAcceptPipedResponse = crate::ipc::decode_payload(&wire).unwrap(); - assert_eq!(answer.socket_id, 0x1122_3344); + fn an_accepts_answer_ends_in_its_streams_nodelay() { + let mut wire = [192, 0, 2, 7, 0x39, 0x30, 0x16, 0, 0, 0, 0, 0]; + let answer: TcpAcceptResponse = crate::ipc::decode_payload(&wire).unwrap(); assert_eq!((answer.remote_addr, answer.remote_port, answer.local_port), ([192, 0, 2, 7], 12345, 22)); - assert!(!answer.options.nodelay()); - assert_eq!(answer.options, TcpOptions::new(false)); - wire[12] = 1; - let answer: TcpAcceptPipedResponse = crate::ipc::decode_payload(&wire).unwrap(); - assert_eq!(answer.options, TcpOptions::new(true)); - wire[12..].copy_from_slice(&[0, 0, 0, 0x80]); - let answer: TcpAcceptPipedResponse = crate::ipc::decode_payload(&wire).unwrap(); - assert!(answer.options.nodelay()); + assert_eq!(answer.nodelay, 0); + wire[8..].copy_from_slice(&[0, 0, 0, 0x80]); + let answer: TcpAcceptResponse = crate::ipc::decode_payload(&wire).unwrap(); + assert_ne!(answer.nodelay, 0); assert!(matches!( - crate::ipc::decode_payload::(&wire[..12]), + crate::ipc::decode_payload::(&wire[..8]), Err(IpcError::Malformed) )); } diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs new file mode 100644 index 00000000000..81cb627e8a1 --- /dev/null +++ b/sdk/toyos/src/net/stream.rs @@ -0,0 +1,772 @@ +//! A TCP stream: one region and one connection to netstack. +//! +//! **The region is the stream's two TCP buffers**, made by the client and lent +//! with the connect or the accept. [`LAYOUT`] is its one declaration: a header +//! of [`WORDS`] words, each on a 64-byte line of its own, then the receive ring +//! netstack fills and the send ring the client fills. The receive ring's `end` +//! is the peer's FIN, [`RESET`], [`TIMED_OUT`] or [`UNREACHABLE`]; the send +//! ring's is the client's shutdown; [`NODELAY`] is the client's Nagle switch +//! (RFC 9293 §3.7.4). +//! +//! **The connection carries the request and its answer, then only doorbells +//! and the hang-up** (virtio 1.2 §2.9). netstack rings a reader on the +//! connection and a writer on the stream's bell, so neither takes the other's +//! wake. A doorbell says "look", never "there are bytes". +//! +//! **A stream belongs to the process that connected or accepted it.** Every +//! handle of its connection in this process shares one end; a handle with no +//! end here is refused by name and nothing is written on it +//! (`issues/a-stream-cannot-move-between-processes.md`). An end goes once no +//! handle and no call is left. The table is the instance std links: a program +//! that also links `toyos` itself and drives these functions holds ends std +//! cannot see. + +use core::fmt; +use core::sync::atomic::{AtomicU32, Ordering}; + +use toyos_abi::syscall::{self, SyscallError}; +use toyos_abi::RawHandle; +use toyos_transport::session::{self, Bells, Doorbell, Layout, Mode, Polled, Receiving, Refused as Answer, Sending, Took, Waited}; +use toyos_transport::stream::Place; +use toyos_transport::Span; + +use crate::ipc::Connection; +use crate::shm::SharedMemory; +use crate::sync::Lock; +use crate::OwnedHandle; + +use super::{MsgType, NetError, NetstackConn, TcpAcceptRequest, TcpAcceptResponse, TcpConnectRequest, TcpConnectResponse, TcpSocketId}; + +pub use toyos_transport::session::Way; + +// ---- the region ---- + +/// The region a stream is: four 2 MiB pages. +pub const REGION_BYTES: usize = 8 << 20; +/// The header's words: its first 4 KiB. +pub const WORDS: usize = 1024; +const HEADER_BYTES: usize = WORDS * 4; +/// The receive ring, after the header. +pub const RECEIVE_BYTES: u32 = 4 << 20; +pub const RECEIVE_RING: usize = HEADER_BYTES; +/// The send ring, the rest. +pub const SEND_BYTES: u32 = (4 << 20) - HEADER_BYTES as u32; +pub const SEND_RING: usize = RECEIVE_RING + RECEIVE_BYTES as usize; + +/// Word `n` of the protocol's, on a 64-byte line of its own. +const fn line(n: usize) -> usize { + n * 16 +} + +/// The client's Nagle's algorithm: any word but 0 turns it off. +pub const NODELAY: usize = line(10); + +pub const LAYOUT: Layout = Layout::new::<{ line(8) }, { line(9) }>( + Place::new::<{ line(0) }, { line(1) }, { line(2) }, { line(3) }, RECEIVE_RING>(), + Place::new::<{ line(4) }, { line(5) }, { line(6) }, { line(7) }, SEND_RING>(), +); + +const _: () = assert!(SEND_RING + SEND_BYTES as usize == REGION_BYTES && NODELAY < WORDS); + +/// The header, as the one type that may alias words a peer writes. +pub type Words = [AtomicU32; WORDS]; + +/// netstack's end of a stream. +pub type Server = session::Server; + +/// The receive ring's `end`: the peer's reset, or netstack's own. +pub const RESET: u32 = 2; +/// The receive ring's `end`: [tcp] gave the connection up, by its user timeout +/// or R2 (RFC 9293 §3.8.3). +pub const TIMED_OUT: u32 = 3; +/// The receive ring's `end`: an ICMP error ended the connection. +pub const UNREACHABLE: u32 = 4; + +// ---- the process's ends ---- + +/// The most streams one process holds: the per-process bound libc's socket +/// table has. An estimate: no measurement sets it. +pub const MAX_STREAMS: usize = 128; +/// The most handles of streams' connections one process holds. +const MAX_HANDLES: usize = 2 * MAX_STREAMS; + +type Rx = Receiving; +type Tx = Sending; + +/// An end's two halves, each behind its own lock. +struct Halves { + rx: Lock>, + tx: Lock>, +} + +static HALVES: [Halves; MAX_STREAMS] = [const { Halves { rx: Lock::new(None), tx: Lock::new(None) } }; MAX_STREAMS]; + +static TABLE: Lock = Lock::new(Table::new()); + +/// What an end holds besides its halves. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct End { + region: RawHandle, + /// The region's mapping. + base: usize, + bell: RawHandle, + peer: ([u8; 4], u16), + local_port: u16, + /// `shutdown` was asked for the receiving half. + read_shut: bool, + handles: u32, + calls: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Slot { + Free, + /// Taken by a connect or an accept that has not answered. + Making, + Live(End), +} + +/// The process's ends, and the handles each is reached by. +struct Table { + slots: [Slot; MAX_STREAMS], + handles: [Option<(RawHandle, usize)>; MAX_HANDLES], +} + +impl Table { + const fn new() -> Self { + Self { slots: [Slot::Free; MAX_STREAMS], handles: [None; MAX_HANDLES] } + } + + /// A free slot made [`Slot::Making`], where a handle has room beside it. + fn reserve(&mut self) -> Option { + self.handles.iter().position(Option::is_none)?; + let slot = self.slots.iter().position(|slot| *slot == Slot::Free)?; + self.slots[slot] = Slot::Making; + Some(slot) + } + + fn unreserve(&mut self, slot: usize) { + self.slots[slot] = Slot::Free; + } + + /// The made end, reached by `handle`. + fn enter(&mut self, slot: usize, end: End, handle: RawHandle) { + let room = self.handles.iter().position(Option::is_none).expect("a reservation keeps a handle's room"); + self.handles[room] = Some((handle, slot)); + self.slots[slot] = Slot::Live(End { handles: 1, calls: 0, ..end }); + } + + fn find(&self, handle: RawHandle) -> Option { + self.handles.iter().flatten().find(|(held, _)| *held == handle).map(|&(_, slot)| slot) + } + + fn end(&mut self, slot: usize) -> &mut End { + match &mut self.slots[slot] { + Slot::Live(end) => end, + Slot::Free | Slot::Making => unreachable!("a handle names an end that is not live"), + } + } + + /// A call through `handle` begins: its end, counted. + fn call(&mut self, handle: RawHandle) -> Option<(usize, End)> { + let slot = self.find(handle)?; + let end = self.end(slot); + end.calls += 1; + Some((slot, *end)) + } + + /// A call ends: the end, where it was its last use. + fn returned(&mut self, slot: usize) -> Option { + let end = self.end(slot); + end.calls -= 1; + self.free_unused(slot) + } + + fn free_unused(&mut self, slot: usize) -> Option { + let end = *self.end(slot); + (end.handles == 0 && end.calls == 0).then(|| { + self.slots[slot] = Slot::Free; + end + }) + } + + /// `copy`, a new handle of `handle`'s connection, reaches its end too; + /// `false` with no room for it. + fn add(&mut self, slot: usize, copy: RawHandle) -> bool { + let Some(room) = self.handles.iter().position(Option::is_none) else { return false }; + self.handles[room] = Some((copy, slot)); + self.end(slot).handles += 1; + true + } + + /// `handle` reaches its end no more: `None` for a handle that reached + /// none, else its end's slot and the end where it was its last use. + fn forget(&mut self, handle: RawHandle) -> Option> { + let at = self.handles.iter().position(|entry| entry.is_some_and(|(held, _)| held == handle))?; + let (_, slot) = self.handles[at].take().expect("found"); + self.end(slot).handles -= 1; + Some(self.free_unused(slot).map(|end| (slot, end))) + } +} + +/// Lets go of an end no handle or call reaches: its halves, its bell and its +/// region, whose mapping goes with its handle. +fn free(slot: usize, end: End) { + *HALVES[slot].rx.lock() = None; + *HALVES[slot].tx.lock() = None; + syscall::close(end.bell); + syscall::close(end.region); +} + +/// Why a stream call refused, in the protocol's words. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refused { + /// A handle this process holds no stream end for. + NotThisProcess(RawHandle), + /// The call does not wait, and its direction is not ready. + WouldBlock, + /// The call's deadline passed. + TimedOut, + /// The peer reset the connection, or netstack did. + Reset, + /// [tcp] gave the connection up. + ConnectionTimedOut, + /// An ICMP error ended the connection. + Unreachable, + /// netstack went away, or wrote what no netstack writes. + Aborted, + /// The sending half was shut down. + Shut, +} + +impl From for Refused { + fn from(answer: Answer) -> Self { + match answer { + Answer::WouldBlock => Self::WouldBlock, + Answer::TimedOut => Self::TimedOut, + Answer::Failed(TIMED_OUT) => Self::ConnectionTimedOut, + Answer::Failed(UNREACHABLE) => Self::Unreachable, + Answer::Failed(_) => Self::Reset, + Answer::Shut => Self::Shut, + Answer::Gone | Answer::Violation(_) => Self::Aborted, + } + } +} + +/// A raw handle used where a stream's cannot be, by name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Misuse { + /// Taken as a stream, and this process holds no end for it. + NotThisProcess(RawHandle), + /// A stream's, taken as a file. + AsFile(RawHandle), + /// A stream's, moved into another process. + Moved(RawHandle), +} + +impl fmt::Display for Misuse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotThisProcess(handle) => write!( + f, + "toyos::net: handle {} is no stream of this process's: a stream belongs to the process that connected or \ + accepted it", + handle.0 + ), + Self::AsFile(handle) => { + write!(f, "toyos::net: handle {} is a stream's, which is read and written through its region, not as a file", handle.0) + } + Self::Moved(handle) => { + write!(f, "toyos::net: handle {} is a stream's, and a stream does not move into another process", handle.0) + } + } + } +} + +/// A call's view of its end, for as long as the call is counted. +struct Call { + slot: usize, + end: End, + /// The handle the call came in through: a handle of the connection. + connection: RawHandle, +} + +impl Call { + fn words(&self) -> &Words { + // SAFETY: the region is mapped at `base`, 2 MiB aligned, for as long as + // a call is counted, and its header is `WORDS` words reached only as + // atomics. + unsafe { &*(self.end.base as *const Words) } + } + + fn halves(&self) -> &'static Halves { + &HALVES[self.slot] + } + + fn bells(&self) -> Doorbells { + Doorbells { connection: self.connection, bell: self.end.bell } + } +} + +/// Runs `f` as a call through `handle`, its end kept until it returns. +fn call(handle: RawHandle, f: impl FnOnce(&Call) -> R) -> Result { + let (slot, end) = TABLE.lock().call(handle).ok_or(Refused::NotThisProcess(handle))?; + let answer = f(&Call { slot, end, connection: handle }); + let freed = TABLE.lock().returned(slot); + if let Some(end) = freed { + free(slot, end); + } + Ok(answer) +} + +/// A half reached through its slot's lock. +struct Half(&'static Lock>); + +impl session::Lock for Half { + fn with(&self, f: impl FnOnce(&mut T) -> R) -> R { + f(self.0.lock().as_mut().expect("a live end has both halves")) + } +} + +/// A call's doorbells: the connection it came in through and the stream's +/// bell. +struct Doorbells { + connection: RawHandle, + bell: RawHandle, +} + +impl Doorbells { + fn handle(&self, bell: Doorbell) -> RawHandle { + match bell { + Doorbell::Connection => self.connection, + Doorbell::Bell => self.bell, + } + } +} + +impl Bells for Doorbells { + /// Nanoseconds since boot. + type Until = u64; + + fn drain(&self, bell: Doorbell) -> Took { + let mut took = Took::Nothing; + let mut bytes = [0u8; 64]; + loop { + match syscall::read_nonblock(self.handle(bell), &mut bytes) { + Ok(0) => return Took::Gone, + Ok(_) => took = Took::Rung, + Err(SyscallError::WouldBlock) => return took, + Err(_) => return Took::Gone, + } + } + } + + fn wait(&self, bell: Doorbell, until: Option) -> Waited { + let handle = self.handle(bell); + let mut bytes = [0u8; 64]; + let Some(until) = until else { + return match syscall::read(handle, &mut bytes) { + Ok(0) | Err(_) => Waited::Gone, + Ok(_) => Waited::Rung, + }; + }; + let left = until.saturating_sub(toyos_abi::clock::nanos_since_boot()); + if left > 0 { + let poller = crate::poller::Poller::new(1); + poller.watch_raw(handle, crate::poller::READABLE, 0); + poller.wait(1, left, |_| {}); + } + match syscall::read_nonblock(handle, &mut bytes) { + Ok(0) => Waited::Gone, + Ok(_) => Waited::Rung, + Err(SyscallError::WouldBlock) => Waited::TimedOut, + Err(_) => Waited::Gone, + } + } + + fn ring(&self) { + // Full is rung already, and gone is the next look's to find. + let _ = syscall::write_nonblock(self.connection, &[1]); + } +} + +/// How a call waits: not at all, or until ready or a deadline in nanoseconds +/// since boot. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Wait { + Nonblocking, + Blocking(Option), +} + +impl Wait { + fn mode(self) -> Mode { + match self { + Self::Nonblocking => Mode::Nonblocking, + Self::Blocking(until) => Mode::Blocking(until), + } + } +} + +/// Reads into `buf`: a count of at least one byte, or 0 for the peer's FIN +/// after its last byte or a receiving half shut down. +pub fn read(handle: RawHandle, buf: &mut [u8], wait: Wait) -> Result { + let max = u32::try_from(buf.len()).unwrap_or(u32::MAX); + call(handle, |call| { + if call.end.read_shut { + return Ok(0); + } + let base = call.end.base; + let out = buf.as_mut_ptr(); + let copy = |span: Span, at: usize| { + // SAFETY: the span lies in the region's receive ring, mapped for the + // call, and `at + span.len` is at most the `max` bytes of `buf` + // the read was handed; the region is never `buf`. + unsafe { core::ptr::copy_nonoverlapping((base + span.offset) as *const u8, out.add(at), span.len) } + }; + let rx = Half(&call.halves().rx); + let read = session::read(call.words(), &rx, &call.bells(), wait.mode(), max, copy); + read.map(|n| n as usize).map_err(Refused::from) + })? +} + +/// Writes from `buf`: a count of at least one byte. A failure netstack stored +/// is answered, and nothing copied, whatever room there is. +pub fn write(handle: RawHandle, buf: &[u8], wait: Wait) -> Result { + let len = u32::try_from(buf.len()).unwrap_or(u32::MAX); + call(handle, |call| { + let base = call.end.base; + let from = buf.as_ptr(); + let copy = |span: Span, at: usize| { + // SAFETY: as in `read`, with the two sides swapped. + unsafe { core::ptr::copy_nonoverlapping(from.add(at), (base + span.offset) as *mut u8, span.len) } + }; + let tx = Half(&call.halves().tx); + let wrote = session::write(call.words(), &tx, &call.bells(), wait.mode(), len, copy); + wrote.map(|n| n as usize).map_err(Refused::from) + })? +} + +/// The sending half sends no more: netstack sends the FIN after its last byte. +pub fn shutdown_write(handle: RawHandle) -> Result<(), Refused> { + call(handle, |call| session::shutdown(call.words(), &Half(&call.halves().tx), &call.bells())) +} + +/// The receiving half reads no more: every read answers 0. +pub fn shutdown_read(handle: RawHandle) -> Result<(), Refused> { + let mut table = TABLE.lock(); + let slot = table.find(handle).ok_or(Refused::NotThisProcess(handle))?; + table.end(slot).read_shut = true; + Ok(()) +} + +/// Nagle's algorithm off or on, netstack rung to read it. +pub fn set_nodelay(handle: RawHandle, nodelay: bool) -> Result<(), Refused> { + call(handle, |call| { + call.words()[NODELAY].store(u32::from(nodelay), Ordering::Relaxed); + call.bells().ring(); + }) +} + +pub fn nodelay(handle: RawHandle) -> Result { + call(handle, |call| call.words()[NODELAY].load(Ordering::Relaxed) != 0) +} + +/// The failure that ended the stream, if one did. +pub fn failure(handle: RawHandle) -> Result, Refused> { + call(handle, |call| match LAYOUT.received_end(call.words()) { + session::OPEN | session::ORDERLY => None, + code => Some(Refused::from(Answer::Failed(code))), + }) +} + +/// The peer's address and port, and this end's port. +pub fn endpoints(handle: RawHandle) -> Result<(([u8; 4], u16), u16), Refused> { + let mut table = TABLE.lock(); + let slot = table.find(handle).ok_or(Refused::NotThisProcess(handle))?; + let end = table.end(slot); + Ok((end.peer, end.local_port)) +} + +/// The handle a watch of `way` is made on, for reading: the connection for +/// [`Way::Receive`], the bell for [`Way::Send`]. +pub fn doorbell(handle: RawHandle, way: Way) -> Result { + call(handle, |call| call.bells().handle(session::doorbell(way))) +} + +/// A source registered for the ways named: each one's want armed, and +/// netstack rung if one is ready, so its first event comes. +pub fn register(handle: RawHandle, receive: bool, send: bool) -> Result<(), Refused> { + call(handle, |call| { + let (rx, tx) = (Half(&call.halves().rx), Half(&call.halves().tx)); + session::register(call.words(), receive.then_some(&rx), send.then_some(&tx), &call.bells()) + .map_err(|_| Refused::Aborted) + })? +} + +/// A poll of one way that found it not ready: its end kept until +/// [`unwatch`]. +#[must_use] +pub struct Watching { + slot: usize, + end: End, + connection: RawHandle, + way: Way, + watching: session::Watching, +} + +/// What a poll's look found. +pub enum Poll { + Ready, + /// Not ready: the caller watches [`doorbell`] for reading, then calls + /// [`unwatch`]. + Watching(Watching), +} + +/// A poll of `way`: ready at once, or counted in, drained, armed and looked at +/// again. +pub fn watch(handle: RawHandle, way: Way) -> Result { + let (slot, end) = TABLE.lock().call(handle).ok_or(Refused::NotThisProcess(handle))?; + let call = Call { slot, end, connection: handle }; + let polled = match way { + Way::Receive => session::watch(call.words(), &Half(&call.halves().rx), &call.bells()), + Way::Send => session::watch(call.words(), &Half(&call.halves().tx), &call.bells()), + }; + match polled { + Ok(Polled::Watching(watching)) => Ok(Poll::Watching(Watching { slot, end, connection: handle, way, watching })), + Ok(Polled::Ready) | Err(_) => { + ended(slot); + Ok(Poll::Ready) + } + } +} + +/// The end of a poll [`watch`] counted in: whether its way is ready now. +pub fn unwatch(watching: Watching) -> bool { + let Watching { slot, end, connection, way, watching } = watching; + let call = Call { slot, end, connection }; + let ready = match way { + Way::Receive => session::unwatch(call.words(), &Half(&call.halves().rx), &call.bells(), watching), + Way::Send => session::unwatch(call.words(), &Half(&call.halves().tx), &call.bells(), watching), + }; + ended(slot); + ready.unwrap_or(true) +} + +fn ended(slot: usize) { + let freed = TABLE.lock().returned(slot); + if let Some(end) = freed { + free(slot, end); + } +} + +// ---- handles ---- + +/// Whether `handle` reaches one of this process's stream ends. A lookup, and +/// nothing is written. +pub fn is_stream(handle: RawHandle) -> bool { + TABLE.lock().find(handle).is_some() +} + +/// A raw handle taken back as a stream: one this process holds an end for +/// shares it, and any other is refused by name, with nothing written on it. +pub fn adopt(handle: RawHandle) -> Result<(), Misuse> { + if is_stream(handle) { Ok(()) } else { Err(Misuse::NotThisProcess(handle)) } +} + +/// A new handle of `handle`'s object: of a stream's connection, one that +/// reaches the same end; of anything else, the bare `dup`. +pub fn duplicate(handle: RawHandle) -> Result { + let copy = syscall::dup(handle)?; + let mut table = TABLE.lock(); + let Some(slot) = table.find(handle) else { return Ok(copy) }; + if table.add(slot, copy) { + return Ok(copy); + } + drop(table); + syscall::close(copy); + Err(SyscallError::ResourceExhausted) +} + +/// Closes `handle`: of a stream's connection, its end goes with the last +/// handle and the last call; of anything else, the bare `close`. +pub fn let_go(handle: RawHandle) { + let forgotten = TABLE.lock().forget(handle); + syscall::close(handle); + if let Some(Some((slot, end))) = forgotten { + free(slot, end); + } +} + +// ---- connect and accept ---- + +/// A stream netstack connected. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Connected { + /// Its connection, which reaches its end. + pub handle: RawHandle, + pub local_port: u16, +} + +/// An active open to `addr`:`port`, answered once the handshake ends or +/// `timeout_ms` passes, 0 for netstack's own bound, with Nagle's algorithm off +/// from the first segment where `nodelay`. Refused [`NetError::ResourceExhausted`] +/// before anything is made where the process holds [`MAX_STREAMS`]. +pub fn connect(addr: [u8; 4], port: u16, timeout_ms: u32, nodelay: bool) -> Result { + let request = TcpConnectRequest { addr, port, _pad: 0, timeout_ms }; + let opened = open(nodelay, (addr, port), |netstack, region| { + netstack.request_with_handles([region], MsgType::TcpConnect, &request)?.answer_with_handle::() + })?; + Ok(Connected { handle: opened.handle, local_port: opened.answer.local_port }) +} + +/// A stream a listener's accept was handed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Accepted { + pub handle: RawHandle, + pub remote_addr: [u8; 4], + pub remote_port: u16, + pub local_port: u16, + /// Whether Nagle's algorithm is off for it, as its listener had it. + pub nodelay: bool, +} + +/// The oldest connection waiting at `listener` as a stream. The caller took +/// one of the listener's wakes first. Refused as [`connect`] is. +pub fn accept(listener: TcpSocketId) -> Result { + let request = TcpAcceptRequest { listener: listener.0 }; + let opened = open(false, ([0; 4], 0), |netstack, region| { + netstack.request_with_handles([region], MsgType::TcpAccept, &request)?.answer_with_handle::() + })?; + let answer = opened.answer; + Ok(Accepted { + handle: opened.handle, + remote_addr: answer.remote_addr, + remote_port: answer.remote_port, + local_port: answer.local_port, + nodelay: answer.nodelay != 0, + }) +} + +/// A request that made a stream, and its answer. +struct Opened { + handle: RawHandle, + answer: A, +} + +/// What an answer says of the stream's endpoints. +trait Answered { + fn local_port(&self) -> u16; + /// The peer, where the answer names it. + fn peer(&self) -> Option<([u8; 4], u16)>; +} + +impl Answered for TcpConnectResponse { + fn local_port(&self) -> u16 { + self.local_port + } + fn peer(&self) -> Option<([u8; 4], u16)> { + None + } +} + +impl Answered for TcpAcceptResponse { + fn local_port(&self) -> u16 { + self.local_port + } + fn peer(&self) -> Option<([u8; 4], u16)> { + Some((self.remote_addr, self.remote_port)) + } +} + +/// Makes a stream: a slot, a region, then `ask`'s request on a new connection +/// to netstack carrying it, whose answer hands over the bell. Nothing made +/// outlives a refusal. +fn open( + nodelay: bool, + peer: ([u8; 4], u16), + ask: impl FnOnce(NetstackConn, OwnedHandle) -> Result<(A, RawHandle, Connection), NetError>, +) -> Result, NetError> { + let slot = TABLE.lock().reserve().ok_or(NetError::ResourceExhausted)?; + let made = (|| { + let region = SharedMemory::create(REGION_BYTES).map_err(|e| match e { + SyscallError::ResourceExhausted => NetError::ResourceExhausted, + _ => NetError::Io, + })?; + // SAFETY: a fresh region of `REGION_BYTES`, 2 MiB aligned, whose header + // is reached only as atomics. + let words = unsafe { &*(region.as_ptr() as *const Words) }; + words[NODELAY].store(u32::from(nodelay), Ordering::Relaxed); + let (rx, tx) = session::client(words, LAYOUT); + let lent = region.share().map_err(|_| NetError::Io)?; + let (answer, bell, connection) = ask(NetstackConn::connect()?, lent)?; + *HALVES[slot].rx.lock() = Some(rx); + *HALVES[slot].tx.lock() = Some(tx); + let (region, base) = region.into_raw(); + let peer = answer.peer().unwrap_or(peer); + let end = End { region, base: base as usize, bell, peer, local_port: answer.local_port(), read_shut: false, handles: 0, calls: 0 }; + Ok((connection.0.into_raw(), end, answer)) + })(); + let mut table = TABLE.lock(); + match made { + Ok((handle, end, answer)) => { + table.enter(slot, end, handle); + Ok(Opened { handle, answer }) + } + Err(refused) => { + table.unreserve(slot); + Err(refused) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn h(n: u32) -> RawHandle { + RawHandle(n) + } + + fn end() -> End { + End { region: h(1), base: 0x20_0000, bell: h(2), peer: ([192, 0, 2, 2], 80), local_port: 49_152, read_shut: false, handles: 0, calls: 0 } + } + + /// A handle given out and taken back reaches the end it was given out of, + /// and the end goes only with its last handle and its last call. + #[test] + fn an_end_goes_with_its_last_handle_and_its_last_call() { + let mut table = Table::new(); + let slot = table.reserve().unwrap(); + table.enter(slot, end(), h(10)); + assert!(table.add(slot, h(11))); + assert_eq!(table.find(h(11)), Some(slot), "a duplicate reaches the end"); + let (called, _) = table.call(h(11)).unwrap(); + assert_eq!(table.forget(h(10)), Some(None), "a handle is left"); + assert_eq!(table.forget(h(11)), Some(None), "a call is left"); + assert_eq!(table.find(h(11)), None, "the handle reaches nothing"); + assert_eq!(table.returned(called), Some(End { handles: 0, calls: 0, ..end() }), "the last call frees it"); + assert_eq!(table.slots[slot], Slot::Free); + } + + /// A handle no end was made for is no stream, and forgetting it frees + /// nothing. + #[test] + fn a_handle_no_end_was_made_for_is_no_stream() { + let mut table = Table::new(); + let slot = table.reserve().unwrap(); + table.enter(slot, end(), h(10)); + assert_eq!((table.find(h(12)), table.call(h(12)), table.forget(h(12))), (None, None, None)); + assert_eq!(table.find(h(10)), Some(slot)); + } + + /// Past [`MAX_STREAMS`] a stream is refused before anything is made, and + /// a reservation given back is a slot again. + #[test] + fn past_the_bound_no_slot_is_reserved() { + let mut table = Table::new(); + let slots: Vec = (0..MAX_STREAMS).map(|_| table.reserve().unwrap()).collect(); + assert_eq!(table.reserve(), None); + table.unreserve(slots[7]); + assert_eq!(table.reserve(), Some(slots[7])); + } +} diff --git a/sdk/toyos/src/poller.rs b/sdk/toyos/src/poller.rs index 5a724a24fc0..79e08c940be 100644 --- a/sdk/toyos/src/poller.rs +++ b/sdk/toyos/src/poller.rs @@ -219,7 +219,7 @@ impl Poller { /// loss reachable while looking like a success. pub fn new(capacity: u32) -> Self { assert!( - capacity >= 1 && capacity <= Self::MAX_HANDLES, + (1..=Self::MAX_HANDLES).contains(&capacity), "Poller::new: {capacity} handles is outside 1..={}; \ bound the watched set below the kernel's deepest ring", Self::MAX_HANDLES, diff --git a/sdk/toyos/src/relocate/mod.rs b/sdk/toyos/src/relocate/mod.rs index 1888678330c..608cef1b3de 100644 --- a/sdk/toyos/src/relocate/mod.rs +++ b/sdk/toyos/src/relocate/mod.rs @@ -292,7 +292,7 @@ impl<'a> Image<'a> { } fn headers(&self) -> impl Iterator + 'a { - self.phdrs.chunks_exact(PHDR_SIZE).filter_map(Phdr::parse) + self.phdrs.as_chunks::().0.iter().filter_map(|phdr| Phdr::parse(phdr)) } fn loads(&self) -> impl Iterator + 'a { @@ -322,7 +322,7 @@ impl<'a> Image<'a> { fn rela(&self, dynamic: &[u8]) -> Result, Refusal> { let (mut rela, mut size, mut jmprel, mut plt) = (None, None, false, None); let aarch64 = self.relative == R_AARCH64_RELATIVE; - for entry in dynamic.chunks_exact(DYN_SIZE) { + for entry in dynamic.as_chunks::().0 { let (Some(tag), Some(value)) = (u64_at(entry, 0), u64_at(entry, 8)) else { break }; match tag { DT_NULL => break, @@ -366,7 +366,7 @@ impl<'a> Image<'a> { /// Write `bias + addend` at every entry of `rela` through `write`, which /// is handed the entry's link-time address; the first refusal stops it. fn apply(&self, rela: &[u8], bias: u64, mut write: impl FnMut(u64, u64)) -> Result<(), Refusal> { - for entry in rela.chunks_exact(RELA_SIZE) { + for entry in rela.as_chunks::().0 { let (Some(offset), Some(info), Some(addend)) = (u64_at(entry, 0), u64_at(entry, 8), u64_at(entry, 16)) else { return Err(Refusal::RelaShape); diff --git a/sdk/toyos/src/shm.rs b/sdk/toyos/src/shm.rs index c408db6fb16..f9edddaa6c5 100644 --- a/sdk/toyos/src/shm.rs +++ b/sdk/toyos/src/shm.rs @@ -74,6 +74,12 @@ impl SharedMemory { self.ptr } + /// Give up ownership: the handle and its mapping stay, and the mapping goes + /// when the caller closes the handle. + pub fn into_raw(self) -> (RawHandle, *mut u8) { + (self.handle.into_raw(), self.ptr) + } + pub fn len(&self) -> usize { self.size } diff --git a/sdk/toyos/src/sync.rs b/sdk/toyos/src/sync.rs new file mode 100644 index 00000000000..76ec06d2328 --- /dev/null +++ b/sdk/toyos/src/sync.rs @@ -0,0 +1,74 @@ +//! A lock over the kernel's futex: one `u32`, 0 free, 1 held, 2 held with +//! waiters, as Drepper's "Futexes Are Tricky" (2011) builds its mutex in +//! §"Mutex, Take 2". A holder that finds 2 at its release wakes one waiter, +//! and a waiter sleeps only while the word still says 2. It allocates nothing, +//! so a `static` can hold one. + +use core::cell::UnsafeCell; +use core::sync::atomic::{AtomicU32, Ordering}; + +use toyos_abi::syscall; + +/// `T` behind a futex lock. +pub struct Lock { + state: AtomicU32, + value: UnsafeCell, +} + +// SAFETY: the value is reached only through a `Held`, which the lock makes one +// thread's at a time. +unsafe impl Sync for Lock {} + +/// The lock held: the value, until this drops. +pub struct Held<'a, T>(&'a Lock); + +impl Lock { + pub const fn new(value: T) -> Self { + Self { state: AtomicU32::new(0), value: UnsafeCell::new(value) } + } + + pub fn lock(&self) -> Held<'_, T> { + lock_word(&self.state); + Held(self) + } +} + +impl core::ops::Deref for Held<'_, T> { + type Target = T; + fn deref(&self) -> &T { + // SAFETY: the lock is held. + unsafe { &*self.0.value.get() } + } +} + +impl core::ops::DerefMut for Held<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: the lock is held. + unsafe { &mut *self.0.value.get() } + } +} + +impl Drop for Held<'_, T> { + fn drop(&mut self) { + unlock_word(&self.0.state); + } +} + +/// Takes the lock `state` is the word of, sleeping while another holds it. +pub fn lock_word(state: &AtomicU32) { + if state.compare_exchange(0, 1, Ordering::Acquire, Ordering::Relaxed).is_ok() { + return; + } + while state.swap(2, Ordering::Acquire) != 0 { + // SAFETY: `state` is a live, aligned u32. + unsafe { syscall::futex_wait(state.as_ptr(), 2, None) }; + } +} + +/// Lets go of the lock `state` is the word of, waking one sleeper if any. +pub fn unlock_word(state: &AtomicU32) { + if state.swap(0, Ordering::Release) == 2 { + // SAFETY: `state` is a live, aligned u32. + unsafe { syscall::futex_wake(state.as_ptr(), 1) }; + } +} diff --git a/sdk/toyos/src/volatile.rs b/sdk/toyos/src/volatile.rs index 41fda4f08c8..912288b67f9 100644 --- a/sdk/toyos/src/volatile.rs +++ b/sdk/toyos/src/volatile.rs @@ -63,7 +63,7 @@ impl Window { self.len ); assert!( - (self.base as usize + offset) % align_of::() == 0, + (self.base as usize + offset).is_multiple_of(align_of::()), "window: a {}-byte access at {offset:#x} is not aligned for it", size_of::() ); @@ -91,12 +91,12 @@ impl Window { /// window's base is word aligned. pub fn copy_out(self, offset: usize, out: &mut [u8]) { let from = self.words(offset, out.len()); - for (i, chunk) in out.chunks_exact_mut(8).enumerate() { + for (i, chunk) in out.as_chunks_mut::<8>().0.iter_mut().enumerate() { // SAFETY: `words` bounded `from .. from + out.len()` inside the // window and aligned it; volatile because the other side may write // the same bytes concurrently. let word = unsafe { from.add(i).read_volatile() }; - chunk.copy_from_slice(&word.to_ne_bytes()); + *chunk = word.to_ne_bytes(); } } @@ -104,8 +104,8 @@ impl Window { /// [`Self::copy_out`]. pub fn copy_in(self, offset: usize, data: &[u8]) { let to = self.words(offset, data.len()); - for (i, chunk) in data.chunks_exact(8).enumerate() { - let word = u64::from_ne_bytes(chunk.try_into().expect("chunks of eight")); + for (i, chunk) in data.as_chunks::<8>().0.iter().enumerate() { + let word = u64::from_ne_bytes(*chunk); // SAFETY: as in `copy_out`. unsafe { to.add(i).write_volatile(word) }; } @@ -113,9 +113,9 @@ impl Window { /// `len` bytes from `offset` as words, bounded and aligned once. fn words(self, offset: usize, len: usize) -> *mut u64 { - assert!(offset % 8 == 0 && len % 8 == 0, "window: a copy of part of a word"); + assert!(offset.is_multiple_of(8) && len.is_multiple_of(8), "window: a copy of part of a word"); let whole = self.sub(offset, len); - assert!(whole.base as usize % align_of::() == 0, "window: a copy from an unaligned window"); + assert!((whole.base as usize).is_multiple_of(align_of::()), "window: a copy from an unaligned window"); whole.base as *mut u64 } diff --git a/system/netstack/Cargo.toml b/system/netstack/Cargo.toml index 73dad107252..d3b72772f70 100644 --- a/system/netstack/Cargo.toml +++ b/system/netstack/Cargo.toml @@ -22,6 +22,7 @@ toyos-net-shard = { path = "shard" } toyos-net-tcp = { path = "shard/tcp" } toyos-net-udp = { path = "udp" } toyos-net-wire = { path = "wire" } +toyos-transport = { path = "../../lib/blockring/transport" } [dev-dependencies] toyos-pci-claim = { path = "../../lib/pci-claim", features = ["host-grant"] } diff --git a/system/netstack/node/Cargo.toml b/system/netstack/node/Cargo.toml index 596c08f26d6..398973ee91b 100644 --- a/system/netstack/node/Cargo.toml +++ b/system/netstack/node/Cargo.toml @@ -18,6 +18,9 @@ toyos-net-shard = { path = "../shard" } toyos-net-tcp = { path = "../shard/tcp" } toyos-net-udp = { path = "../udp" } toyos-net-wire = { path = "../wire" } +# The stream protocol's one declaration, the region's layout, and the session's server end. +toyos = { path = "../../../sdk/toyos" } +toyos-transport = { path = "../../../lib/blockring/transport" } [dev-dependencies] # The outside reader of every frame the node emits: a published Ethernet, ARP, diff --git a/system/netstack/node/src/lease/tcp.rs b/system/netstack/node/src/lease/tcp.rs index bd9bcbd731c..a27c334deed 100644 --- a/system/netstack/node/src/lease/tcp.rs +++ b/system/netstack/node/src/lease/tcp.rs @@ -4,7 +4,9 @@ use core::net::Ipv4Addr; use toyos_net_shard::{ConnectError, ListenError}; -use toyos_net_tcp::{ConnId, Endpoint, Error, ListenerId, Options, Received, Status, Tuple}; +use alloc::boxed::Box; + +use toyos_net_tcp::{ConnId, Endpoint, Error, Lent, ListenerId, Options, Status, Tuple}; use toyos_net_wire::{Instant, Port}; use super::Stack; @@ -31,17 +33,19 @@ impl Stack { held(self.shard.tuple(id)).local.port } - /// Queues `data`, all of it: the caller offers no more than [`Status::writable`]. - pub(crate) fn tcp_send(&mut self, now: Instant, id: ConnId, data: &[u8]) { - let taken = held(self.shard.send(now, id, data)); - if taken != data.len() { - unreachable!("[tcp] took {taken} of {} bytes it had room for", data.len()); - } + /// [`toyos_net_tcp::Tcp::lend`]. + pub(crate) fn tcp_lend(&mut self, id: ConnId, rx: Box, tx: Box) -> Result<(), Error> { + self.shard.lend(id, rx, tx) + } + + /// The client read `n` bytes out of its lent receive buffer. + pub(crate) fn tcp_consume(&mut self, now: Instant, id: ConnId, n: u32) { + held(self.shard.consume(now, id, usize::try_from(n).unwrap_or(usize::MAX))); } - /// [`toyos_net_tcp::Tcp::recv_with`]. - pub(crate) fn tcp_recv(&mut self, now: Instant, id: ConnId, take: impl FnOnce(&[u8]) -> usize) -> Result { - self.shard.recv_with(now, id, take) + /// The client placed `n` more bytes in its lent send buffer: how many [tcp] had room for. + pub(crate) fn tcp_queue(&mut self, now: Instant, id: ConnId, n: u32) -> Result { + self.shard.queue(now, id, usize::try_from(n).unwrap_or(usize::MAX)) } /// The connection's FIN, after what is queued. Of a synchronized connection that has not diff --git a/system/netstack/node/src/lib.rs b/system/netstack/node/src/lib.rs index 060f818de20..dc4c049a387 100644 --- a/system/netstack/node/src/lib.rs +++ b/system/netstack/node/src/lib.rs @@ -6,8 +6,8 @@ //! //! The node carries out what the client asks and tells it what became of its address. What the //! interface holds is `lease`'s to write and nobody else's: see that module for the three rules -//! its types keep. A client's TCP connection and its two pipes are `streams`', a passive open and -//! its owner's wakes `listeners`'. +//! its types keep. A client's TCP connection and the region it lent are `streams`', a passive open +//! and its owner's wakes `listeners`'. //! //! A client's datagram sockets are `datagram`'s, the machine's `.local` name is `name`'s //! and the lookups of other machines' names are `resolve`'s: each reads the lease and writes none @@ -46,8 +46,8 @@ mod places; mod resolve; mod streams; -pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerId, Wake}; -pub use streams::{ConnectRefused, FromClient, PipeEnd, Pipes, ReadRefusal, StreamEvent, StreamId, ToClient, Watch, WriteRefusal}; +pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerId, Wake, WriteRefusal}; +pub use streams::{ConnectRefused, Session, StreamEvent, StreamId}; use alloc::vec; use alloc::vec::Vec; diff --git a/system/netstack/node/src/listeners.rs b/system/netstack/node/src/listeners.rs index 36e65b65c09..0a4039093a5 100644 --- a/system/netstack/node/src/listeners.rs +++ b/system/netstack/node/src/listeners.rs @@ -1,7 +1,8 @@ //! Listeners: a passive open of [tcp]'s and the pipe its owner reads its wakes from. The queues //! are [tcp]'s: handshakes in progress, and connections that finished theirs and wait, oldest -//! first. A connection that waits is nobody's stream yet; an accept makes the oldest one a -//! stream on the pipes the accept hands over. +//! first. A connection that waits is nobody's stream yet, its text held in [tcp]'s own storage; +//! an accept makes the oldest one a stream on the region the accept lends, and what it held moves +//! there at its offsets, text past a hole included, the FIN staying [tcp]'s state. //! //! **The owner holds one unspent wake for each connection that waits and that there is a place //! for, and an accept spends one whatever it answers.** The owner reads a wake and then accepts, @@ -39,9 +40,19 @@ use toyos_net_shard::ListenError; use toyos_net_tcp::{Endpoint, Error, Options}; use toyos_net_wire::{Instant, Port}; -use crate::streams::{Pipes, StreamId, WriteRefusal}; +use crate::streams::{Session, StreamId}; use crate::Node; +/// Why the write end of a listener's wake pipe took nothing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WriteRefusal { + Full, + /// Nobody holds the read end. + Gone, + /// Anything else: the handle is not the pipe end its owner said it was. + Broken, +} + /// The write end of the pipe a listener's owner reads its wakes from. Dropped, the owner reads /// the end. pub trait Wake { @@ -76,13 +87,13 @@ pub struct Accepted { pub nodelay: bool, } -/// Why an accept took no connection. Whatever waits still waits, and the pipe ends are dropped. +/// Why an accept took no connection. Whatever waits still waits, and the session is dropped. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AcceptRefused { /// The id names no listener. NoListener, - /// The request carried no pipes. - NoPipes, + /// The request carried no region. + NoRegion, /// The node holds all it has places for (`places`), whatever waits. Full, /// No connection waits. @@ -149,25 +160,24 @@ impl Node { true } - /// The owner's accept: the oldest connection waiting at `id` becomes a stream on `pipes`, - /// and what it already received moves at once. - pub fn accept(&mut self, now: Instant, id: ListenerId, pipes: Option) -> Result { + /// The owner's accept: the oldest connection waiting at `id` becomes a stream on `session`'s + /// region, and what it already received moves there at once. + pub fn accept(&mut self, now: Instant, id: ListenerId, session: Option>) -> Result { let Some(listener) = self.listeners.live.get_mut(&id) else { return Err(AcceptRefused::NoListener) }; listener.unspent = listener.unspent.saturating_sub(1); let bound = listener.bound; - let answer = self.take(bound, pipes); + let answer = self.take(bound, session); self.pass(now, false); answer } - fn take(&mut self, bound: toyos_net_tcp::ListenerId, pipes: Option) -> Result { - let Some(pipes) = pipes else { return Err(AcceptRefused::NoPipes) }; + fn take(&mut self, bound: toyos_net_tcp::ListenerId, session: Option>) -> Result { + let Some(session) = session else { return Err(AcceptRefused::NoRegion) }; if self.room() == 0 { return Err(AcceptRefused::Full); } let Some((conn, tuple, options)) = self.stack.tcp_accept(bound) else { return Err(AcceptRefused::Nothing) }; - // The peer's address is what `streams` counts a stream its client can see no more by. - let id = self.streams.accepted(conn, tuple.remote.addr, options, pipes); + let id = self.streams.accepted(&mut self.stack, conn, options, session); Ok(Accepted { id, remote: tuple.remote, local: tuple.local.port, nodelay: options.nodelay }) } diff --git a/system/netstack/node/src/places.rs b/system/netstack/node/src/places.rs index ce45b8c1870..869eaa3904b 100644 --- a/system/netstack/node/src/places.rs +++ b/system/netstack/node/src/places.rs @@ -1,15 +1,10 @@ //! Places: the one bound on the streams, listeners and datagram sockets clients make the node -//! hold. A stream holds a place from its -//! connect or its accept until the node lets it go, a listener from its listen until it is -//! closed, a datagram socket from its bind until it is closed, and a connection the node closed -//! holds the place its stream had until [tcp] has finished it, because it is still two buffers -//! of [tcp]'s and its peer decides for how long. A stream its client can see no more, which its -//! peer keeps alive, holds its place through both: it is a stream while its pipe holds bytes, -//! and [tcp]'s to finish after. `streams` lets one peer address keep at most -//! `OWNERLESS_PER_PEER` of them alive and counts no addresses, so that is each address's share -//! of the cut's clock and no bound on the node: peers at however many addresses hold no more of -//! them than there are places. With no place left a connect, a listen, an accept and a bind are -//! refused with nothing made. +//! hold. A stream holds a place from its connect or its accept until the node lets it go, a +//! listener from its listen until it is closed, a datagram socket from its bind until it is +//! closed, and a connection the node closed holds the place its stream had until [tcp] has +//! finished it, because its buffers are still the region its client lent and its peer decides +//! for how long, within [tcp]'s bounds on a connection nobody holds. With no place left a +//! connect, a listen, an accept and a bind are refused with nothing made. //! //! **The node's own sockets stand outside the places.** The responder's socket for the //! machine's name and the socket of each query `resolve` has out are bound past @@ -17,10 +12,10 @@ //! bound refuse no lookup and no answer for the name. A lookup is a client's too, and its bound //! is `resolve`'s own: `toyos_dns::MAX_LOOKUPS` lookups, each with the queries of its rounds. //! -//! The number is the shell's: each stream is two of its client's pipes kept alive and watched, -//! each listener one. A new node has none. +//! The number is the shell's: each stream is its client's region, connection and bell kept +//! alive and its connection watched, each listener its wake pipe. A new node has none. //! -//! **A place is not one size.** A stream or a connection [tcp] finishes is [tcp]'s two buffers, +//! **A place is not one size.** A stream or a connection [tcp] finishes is its client's region, //! and a datagram socket [udp]'s two queues of `limits::RX_DATAGRAMS` and `limits::TX_DATAGRAMS` //! datagrams. A listener's place stands for all a peer can make it hold without a client //! asking, which [tcp] bounds for each listener: `limits::LISTEN_PENDING` handshakes in diff --git a/system/netstack/node/src/streams.rs b/system/netstack/node/src/streams.rs index 1201fa2e44b..ffee6ac25de 100644 --- a/system/netstack/node/src/streams.rs +++ b/system/netstack/node/src/streams.rs @@ -1,108 +1,49 @@ -//! Streams: a connection of [tcp]'s and the two pipes its client reads and writes it by. The pipe -//! ends are traits, so the kernel's calls stay in netstack and the tests fake them. +//! Streams: a connection of [tcp]'s whose two buffers are the region its client lent +//! (`toyos::net::stream`), and the session's server end over the region's words. [`Session`] +//! keeps the mapping and the doorbells in netstack, so tests lend a region on the heap. //! -//! **The node holds a stream for as long as it holds one of its pipes.** A pass -//! ([`Node::bridge`]) moves what each side takes and lets go of an end that is finished: the -//! to-client end once the peer's FIN or a failure has been read through it or its reader is gone, -//! the from-client end once the connection failed, or its writer is gone and its last byte is -//! queued with the FIN after it. With neither end left the node closes the connection, which -//! [tcp] then finishes alone by its own rules for a user who let go (a reset if text is unread, -//! RFC 9293 §3.6.1), and the stream's id names nothing. +//! **A pass** ([`Node::bridge`]) reads the client's head into [tcp]'s window (RFC 9293 +//! §3.8.6.2.2), publishes what [tcp] holds in order and the peer's FIN after it, gives +//! acknowledged bytes back as room, queues what the client published with no copy, and turns the +//! client's shutdown into the FIN after its last byte. A failure is stored as the receive ring's +//! `end`, which wakes the client either way. netstack arms `tx_data` while it took everything and +//! [tcp] has room, and `rx_room` only while the window last offered is below the update +//! threshold. //! -//! **How a stream ended is the order its ends go in**, which is all a pipe can say: a client that -//! reads the end of the to-client pipe and finds the from-client pipe still read had the peer's -//! FIN, and one that finds it gone had a failure. So a failure lets the from-client end go before -//! the to-client end, and an orderly end keeps a from-client end whose last byte and FIN are -//! queued, read no more, until its writer leaves or the client lets go of the stream. +//! **Every word read from the region is the client's, and judged**: a cursor past its bound, or +//! text published after its own shutdown, is a [`Violation`] that resets that stream alone. //! -//! **Nothing leaves the stack that the client's pipe did not take**, and **nothing leaves the -//! pipe that the stack will not take**: a byte moved is a byte acknowledged to whoever sent it, -//! so one the other side refused would be cut out of the middle of the stream with nothing -//! saying so. What a full pipe refuses stays in [tcp], whose window closes on the peer; what -//! [tcp] has no room for stays in the pipe, which blocks the client. -//! -//! **A stream its client can see no more leaves its unsent bytes [`OWNERLESS_LIFE`] without -//! progress.** That is a stream whose to-client end the node let go, whichever way, and whose -//! client writes no more: no byte and no end reaches the client through it again, and nothing -//! tells the node whether that client still lives. The peer's FIN read through counts as much as -//! a reader that left, since behind it the node cannot learn of the leaving. The bytes the pipe -//! still holds are sent as [tcp] makes room, and the connection is reset once the pipe has given -//! up no byte for [`OWNERLESS_LIFE`]. A peer that takes a byte now and then would so hold such a -//! connection for as long as it liked (RFC 9293 §3.8.6.1 lets one be reclaimed), so a peer -//! address restarts that clock for at most [`OWNERLESS_PER_PEER`] streams at once; any other has -//! [`OWNERLESS_LIFE`] from the pass that first found it so, whatever it gives up, until it -//! becomes one of them: a pass over the streams that begins with fewer makes it one, the oldest -//! stream first. The count is an address's and addresses are not counted: how many such streams -//! the node holds in all is `places`' to bound, of which each holds one, and how slowly a peer -//! may take is bounded nowhere. A client that still holds its reading end, or may still write, -//! is never timed here: what it cannot send is [tcp]'s to give up on. -//! -//! Every call that can move a stream ends in a pass: a batch of frames, a deadline, and each call -//! here. A transmit opportunity can only fail a connect, whose next hop it found to answer -//! nobody, and ends in a pass over the connects. -//! -//! **A send pipe found empty is read again only once the kernel says it holds bytes** -//! ([`Node::bridge`]'s `readable`), or once its client writes no more, when an empty pipe is the -//! end. netstack watches such a pipe for bytes as long as [tcp] has room for them -//! ([`Watch::readable`]), and the kernel answers a watch of a pipe that already holds some at -//! once, so no byte waits on a pass that never reads. +//! **A stream ends with its connection or its client.** Ended, the stream is let go and the +//! client reads what the region holds. The client's hang-up ([`Node::left`]) closes the +//! connection, which [tcp] finishes alone: what was published is sent before the FIN (§3.10.4), +//! or a reset is sent where text is unread (§3.6.1). use alloc::boxed::Box; use alloc::collections::{BTreeMap, VecDeque}; -use core::net::Ipv4Addr; +use core::sync::atomic::Ordering; use core::time::Duration; +use toyos::net::stream::{Server, Words, LAYOUT, NODELAY, RECEIVE_BYTES, RESET, TIMED_OUT, UNREACHABLE}; use toyos_net_shard::ConnectError; -use toyos_net_tcp::{ConnId, Endpoint, Error, Failure, Options, Received, State}; +use toyos_net_tcp::{ConnId, Endpoint, Error, Failure, Lent, Options, State, Status}; use toyos_net_wire::{Instant, Port}; +use toyos_transport::session::Rings; +use toyos_transport::Violation; use crate::lease::Stack; use crate::Node; -/// How long the pipe of a stream its client can see no more may give up no byte: R2, the time -/// RFC 9293 §3.8.3 gives a segment's retransmission before the connection is closed, at the 100 -/// seconds it asks for at least. -const OWNERLESS_LIFE: Duration = Duration::from_secs(100); - -/// How many such streams one peer address keeps alive at once by taking their bytes. An -/// estimate: no measurement sets it. It bounds an address, not the node, and not how slowly a -/// peer may take. -const OWNERLESS_PER_PEER: usize = 16; - -/// The most one read of a client's pipe takes. -const CHUNK: usize = 4096; - -/// Why the write end of a pipe took nothing. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum WriteRefusal { - Full, - /// Nobody holds the read end. - Gone, - /// Anything else: the handle is not the pipe end its client said it was. - Broken, -} - -/// Why the read end of a pipe gave nothing. A pipe with no writer left is no refusal: it gives -/// what it holds and then the end. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ReadRefusal { - /// Nothing to read, and a writer is left. - Empty, - /// Anything else: the handle is not the pipe end its client said it was. - Broken, -} - -/// The write end of the pipe a client reads its stream from. Dropped, the client reads the end. -pub trait ToClient { - /// Takes a prefix of `bytes` and answers its length. - fn write(&mut self, bytes: &[u8]) -> Result; -} - -/// The read end of the pipe a client writes its stream to. Dropped, the client's writes fail. -pub trait FromClient { - /// Fills a prefix of `out` and answers its length. 0 is the end: the pipe is empty and no - /// writer is left. - fn read(&mut self, out: &mut [u8]) -> Result; +/// A client's session: the region it lent and its two doorbells. Dropped, the node holds +/// neither, and the client's next look finds the `end` or the hang-up. +pub trait Session { + /// The region's header. + fn words(&self) -> &Words; + /// The region's receive ring, as [tcp]'s receive buffer. + fn receive(&self) -> Box; + /// The region's send ring, as [tcp]'s send buffer. + fn send(&self) -> Box; + /// Rings the client's doorbells `rings` names, never waiting. + fn ring(&self, rings: Rings); } /// Why a connect made no stream. @@ -113,19 +54,6 @@ pub enum ConnectRefused { Stack(ConnectError), } -/// The two ends a client's connect or accept hands over. -pub struct Pipes { - pub to_client: Box, - pub from_client: Box, -} - -/// One of a stream's two pipe ends. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum PipeEnd { - ToClient, - FromClient, -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct StreamId(u64); @@ -137,88 +65,68 @@ pub enum StreamEvent { Failed { id: StreamId, failure: Failure }, /// The answer to a connect: its deadline passed first. TimedOut { id: StreamId }, - /// The answer to a connect: its stream was closed first. + /// The answer to a connect: its client left first. Closed { id: StreamId }, - /// A line for the log: reset with its bytes unsent and nobody left to see it, at the end of - /// its [`OWNERLESS_LIFE`]. - Cut { id: StreamId }, + /// A line for the log: the client wrote what no client writes, and its stream was reset. + Violated { id: StreamId, violation: Violation }, } -/// What netstack asks the kernel about an established stream's pipes until the next pass. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Watch { - /// Bytes in the from-client pipe: [tcp] has room for them. - pub readable: bool, - /// The from-client pipe's writer leaving, reported by [`Node::pipe_gone`]. - pub writer: bool, - /// Room in the to-client pipe: it refused bytes [tcp] still holds. - pub writable: bool, - /// The to-client pipe's reader leaving, reported by [`Node::pipe_gone`]. - pub reader: bool, +/// The receive ring's `end` for a failure. +const fn code(failure: Failure) -> u32 { + match failure { + Failure::TimedOut => TIMED_OUT, + Failure::Unreachable(_) | Failure::Prohibited => UNREACHABLE, + Failure::Refused | Failure::Reset => RESET, + } +} + +/// Whether the peer's FIN came after every byte before it. +fn fin_received(status: &Status) -> bool { + match status.state { + State::CloseWait | State::Closing | State::LastAck | State::TimeWait => true, + State::Closed => status.failure.is_none(), + State::SynSent | State::SynReceived | State::Established | State::FinWait1 | State::FinWait2 => false, + } +} + +fn count(n: usize) -> u32 { + u32::try_from(n).unwrap_or(u32::MAX) } struct Stream { conn: ConnId, - /// The peer's address: what [`OWNERLESS_PER_PEER`] counts by. - remote: Ipv4Addr, - /// Before `to_client`, so a stream dropped whole lets its send pipe go first: the order a - /// failure owes its client. - from_client: Option>, - to_client: Option>, - /// The connect is not answered yet, and no byte moves. - connecting: bool, - /// The connect's, while connecting; then the cut's, once the client can see the stream no - /// more. + session: Box, + /// The session's server end, once the connection is established and its buffers are the + /// region's; `None` while a connect is not answered. + server: Option, + /// The connect's, while connecting. deadline: Option, - /// The client writes no more: its writer is gone, it shut its sending half down or it - /// closed. An empty pipe is then the end. - done_writing: bool, - /// The FIN is queued after the client's last byte. A from-client end still held is kept for - /// its client to find there, until its writer leaves. + /// Bytes published that the client had not released, as the last pass read them. + held: u32, + /// Bytes of the send ring [tcp] holds, as the last pass counted them. + queued: u32, + /// The client's FIN is queued after its last byte. fin_queued: bool, - /// The client let go of the stream, or nobody reads it: nobody asks how it ended. - left: bool, - /// One of its peer address's [`OWNERLESS_PER_PEER`]: a byte given up restarts the cut's clock. - extended: bool, - /// The to-client pipe refused bytes [tcp] holds. - held: bool, - /// [tcp] had room for the client's bytes when the last pass ended. - room: bool, - /// The last read of the from-client pipe found it empty, and the kernel has not said since - /// that it holds bytes. - drained: bool, /// The options last written to [tcp]. options: Options, + /// The client rang since the last pass. + rung: bool, } impl Stream { - /// An established connection on its client's pipes. `options` are the ones [tcp] holds for - /// it. - fn established(conn: ConnId, remote: Ipv4Addr, options: Options, pipes: Pipes) -> Self { - Self { - conn, - remote, - to_client: Some(pipes.to_client), - from_client: Some(pipes.from_client), - connecting: false, - deadline: None, - done_writing: false, - fin_queued: false, - left: false, - extended: false, - held: false, - room: false, - drained: false, - options, + /// A connection whose buffers are the region's from here on, and its server end. + fn lent(&mut self, stack: &mut Stack) { + let session = &self.session; + if stack.tcp_lend(self.conn, session.receive(), session.send()).is_err() { + unreachable!("[tcp] would not lend an established connection its client's buffers"); } + self.server = Some(Server::new(session.words(), LAYOUT)); } - /// One pass. `false` lets the stream go: its pipe ends drop with it, and its connection is - /// [tcp]'s alone or gone. `extended` is how many streams each peer address kept alive when - /// the pass over the streams began, and those it has made so since. - fn pass(&mut self, id: StreamId, now: Instant, stack: &mut Stack, events: &mut VecDeque, extended: &mut BTreeMap) -> bool { + /// One pass. `false` lets the stream go: its connection is [tcp]'s alone or gone. + fn pass(&mut self, id: StreamId, now: Instant, stack: &mut Stack, events: &mut VecDeque) -> bool { let conn = self.conn; - if self.connecting { + if self.server.is_none() { let status = stack.tcp_status(conn); let event = match (status.state, status.failure) { (_, Some(failure)) => StreamEvent::Failed { id, failure }, @@ -231,124 +139,99 @@ impl Stream { stack.tcp_abort(now, conn); return false; } - self.connecting = false; self.deadline = None; + self.lent(stack); } - - self.held = false; - while let Some(pipe) = self.to_client.as_mut() { - let mut refused = None; - let received = stack.tcp_recv(now, conn, |bytes| match pipe.write(bytes) { - Ok(taken) => taken, - Err(refusal) => { - refused = Some(refusal); - 0 - } - }); - match (received, refused) { - (Ok(Received::Data(taken)), None) if taken > 0 => {} - (Ok(Received::Data(_)), None | Some(WriteRefusal::Full)) => { - self.held = true; - break; - } - // Nobody reads what the peer sends. - (Ok(Received::Data(_)), Some(WriteRefusal::Gone)) => { - self.to_client = None; - self.left = true; - } - (Ok(Received::Data(_)), Some(WriteRefusal::Broken)) => { - stack.tcp_abort(now, conn); - return false; - } - // The peer's FIN after its last byte: the client reads the end, and its send pipe - // stays. - (Ok(Received::End), _) => self.to_client = None, - // The connection's failure: the client's send pipe goes first, so that the end it - // reads finds no reader behind it. - (Err(Error::Failed(_)), _) => { - self.from_client = None; - self.to_client = None; + match self.moved(now, stack) { + Ok(true) => true, + Ok(false) => { + stack.tcp_close(now, conn); + false + } + Err(violation) => { + if let Some(server) = self.server.as_mut() { + self.session.ring(server.fail(self.session.words(), RESET)); } - (Err(Error::WouldBlock), _) => break, - (Err(refusal), _) => unreachable!("[tcp] refused the holder of a connection a read: {refusal:?}"), + stack.tcp_abort(now, conn); + events.push_back(StreamEvent::Violated { id, violation }); + false } } + } - let mut gave_up = false; - while let Some(pipe) = self.from_client.as_mut() { - if self.drained && !self.done_writing { - break; + /// Moves what changed, round after round until a look after netstack's arms finds nothing + /// more, and rings the client once: `false` once the connection ended. + fn moved(&mut self, now: Instant, stack: &mut Stack) -> Result { + let conn = self.conn; + let Some(server) = self.server.as_mut() else { unreachable!("a stream moves only once established") }; + let words = self.session.words(); + let mut rings = Rings::default(); + let keep = loop { + let held = server.held(words)?; + let released = self.held.checked_sub(held).ok_or(Violation::HeadPastTail)?; + if released > 0 { + stack.tcp_consume(now, conn, released); } - // Never more than [tcp] has room for, and so never a read of no bytes, whose answer - // would be the end's; [tcp] has none once the FIN is queued. - let room = stack.tcp_status(conn).writable.min(CHUNK); - if room == 0 { - break; + let status = stack.tcp_status(conn); + let unread = count(status.readable); + if let Some(new) = unread.checked_sub(held).filter(|&new| new > 0) { + rings = rings.and(server.publish(words, new)?); } - let mut chunk = [0u8; CHUNK]; - let Some(space) = chunk.get_mut(..room) else { unreachable!("room is at most a chunk") }; - // The read, and whether the pipe's writer is gone, which a read of 0 is. - let (read, writer_gone) = match pipe.read(space) { - Ok(read) => (read, read == 0), - Err(ReadRefusal::Empty) if self.done_writing => (0, false), - Err(ReadRefusal::Empty) => { - self.drained = true; - break; - } - Err(ReadRefusal::Broken) => { - stack.tcp_abort(now, conn); - return false; + self.held = unread; + if fin_received(&status) { + rings = rings.and(server.finish(words)); + } + + let queued = count(status.queued); + if let Some(acked) = self.queued.checked_sub(queued).filter(|&acked| acked > 0) { + rings = rings.and(server.release(words, acked)?); + } + self.queued = queued; + let shut = server.shut(words); + let fresh = server.sent(words)?.checked_sub(queued).ok_or(Violation::TailPastDepth)?; + if fresh > 0 && status.failure.is_none() { + match stack.tcp_queue(now, conn, fresh) { + // [tcp]'s send buffer is the whole ring, so what the ring held fits it. + Ok(taken) if count(taken) == fresh => self.queued = self.queued.saturating_add(fresh), + // Text published past the client's own shutdown. + Err(Error::Closing) => return Err(Violation::TailPastDepth), + Err(Error::Failed(_)) => {} + answer => unreachable!("[tcp] queued {answer:?} of {fresh} bytes its client's ring held"), } - }; - if read == 0 { + } + if shut && !self.fin_queued && status.failure.is_none() { stack.tcp_shutdown_write(now, conn); self.fin_queued = true; - if writer_gone { - self.from_client = None; - } - break; } - let Some(bytes) = space.get(..read) else { unreachable!("a pipe read {read} bytes into room for {room}") }; - stack.tcp_send(now, conn, bytes); - gave_up = true; - } + let nodelay = words[NODELAY].load(Ordering::Relaxed) != 0; + if nodelay != self.options.nodelay && status.failure.is_none() && status.state != State::Closed { + self.options.nodelay = nodelay; + stack.tcp_set_options(now, conn, self.options); + } - let status = stack.tcp_status(conn); - // A connection that failed takes no more of the client's bytes: its writes fail rather - // than fill a pipe nobody drains, and its end reads as a failure. One that ended in order - // ended after the client's FIN, so its writing was over already. - if status.failure.is_some() { - self.from_client = None; - } - // A finished send pipe is kept only for a client that can still read the end and look - // behind it. - if self.fin_queued && self.left { - self.from_client = None; - } - self.room = status.writable > 0; - if self.to_client.is_none() && self.from_client.is_none() { - stack.tcp_close(now, conn); - return false; - } - // Nothing of the stream reaches its client again, alive or not, and its pipe still holds - // what may be bytes to send. - if self.to_client.is_none() && self.done_writing && !self.fin_queued { - if !self.extended { - let kept = extended.entry(self.remote).or_insert(0); - if *kept < OWNERLESS_PER_PEER { - *kept = kept.saturating_add(1); - self.extended = true; - } + if let Some(failure) = status.failure { + rings = rings.and(server.fail(words, code(failure))); + break false; } - if self.deadline.is_none() || (self.extended && gave_up) { - self.deadline = Some(now.after(OWNERLESS_LIFE)); - } else if self.deadline.is_some_and(|at| at <= now) { - stack.tcp_abort(now, conn); - events.push_back(StreamEvent::Cut { id }); - return false; + if matches!(status.state, State::Closed | State::TimeWait) { + break false; } - } - true + if core::mem::take(&mut self.rung) { + rings = rings.and(server.rung(words)?); + } + let mut more = false; + if !shut && status.writable > 0 { + more |= server.want_data(words)? > self.queued; + } + if status.starved { + more |= RECEIVE_BYTES.saturating_sub(server.want_room(words)?) < self.held; + } + if !more { + break true; + } + }; + self.session.ring(rings); + Ok(keep) } } @@ -369,10 +252,14 @@ impl Streams { id } - /// Holds a connection a listener's owner accepted, established, on the pipes its accept - /// handed over. `options` are the ones [tcp] says it has. - pub(crate) fn accepted(&mut self, conn: ConnId, remote: Ipv4Addr, options: Options, pipes: Pipes) -> StreamId { - self.hold(Stream::established(conn, remote, options, pipes)) + /// Holds a connection a listener's owner accepted, established, its buffers the region of + /// `session` from here on, what it held moved there. `options` are the ones [tcp] says it + /// has, `nodelay` among them written into the region's header. + pub(crate) fn accepted(&mut self, stack: &mut Stack, conn: ConnId, options: Options, session: Box) -> StreamId { + session.words()[NODELAY].store(u32::from(options.nodelay), Ordering::Relaxed); + let mut stream = Stream { conn, session, server: None, deadline: None, held: 0, queued: 0, fin_queued: false, options, rung: false }; + stream.lent(stack); + self.hold(stream) } pub(crate) fn next_deadline(&self) -> Option { @@ -381,25 +268,26 @@ impl Streams { } impl Node { - /// An active open to `remote`, answered by a [`StreamEvent`] once the handshake ends or - /// `timeout` passes. Refused, nothing was sent and the pipe ends are dropped. - pub fn connect(&mut self, now: Instant, remote: Endpoint, timeout: Option, pipes: Pipes) -> Result { + /// An active open to `remote` on `session`'s region, with Nagle's algorithm as its `nodelay` + /// word says, answered by a [`StreamEvent`] once the handshake ends or `timeout` passes. + /// Refused, nothing was sent and `session` is dropped. + pub fn connect(&mut self, now: Instant, remote: Endpoint, timeout: Option, session: Box) -> Result { if self.room() == 0 { return Err(ConnectRefused::Full); } let conn = self.stack.tcp_connect(now, remote).map_err(ConnectRefused::Stack)?; + let options = Options { nodelay: session.words()[NODELAY].load(Ordering::Relaxed) != 0, ..Options::default() }; + self.stack.tcp_set_options(now, conn, options); let deadline = timeout.map(|within| now.after(within)); - // An active open has [tcp]'s defaults until `set_nodelay`. - Ok(self.streams.hold(Stream { connecting: true, deadline, ..Stream::established(conn, remote.addr, Options::default(), pipes) })) + Ok(self.streams.hold(Stream { conn, session, server: None, deadline, held: 0, queued: 0, fin_queued: false, options, rung: false })) } - /// One pass over every stream: netstack calls it once the kernel has answered pipes it - /// watches ready as [`Self::watches`] asked, and `readable` are the streams whose from-client - /// pipe it said holds bytes. - pub fn bridge(&mut self, now: Instant, readable: impl IntoIterator) { - for id in readable { + /// One pass over every stream: netstack calls it once the kernel answered the connections it + /// watches, `rung` being the streams whose client rang. + pub fn bridge(&mut self, now: Instant, rung: impl IntoIterator) { + for id in rung { if let Some(stream) = self.streams.live.get_mut(&id) { - stream.drained = false; + stream.rung = true; } } self.pass(now, false); @@ -409,93 +297,29 @@ impl Node { pub(crate) fn pass(&mut self, now: Instant, connects: bool) { let Streams { live, events, .. } = &mut self.streams; let stack = &mut self.stack; - let mut extended = BTreeMap::new(); - for stream in live.values().filter(|stream| stream.extended) { - let peers = extended.entry(stream.remote).or_insert(0usize); - *peers = peers.saturating_add(1); - } - live.retain(|id, stream| (connects && !stream.connecting) || stream.pass(*id, now, stack, events, &mut extended)); + live.retain(|id, stream| (connects && stream.server.is_some()) || stream.pass(*id, now, stack, events)); self.wake_owners(now); } - /// The client lets go of the stream: nobody reads it, and what its pipe still holds is sent - /// before the FIN. A connect not yet answered is answered [`StreamEvent::Closed`]. - pub fn close(&mut self, now: Instant, id: StreamId) { - let Some(stream) = self.streams.live.get_mut(&id) else { return }; - if stream.connecting { + /// The client hung up its connection: what it published is queued and [tcp] closes the + /// connection. A connect not yet answered is answered [`StreamEvent::Closed`]. + pub fn left(&mut self, now: Instant, id: StreamId) { + let Some(mut stream) = self.streams.live.remove(&id) else { return }; + if stream.server.is_none() { self.stack.tcp_abort(now, stream.conn); - self.streams.live.remove(&id); self.streams.events.push_back(StreamEvent::Closed { id }); - self.wake_owners(now); - return; - } - stream.to_client = None; - stream.done_writing = true; - stream.left = true; - self.pass(now, false); - } - - /// The client sends no more: what its pipe still holds is sent, then the FIN. `false` is an - /// id that names no established stream. - pub fn shutdown_write(&mut self, now: Instant, id: StreamId) -> bool { - let Some(stream) = self.streams.live.get_mut(&id).filter(|stream| !stream.connecting) else { return false }; - stream.done_writing = true; - self.pass(now, false); - true - } - - /// Nagle's algorithm off or on (RFC 9293 §3.7.4). `false` is an id that names no stream. - pub fn set_nodelay(&mut self, now: Instant, id: StreamId, nodelay: bool) -> bool { - let Some(stream) = self.streams.live.get_mut(&id) else { return false }; - stream.options.nodelay = nodelay; - self.stack.tcp_set_options(now, stream.conn, stream.options); - true - } - - /// The kernel said nobody holds the other end of one of an established stream's pipes: the - /// to-client pipe has no reader, or the from-client pipe no writer, whatever it still holds. - pub fn pipe_gone(&mut self, now: Instant, id: StreamId, end: PipeEnd) { - let Some(stream) = self.streams.live.get_mut(&id).filter(|stream| !stream.connecting) else { return }; - match end { - PipeEnd::ToClient => { - stream.to_client = None; - stream.left = true; + } else { + match stream.moved(now, &mut self.stack) { + Ok(_) => self.stack.tcp_close(now, stream.conn), + Err(violation) => { + self.stack.tcp_abort(now, stream.conn); + self.streams.events.push_back(StreamEvent::Violated { id, violation }); + } } - // A finished send pipe was kept for its writer alone. - PipeEnd::FromClient if stream.fin_queued => stream.from_client = None, - PipeEnd::FromClient => stream.done_writing = true, } self.pass(now, false); } - /// The kernel refused netstack's watch of a pipe end the node still holds: the handle is no - /// pipe end, and its client's connection is reset. - pub fn pipe_broken(&mut self, now: Instant, id: StreamId, end: PipeEnd) { - let Some(stream) = self.streams.live.get(&id).filter(|stream| !stream.connecting) else { return }; - let held = match end { - PipeEnd::ToClient => stream.to_client.is_some(), - PipeEnd::FromClient => stream.from_client.is_some(), - }; - if held { - self.stack.tcp_abort(now, stream.conn); - self.streams.live.remove(&id); - self.wake_owners(now); - } - } - - /// What to ask the kernel about each established stream's pipes, as the last pass left them. - pub fn watches(&self) -> impl Iterator + '_ { - self.streams.live.iter().filter(|(_, stream)| !stream.connecting).map(|(id, stream)| { - let (from, to) = (stream.from_client.is_some(), stream.to_client.is_some()); - (*id, Watch { - readable: from && stream.room, - writer: from && (!stream.done_writing || stream.fin_queued), - writable: to && stream.held, - reader: to, - }) - }) - } - /// The streams the node holds, connecting ones included. pub fn streams(&self) -> usize { self.streams.live.len() diff --git a/system/netstack/node/tests/common/client.rs b/system/netstack/node/tests/common/client.rs new file mode 100644 index 00000000000..a952dbcba0d --- /dev/null +++ b/system/netstack/node/tests/common/client.rs @@ -0,0 +1,233 @@ +//! A client's end of a stream: `toyos_transport::session`'s own client sequences over a region +//! on the heap, laid out as `toyos::net::stream` lays one out, and the session the node is lent. +//! No call waits: a doorbell is a count, the client's ring of netstack a flag the test reads to +//! say which streams a pass is owed. + +use std::cell::{Cell, RefCell}; +use std::rc::Rc; +use std::sync::atomic::{AtomicU32, Ordering}; + +use toyos::net::stream::{Words, LAYOUT, NODELAY, RECEIVE_BYTES, RECEIVE_RING, REGION_BYTES, SEND_BYTES, SEND_RING}; +use toyos_net_node::Session; +use toyos_net_tcp::Lent; +use toyos_transport::session::{self, Bells, Doorbell, Lock, Mode, Receiving, Refused, Rings, Sending, Took, Waited}; + +/// The region and its doorbells, which the client and the node share. +pub struct Region { + words: Box, + bytes: RefCell>, + /// netstack's rings of the client's connection and bell, not yet drained. + connection: Cell, + bell: Cell, + /// Every ring of netstack's, of either doorbell. + rung: Cell, + /// The client rang netstack since the test last asked. + rang: Cell, + /// The node lets go of the session. + dropped: Cell, +} + +struct Held(RefCell); + +impl Lock for Held { + fn with(&self, f: impl FnOnce(&mut T) -> R) -> R { + f(&mut self.0.borrow_mut()) + } +} + +impl Bells for Region { + type Until = (); + + fn drain(&self, bell: Doorbell) -> Took { + let count = match bell { + Doorbell::Connection => &self.connection, + Doorbell::Bell => &self.bell, + }; + if count.replace(0) > 0 { Took::Rung } else { Took::Nothing } + } + + fn wait(&self, _: Doorbell, _: Option<()>) -> Waited { + unreachable!("a test's client never waits") + } + + fn ring(&self) { + self.rang.set(true); + } +} + +pub struct Client { + pub region: Rc, + rx: Held>, + tx: Held>, +} + +/// One ring of the region, as [tcp]'s storage. +#[derive(Debug)] +struct Ring { + region: Rc, + base: usize, + len: usize, +} + +impl std::fmt::Debug for Region { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Region") + } +} + +impl Lent for Ring { + fn len(&self) -> usize { + self.len + } + + fn put(&mut self, at: usize, data: &[u8]) { + assert!(at + data.len() <= self.len, "a put past the ring"); + self.region.bytes.borrow_mut()[self.base + at..][..data.len()].copy_from_slice(data); + } + + fn take(&self, at: usize, out: &mut [u8]) { + assert!(at + out.len() <= self.len, "a take past the ring"); + out.copy_from_slice(&self.region.bytes.borrow()[self.base + at..][..out.len()]); + } +} + +/// What the node is lent. +struct Lend(Rc); + +impl Session for Lend { + fn words(&self) -> &Words { + &self.0.words + } + + fn receive(&self) -> Box { + Box::new(Ring { region: self.0.clone(), base: RECEIVE_RING, len: RECEIVE_BYTES as usize }) + } + + fn send(&self) -> Box { + Box::new(Ring { region: self.0.clone(), base: SEND_RING, len: SEND_BYTES as usize }) + } + + fn ring(&self, rings: Rings) { + let region = &self.0; + for (rung, count) in [(rings.connection, ®ion.connection), (rings.bell, ®ion.bell)] { + if rung { + count.set(count.get() + 1); + region.rung.set(region.rung.get() + 1); + } + } + } +} + +impl Drop for Lend { + fn drop(&mut self) { + self.0.dropped.set(true); + } +} + +impl std::fmt::Debug for Client { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Client") + } +} + +impl Client { + /// A client with a fresh region, and the session a connect or an accept lends the node. + pub fn new() -> (Rc, Box) { + let region = Rc::new(Region { + words: Box::new(std::array::from_fn(|_| AtomicU32::new(0))), + bytes: RefCell::new(vec![0; REGION_BYTES]), + connection: Cell::new(0), + bell: Cell::new(0), + rung: Cell::new(0), + rang: Cell::new(false), + dropped: Cell::new(false), + }); + let (rx, tx) = session::client(®ion.words, LAYOUT); + let client = Client { region: region.clone(), rx: Held(RefCell::new(rx)), tx: Held(RefCell::new(tx)) }; + (Rc::new(client), Box::new(Lend(region))) + } + + /// A client that set Nagle's algorithm off before it connected. + pub fn nodelay() -> (Rc, Box) { + let (client, session) = Self::new(); + client.region.words[NODELAY].store(1, Ordering::Relaxed); + (client, session) + } + + /// A read of at most `max` bytes that does not wait. + pub fn read(&self, max: usize) -> Result, Refused> { + let mut out = vec![0; max]; + let region = &self.region; + let n = session::read(®ion.words, &self.rx, &**region, Mode::Nonblocking, u32::try_from(max).unwrap(), |span, at| { + out[at..at + span.len].copy_from_slice(®ion.bytes.borrow()[span.offset..][..span.len]); + })?; + out.truncate(n as usize); + Ok(out) + } + + /// Everything readable now, and how the reading stopped: `WouldBlock`, or the end. + pub fn read_all(&self) -> (Vec, Result<(), Refused>) { + let mut all = Vec::new(); + loop { + match self.read(1 << 20) { + Ok(bytes) if bytes.is_empty() => return (all, Ok(())), + Ok(bytes) => all.extend(bytes), + Err(refused) => return (all, Err(refused)), + } + } + } + + /// A write that does not wait: how much of `data` the send ring took. + pub fn write(&self, data: &[u8]) -> Result { + let region = &self.region; + let n = session::write(®ion.words, &self.tx, &**region, Mode::Nonblocking, u32::try_from(data.len()).unwrap(), |span, at| { + region.bytes.borrow_mut()[span.offset..][..span.len].copy_from_slice(&data[at..at + span.len]); + })?; + Ok(n as usize) + } + + pub fn shutdown(&self) { + session::shutdown(&self.region.words, &self.tx, &*self.region); + } + + /// Nagle's algorithm off or on, and netstack rung to read it. + pub fn set_nodelay(&self, nodelay: bool) { + self.region.words[NODELAY].store(u32::from(nodelay), Ordering::Relaxed); + self.region.ring(); + } + + /// The source registered for both ways. + pub fn register(&self) { + session::register(&self.region.words, Some(&self.rx), Some(&self.tx), &*self.region).expect("netstack keeps the protocol"); + } + + /// Whether the client rang netstack since the last call. + pub fn rang(&self) -> bool { + self.region.rang.replace(false) + } + + /// netstack's rings of either doorbell so far. + pub fn rung(&self) -> u32 { + self.region.rung.get() + } + + /// netstack's rings of the bell, which a writer waits on, not yet drained. + pub fn bell(&self) -> u32 { + self.region.bell.get() + } + + /// The node let go of the session. + pub fn dropped(&self) -> bool { + self.region.dropped.get() + } + + /// Rings netstack as the client's own calls do, for a word written past them. + pub fn region_ring(&self) { + self.region.ring(); + } + + /// The region's word `at`, which a hostile client may write. + pub fn word(&self, at: usize) -> &AtomicU32 { + &self.region.words[at] + } +} diff --git a/system/netstack/node/tests/common/mod.rs b/system/netstack/node/tests/common/mod.rs index f5114b2517c..f516505b957 100644 --- a/system/netstack/node/tests/common/mod.rs +++ b/system/netstack/node/tests/common/mod.rs @@ -9,6 +9,8 @@ #![allow(dead_code)] +pub mod client; + use std::net::Ipv4Addr; use std::time::Duration; diff --git a/system/netstack/node/tests/listeners.rs b/system/netstack/node/tests/listeners.rs index f6f59cf7bd4..8861d4cf57c 100644 --- a/system/netstack/node/tests/listeners.rs +++ b/system/netstack/node/tests/listeners.rs @@ -1,5 +1,6 @@ //! Listeners and the node's places, against peers the test scripts, at 192.0.2.2 unless a test -//! puts one elsewhere, and pipe ends it fakes. No scenario ids: the specifications' listening +//! puts one elsewhere, and clients that are `toyos_transport::session`'s own client end over a +//! region on the heap (`common::client`). No scenario ids: the specifications' listening //! scenarios are `toyos-net-tcp`'s; these are what the node does between [tcp]'s queues, an //! owner's wakes and its accepts. //! @@ -13,14 +14,18 @@ mod common; use std::cell::RefCell; -use std::collections::{BTreeSet, VecDeque}; +use std::collections::BTreeSet; use std::net::Ipv4Addr; use std::rc::Rc; +use std::sync::atomic::Ordering; use std::time::Duration; +use common::client::Client; use common::{arp, batch, terms, Segment, Wire, A, BUDGET, ELSEWHERE, MAC, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, Refused, StreamEvent, StreamId, ToClient, Wake, WriteRefusal}; +use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, ListenRefused, ListenerId, Node, Refused, StreamEvent, StreamId, Wake, WriteRefusal}; +use toyos_transport::session::Refused as Read; +use toyos_transport::Violation; use toyos_net_tcp::{limits, Counter, Endpoint}; use toyos_net_wire::{Instant, Port}; @@ -61,8 +66,6 @@ const P3: Peer = Peer { addr: B, port: 40_003 }; const WEB: Peer = Peer { addr: B, port: 80 }; /// Every peer's initial sequence number. const ISS: u32 = 5000; -/// [tcp]'s send buffer, as `common::Wire` configures it. -const SEND_BUFFER: usize = 65_535; const SYN: u8 = 1; const FIN: u8 = 2; @@ -104,60 +107,13 @@ fn wakes(owner: &Owner) -> usize { owner.borrow().wakes } -// ---- a client's pipes ---- +// ---- a client ---- -#[derive(Debug, Default)] -struct Ends { - /// What the node wrote: the pipe takes everything. - inbox: Vec, - /// What the client wrote and the node has not read. - outbox: VecDeque, - /// How many of the two ends the node dropped. - dropped: usize, -} - -type Client = Rc>; - -struct WriteEnd(Client); -struct ReadEnd(Client); - -impl ToClient for WriteEnd { - fn write(&mut self, bytes: &[u8]) -> Result { - self.0.borrow_mut().inbox.extend_from_slice(bytes); - Ok(bytes.len()) - } -} - -impl FromClient for ReadEnd { - fn read(&mut self, out: &mut [u8]) -> Result { - let mut ends = self.0.borrow_mut(); - let read = out.len().min(ends.outbox.len()); - if read == 0 { - return Err(ReadRefusal::Empty); - } - for slot in &mut out[..read] { - *slot = ends.outbox.pop_front().unwrap(); - } - Ok(read) - } -} - -impl Drop for WriteEnd { - fn drop(&mut self) { - self.0.borrow_mut().dropped += 1; - } -} - -impl Drop for ReadEnd { - fn drop(&mut self) { - self.0.borrow_mut().dropped += 1; - } -} - -fn client() -> (Client, Pipes) { - let client = Client::default(); - let pipes = Pipes { to_client: Box::new(WriteEnd(client.clone())), from_client: Box::new(ReadEnd(client.clone())) }; - (client, pipes) +/// What the client reads now, short of the end. +fn inbox(client: &Client) -> Vec { + let (bytes, stopped) = client.read_all(); + assert_eq!(stopped, Err(Read::WouldBlock), "the stream did not end"); + bytes } // ---- the peers ---- @@ -375,44 +331,43 @@ impl Net { (id, owner) } - /// The owner's accept, on a new client's pipes. - fn accept(&mut self, id: ListenerId) -> Result<(Accepted, Client), AcceptRefused> { - let (client, pipes) = client(); - let answer = self.node.accept(self.now, id, Some(pipes)); + /// The owner's accept, on a new client's region. + fn accept(&mut self, id: ListenerId) -> Result<(Accepted, Rc), AcceptRefused> { + let (client, session) = Client::new(); + let answer = self.node.accept(self.now, id, Some(session)); self.pump(); answer.map(|accepted| (accepted, client)) } /// An accept that takes `peer`'s connection. - fn accepts(&mut self, id: ListenerId, peer: Peer) -> (StreamId, Client) { + fn accepts(&mut self, id: ListenerId, peer: Peer) -> (StreamId, Rc) { let (accepted, client) = self.accept(id).expect("a connection and a place"); assert_eq!((accepted.remote, accepted.local), (peer.endpoint(), Port::new(SSH).unwrap())); (accepted.id, client) } /// A connect to [`WEB`]. - fn connect(&mut self) -> (Result, Client) { - let (client, pipes) = client(); - let answer = self.node.connect(self.now, WEB.endpoint(), None, pipes); + fn connect(&mut self) -> (Result, Rc) { + let (client, session) = Client::new(); + let answer = self.node.connect(self.now, WEB.endpoint(), None, session); self.pump(); (answer, client) } - /// The client of stream `id` writes `text`, and the pass netstack runs when its pipe has - /// bytes. + /// The client of stream `id` writes `text`, and the pass its ring brings. fn says(&mut self, id: StreamId, client: &Client, text: &[u8]) { - client.borrow_mut().outbox.extend(text); + assert_eq!(client.write(text), Ok(text.len())); self.node.bridge(self.now, [id]); self.pump(); } - /// The stream's client writes 100,000 bytes and lets go of it: [tcp] has room for - /// [`SEND_BUFFER`] of them, and the rest stays in its pipe. + /// The stream's client writes 100,000 bytes and leaves: [tcp] queues them all, its send + /// buffer being the client's ring, and finishes the connection alone. fn departs(&mut self, id: StreamId, client: &Client) { - client.borrow_mut().outbox.extend(vec![7u8; 100_000]); - self.node.close(self.now, id); + assert_eq!(client.write(&[7u8; 100_000]), Ok(100_000)); + self.node.left(self.now, id); self.pump(); - assert_eq!(client.borrow().outbox.len(), 100_000 - SEND_BUFFER); + assert!(client.dropped()); } fn events(&mut self) -> Vec { @@ -449,17 +404,36 @@ fn a_listener_answers_a_syn_and_wakes_its_owner_when_the_handshake_ends() { net.text(P1, b"hello"); assert_eq!((net.node.streams(), net.node.listeners(), net.node.held()), (0, 1, 1), "a connection that waits is nobody's stream"); - let (client, pipes) = client(); - let accepted = net.node.accept(net.now, id, Some(pipes)).unwrap(); + let (client, session) = Client::new(); + let accepted = net.node.accept(net.now, id, Some(session)).unwrap(); assert_eq!((accepted.remote, accepted.local), (P1.endpoint(), Port::new(SSH).unwrap())); - assert_eq!(client.borrow().inbox, b"hello", "what arrived before the accept moves in it"); + assert_eq!(inbox(&client), b"hello", "what arrived before the accept moves in it"); assert_eq!((net.node.streams(), net.node.listeners(), net.node.held()), (1, 1, 2)); - client.borrow_mut().outbox.extend(b"welcome"); + assert_eq!(client.write(b"welcome"), Ok(7)); net.node.bridge(net.now, [accepted.id]); net.pump(); let said = net.last(P1); assert_eq!((said.text.as_slice(), said.seq), (&b"welcome"[..], synack.seq.wrapping_add(1))); - assert_eq!((wakes(&owner), client.borrow().dropped), (1, 0)); + assert_eq!((wakes(&owner), client.dropped()), (1, false)); +} + +// RFC 9293 §3.10.7.4 holds text past RCV.NXT for later processing and takes a FIN only once every +// byte before it is in. A child holds, before its accept, text in order, text past a hole and a +// FIN behind that; the hole is filled after the accept, and the client reads all of it in order, +// then the end, and the FIN is acknowledged. +#[test] +fn what_a_child_held_in_order_past_a_hole_and_its_fin_reaches_its_stream() { + let mut net = Net::new(); + let (id, _owner) = net.listen(SSH); + net.handshake(P1); + let ack = net.iss(P1).wrapping_add(1); + net.deliver(&frame(P1, SSH, ISS + 1, Some(ack), 0, b"he")); + net.deliver(&frame(P1, SSH, ISS + 4, Some(ack), FIN, b"lo")); + assert_eq!(net.last(P1).ack, Some(ISS + 3), "a hole at the third byte"); + let (_stream, client) = net.accepts(id, P1); + net.deliver(&frame(P1, SSH, ISS + 3, Some(ack), 0, b"l")); + assert_eq!(client.read_all(), (b"hello".to_vec(), Ok(())), "every byte, in order, then the end"); + assert_eq!(net.last(P1).ack, Some(ISS + 7), "the FIN acknowledged"); } // One SYN and nothing more: the next peer is answered at once, and the first handshake is given up within [tcp]'s bound, after which its @@ -528,8 +502,8 @@ fn an_accept_spends_a_wake_whatever_it_answers() { net.handshake(P2); assert_eq!(wakes(&owner), 2, "the wake that accept spent is not counted against the next connection"); - assert_eq!(net.node.accept(net.now, id, None), Err(AcceptRefused::NoPipes)); - assert_eq!(wakes(&owner), 3, "the connection an accept without pipes left is announced again"); + assert_eq!(net.node.accept(net.now, id, None), Err(AcceptRefused::NoRegion)); + assert_eq!(wakes(&owner), 3, "the connection an accept without a region left is announced again"); net.accepts(id, P2); assert_eq!(wakes(&owner), 3); } @@ -561,7 +535,7 @@ fn a_wake_is_owed_only_for_a_connection_there_is_a_place_for() { assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::Full); assert_eq!((wakes(&owner), net.node.held()), (1, 2), "no place, no wake"); - net.node.close(net.now, first); + net.node.left(net.now, first); net.pump(); assert!(net.last(P1).fin); assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (0, 2, 1), "a connection [tcp] is finishing holds its place"); @@ -570,23 +544,23 @@ fn a_wake_is_owed_only_for_a_connection_there_is_a_place_for() { net.accepts(id, P2); } -// A client that left with bytes its peer never takes is still a stream, by `streams`' rule for a -// departed client, and holds its place until that rule cuts it. +// A client that left with bytes its peer never takes is [tcp]'s to finish, and its connection +// holds the place its stream had until [tcp] gives it up, `limits::ORPHAN_IDLE` without +// progress. #[test] -fn a_departed_clients_connection_holds_its_place_until_it_is_cut() { +fn a_departed_clients_connection_holds_its_place_until_tcp_gives_it_up() { let mut net = with_places(2); let (id, owner) = net.listen(SSH); net.handshake(P1); let (stream, client) = net.accepts(id, P1); net.handshake(P2); - client.borrow_mut().outbox.extend(vec![7u8; 100_000]); - net.node.close(net.now, stream); - net.pump(); - assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (1, 2, 1), "its pipe still holds what [tcp] had no room for"); + net.departs(stream, &client); + assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (0, 2, 1), "a connection [tcp] is finishing holds its place"); let (refused, _other) = net.connect(); assert_eq!(refused, Err(ConnectRefused::Full)); - net.run(Duration::from_secs(101)); - assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (0, 1, 2), "cut, and its place is the connection's that waited"); + net.run(limits::ORPHAN_IDLE + Duration::from_secs(1)); + assert!(net.last(P1).rst, "given up"); + assert_eq!((net.node.held(), wakes(&owner)), (1, 2), "its place is the connection's that waited"); } #[test] @@ -596,19 +570,21 @@ fn closing_a_connect_gives_its_place_to_a_connection_that_waits() { let (connecting, _client) = net.connect(); net.handshake(P1); assert_eq!(wakes(&owner), 0); - net.node.close(net.now, connecting.unwrap()); + net.node.left(net.now, connecting.unwrap()); assert_eq!(wakes(&owner), 1); } #[test] -fn a_stream_reset_for_its_pipe_gives_its_place_to_a_connection_that_waits() { +fn a_stream_reset_for_its_clients_violation_gives_its_place_to_a_connection_that_waits() { let mut net = with_places(2); let (id, owner) = net.listen(SSH); net.handshake(P1); - let (stream, _client) = net.accepts(id, P1); + let (stream, client) = net.accepts(id, P1); net.handshake(P2); assert_eq!(wakes(&owner), 1); - net.node.pipe_broken(net.now, stream, PipeEnd::ToClient); + client.word(0).store(7, Ordering::Release); + net.node.bridge(net.now, [stream]); + assert_eq!(net.events(), [StreamEvent::Violated { id: stream, violation: Violation::HeadPastTail }]); assert_eq!(wakes(&owner), 2); } @@ -641,11 +617,11 @@ fn a_connect_past_the_places_is_refused_and_sends_nothing() { assert!(first.is_ok() && second.is_ok(), "{first:?} {second:?}"); let (third, client) = net.connect(); assert_eq!(third, Err(ConnectRefused::Full), "a connect not yet answered holds a place"); - assert_eq!((client.borrow().dropped, net.node.held()), (2, 2)); + assert_eq!((client.dropped(), net.node.held()), (true, 2)); let sources: BTreeSet = net.to(WEB).iter().map(|segment| segment.from_port).collect(); assert_eq!(sources.len(), 2, "no SYN left for the connect refused: {:?}", net.heard); - net.node.close(net.now, first.unwrap()); + net.node.left(net.now, first.unwrap()); let (fourth, _c) = net.connect(); assert!(fourth.is_ok(), "{fourth:?}"); } @@ -864,10 +840,12 @@ fn a_stream_starts_with_the_options_its_connection_took_from_its_listener() { assert_eq!(net.texts(P2), [b"a".to_vec(), b"b".to_vec()], "the second write waits for no acknowledgment"); // An accepted stream's options are its own from then on, and the listener's its own. - assert!(net.node.set_nodelay(net.now, second, false)); + b.set_nodelay(false); + net.node.bridge(net.now, [second]); net.says(second, &b, b"c"); assert_eq!(net.texts(P2), [b"a".to_vec(), b"b".to_vec()], "the third write waits"); - assert!(net.node.set_nodelay(net.now, first, true)); + a.set_nodelay(true); + net.node.bridge(net.now, [first]); net.pump(); assert_eq!(net.texts(P1), [b"a".to_vec(), b"b".to_vec()], "and the first stream's second waits no longer"); @@ -932,103 +910,39 @@ fn a_handshake_reset_before_it_ends_leaves_the_next_connection_its_listeners_opt } } -/// The write end of a client's pipe whose reader is gone for good: the node resets the stream. -struct BrokenEnd; - -impl ToClient for BrokenEnd { - fn write(&mut self, _: &[u8]) -> Result { - Err(WriteRefusal::Broken) - } -} - -// The accept's own pass moves what the connection already received, and a pipe that refuses it -// for good ends the stream there (RFC 9293 §3.10.7.4's reset is the peer's notice). The answer -// still says what the stream was handed over with, though no stream is left to answer for it. -#[test] -fn an_accept_answers_the_option_of_a_stream_its_own_pass_let_go() { - let mut net = Net::new(); - let (id, _owner) = net.listen_with(SSH, true); - net.handshake(P1); - net.text(P1, b"hello"); - let (client, Pipes { from_client, .. }) = client(); - let accepted = net.node.accept(net.now, id, Some(Pipes { to_client: Box::new(BrokenEnd), from_client })).unwrap(); - net.pump(); - assert!(net.last(P1).rst, "{:?}", net.heard); - assert_eq!((accepted.remote, accepted.nodelay), (P1.endpoint(), true)); - assert_eq!((net.node.streams(), client.borrow().dropped), (0, 2)); -} - -// `streams` counts a stream its client can see no more by its peer's address, and lets an -// address keep sixteen alive at once. Seventeen are accepted from 192.0.2.2 and one from -// 192.0.2.3, each client leaves bytes behind, and each peer takes a segment of them fifty -// seconds on: only the seventeenth from 192.0.2.2 has no more than its 100 seconds. -#[test] -fn an_accepted_stream_is_one_of_its_peers_addresss_sixteen() { - let mut net = Net::new(); - let (id, _owner) = net.listen(SSH); - let peers: Vec = (0..17).map(|nth| Peer { addr: B, port: 41_000 + nth }).chain([Peer { addr: C, port: 41_000 }]).collect(); - let mut streams = Vec::new(); - for peer in &peers { - net.handshake(*peer); - let (stream, client) = net.accepts(id, *peer); - net.departs(stream, &client); - streams.push(stream); - } - assert_eq!((net.node.streams(), net.events()), (18, vec![])); - - net.run(Duration::from_secs(50)); - for peer in &peers { - net.takes(*peer, 1_460); - } - net.run(Duration::from_secs(51)); - assert_eq!((net.node.streams(), net.events()), (17, vec![StreamEvent::Cut { id: streams[16] }])); - let reset: Vec = net.heard.iter().filter(|segment| segment.rst).map(|segment| Peer { addr: segment.to, port: segment.to_port }).collect(); - assert_eq!(reset, [peers[16]]); -} - -// `streams` cuts a stream whose client still holds it, once the peer's FIN ended its reading -// and it shut its writing down: what that client asks of the stream afterwards names nothing, -// and not the stream that took its place either, since no id is used twice. +// A stream the node let go, its connection ended by both FINs, names nothing from then: its +// client's hang-up does nothing to the stream that took its place, since no id is used twice. #[test] -fn a_request_for_a_stream_that_was_cut_names_nothing() { +fn a_hang_up_of_a_stream_the_node_let_go_names_nothing() { let mut net = with_places(2); let (id, owner) = net.listen(SSH); net.handshake(P1); - let (cut, client) = net.accepts(id, P1); + let (ended, client) = net.accepts(id, P1); net.handshake(P2); - let ack = net.iss(P1).wrapping_add(1); - net.deliver(&frame(P1, SSH, ISS + 1, Some(ack), FIN, &[])); - client.borrow_mut().outbox.extend(vec![7u8; 100_000]); - assert!(net.node.shutdown_write(net.now, cut)); + client.shutdown(); + net.node.bridge(net.now, [ended]); net.pump(); - assert_eq!((net.node.streams(), client.borrow().dropped, wakes(&owner)), (1, 1, 1), "the client reads the end and may still close"); - - net.run(Duration::from_secs(101)); - assert_eq!(net.events(), [StreamEvent::Cut { id: cut }]); - assert!(net.last(P1).rst); - assert_eq!((net.node.streams(), net.node.held(), client.borrow().dropped, wakes(&owner)), (0, 1, 2, 2)); + assert!(net.last(P1).fin); + net.fin(P1); + assert_eq!(client.read_all(), (Vec::new(), Ok(())), "the client reads the end"); + assert_eq!((net.node.streams(), client.dropped(), wakes(&owner)), (0, true, 2)); let (next, other) = net.accepts(id, P2); - assert_ne!(next, cut); + assert_ne!(next, ended); let said = net.heard.len(); - net.node.close(net.now, cut); - assert!(!net.node.shutdown_write(net.now, cut)); - assert!(!net.node.set_nodelay(net.now, cut, true)); - net.node.pipe_gone(net.now, cut, PipeEnd::FromClient); - net.node.pipe_broken(net.now, cut, PipeEnd::FromClient); + net.node.left(net.now, ended); net.pump(); - assert_eq!((net.node.streams(), net.node.held(), other.borrow().dropped, wakes(&owner)), (1, 2, 0, 2), "the stream in its place stands"); + assert_eq!((net.node.streams(), net.node.held(), other.dropped(), wakes(&owner)), (1, 2, false, 2), "the stream in its place stands"); assert_eq!((net.heard.len(), net.events()), (said, vec![])); } -// The track's exit for the bound across peer addresses. `streams` lets each address keep -// sixteen streams their clients can see no more alive and counts no addresses, so the places -// are what bounds the node. Four addresses, more than the node has places for, bring sixteen -// connections each, and every peer whose connection is accepted keeps its departed client's -// bytes coming for 250 seconds: the node holds two streams at a time, announces no connection -// it has no place for, and refuses every accept and connect past them. +// The places bound what peers at however many addresses make the node hold. Four addresses, more +// than the node has places for, bring sixteen connections each, and every peer whose connection +// is accepted takes its departed client's bytes for 250 seconds, keeping [tcp]'s connection alive: +// the node holds two at a time, announces no connection it has no place for, and refuses every +// accept and connect past them, until [tcp] gives up a connection whose peer takes no more. #[test] -fn peers_at_more_addresses_than_there_are_places_hold_no_stream_past_them() { +fn peers_at_more_addresses_than_there_are_places_hold_no_connection_past_them() { let mut net = with_places(3); let (id, owner) = net.listen(SSH); for nth in 0..16 { @@ -1040,15 +954,14 @@ fn peers_at_more_addresses_than_there_are_places_hold_no_stream_past_them() { for (round, pair) in [[B, C], [D, E]].into_iter().enumerate() { let peers = pair.map(|addr| Peer { addr, port: 42_000 }); - let held = peers.map(|peer| { + for peer in peers { let (stream, client) = net.accepts(id, peer); net.departs(stream, &client); - (stream, client) - }); + } let full = |net: &mut Net| { assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::Full, "round {round}"); assert_eq!(net.connect().0, Err(ConnectRefused::Full), "round {round}"); - assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (2, 3, 2 + 2 * round)); + assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (0, 3, 2 + 2 * round)); }; full(&mut net); for step in 1..=5 { @@ -1058,13 +971,11 @@ fn peers_at_more_addresses_than_there_are_places_hold_no_stream_past_them() { } full(&mut net); } - assert!(held.iter().all(|(_, client)| client.borrow().outbox.len() < 100_000 - SEND_BUFFER), "round {round}: each pipe gave bytes up"); - // The peers take no more: both streams are cut, and two of the connections that wait - // are announced. - net.run(Duration::from_secs(101)); - let cut: Vec = held.iter().map(|(stream, _)| StreamEvent::Cut { id: *stream }).collect(); - assert_eq!(net.events(), cut, "round {round}"); + // The peers take no more: [tcp] gives both connections up, and two of the connections + // that wait are announced. + net.run(limits::ORPHAN_IDLE + Duration::from_secs(1)); + assert_eq!(net.events(), vec![], "round {round}"); assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (0, 1, 4 + 2 * round)); } } diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index 61f7a8991eb..7e26bf820f2 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -1,6 +1,7 @@ //! Streams on the node, against peers the test scripts, at 192.0.2.2:80 unless a test puts one -//! elsewhere, and pipe ends it fakes. No scenario ids: the specifications' TCP scenarios are -//! `toyos-net-tcp`'s; these are what the node does between [tcp] and a client's two pipes. +//! elsewhere, and clients that are `toyos_transport::session`'s own client end over a region on +//! the heap (`common::client`). No scenario ids: the specifications' TCP scenarios are +//! `toyos-net-tcp`'s; these are what the node does between [tcp] and a client's region. //! //! What is not ours: every segment the node emits is read by `etherparse` as it leaves //! ([`Far::hears`]), its IPv4 and TCP checksums and lengths that crate's sums, and every segment @@ -10,18 +11,21 @@ mod common; -use std::cell::RefCell; -use std::collections::VecDeque; use std::net::Ipv4Addr; use std::rc::Rc; +use std::sync::atomic::Ordering; use std::time::Duration; +use common::client::Client; use common::{arp, batch, terms, Segment, Wire, A, BUDGET, MAC, MAC_B, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos_net_node::{ConnectRefused, FromClient, Node, PipeEnd, Pipes, ReadRefusal, StreamEvent, StreamId, ToClient, Watch, WriteRefusal}; +use toyos::net::stream::RESET; +use toyos_net_node::{ConnectRefused, Node, StreamEvent, StreamId}; use toyos_net_shard::ConnectError; use toyos_net_tcp::{Counter, Endpoint, Failure}; use toyos_net_wire::{Instant, Port}; +use toyos_transport::session::Refused; +use toyos_transport::Violation; const B: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 2); /// A second peer's address, and its MAC. @@ -30,8 +34,10 @@ const MAC_C: [u8; 6] = [2, 0, 0, 0, 0, 0x0c]; const PORT: u16 = 80; /// The peer's initial sequence number. const ISS: u32 = 5000; -/// [tcp]'s send buffer, as `common::Wire` configures it. -const SEND_BUFFER: usize = 65_535; +/// The region's header words a hostile client writes: the receive ring's head and the send +/// ring's tail (`toyos::net::stream::LAYOUT`). +const RX_HEAD: usize = 0; +const TX_TAIL: usize = 5 * 16; fn peer() -> Endpoint { Endpoint { addr: B, port: Port::new(PORT).unwrap() } @@ -42,104 +48,16 @@ fn text(len: usize) -> Vec { (0..len).map(|i| (i % 251) as u8 ^ (i / 251) as u8).collect() } -// ---- the client's pipes ---- - -#[derive(Default)] -struct Ends { - /// What the client wrote and the node has not read. - outbox: VecDeque, - /// The client holds no write end: the outbox, once empty, reads as the end. - writer_gone: bool, - /// What the node wrote and the client has not read: at most `room` bytes. - inbox: Vec, - room: usize, - /// The client holds no read end. - reader_gone: bool, - /// The handle the client gave for an end is no pipe end. - write_broken: bool, - read_broken: bool, - to_dropped: bool, - from_dropped: bool, - /// The ends the node let go of, in the order it did. - let_go: Vec, - /// The node's calls of each end: each is a syscall in netstack. - writes: usize, - reads: usize, +/// What the client reads now: every byte ready, and `Ok` where the end came after them. +fn reads(client: &Client) -> (Vec, Result<(), Refused>) { + client.read_all() } -type Client = Rc>; - -struct WriteEnd(Client); -struct ReadEnd(Client); - -impl ToClient for WriteEnd { - fn write(&mut self, bytes: &[u8]) -> Result { - let mut ends = self.0.borrow_mut(); - ends.writes += 1; - if ends.write_broken { - return Err(WriteRefusal::Broken); - } - if ends.reader_gone { - return Err(WriteRefusal::Gone); - } - let taken = bytes.len().min(ends.room - ends.inbox.len()); - if taken == 0 { - return Err(WriteRefusal::Full); - } - ends.inbox.extend_from_slice(&bytes[..taken]); - Ok(taken) - } -} - -impl Drop for WriteEnd { - fn drop(&mut self) { - let mut ends = self.0.borrow_mut(); - ends.to_dropped = true; - ends.let_go.push(PipeEnd::ToClient); - } -} - -impl FromClient for ReadEnd { - fn read(&mut self, out: &mut [u8]) -> Result { - let mut ends = self.0.borrow_mut(); - ends.reads += 1; - if ends.read_broken { - return Err(ReadRefusal::Broken); - } - let read = out.len().min(ends.outbox.len()); - if read == 0 { - return if ends.writer_gone { Ok(0) } else { Err(ReadRefusal::Empty) }; - } - for slot in &mut out[..read] { - *slot = ends.outbox.pop_front().unwrap(); - } - Ok(read) - } -} - -impl Drop for ReadEnd { - fn drop(&mut self) { - let mut ends = self.0.borrow_mut(); - ends.from_dropped = true; - ends.let_go.push(PipeEnd::FromClient); - } -} - -/// A client whose receive pipe holds 65,536 bytes, and the two ends it hands over. -fn client() -> (Client, Pipes) { - let client = Rc::new(RefCell::new(Ends { room: 65_536, ..Ends::default() })); - let pipes = Pipes { to_client: Box::new(WriteEnd(client.clone())), from_client: Box::new(ReadEnd(client.clone())) }; - (client, pipes) -} - -fn dropped(client: &Client) -> (bool, bool) { - let ends = client.borrow(); - (ends.to_dropped, ends.from_dropped) -} - -/// The ends the node let go of, in the order it did. -fn let_go(client: &Client) -> Vec { - client.borrow().let_go.clone() +/// The bytes the client can read now, which must end short of the end. +fn inbox(client: &Client) -> Vec { + let (bytes, stopped) = client.read_all(); + assert_eq!(stopped, Err(Refused::WouldBlock), "the stream did not end"); + bytes } // ---- the far end ---- @@ -150,9 +68,6 @@ enum Manner { Answers, /// Says nothing to any segment. Deaf, - /// Answers, and takes text only as far as the window it offers, which it then shuts: a - /// reader that reads when the test says so ([`Net::run_slowly_until`]). - Slow, /// Resets a SYN (RFC 9293 §3.10.7.1). Refuses, } @@ -166,8 +81,6 @@ struct Far { /// The test turned to another peer ([`Net::turn_to`]): of the node's segments this one hears /// its own connection's and no other, a SYN least of all. parked: bool, - /// The window the peer's segments offer. - offers: u16, /// The node's port, from its last SYN to this address: segments from another are heard and /// not answered. port: u16, @@ -203,7 +116,6 @@ impl Far { manner: Manner::Answers, absent: false, parked: false, - offers: 65_535, port: 0, snd_nxt: ISS, rcv_nxt: 0, @@ -218,7 +130,7 @@ impl Far { /// A segment from the peer in its frame, built by `etherparse`: PSH with text. fn frame(&self, seq: u32, ack: Option, syn: bool, fin: bool, rst: bool, text: &[u8]) -> Vec { - let mut step = PacketBuilder::ethernet2(self.mac, MAC).ipv4(self.addr.octets(), A.octets(), 64).tcp(PORT, self.port, seq, self.offers); + let mut step = PacketBuilder::ethernet2(self.mac, MAC).ipv4(self.addr.octets(), A.octets(), 64).tcp(PORT, self.port, seq, 65_535); if syn { step = step.syn().options(&[TcpOptionElement::MaximumSegmentSize(1460)]).unwrap(); } @@ -258,16 +170,6 @@ impl Far { self.frame(self.snd_nxt, Some(self.rcv_nxt), false, false, true, &[]) } - /// An acknowledgment that says nothing new but the window the peer offers. - fn update(&self) -> Vec { - self.frame(self.snd_nxt, Some(self.rcv_nxt), false, false, false, &[]) - } - - /// The peer offers a window of `bytes` from its next segment on. - fn offer(&mut self, bytes: u16) { - self.offers = bytes; - } - /// How many bytes the node's window still lets the peer send. fn open(&self) -> usize { self.acked.wrapping_add(u32::from(self.window)).wrapping_sub(self.snd_nxt) as usize @@ -322,18 +224,14 @@ impl Far { match self.manner { Manner::Deaf => return Vec::new(), Manner::Refuses => return vec![self.frame(0, Some(segment.seq.wrapping_add(1)), false, false, true, &[])], - Manner::Answers | Manner::Slow => {} + Manner::Answers => {} } if segment.syn { self.rcv_nxt = segment.seq.wrapping_add(1); self.snd_nxt = ISS.wrapping_add(1); return vec![self.frame(ISS, Some(self.rcv_nxt), true, false, false, &[])]; } - let fits = self.manner != Manner::Slow || segment.text.len() <= usize::from(self.offers); - if segment.seq == self.rcv_nxt && fits { - if self.manner == Manner::Slow && !segment.text.is_empty() { - self.offers = 0; - } + if segment.seq == self.rcv_nxt { self.received.extend_from_slice(&segment.text); self.rcv_nxt = self.rcv_nxt.wrapping_add(u32::try_from(segment.text.len()).unwrap()); if segment.fin { @@ -362,8 +260,8 @@ struct Net { far: Far, /// The peers the test turned from, each with its connection. parked: Vec, - /// Each stream's client, for the kernel's answers [`Self::bridge`] gives. - clients: Vec<(StreamId, Client)>, + /// Each stream's client, whose rings [`Self::bridge`] reads. + clients: Vec<(StreamId, Rc)>, } fn draw(draws: &mut u32) -> impl FnMut() -> u32 + '_ { @@ -435,19 +333,11 @@ impl Net { self.pump(); } - /// A pass, as netstack runs one when a pipe it watches is ready, and what it sends. The - /// kernel answers a send pipe readable that holds bytes, has no writer left or is no pipe. + /// A pass, as netstack runs one when the connections it watches answer, each stream whose + /// client rang among them, and what it sends. fn bridge(&mut self) { - let readable: Vec = self - .clients - .iter() - .filter(|(_, client)| { - let ends = client.borrow(); - !ends.outbox.is_empty() || ends.writer_gone || ends.read_broken - }) - .map(|(id, _)| *id) - .collect(); - self.node.bridge(self.now, readable); + let rung: Vec = self.clients.iter().filter(|(_, client)| client.rang()).map(|(id, _)| *id).collect(); + self.node.bridge(self.now, rung); self.pump(); } @@ -474,42 +364,15 @@ impl Net { panic!("100,000 deadlines without the clock passing {limit:?}"); } - /// [`Self::run_until`] with every [`Manner::Slow`] peer reading 1,460 bytes a second: each - /// second one whose connection has had neither the node's FIN nor its reset offers that - /// window again. - fn run_slowly_until(&mut self, limit: Duration, done: impl Fn(&Net) -> bool) -> bool { - let end = self.now.after(limit); - let mut reads = self.now.after(Duration::from_secs(1)); - for _ in 0..100_000 { - if done(self) { - return true; - } - let at = self.node.next_deadline().map_or(reads, |at| at.min(reads)); - if at > end { - return false; - } - self.fire(at); - if self.now >= reads { - reads = self.now.after(Duration::from_secs(1)); - let mut updates = Vec::new(); - for far in self.parked.iter_mut().chain([&mut self.far]) { - if far.manner == Manner::Slow && !far.fin && far.resets.is_empty() { - far.offer(1_460); - updates.push(far.update()); - } - } - for update in &updates { - self.deliver(update); - } - } - } - panic!("100,000 deadlines without the clock passing {limit:?}"); + fn connect(&mut self, timeout: Option) -> (StreamId, Rc) { + self.connect_as(Client::new(), timeout) } - fn connect(&mut self, timeout: Option) -> (StreamId, Client) { - let (client, pipes) = client(); + /// A connect on a client's region, set up as `made` set it. + fn connect_as(&mut self, made: (Rc, Box), timeout: Option) -> (StreamId, Rc) { + let (client, session) = made; let remote = Endpoint { addr: self.far.addr, ..peer() }; - let id = self.node.connect(self.now, remote, timeout, pipes).expect("a route to the peer"); + let id = self.node.connect(self.now, remote, timeout, session).expect("a route to the peer"); self.clients.push((id, client.clone())); self.pump(); (id, client) @@ -519,24 +382,36 @@ impl Net { self.node.drain_stream_events().collect() } - fn watch(&self, id: StreamId) -> Option { - self.node.watches().find(|(watched, _)| *watched == id).map(|(_, watch)| watch) - } - /// The sequence number of the node's first byte of text: its SYN's, plus one. fn first(&self) -> u32 { let syn = self.far.segments.iter().find(|segment| segment.syn).expect("a SYN"); syn.seq.wrapping_add(1) } + + /// The client leaves: its connection's hang-up, and what the node sends. + fn leaves(&mut self, id: StreamId) { + self.node.left(self.now, id); + self.pump(); + } + + /// Everything the client writes of `text`, a pass after each ring, until it is all in the + /// send ring. + fn writes(&mut self, client: &Client, text: &[u8]) { + let mut written = 0; + while written < text.len() { + written += client.write(&text[written..]).expect("room in the send ring"); + self.bridge(); + } + } } /// A stream whose connect was answered `Connected`. -fn established() -> (Net, StreamId, Client) { +fn established() -> (Net, StreamId, Rc) { established_within(None) } /// A stream whose connect, with `timeout`, was answered `Connected`. -fn established_within(timeout: Option) -> (Net, StreamId, Client) { +fn established_within(timeout: Option) -> (Net, StreamId, Rc) { let mut net = Net::new(); let (id, client) = net.connect(timeout); let local = Port::new(net.far.port).expect("the SYN's source port"); @@ -544,30 +419,27 @@ fn established_within(timeout: Option) -> (Net, StreamId, Client) { (net, id, client) } -const IDLE: Watch = Watch { readable: true, writer: true, writable: false, reader: true }; - // ---- connect ---- // RFC 9293 §3.5, figure 6: a SYN that acknowledges nothing, and after the peer's SYN-ACK an ACK // of its sequence number plus one from ours plus one. #[test] fn a_connect_is_a_handshake_and_its_answer_names_the_port() { - let (net, id, client) = established(); + let (net, _, client) = established(); let [syn, ack] = &net.far.segments[..] else { panic!("a SYN and an ACK, not {:?}", net.far.segments) }; assert!(syn.syn && syn.ack.is_none() && syn.text.is_empty() && !syn.fin && !syn.rst, "{syn:?}"); assert_eq!((ack.syn, ack.seq, ack.ack, ack.text.len()), (false, syn.seq.wrapping_add(1), Some(ISS + 1), 0)); - assert_eq!((net.node.streams(), net.watch(id)), (1, Some(IDLE))); - assert_eq!(dropped(&client), (false, false)); + assert_eq!((net.node.streams(), client.dropped()), (1, false)); } #[test] fn a_connect_before_the_lease_is_refused_and_sends_nothing() { let mut wire = Wire::new(); wire.link(true); - let (client, pipes) = client(); - let refused = wire.node.connect(wire.now, peer(), None, pipes); + let (client, session) = Client::new(); + let refused = wire.node.connect(wire.now, peer(), None, session); assert!(matches!(refused, Err(ConnectRefused::Stack(ConnectError::Route(_)))), "{refused:?}"); - assert_eq!((wire.node.streams(), dropped(&client)), (0, (true, true))); + assert_eq!((wire.node.streams(), client.dropped()), (0, true)); // `common::outside` takes no TCP segment. wire.pump(); } @@ -579,7 +451,7 @@ fn a_refused_connect_is_answered_with_the_refusal() { net.far.manner = Manner::Refuses; let (id, client) = net.connect(None); assert_eq!(net.events(), [StreamEvent::Failed { id, failure: Failure::Refused }]); - assert_eq!((net.node.streams(), dropped(&client)), (0, (true, true))); + assert_eq!((net.node.streams(), client.dropped()), (0, true)); } // The deadline is the node's own and falls between two of the SYN's retransmissions; RFC 9293 @@ -594,7 +466,7 @@ fn a_connect_past_its_deadline_is_timed_out_at_the_deadline() { assert!(net.run_until(Duration::from_secs(10), |net| net.node.streams() == 0), "the connect ends"); assert_eq!(net.now, start.after(Duration::from_millis(2_500))); assert_eq!(net.events(), [StreamEvent::TimedOut { id }]); - assert_eq!(dropped(&client), (true, true)); + assert!(client.dropped()); assert!(!net.far.segments.is_empty() && net.far.segments.iter().all(|segment| segment.syn), "{:?}", net.far.segments); // The connection is nobody's from here: its SYN is not sent again. let sent = net.far.segments.len(); @@ -623,7 +495,7 @@ fn a_connect_nobody_answers_arp_for_fails_when_tcp_knows() { } let events = net.events(); assert!(matches!(events[..], [StreamEvent::Failed { id: failed, failure: Failure::Unreachable(_) }] if failed == id), "{events:?}"); - assert_eq!((dropped(&client), net.far.segments.len()), ((true, true), 0)); + assert_eq!((client.dropped(), net.far.segments.len()), (true, 0)); } // A connect's deadline ends with its answer: the stream has none while its client holds it. @@ -636,15 +508,13 @@ fn an_established_streams_connect_deadline_is_gone() { } #[test] -fn a_connect_closed_before_its_answer_is_answered_closed() { +fn a_connect_whose_client_left_before_its_answer_is_answered_closed() { let mut net = Net::new(); net.far.manner = Manner::Deaf; let (id, client) = net.connect(None); - assert!(!net.node.shutdown_write(net.now, id), "not established"); - assert_eq!(net.watch(id), None); - net.node.close(net.now, id); + net.leaves(id); assert_eq!(net.events(), [StreamEvent::Closed { id }]); - assert_eq!((net.node.streams(), dropped(&client)), (0, (true, true))); + assert_eq!((net.node.streams(), client.dropped()), (0, true)); // The connection is nobody's from here: its SYN is not sent again. let sent = net.far.segments.len(); assert!(!net.run_until(Duration::from_secs(60), |_| false)); @@ -653,17 +523,16 @@ fn a_connect_closed_before_its_answer_is_answered_closed() { // A batch of frames and a deadline each end in a pass of their own: an opportunity's pass is over // the connects only, so neither test offers one. The batch's pass is one, after its last frame: -// what the batch brought is one write to the client's pipe, and its send pipe, found empty as the -// connect was answered, is not asked. +// what the batch brought is one publish, one ring of the client that waits. #[test] fn a_batch_of_frames_moves_an_established_stream_once_with_no_opportunity_after_it() { let (mut net, _, client) = established(); + assert_eq!(client.read(64), Err(Refused::WouldBlock), "the read arms its want"); let frames = [net.far.text(b"all "), net.far.text(b"at "), net.far.text(b"once")]; - let (writes, reads) = (client.borrow().writes, client.borrow().reads); + let rung = client.rung(); net.node.receive(net.now, BUDGET, batch(&[&frames[0], &frames[1], &frames[2]]), draw(&mut net.draws)); - let ends = client.borrow(); - assert_eq!(ends.inbox, b"all at once"); - assert_eq!((ends.writes - writes, ends.reads - reads), (1, 0), "the pipes' calls in the batch's pass"); + assert_eq!(client.rung() - rung, 1, "one ring for the batch"); + assert_eq!(inbox(&client), b"all at once"); } // A device that refills its ring as fast as it is read never says it has no frame: a batch @@ -681,92 +550,68 @@ fn a_batch_stops_at_its_budget_and_says_a_pass_is_owed() { }; let owed = net.node.receive(net.now, BUDGET, endless, draw(&mut net.draws)); assert_eq!((owed, handed), (true, BUDGET), "the batch is the budget, and a pass is owed"); - assert_eq!(client.borrow().inbox, b"again", "the rest of the batch was the same segment again"); + assert_eq!(inbox(&client), b"again", "the rest of the batch was the same segment again"); let next = net.far.text(b"once"); let short: Vec<&[u8]> = vec![&next; BUDGET - 1]; assert!(!net.node.receive(net.now, BUDGET, batch(&short), draw(&mut net.draws)), "the ring was emptied"); - assert_eq!(client.borrow().inbox, b"againonce"); + assert_eq!(inbox(&client), b"once"); } -// A send pipe found empty is asked again only once the kernel says it holds bytes: a batch of -// frames, a deadline and the to-client pipe's room each pass over the stream without reading it. +// netstack's want for the client's bytes is armed whenever it took them all and [tcp] has room: +// a write rings it, and the pass the ring brings takes the bytes. #[test] -fn a_send_pipe_found_empty_is_read_again_only_once_the_kernel_says_it_holds_bytes() { - let (mut net, id, client) = established(); - let reads = client.borrow().reads; - client.borrow_mut().outbox.extend(b"request"); - let frame = net.far.text(b"response"); - net.deliver(&frame); - net.fire(net.now.after(Duration::from_secs(1))); - net.node.bridge(net.now, []); +fn a_write_rings_netstack_and_the_pass_it_brings_takes_the_bytes() { + let (mut net, _, client) = established(); + assert_eq!(client.write(b"request"), Ok(7)); + assert!(client.rang(), "netstack wanted the client's bytes"); net.pump(); - assert_eq!((client.borrow().reads - reads, net.far.received.len()), (0, 0)); - - net.node.bridge(net.now, [id]); + assert!(net.far.received.is_empty(), "nothing is taken before the pass"); + net.node.bridge(net.now, []); net.pump(); assert_eq!(net.far.received, b"request"); - assert_eq!(client.borrow().reads - reads, 2, "the request, then the pipe found empty again"); + assert_eq!(client.write(b"again"), Ok(5)); + assert!(client.rang(), "armed again by the pass that took the first"); } +// Every pass moves every established stream: a deadline's takes what its client published +// with no ring delivered. #[test] fn a_deadline_moves_an_established_stream_with_no_opportunity_after_it() { - let mut net = Net::new(); - let id = departed(&mut net, 100_000, false); - net.events(); - let cut = net.now.after(Duration::from_secs(100)); - net.node.fire(cut, draw(&mut net.draws)); - assert_eq!((net.node.streams(), net.events()), (0, vec![StreamEvent::Cut { id }])); + let (mut net, _, client) = established(); + assert_eq!(client.write(b"unrung"), Ok(6)); + let at = net.now.after(Duration::from_secs(1)); + net.node.fire(at, draw(&mut net.draws)); + net.pump(); + assert_eq!(net.far.received, b"unrung"); } -// ---- the bridge ---- +// ---- the stream's bytes ---- +// [tcp]'s send buffer is the client's whole send ring: a write larger than the peer's window is +// queued whole, with no copy, and the peer's stream is the client's, byte for byte. #[test] -fn nothing_leaves_the_pipe_that_the_stack_will_not_take() { +fn a_write_larger_than_the_window_is_queued_whole_and_arrives_whole() { let (mut net, id, client) = established(); let sent = text(100_000); - client.borrow_mut().outbox.extend(&sent); - // A pass with no transmit opportunity after it: the send buffer fills and nothing leaves it. + assert_eq!(client.write(&sent), Ok(sent.len()), "the send ring holds all of it"); net.node.bridge(net.now, [id]); - assert_eq!(client.borrow().outbox.len(), sent.len() - SEND_BUFFER, "what [tcp] had no room for is still the pipe's"); - assert_eq!(net.watch(id), Some(Watch { readable: false, ..IDLE })); - net.pump(); assert!(net.run_until(Duration::from_secs(5), |net| net.far.received.len() >= sent.len()), "{} bytes arrived", net.far.received.len()); assert!(net.far.received == sent, "the peer's stream is the client's, byte for byte"); - assert!(client.borrow().outbox.is_empty()); - assert_eq!(net.watch(id), Some(IDLE)); -} - -#[test] -fn a_full_pipe_keeps_the_rest_in_the_stack_and_is_watched_for_room() { - let (mut net, id, client) = established(); - client.borrow_mut().room = 100; - let sent = text(300); - let frame = net.far.text(&sent); - net.deliver(&frame); - for taken in [0, 100, 200] { - assert!(client.borrow().inbox == sent[taken..taken + 100], "after {taken} bytes"); - // Watched for room only while the stack holds bytes it refused. - assert_eq!(net.watch(id), Some(Watch { writable: taken < 200, ..IDLE }), "after {taken} bytes"); - client.borrow_mut().inbox.clear(); - net.bridge(); - } - assert!(client.borrow().inbox.is_empty()); - assert_eq!(net.watch(id), Some(IDLE)); } +// RFC 9293 §3.8.6: the peer sends what the node's window allows, and the client reads it as it +// comes: the client's stream is the peer's, byte for byte, the ring wrapped many times over. #[test] -fn nothing_leaves_the_stack_that_the_pipe_did_not_take() { +fn the_clients_stream_is_the_peers_byte_for_byte() { let (mut net, _, client) = established(); - client.borrow_mut().room = 1_000; let sent = text(200_000); let (mut read, mut offset) = (Vec::new(), 0); for _ in 0..10_000 { if read.len() == sent.len() { break; } - // The peer sends what the node's window allows (RFC 9293 §3.8.6), and no more. let len = net.far.open().min(1_460).min(sent.len() - offset); if len > 0 { let frame = net.far.text(&sent[offset..offset + len]); @@ -774,8 +619,7 @@ fn nothing_leaves_the_stack_that_the_pipe_did_not_take() { net.deliver(&frame); continue; } - // The window is shut or the text all sent: the client reads what its pipe holds. - let taken = std::mem::take(&mut client.borrow_mut().inbox); + let taken = inbox(&client); if taken.is_empty() { let at = net.node.next_deadline().expect("the lease's renewal, if nothing else"); net.fire(at); @@ -788,12 +632,12 @@ fn nothing_leaves_the_stack_that_the_pipe_did_not_take() { assert!(read == sent, "the client's stream is the peer's, byte for byte: {} of {} bytes", read.len(), sent.len()); } -// RFC 9293 §3.8.6.2.2: the window a full pipe shut is opened again by an acknowledgment once the -// pipe has taken what the stack held, at the first opportunity after the pass that moved it. +// RFC 9293 §3.8.6.2.2: text the client has not read shuts the window, and its read, with the +// window last offered below the threshold, rings netstack, which opens it at the next +// opportunity. #[test] -fn room_in_the_pipe_opens_the_window_at_the_next_opportunity() { +fn a_read_opens_a_shut_window_at_the_next_opportunity() { let (mut net, _, client) = established(); - client.borrow_mut().room = 0; let sent = text(65_535); let mut offset = 0; for _ in 0..1_000 { @@ -813,9 +657,10 @@ fn room_in_the_pipe_opens_the_window_at_the_next_opportunity() { let shut = |net: &Net| (net.far.acked, net.far.window) == (net.far.snd_nxt, 0); assert!(net.run_until(Duration::from_secs(1), shut), "all of it acknowledged, and the window shut: {} {}", net.far.acked, net.far.window); - client.borrow_mut().room = 65_536; - net.node.bridge(net.now, []); - assert!(client.borrow().inbox == sent); + assert!(inbox(&client) == sent); + assert!(client.rang(), "netstack wanted the room a read gives"); + let rung: Vec = net.clients.iter().map(|(id, _)| *id).collect(); + net.node.bridge(net.now, rung); assert_eq!(net.opportunity(), 1, "the window update"); let update = net.far.segments.last().expect("a segment"); assert_eq!((update.ack, update.window, update.text.len()), (Some(net.far.snd_nxt), 65_535, 0)); @@ -827,9 +672,9 @@ fn room_in_the_pipe_opens_the_window_at_the_next_opportunity() { #[test] fn an_echo_past_both_windows_arrives_whole_both_ways_and_ends_in_both_fins() { let (mut net, _, client) = established(); - client.borrow_mut().room = 4_096; let sent = text(200_000); let mut offset = 0; + let mut echo = Vec::new(); for _ in 0..100_000 { if net.far.received.len() == sent.len() { break; @@ -841,12 +686,13 @@ fn an_echo_past_both_windows_arrives_whole_both_ways_and_ends_in_both_fins() { net.deliver(&frame); continue; } - // The client echoes what its pipe holds; with nothing to echo the clock moves. - let taken = std::mem::take(&mut client.borrow_mut().inbox); - client.borrow_mut().outbox.extend(taken); - let before = (net.far.received.len(), client.borrow().outbox.len()); + // The client echoes what it reads; with nothing to echo the clock moves. + echo.extend(inbox(&client)); + let wrote = client.write(&echo).unwrap_or(0); + echo.drain(..wrote); + let before = net.far.received.len(); net.bridge(); - if (net.far.received.len(), client.borrow().outbox.len()) == before { + if net.far.received.len() == before && wrote == 0 { let at = net.node.next_deadline().expect("the lease's renewal, if nothing else"); net.fire(at); } @@ -856,36 +702,64 @@ fn an_echo_past_both_windows_arrives_whole_both_ways_and_ends_in_both_fins() { let frame = net.far.fin(); net.deliver(&frame); - client.borrow_mut().writer_gone = true; + assert_eq!(reads(&client), (Vec::new(), Ok(())), "the peer's FIN is the end"); + client.shutdown(); net.bridge(); assert!(net.far.fin, "the client's end is a FIN"); assert_eq!(net.far.acked, net.far.snd_nxt, "the peer's FIN is acknowledged"); - assert_eq!((dropped(&client), net.node.streams()), ((true, true), 0)); + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); assert!(net.far.resets.is_empty(), "{:?}", net.far.resets); } +// A writer over a full send ring sleeps on its bell, and the room the peer's acknowledgments give +// back rings it there. +#[test] +fn room_released_rings_a_writer_over_a_full_send_ring_on_its_bell() { + let (mut net, _, client) = established(); + net.far.manner = Manner::Deaf; + let mut written = 0; + loop { + match client.write(&text(1 << 20)) { + Ok(n) => written += n, + Err(Refused::WouldBlock) => break, + Err(other) => panic!("{other:?}"), + } + net.bridge(); + } + assert_eq!(written, toyos::net::stream::SEND_BYTES as usize, "the ring is full"); + let bell = client.bell(); + // The peer acknowledges every byte it heard of the node's. + let heard = u32::try_from(net.far.segments.iter().map(|segment| segment.text.len()).sum::()).unwrap(); + let frame = net.far.frame(net.far.snd_nxt, Some(net.first().wrapping_add(heard)), false, false, false, &[]); + net.deliver(&frame); + assert!(client.bell() > bell, "the bell is rung for the room"); + assert!(client.write(b"more").is_ok()); +} + // ---- the stream's ends ---- // RFC 9293 §3.6, case 1: our FIN follows our last byte, and in FIN-WAIT-2 the peer's text is -// still received; its FIN is acknowledged and the connection is the stack's to wait out. +// still received; its FIN is acknowledged and ends the stream, which the client reads after the +// text. #[test] fn the_end_of_the_clients_writing_is_a_fin_and_the_peer_still_sends() { - let (mut net, id, client) = established(); - client.borrow_mut().outbox.extend(b"request"); - client.borrow_mut().writer_gone = true; + let (mut net, _, client) = established(); + net.writes(&client, b"request"); + client.shutdown(); net.bridge(); assert_eq!((net.far.received.as_slice(), net.far.fin), (&b"request"[..], true)); let fin = net.far.segments.iter().find(|segment| segment.fin).expect("a FIN"); assert_eq!(fin.seq.wrapping_add(u32::try_from(fin.text.len()).unwrap()), net.first().wrapping_add(7)); - assert_eq!((dropped(&client), net.node.streams()), ((false, true), 1)); - assert_eq!(net.watch(id), Some(Watch { readable: false, writer: false, writable: false, reader: true })); + assert_eq!((client.dropped(), net.node.streams()), (false, 1)); + assert_eq!(client.write(b"late"), Err(Refused::Shut)); let frame = net.far.text(b"response"); net.deliver(&frame); - assert_eq!(client.borrow().inbox, b"response"); + assert_eq!(inbox(&client), b"response"); let frame = net.far.fin(); net.deliver(&frame); - assert_eq!((dropped(&client), net.node.streams()), ((true, true), 0)); + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); + assert_eq!(reads(&client), (Vec::new(), Ok(())), "the end"); assert_eq!(net.far.acked, net.far.snd_nxt, "the peer's FIN is acknowledged"); assert!(net.far.resets.is_empty() && net.events().is_empty()); } @@ -899,28 +773,26 @@ fn the_peers_fin_ends_the_clients_reading_and_not_its_writing() { net.deliver(&frame); let frame = net.far.fin(); net.deliver(&frame); - assert_eq!(client.borrow().inbox, b"all there is"); - assert_eq!((dropped(&client), net.node.streams()), ((true, false), 1)); + assert_eq!(reads(&client), (b"all there is".to_vec(), Ok(()))); + assert_eq!((client.dropped(), net.node.streams()), (false, 1)); // CLOSE-WAIT lasts as long as the client likes. net.fire(net.now.after(Duration::from_secs(150))); assert_eq!((net.node.streams(), net.events()), (1, vec![])); - client.borrow_mut().outbox.extend(b"noted"); - client.borrow_mut().writer_gone = true; + net.writes(&client, b"noted"); + client.shutdown(); net.bridge(); assert_eq!((net.far.received.as_slice(), net.far.fin), (&b"noted"[..], true)); - assert_eq!((dropped(&client), net.node.streams()), ((true, true), 0)); + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); assert!(net.far.resets.is_empty()); } -// RFC 9293 §3.6: both FINs end the connection, not the client's reading: text its pipe had no -// room for when the peer's FIN arrived is still its to read, and the end comes after it. +// RFC 9293 §3.6: both FINs end the connection, not the client's reading: text it had not read when +// the stream ended is still its to read, and the end comes after it. #[test] -fn text_the_pipe_had_no_room_for_outlives_both_fins() { +fn text_unread_when_both_fins_ended_the_connection_is_still_the_clients() { let (mut net, _, client) = established(); - client.borrow_mut().room = 0; - client.borrow_mut().outbox.extend(b"request"); - client.borrow_mut().writer_gone = true; + client.shutdown(); net.bridge(); assert!(net.far.fin); let frame = net.far.text(b"held back"); @@ -928,500 +800,176 @@ fn text_the_pipe_had_no_room_for_outlives_both_fins() { let frame = net.far.fin(); net.deliver(&frame); assert_eq!(net.far.acked, net.far.snd_nxt, "the peer's FIN is acknowledged"); - assert_eq!((client.borrow().inbox.len(), dropped(&client), net.node.streams()), (0, (false, true), 1)); - - client.borrow_mut().room = 65_536; - net.bridge(); - assert_eq!(client.borrow().inbox, b"held back"); - assert_eq!((dropped(&client), net.node.streams()), ((true, true), 0)); + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); + assert_eq!(reads(&client), (b"held back".to_vec(), Ok(()))); } -// A client that shut its writing down and still reads is alive: what it cannot send is [tcp]'s -// to give up on (R2, RFC 9293 §3.8.3), not the node's to cut. #[test] -fn a_live_client_that_is_done_writing_is_not_timed() { - let (mut net, id, client) = established(); - net.far.manner = Manner::Deaf; - client.borrow_mut().outbox.extend(text(100_000)); - assert!(net.node.shutdown_write(net.now, id)); - net.pump(); - assert!(!net.run_until(Duration::from_secs(150), |_| false)); - assert_eq!((dropped(&client), net.node.streams(), net.far.resets.len()), ((false, false), 1, 0)); - assert!(net.events().is_empty()); -} - -#[test] -fn a_shutdown_sends_what_the_pipe_held_and_then_the_fin() { - let (mut net, id, client) = established(); - client.borrow_mut().outbox.extend(b"written before the shutdown"); - assert!(net.node.shutdown_write(net.now, id)); - net.pump(); +fn a_shutdown_sends_what_was_written_and_then_the_fin() { + let (mut net, _, client) = established(); + assert_eq!(client.write(b"written before the shutdown"), Ok(27)); + client.shutdown(); + net.bridge(); assert_eq!((net.far.received.as_slice(), net.far.fin), (&b"written before the shutdown"[..], true)); - assert_eq!((dropped(&client), net.node.streams()), ((false, false), 1), "the send pipe is kept for its writer"); - assert_eq!(net.watch(id), Some(Watch { readable: false, writer: true, writable: false, reader: true })); -} - -// RFC 9293 §3.6, case 1, as a client that shut its sending half down reads it: the peer's FIN -// after its last byte ends the to-client pipe while the from-client pipe is still there, which is -// what an orderly end is to std's `ended`. The send pipe goes when its -// writer does, and not before, however long that is. -#[test] -fn the_peers_fin_after_the_clients_shutdown_leaves_the_send_pipe_to_its_writer() { - let (mut net, id, client) = established(); - client.borrow_mut().outbox.extend(b"request"); - assert!(net.node.shutdown_write(net.now, id)); - net.pump(); - assert_eq!((net.far.received.as_slice(), net.far.fin), (&b"request"[..], true)); - let frame = net.far.text(b"response"); - net.deliver(&frame); - let frame = net.far.fin(); - net.deliver(&frame); - assert_eq!(client.borrow().inbox, b"response"); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::ToClient], 1)); - assert_eq!(net.watch(id), Some(Watch { readable: false, writer: true, writable: false, reader: false })); - - net.fire(net.now.after(Duration::from_secs(300))); - assert_eq!((let_go(&client), net.node.streams(), net.events()), (vec![PipeEnd::ToClient], 1, vec![]), "past TIME-WAIT"); - client.borrow_mut().writer_gone = true; - net.node.pipe_gone(net.now, id, PipeEnd::FromClient); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::ToClient, PipeEnd::FromClient], 0)); - assert!(net.far.resets.is_empty()); -} - -// RFC 9293 §3.6, case 1, with the client's writer gone after its shutdown and before the peer's -// FIN: the finished send pipe goes with its writer, and the stream lives on its to-client pipe, -// which still carries the peer's text and then its end. -#[test] -fn a_writer_that_leaves_after_the_shutdown_leaves_the_stream_to_its_reader() { - let (mut net, id, client) = established(); - assert!(net.node.shutdown_write(net.now, id)); - net.pump(); - assert!(net.far.fin); - client.borrow_mut().writer_gone = true; - net.node.pipe_gone(net.now, id, PipeEnd::FromClient); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::FromClient], 1)); - assert_eq!(net.watch(id), Some(Watch { readable: false, writer: false, writable: false, reader: true })); - - let frame = net.far.text(b"late"); - net.deliver(&frame); - let frame = net.far.fin(); - net.deliver(&frame); - assert_eq!(client.borrow().inbox, b"late"); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::FromClient, PipeEnd::ToClient], 0)); - assert!(net.far.resets.is_empty() && net.events().is_empty()); + assert_eq!((client.dropped(), net.node.streams()), (false, 1)); } -// RFC 9293 §3.6, case 2, with the client's end a shutdown: the connection is over in order, and -// the send pipe is still there for the client that reads the end after it. +// RFC 9293 §3.10.7.4 after the client's FIN: a reset is the end the client reads, not a FIN. #[test] -fn a_shutdown_after_the_peers_fin_keeps_the_send_pipe_past_the_connections_end() { - let (mut net, id, client) = established(); - let frame = net.far.fin(); - net.deliver(&frame); - assert_eq!(let_go(&client), vec![PipeEnd::ToClient]); - client.borrow_mut().outbox.extend(b"noted"); - assert!(net.node.shutdown_write(net.now, id)); - net.pump(); - assert_eq!((net.far.received.as_slice(), net.far.fin), (&b"noted"[..], true)); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::ToClient], 1)); - client.borrow_mut().writer_gone = true; - net.node.pipe_gone(net.now, id, PipeEnd::FromClient); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::ToClient, PipeEnd::FromClient], 0)); - assert!(net.far.resets.is_empty()); -} - -// RFC 9293 §3.10.7.4 after the client's FIN: the reset lets the send pipe go before the to-client -// pipe ends, so a client that shut its sending half down reads a failure and not a FIN. -#[test] -fn a_reset_after_the_clients_shutdown_lets_the_send_pipe_go_first() { - let (mut net, id, client) = established(); - assert!(net.node.shutdown_write(net.now, id)); - net.pump(); +fn a_reset_after_the_clients_shutdown_is_the_end_the_client_reads() { + let (mut net, _, client) = established(); + client.shutdown(); + net.bridge(); assert!(net.far.fin); net.far.manner = Manner::Deaf; let frame = net.far.rst(); net.deliver(&frame); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::FromClient, PipeEnd::ToClient], 0)); -} - -// A client whose reader is gone reads no end, so its finished send pipe goes with its reader, -// however that was found gone. -#[test] -fn a_finished_send_pipe_goes_with_its_reader() { - for kernel in [true, false] { - let (mut net, id, client) = established(); - assert!(net.node.shutdown_write(net.now, id)); - net.pump(); - assert_eq!(dropped(&client), (false, false), "kernel: {kernel}"); - client.borrow_mut().reader_gone = true; - if kernel { - net.node.pipe_gone(net.now, id, PipeEnd::ToClient); - } else { - let frame = net.far.text(b"to nobody"); - net.deliver(&frame); - } - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::ToClient, PipeEnd::FromClient], 0), "kernel: {kernel}"); - } + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); + assert_eq!(reads(&client), (Vec::new(), Err(Refused::Failed(RESET)))); } -// A reset after the peer's FIN ends the send pipe its client still writes into. +// RFC 9293 §3.10.7.4 and §3.10.2: a reset after the peer's FIN flushes what was queued, and a +// write after it, with room in the send ring, answers the reset and copies nothing. #[test] -fn a_reset_after_the_peers_fin_ends_the_send_pipe() { +fn a_reset_after_the_peers_fin_fails_the_clients_writes() { let (mut net, _, client) = established(); let frame = net.far.fin(); net.deliver(&frame); + assert_eq!(reads(&client), (Vec::new(), Ok(()))); net.far.manner = Manner::Deaf; - client.borrow_mut().outbox.extend(b"late"); - net.bridge(); + net.writes(&client, b"late"); let frame = net.far.rst(); net.deliver(&frame); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::ToClient, PipeEnd::FromClient], 0)); + assert_eq!(client.write(b"later"), Err(Refused::Failed(RESET))); + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); } +// RFC 9293 §3.10.4: the client's leaving is a CLOSE, and what it wrote leaves before the FIN. #[test] -fn a_close_sends_what_the_pipe_held_and_then_the_fin() { +fn a_client_that_leaves_has_what_it_wrote_sent_and_then_the_fin() { let (mut net, id, client) = established(); - client.borrow_mut().outbox.extend(b"written and dropped"); - net.node.close(net.now, id); - net.pump(); - assert_eq!((net.far.received.as_slice(), net.far.fin), (&b"written and dropped"[..], true)); - assert_eq!((dropped(&client), net.node.streams()), ((true, true), 0)); + assert_eq!(client.write(b"written and left"), Ok(16)); + net.leaves(id); + assert_eq!((net.far.received.as_slice(), net.far.fin), (&b"written and left"[..], true)); + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); assert!(net.far.resets.is_empty()); - // The id names nothing. - net.node.close(net.now, id); - assert_eq!((net.node.set_nodelay(net.now, id, true), net.node.shutdown_write(net.now, id)), (false, false)); + let frame = net.far.fin(); + net.deliver(&frame); + assert_eq!(net.far.acked, net.far.snd_nxt, "the peer's FIN is acknowledged by a connection nobody holds"); } -// RFC 9293 §3.6.1 (SHLD-3): a close with text unread is a reset, which shows the peer it was -// lost. The rule is [tcp]'s; the node's is to hand it the close. +// RFC 9293 §3.6.1 (SHLD-3): a client that leaves with text unread resets, which shows the peer +// it was lost. The rule is [tcp]'s; the node's is to hand it the close. #[test] -fn a_close_with_text_unread_resets() { - let (mut net, id, client) = established(); - client.borrow_mut().room = 0; +fn a_client_that_leaves_with_text_unread_resets() { + let (mut net, id, _client) = established(); let frame = net.far.text(b"never read"); net.deliver(&frame); - net.node.close(net.now, id); - net.pump(); + net.leaves(id); assert_eq!((net.far.resets.len(), net.node.streams()), (1, 0)); } -// RFC 9293 §3.10.7.4: a reset in sequence ends the connection, and the client learns it as the -// end of both pipes. +// RFC 9293 §3.10.7.4: a reset in sequence ends the connection, and the client reads it as the +// failure it was. #[test] -fn a_reset_ends_both_pipes_and_the_stream() { +fn a_reset_is_the_end_the_client_reads_and_lets_the_stream_go() { let (mut net, _, client) = established(); - client.borrow_mut().outbox.extend(b"never sent"); net.far.manner = Manner::Deaf; + net.writes(&client, b"never sent"); let frame = net.far.rst(); net.deliver(&frame); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::FromClient, PipeEnd::ToClient], 0), "the send pipe first"); + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); + assert_eq!(reads(&client), (Vec::new(), Err(Refused::Failed(RESET)))); assert!(net.events().is_empty() && net.far.resets.is_empty()); } // Text and the reset after it in one batch: the client reads the text before the failure, as it -// would had they come in passes of their own (Linux and the BSDs read what arrived before the -// reset ahead of ECONNRESET). +// would had they come in passes of their own. #[test] -fn a_reset_in_the_batch_of_the_text_before_it_ends_the_pipes_after_the_text() { +fn a_reset_in_the_batch_of_the_text_before_it_is_read_after_the_text() { let (mut net, _, client) = established(); let text = net.far.text(b"last"); let rst = net.far.rst(); net.node.receive(net.now, BUDGET, batch(&[&text, &rst]), draw(&mut net.draws)); - assert_eq!((client.borrow().inbox.clone(), let_go(&client)), (b"last".to_vec(), vec![PipeEnd::FromClient, PipeEnd::ToClient])); + assert_eq!(reads(&client), (b"last".to_vec(), Err(Refused::Failed(RESET)))); } -// The reset's text outlives the pass that saw the reset while the client's pipe has no room for -// it, and the failure is read after it. +// A writer asleep over a full send ring wakes for a reset (ruling 2): the failure makes the send +// direction ready and rings its bell, and the write answers it. #[test] -fn text_the_pipe_had_no_room_for_outlives_the_reset() { +fn a_reset_rings_a_writer_over_a_full_send_ring_on_its_bell() { let (mut net, _, client) = established(); - client.borrow_mut().room = 0; - let text = net.far.text(b"held back"); - let rst = net.far.rst(); - net.node.receive(net.now, BUDGET, batch(&[&text, &rst]), draw(&mut net.draws)); - assert_eq!((client.borrow().inbox.len(), let_go(&client), net.node.streams()), (0, vec![PipeEnd::FromClient], 1)); - - client.borrow_mut().room = 65_536; - net.bridge(); - assert_eq!(client.borrow().inbox, b"held back"); - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::FromClient, PipeEnd::ToClient], 0)); -} - -// ---- a client that leaves ---- - -#[test] -fn a_client_that_left_is_finished_by_the_stack() { - let (mut net, id, client) = established(); - client.borrow_mut().outbox.extend(b"last words"); - client.borrow_mut().writer_gone = true; - // The kernel's two answers to a client's leaving, one an end. - net.node.pipe_gone(net.now, id, PipeEnd::ToClient); - net.node.pipe_gone(net.now, id, PipeEnd::FromClient); - assert_eq!((dropped(&client), net.node.streams()), ((true, true), 0), "nothing of the client's is left to serve"); - net.pump(); - assert_eq!((net.far.received.as_slice(), net.far.fin), (&b"last words"[..], true)); - let frame = net.far.fin(); - net.deliver(&frame); - assert_eq!(net.far.acked, net.far.snd_nxt, "the peer's FIN is acknowledged by a connection nobody holds"); - assert!(net.far.resets.is_empty() && net.events().is_empty()); -} - -#[test] -fn a_reader_that_left_is_not_written_to_again() { - let (mut net, id, client) = established(); - client.borrow_mut().reader_gone = true; - let frame = net.far.text(b"to nobody"); - net.deliver(&frame); - assert_eq!((dropped(&client), net.node.streams()), ((true, false), 1), "its writer may still write"); - assert_eq!(net.watch(id), Some(Watch { reader: false, ..IDLE })); - net.fire(net.now.after(Duration::from_secs(150))); - assert_eq!((net.node.streams(), net.events(), net.far.resets.len()), (1, vec![], 0), "and for as long as it likes"); -} - -/// How a client's reader is found gone. -#[derive(Clone, Copy, Debug)] -enum Leaving { - /// The kernel says so of the to-client pipe. - Kernel, - /// A write to it is refused for it. - Write, - /// The client closes the stream. - Close, -} - -// R2 of RFC 9293 §3.8.3, at the 100 seconds it asks for at least, from the moment nobody is left -// when the pipe gives up nothing after; the reset is §3.10.5's. -#[test] -fn a_departed_clients_unsent_bytes_have_100_seconds() { - for leaving in [Leaving::Kernel, Leaving::Write, Leaving::Close] { - let (mut net, id, client) = established_within(Some(Duration::from_secs(30))); - match leaving { - Leaving::Kernel => net.node.pipe_gone(net.now, id, PipeEnd::ToClient), - Leaving::Write => { - client.borrow_mut().reader_gone = true; - let frame = net.far.text(b"to nobody"); - net.deliver(&frame); - } - Leaving::Close => {} - } - net.far.manner = Manner::Deaf; - client.borrow_mut().outbox.extend(text(100_000)); + net.far.manner = Manner::Deaf; + while client.write(&text(1 << 20)).is_ok() { net.bridge(); - assert_eq!(client.borrow().outbox.len(), 100_000 - SEND_BUFFER, "{leaving:?}"); - - net.fire(net.now.after(Duration::from_secs(5))); - assert_eq!(net.node.streams(), 1, "{leaving:?}: its writer is still there"); - let left = net.now; - match leaving { - Leaving::Close => net.node.close(left, id), - Leaving::Kernel | Leaving::Write => { - client.borrow_mut().writer_gone = true; - net.node.pipe_gone(left, id, PipeEnd::FromClient); - } - } - net.pump(); - assert_eq!((dropped(&client), net.node.streams()), ((true, false), 1), "{leaving:?}"); - assert_eq!(net.watch(id), Some(Watch { readable: false, writer: false, writable: false, reader: false }), "{leaving:?}"); - - assert!(net.run_until(Duration::from_secs(200), |net| net.node.streams() == 0), "{leaving:?}: the stream is let go"); - assert_eq!(net.now, left.after(Duration::from_secs(100)), "{leaving:?}: from the moment nobody was left"); - assert_eq!(net.events(), [StreamEvent::Cut { id }], "{leaving:?}"); - assert_eq!((dropped(&client), net.far.resets.len()), ((true, true), 1), "{leaving:?}"); - assert_eq!(client.borrow().outbox.len(), 100_000 - SEND_BUFFER, "{leaving:?}: and no byte more was taken"); - } -} - -/// A stream whose client wrote `len` bytes and is gone, its writer and its reader, at the -/// clock's present. Its peer hears nothing after the handshake, or is a [`Manner::Slow`] reader -/// offering 1,460 bytes. -fn departed(net: &mut Net, len: usize, slow: bool) -> StreamId { - net.far.manner = Manner::Answers; - net.far.offer(65_535); - let (id, client) = net.connect(None); - if slow { - net.far.manner = Manner::Slow; - net.far.offer(1_460); - let update = net.far.update(); - net.deliver(&update); - } else { - net.far.manner = Manner::Deaf; } - client.borrow_mut().outbox.extend(text(len)); - client.borrow_mut().writer_gone = true; - net.bridge(); - net.node.pipe_gone(net.now, id, PipeEnd::FromClient); - net.node.pipe_gone(net.now, id, PipeEnd::ToClient); - id -} - -// A peer that takes one segment a second, past the 100 seconds, sees a departed client's whole -// tail and its FIN: the cut's clock runs from the last byte the pipe gave up. -#[test] -fn a_departed_clients_tail_arrives_whole_while_its_peer_takes_it() { - let mut net = Net::new(); - let sent = text(250_000); - let id = departed(&mut net, sent.len(), true); - let left = net.now; - assert_eq!((net.node.streams(), net.events()), (1, vec![StreamEvent::Connected { id, local: Port::new(net.far.port).unwrap() }])); - - assert!(net.run_slowly_until(Duration::from_secs(1_500), |net| net.far.fin), "the FIN arrives: {} bytes did", net.far.received.len()); - assert!(net.now > left.after(Duration::from_secs(100)), "{:?} after {left:?}", net.now); - assert!(net.far.received == sent, "the peer's stream is the client's, byte for byte"); - assert_eq!((net.node.streams(), net.events(), net.far.resets.len()), (0, vec![], 0)); -} - -// Once the peer's FIN is read through and the client writes no more, nothing of the stream -// reaches the client again and nothing tells the node whether it lives: its unsent bytes have -// the 100 seconds of a departed client's. The peer offers no window and answers every probe -// (RFC 9293 §3.8.6.1), which keeps [tcp] from giving the connection up, so the cut is the node's. -#[test] -fn a_client_done_writing_after_the_peers_fin_has_100_seconds_without_progress() { - let (mut net, id, client) = established(); - net.far.manner = Manner::Slow; - net.far.offer(0); - let frame = net.far.fin(); + assert_eq!(client.write(b"x"), Err(Refused::WouldBlock), "full, and the writer's want armed"); + let bell = client.bell(); + let frame = net.far.rst(); net.deliver(&frame); - assert_eq!((dropped(&client), net.node.streams()), ((true, false), 1), "the client reads the end"); - - client.borrow_mut().outbox.extend(text(100_000)); - net.bridge(); - assert_eq!(client.borrow().outbox.len(), 100_000 - SEND_BUFFER); - let left = net.now; - client.borrow_mut().writer_gone = true; - net.node.pipe_gone(left, id, PipeEnd::FromClient); - net.pump(); - assert_eq!((net.node.streams(), net.events()), (1, vec![])); - - assert!(net.run_until(Duration::from_secs(200), |net| net.node.streams() == 0), "the stream is let go"); - assert_eq!(net.now, left.after(Duration::from_secs(100)), "from the moment its client wrote no more"); - assert_eq!(net.events(), [StreamEvent::Cut { id }]); - assert_eq!((dropped(&client), net.far.resets.len()), ((true, true), 1)); - assert_eq!((client.borrow().outbox.len(), net.far.received.len()), (100_000 - SEND_BUFFER, 0), "no byte was given up, and none taken"); - // The peer's script acknowledges every segment that carries text, taken or not. - let probes = net.far.texts(); - assert!(probes.len() > 2, "the peer answered probes: {probes:?}"); + assert!(client.bell() > bell, "the reset rings the bell"); + assert_eq!(client.write(b"x"), Err(Refused::Failed(RESET))); } -// Sixteen connections of departed clients are a peer's to keep alive, all at once; the -// seventeenth has its 100 seconds from the leaving, whatever the peer takes of it. -#[test] -fn a_peer_keeps_sixteen_departed_clients_connections_alive_and_no_more() { - let mut net = Net::new(); - let sent = text(250_000); - for _ in 0..16 { - departed(&mut net, sent.len(), true); - net.turn_to(Far::new()); - } - let id = departed(&mut net, sent.len(), true); - let left = net.now; - assert_eq!(net.node.streams(), 17); - net.events(); - - assert!(net.run_slowly_until(Duration::from_secs(1_500), |net| net.node.streams() < 17), "a stream is let go"); - assert_eq!(net.now, left.after(Duration::from_secs(100)), "the peer took {} bytes of the seventeenth", net.far.received.len()); - assert_eq!((net.node.streams(), net.events(), net.far.resets.len()), (16, vec![StreamEvent::Cut { id }], 1)); - assert!(net.far.received.len() > 1_460, "and it did take some"); - - assert!(net.run_slowly_until(Duration::from_secs(1_500), |net| net.parked.iter().all(|far| far.fin)), "the sixteen FINs arrive"); - for (nth, far) in net.parked.iter().enumerate() { - assert!(far.received == sent, "connection {nth}: the peer's stream is the client's, byte for byte"); - assert!(far.resets.is_empty(), "connection {nth}"); - } - assert_eq!((net.node.streams(), net.events()), (0, vec![])); -} +// ---- a ring back ---- -// The seventeenth becomes one of the sixteen when one of them is done: from then its peer keeps -// it alive too, past the 100 seconds it had from its client's leaving. +// A source registered while its send ring has room rings netstack, which rings it back on the +// bell (amendment D): its first `WRITABLE` event comes. #[test] -fn a_departed_clients_connection_past_the_sixteen_is_one_of_them_once_one_is_done() { - let mut net = Net::new(); - for _ in 0..16 { - departed(&mut net, 100_000, true); - net.turn_to(Far::new()); - } - let sent = text(250_000); - departed(&mut net, sent.len(), true); - let left = net.now; - assert_eq!(net.node.streams(), 17); - net.events(); - - // The sixteen peers read all there is, at once. - let mut updates = Vec::new(); - for far in &mut net.parked { - far.manner = Manner::Answers; - far.offer(65_535); - updates.push(far.update()); - } - for update in &updates { - net.deliver(update); - } - let done = |net: &Net| net.node.streams() == 1 && net.parked.iter().all(|far| far.fin && far.received.len() == 100_000); - assert!(net.run_until(Duration::from_secs(10), done), "the sixteen are done: {} streams are left", net.node.streams()); - assert!(net.now < left.after(Duration::from_secs(100)), "before the seventeenth's 100 seconds: {:?} after {left:?}", net.now); - - assert!(net.run_slowly_until(Duration::from_secs(1_500), |net| net.far.fin), "the FIN arrives: {} bytes did", net.far.received.len()); - assert!(net.now > left.after(Duration::from_secs(100)), "{:?} after {left:?}", net.now); - assert!(net.far.received == sent, "the peer's stream is the client's, byte for byte"); - assert_eq!((net.node.streams(), net.events(), net.far.resets.len()), (0, vec![], 0)); +fn a_source_registered_with_room_is_rung_back_on_its_bell() { + let (mut net, _, client) = established(); + let bell = client.bell(); + client.register(); + assert!(client.rang()); + let rung: Vec = net.clients.iter().map(|(id, _)| *id).collect(); + net.node.bridge(net.now, rung); + assert_eq!(client.bell(), bell + 1); } -// The sixteen are an address's: with sixteen held at one address, a peer at another keeps its -// own departed client's connection alive. +// ---- a hostile client ---- + +// A head the client stored past what was published is a violation of its own stream alone: it +// is reset, with RFC 9293 §3.10.5's reset, and another client's is untouched. #[test] -fn a_peer_at_another_address_keeps_its_own_sixteen() { - let mut net = Net::new(); - for _ in 0..16 { - departed(&mut net, 100_000, false); - } +fn a_head_past_what_was_published_resets_that_stream_alone() { + let (mut net, id, client) = established(); net.turn_to(Far::at(C, MAC_C)); - let sent = text(250_000); - departed(&mut net, sent.len(), true); - let left = net.now; - assert_eq!(net.node.streams(), 17); + let (other, bystander) = net.connect(None); net.events(); - - assert!(net.run_slowly_until(Duration::from_secs(1_500), |net| net.far.fin), "the FIN arrives: {} bytes did", net.far.received.len()); - assert!(net.now > left.after(Duration::from_secs(100)), "{:?} after {left:?}", net.now); - assert!(net.far.received == sent, "the peer's stream is the client's, byte for byte"); - // The sixteen at the first address gave up nothing and were cut. - let cuts = net.events(); - assert!(cuts.len() == 16 && cuts.iter().all(|event| matches!(event, StreamEvent::Cut { .. })), "{cuts:?}"); - assert_eq!((net.node.streams(), net.far.resets.len()), (0, 0)); + client.word(RX_HEAD).store(7, Ordering::Release); + net.bridge(); + assert_eq!(net.events(), [StreamEvent::Violated { id, violation: Violation::HeadPastTail }]); + assert_eq!((client.dropped(), bystander.dropped(), net.node.streams()), (true, false, 1)); + let reset = net.parked[0].resets.first().expect("the reset"); + assert_eq!(reset.seq, net.parked[0].segments.iter().find(|segment| segment.syn).unwrap().seq.wrapping_add(1)); + net.writes(&bystander, b"still here"); + assert_eq!(net.far.received, b"still here"); + let _ = other; } -// ---- pipes that refuse the node ---- - -// RFC 9293 §3.10.5: the reset an abort sends is at SND.NXT. +// A tail the client stored past its ring's room is a violation of its own stream alone. #[test] -fn a_pipe_that_is_no_pipe_resets_its_connection() { - for write in [true, false] { - let (mut net, _, client) = established(); - if write { - client.borrow_mut().write_broken = true; - let frame = net.far.text(b"to a handle that is no pipe"); - net.deliver(&frame); - } else { - client.borrow_mut().read_broken = true; - net.bridge(); - } - assert_eq!((let_go(&client), net.node.streams()), (vec![PipeEnd::FromClient, PipeEnd::ToClient], 0), "write: {write}"); - let [reset] = &net.far.resets[..] else { panic!("one reset, not {:?}", net.far.resets) }; - assert_eq!(reset.seq, net.first(), "write: {write}"); - } +fn a_tail_past_the_send_rings_room_resets_that_stream() { + let (mut net, id, client) = established(); + client.word(TX_TAIL).store(u32::MAX - 3, Ordering::Release); + client.region_ring(); + net.bridge(); + assert_eq!(net.events(), [StreamEvent::Violated { id, violation: Violation::TailPastDepth }]); + assert_eq!((client.dropped(), net.node.streams(), net.far.resets.len()), (true, 0, 1)); } +// Text published after the client's own shutdown is past the FIN [tcp] queued: a violation. #[test] -fn a_refused_watch_resets_only_a_stream_that_holds_the_end() { +fn text_published_after_the_shutdown_resets_that_stream() { let (mut net, id, client) = established(); - client.borrow_mut().writer_gone = true; + client.shutdown(); net.bridge(); - assert_eq!(dropped(&client), (false, true)); - net.node.pipe_broken(net.now, id, PipeEnd::FromClient); - net.pump(); - assert_eq!((net.node.streams(), net.far.resets.len()), (1, 0), "the end was let go before its watch answered"); - net.node.pipe_broken(net.now, id, PipeEnd::ToClient); - net.pump(); - assert_eq!((dropped(&client), net.node.streams(), net.far.resets.len()), ((true, true), 0, 1)); + assert!(net.far.fin); + client.word(TX_TAIL).store(5, Ordering::Release); + client.region_ring(); + net.bridge(); + assert_eq!(net.events(), [StreamEvent::Violated { id, violation: Violation::TailPastDepth }]); + assert_eq!((client.dropped(), net.node.streams()), (true, 0)); } // ---- options ---- @@ -1430,23 +978,33 @@ fn a_refused_watch_resets_only_a_stream_that_holds_the_end() { // algorithm off it leaves (TX-03). #[test] fn nodelay_reaches_the_stack() { - let (mut net, id, client) = established(); + let (mut net, _, client) = established(); net.far.manner = Manner::Deaf; - client.borrow_mut().outbox.extend(text(100)); - net.bridge(); + net.writes(&client, &text(100)); assert_eq!(net.far.texts(), [100]); - client.borrow_mut().outbox.extend(text(50)); - net.bridge(); + net.writes(&client, &text(50)); assert_eq!(net.far.texts(), [100], "the second write waits for the first's acknowledgment"); - assert!(net.node.set_nodelay(net.now, id, true)); - net.pump(); + client.set_nodelay(true); + net.bridge(); + assert_eq!(net.far.texts(), [100, 50]); +} + +// A client's `nodelay` written before its connect holds from the first segment. +#[test] +fn nodelay_set_before_the_connect_holds_from_the_first_segment() { + let mut net = Net::new(); + let (_, client) = net.connect_as(Client::nodelay(), None); + net.events(); + net.far.manner = Manner::Deaf; + net.writes(&client, &text(100)); + net.writes(&client, &text(50)); assert_eq!(net.far.texts(), [100, 50]); } // ---- the wire is not trusted ---- /// An established stream, and a frame of the peer's carrying `text`. -fn expecting(text: &[u8]) -> (Net, Client, Vec) { +fn expecting(text: &[u8]) -> (Net, Rc, Vec) { let (mut net, _, client) = established(); let frame = net.far.text(text); (net, client, frame) @@ -1458,9 +1016,9 @@ fn no_cut_of_a_segment_is_a_segment() { for len in 0..frame.len() { let (mut net, client, frame) = expecting(b"whole or nothing"); net.deliver(&frame[..len]); - assert!(client.borrow().inbox.is_empty(), "cut to {len} bytes"); + assert!(inbox(&client).is_empty(), "cut to {len} bytes"); net.deliver(&frame); - assert_eq!(client.borrow().inbox, b"whole or nothing", "whole, after the cut to {len} bytes"); + assert_eq!(inbox(&client), b"whole or nothing", "whole, after the cut to {len} bytes"); } } @@ -1474,7 +1032,7 @@ fn no_flipped_bit_of_a_segment_delivers_other_bytes() { net.deliver(&frame); // A flip no checksum covers is in the link addresses: taken or refused, the client // reads the peer's text or nothing. - let inbox = client.borrow().inbox.clone(); + let inbox = inbox(&client); assert!(inbox.is_empty() || inbox == b"whole or nothing", "byte {index} bit {bit}: {inbox:?}"); assert_eq!(net.node.streams(), 1, "byte {index} bit {bit}"); } diff --git a/system/netstack/shard/src/lib.rs b/system/netstack/shard/src/lib.rs index 9a3a7d90c66..23e82fcb1e5 100644 --- a/system/netstack/shard/src/lib.rs +++ b/system/netstack/shard/src/lib.rs @@ -48,7 +48,7 @@ use alloc::vec::Vec; use core::net::Ipv4Addr; use toyos_net_ip::{Advice, Delivery, ErrorKind, IfIndex, Ip, Limiter, NextHop, Nud, Resolution, Sent, Source, Transport, TransportError, FRAME}; -use toyos_net_tcp::{ConnId, Endpoint, Hop, IcmpError, IcmpKind, ListenerId, Outgoing, Received, Seq, Served, Status, Tcp, Tuple}; +use toyos_net_tcp::{ConnId, Endpoint, Hop, IcmpError, IcmpKind, Lent, ListenerId, Outgoing, Received, Seq, Served, Status, Tcp, Tuple}; use toyos_net_udp::{Offer, Sender, SocketId, Udp, Verdict}; use toyos_net_udp::Served as UdpServed; use toyos_net_wire::ethernet::{FrameBuilder, IndividualMac, MacAddr}; @@ -542,9 +542,21 @@ impl Shard { done } - /// [`Tcp::recv_with`]: `take` is shown the oldest bytes held and answers how many it took. - pub fn recv_with(&mut self, now: Instant, id: ConnId, take: impl FnOnce(&[u8]) -> usize) -> Result { - let done = self.tcp.recv_with(now, id, take); + /// [`Tcp::lend`]. + pub fn lend(&mut self, id: ConnId, rx: Box, tx: Box) -> Result<(), toyos_net_tcp::Error> { + self.tcp.lend(id, rx, tx) + } + + /// [`Tcp::consume`]. + pub fn consume(&mut self, now: Instant, id: ConnId, n: usize) -> Result { + let done = self.tcp.consume(now, id, n); + self.settle(now); + done + } + + /// [`Tcp::queue`]. + pub fn queue(&mut self, now: Instant, id: ConnId, n: usize) -> Result { + let done = self.tcp.queue(now, id, n); self.settle(now); done } diff --git a/system/netstack/shard/tcp/src/conn.rs b/system/netstack/shard/tcp/src/conn.rs index b2cadac0b0a..0b9e181dc2d 100644 --- a/system/netstack/shard/tcp/src/conn.rs +++ b/system/netstack/shard/tcp/src/conn.rs @@ -2,6 +2,7 @@ //! ([`screen`]), then ESTABLISHED through LAST-ACK in RFC 9293 §3.10.7.4's order, and the segment //! each transmit opportunity builds. +use alloc::boxed::Box; use core::num::NonZeroU16; use core::time::Duration; @@ -9,7 +10,7 @@ use toyos_net_wire::tcp::{SackBlock, SynOptions, TcpFlags, TcpOptions, TcpSegmen use crate::cc::Cc; use crate::counters::{Counter, Log}; -use crate::ring::Ring; +use crate::ring::{Lent, Ring}; use crate::rtt::{Rtt, RTO_AFTER_HANDSHAKE_LOSS, RTO_MAX}; use crate::rx::{Placed, Rx}; use crate::seq::{Seq, Stamp}; @@ -201,8 +202,16 @@ pub struct Out { pub sack: Blocks, } +/// A segment's text: `len` bytes from `offset` of a send ring. +#[derive(Clone, Copy, Debug)] +pub struct Payload<'a> { + pub ring: Option<&'a Ring>, + pub offset: usize, + pub len: usize, +} + /// The payload of a segment that carries none. -pub const NO_PAYLOAD: (&[u8], &[u8]) = (&[], &[]); +pub const NO_PAYLOAD: Payload<'static> = Payload { ring: None, offset: 0, len: 0 }; impl Out { pub const fn rst(rst: &Rst) -> Self { @@ -926,16 +935,39 @@ impl Sync { } } - /// [`Self::recv`] for a reader that may take less than it is shown. - pub fn recv_with(&mut self, take: impl FnOnce(&[u8]) -> usize, now: Instant) -> Result { - if self.rx.unread() == 0 { - return if self.rx.closed { Ok(Received::End) } else { Err(Error::WouldBlock) }; - } - let n = self.rx.read_with(take); + /// The user read the `n` oldest bytes held out of the lent receive ring itself. + pub fn consume(&mut self, n: usize, now: Instant) -> usize { + let n = self.rx.consume(n); if n > 0 { self.rx.after_read(n, now, self.smss()); } - Ok(Received::Data(n)) + n + } + + /// The user placed `n` more bytes behind those queued, in the lent send ring itself: they + /// are queued as [`Self::send`] queues its copy. Answers how many the buffer had room for. + pub fn queue(&mut self, n: usize, now: Instant) -> Result { + if self.tx.fin.is_some() { + return Err(Error::Closing); + } + let n = self.tx.buf.commit(n); + self.refresh(now); + self.update_persist(now); + Ok(n) + } + + /// Both buffers are the user's from here on: what each holds moves there at its offset, and + /// the send buffer is the whole of its ring, so whatever the user places in the ring fits. + /// `false`, and nothing lent, where either is lent already or shorter than what it stores. + pub fn lend(&mut self, rx: Box, tx: Box) -> bool { + if !(self.rx.lendable(rx.len()) && self.tx.buf.lendable(tx.len())) { + return false; + } + let ring = tx.len(); + self.rx.lend(rx); + self.tx.buf.lend(tx); + self.tx.buf.grow(ring); + true } pub fn shutdown_write(&mut self, now: Instant) { @@ -1127,7 +1159,7 @@ impl Sync { let ts = self.ts.map(|ts| ts.option(now)); let out = Out { seq, kind: Kind::Ack { ack: self.rx.next, push, fin }, window, ts, sack }; let offset = self.tx.offset(seq).min(self.tx.buf.len()); - exit.send(via, &out, self.tx.buf.slices(offset, usize::try_from(len).unwrap_or(0))); + exit.send(via, &out, Payload { ring: Some(&self.tx.buf), offset, len: usize::try_from(len).unwrap_or(0) }); Ok(()) } diff --git a/system/netstack/shard/tcp/src/lib.rs b/system/netstack/shard/tcp/src/lib.rs index 0cbb12006d7..7472ed7aa85 100644 --- a/system/netstack/shard/tcp/src/lib.rs +++ b/system/netstack/shard/tcp/src/lib.rs @@ -54,6 +54,7 @@ use toyos_net_wire::siphash; use toyos_net_wire::Port; pub use counters::{Counter, Counters, Refusal, RefusalLog}; +pub use ring::Lent; pub use seq::Seq; pub use stack::{ConnId, Info, ListenerId, Outgoing, Served, Tcp}; pub use toyos_net_wire::siphash::Key; @@ -245,6 +246,11 @@ pub struct Status { pub state: State, pub readable: usize, pub writable: usize, + /// The user's bytes held from SND.UNA on: in flight, or not yet sent. + pub queued: usize, + /// The window last offered is below the update threshold, so a read owes the peer the + /// segment that opens it (RFC 9293 §3.8.6.2.2). + pub starved: bool, pub failure: Option, pub soft_error: Option, /// R1 reached: three expiries without progress (RFC 9293 SHLD-9). @@ -283,7 +289,7 @@ pub enum Hop { /// committed. A next hop not ready leaves everything owed as it was. pub(crate) trait Exit { fn ask(&mut self) -> Result; - fn send(&mut self, via: T, segment: &conn::Out, payload: (&[u8], &[u8])); + fn send(&mut self, via: T, segment: &conn::Out, payload: conn::Payload<'_>); } /// Why a due segment did not leave. diff --git a/system/netstack/shard/tcp/src/ring.rs b/system/netstack/shard/tcp/src/ring.rs index aaf45daab7f..59accbd06d6 100644 --- a/system/netstack/shard/tcp/src/ring.rs +++ b/system/netstack/shard/tcp/src/ring.rs @@ -1,9 +1,24 @@ -//! A byte ring whose storage grows, by doubling, with the furthest offset written and never past -//! the capacity, so a connection holds memory for what it holds and not for what it may. The -//! capacity only rises. Offsets count from the oldest byte held. - +//! A byte ring over storage of its own or a lent one. Its own grows, by doubling, with the +//! furthest offset written and never past the capacity, so a connection holds memory for what it +//! holds and not for what it may. A lent one is a fixed length another party holds, bytes are +//! copied into and out of it and never borrowed from it, and a ring is lent at most once. The +//! capacity only rises, and never past a lent ring's length. Offsets count from the oldest byte +//! held. + +use alloc::boxed::Box; use alloc::vec::Vec; +/// Storage another party holds: a ring's bytes, reached only by copies. +#[allow(clippy::len_without_is_empty, reason = "storage is never empty, and nothing asks")] +pub trait Lent: core::fmt::Debug { + /// The bytes it holds. + fn len(&self) -> usize; + /// Copies `data` in at `at`, which with `data`'s length lies within [`Self::len`]. + fn put(&mut self, at: usize, data: &[u8]); + /// Copies `out.len()` bytes out from `at`, which with that length lies within [`Self::len`]. + fn take(&self, at: usize, out: &mut [u8]); +} + /// Copies the common prefix of `from` into `to`; the count is how much. fn copy(to: &mut [u8], from: &[u8]) -> usize { let n = to.len().min(from.len()); @@ -13,9 +28,15 @@ fn copy(to: &mut [u8], from: &[u8]) -> usize { n } +#[derive(Debug)] +enum Storage { + Own(Vec), + Lent(Box), +} + #[derive(Debug)] pub struct Ring { - bytes: Vec, + bytes: Storage, capacity: usize, head: usize, len: usize, @@ -23,7 +44,7 @@ pub struct Ring { impl Ring { pub const fn new(capacity: usize) -> Self { - Self { bytes: Vec::new(), capacity, head: 0, len: 0 } + Self { bytes: Storage::Own(Vec::new()), capacity, head: 0, len: 0 } } pub const fn len(&self) -> usize { @@ -38,25 +59,75 @@ impl Ring { self.capacity.saturating_sub(self.len) } - /// Raises the capacity to `capacity`; a lower one is no change. + /// Raises the capacity to `capacity`, never past a lent ring's length; a lower one is no + /// change. pub fn grow(&mut self, capacity: usize) { + let capacity = match &self.bytes { + Storage::Own(_) => capacity, + Storage::Lent(lent) => capacity.min(lent.len()), + }; self.capacity = self.capacity.max(capacity); } - /// The physical index of `offset`, which is below the storage's length. + /// How many bytes the storage has. + fn stored(&self) -> usize { + match &self.bytes { + Storage::Own(bytes) => bytes.len(), + Storage::Lent(lent) => lent.len(), + } + } + + /// The storage index of `offset`, which is below the storage's length. fn at(&self, offset: usize) -> usize { let index = self.head.saturating_add(offset); - index.checked_sub(self.bytes.len()).unwrap_or(index) + index.checked_sub(self.stored()).unwrap_or(index) } /// Storage for every offset below `end`, at most the capacity: each byte stored keeps its - /// offset, laid out again from the head. + /// offset, laid out again from the head. A lent ring has it already. fn reserve(&mut self, end: usize) { - if end > self.bytes.len() { - self.bytes.rotate_left(self.head); + let Storage::Own(bytes) = &mut self.bytes else { return }; + if end > bytes.len() { + bytes.rotate_left(self.head); self.head = 0; - let len = end.max(self.bytes.len().saturating_mul(2)).min(self.capacity); - self.bytes.resize(len, 0); + let len = end.max(bytes.len().saturating_mul(2)).min(self.capacity); + bytes.resize(len, 0); + } + } + + /// Whether the ring can be lent storage of `len` bytes: it is not lent already, and + /// stores no more. + pub fn lendable(&self, len: usize) -> bool { + matches!(&self.bytes, Storage::Own(bytes) if bytes.len() <= len) + } + + /// The ring's bytes from here on are `lent`'s, at the offsets they had: everything stored is + /// copied in, the oldest byte held at its start, and the capacity is held to its length. + /// Only where [`Self::lendable`] says so. + pub fn lend(&mut self, mut lent: Box) { + let Storage::Own(bytes) = &self.bytes else { return }; + let (newer, older) = bytes.split_at(self.head.min(bytes.len())); + lent.put(0, older); + lent.put(older.len(), newer); + self.capacity = self.capacity.min(lent.len()); + self.bytes = Storage::Lent(lent); + self.head = 0; + } + + /// Copies `data` in from `storage` index `start`, wrapping at the storage's end. + fn put(&mut self, start: usize, data: &[u8]) -> usize { + match &mut self.bytes { + Storage::Own(bytes) => { + let first = bytes.get_mut(start..).map_or(0, |to| copy(to, data)); + let (_, rest) = data.split_at(first); + first.saturating_add(copy(bytes, rest)) + } + Storage::Lent(lent) => { + let (first, rest) = data.split_at(lent.len().saturating_sub(start).min(data.len())); + lent.put(start, first); + lent.put(0, rest); + data.len() + } } } @@ -69,23 +140,22 @@ impl Ring { self.reserve(offset.saturating_add(fits)); let start = self.at(offset); let (data, _) = data.split_at(fits); - let first = self.bytes.get_mut(start..).map_or(0, |to| copy(to, data)); - let (_, rest) = data.split_at(first); - let second = copy(&mut self.bytes, rest); - first.saturating_add(second) + self.put(start, data) } - /// Counts `n` more bytes held: bytes [`write_at`](Self::write_at) placed at the end. - pub fn commit(&mut self, n: usize) { - self.len = self.len.saturating_add(n).min(self.capacity); + /// Counts `n` more bytes held: bytes [`write_at`](Self::write_at) placed at the end, or a + /// lent ring's holder did. Answers how many fit the capacity. + pub fn commit(&mut self, n: usize) -> usize { + let n = n.min(self.room()); + self.len = self.len.saturating_add(n); + n } pub fn push(&mut self, data: &[u8]) -> usize { let n = data.len().min(self.room()); let (data, _) = data.split_at(n); let written = self.write_at(self.len, data); - self.commit(written); - written + self.commit(written) } pub fn consume(&mut self, n: usize) { @@ -94,22 +164,38 @@ impl Ring { self.len = self.len.saturating_sub(n); } - /// The held bytes in `[offset, offset + len)`, clamped to what is held, in at most two parts. - pub fn slices(&self, offset: usize, len: usize) -> (&[u8], &[u8]) { + /// The held bytes in `[offset, offset + len)`, clamped to what is held: borrowed where they lie + /// in one piece of a ring's own storage, and otherwise copied into `scratch`, which a lent + /// ring's bytes always are. + pub fn payload<'a>(&'a self, offset: usize, len: usize, scratch: &'a mut Vec) -> &'a [u8] { let offset = offset.min(self.len); let len = len.min(self.len.saturating_sub(offset)); let start = self.at(offset); - let first_len = len.min(self.bytes.len().saturating_sub(start)); - let first = self.bytes.get(start..start.saturating_add(first_len)).unwrap_or_default(); - let second = self.bytes.get(..len.saturating_sub(first_len)).unwrap_or_default(); - (first, second) + let first = len.min(self.stored().saturating_sub(start)); + match &self.bytes { + Storage::Own(bytes) if first == len => bytes.get(start..start.saturating_add(len)).unwrap_or_default(), + Storage::Own(bytes) => { + scratch.clear(); + scratch.extend_from_slice(bytes.get(start..start.saturating_add(first)).unwrap_or_default()); + scratch.extend_from_slice(bytes.get(..len.saturating_sub(first)).unwrap_or_default()); + scratch + } + Storage::Lent(lent) => { + scratch.clear(); + scratch.resize(len, 0); + let (to_first, to_rest) = scratch.split_at_mut(first); + lent.take(start, to_first); + lent.take(0, to_rest); + scratch + } + } } pub fn read(&mut self, out: &mut [u8]) -> usize { - let (first, second) = self.slices(0, out.len()); - let n = copy(out, first); - let n = n.saturating_add(out.get_mut(n..).map_or(0, |out| copy(out, second))); - self.consume(n); - n + let n = out.len().min(self.len); + let mut scratch = Vec::new(); + let copied = copy(out, self.payload(0, n, &mut scratch)); + self.consume(copied); + copied } } diff --git a/system/netstack/shard/tcp/src/rx.rs b/system/netstack/shard/tcp/src/rx.rs index ec4873b08b8..7f0903320ae 100644 --- a/system/netstack/shard/tcp/src/rx.rs +++ b/system/netstack/shard/tcp/src/rx.rs @@ -22,12 +22,13 @@ //! shorter would never be sampled. Without timestamps, the time the peer took to fill the window //! offered is at least one round trip, and the least such time is kept. +use alloc::boxed::Box; use alloc::vec::Vec; use core::time::Duration; use toyos_net_wire::tcp::SackBlock; -use crate::ring::Ring; +use crate::ring::{Lent, Ring}; use crate::seq::Seq; use crate::Instant; @@ -446,15 +447,31 @@ impl Rx { self.buf.read(out) } - /// Shows `take` the oldest bytes held, as far as they lie in one piece, and lets go of as - /// many of them as it answers it took. - pub fn read_with(&mut self, take: impl FnOnce(&[u8]) -> usize) -> usize { - let (held, _) = self.buf.slices(0, self.buf.len()); - let n = take(held).min(held.len()); + /// The user read the `n` oldest bytes held out of a lent ring itself: they are let go of. + pub fn consume(&mut self, n: usize) -> usize { + let n = n.min(self.buf.len()); self.buf.consume(n); n } + /// Received text goes into `lent` from here on, everything held moved there at its offset: + /// in-order text and out-of-order ranges, the FIN staying this side's state. The capacity + /// grows no further than its length. Only where [`Self::lendable`] says so. + pub fn lend(&mut self, lent: Box) { + self.max = self.max.min(lent.len()); + self.buf.lend(lent); + } + + pub fn lendable(&self, len: usize) -> bool { + self.buf.lendable(len) + } + + /// The window last offered is below the update threshold, so a read owes the peer the + /// segment that opens it (RFC 9293 §3.8.6.2.2). + pub fn starved(&self, mss: u32) -> bool { + self.last_window < self.threshold(mss) + } + /// `shutdown_read`: what is held is dropped, and later text is dropped as if read. pub fn stop_reading(&mut self) { self.discard = true; diff --git a/system/netstack/shard/tcp/src/stack.rs b/system/netstack/shard/tcp/src/stack.rs index 6fdd2cf6a83..b7cdc101d88 100644 --- a/system/netstack/shard/tcp/src/stack.rs +++ b/system/netstack/shard/tcp/src/stack.rs @@ -16,7 +16,8 @@ use toyos_net_wire::icmp::UnreachableCode; use toyos_net_wire::tcp::{Control, EstablishedOptions, RawWindow, TcpBuilder, TcpSegment}; use toyos_net_wire::Port; -use crate::conn::{screen, Ctx, In, Kind, Out, Rst, Screened, Sync, Tick, Ts, Verdict, NO_PAYLOAD}; +use crate::conn::{screen, Ctx, In, Kind, Out, Payload, Rst, Screened, Sync, Tick, Ts, Verdict, NO_PAYLOAD}; +use crate::ring::Lent; use crate::counters::{Counter, Counters, Log}; use crate::open::{negotiate, refuse_syn_extras, Local, Origin, Rcvd, Sent, SynRcvd, SynSent}; use crate::rx::Rx; @@ -191,16 +192,9 @@ impl Hop, S: FnMut(&Outgoing<'_>, T)> Exit for Way< (self.hop)(&self.tuple).ready() } - fn send(&mut self, via: T, segment: &Out, (first, second): (&[u8], &[u8])) { + fn send(&mut self, via: T, segment: &Out, payload: Payload<'_>) { let Self { tuple, sink, scratch, .. } = self; - let payload = if second.is_empty() { - first - } else { - scratch.clear(); - scratch.extend_from_slice(first); - scratch.extend_from_slice(second); - scratch.as_slice() - }; + let payload = payload.ring.map_or(&[][..], |ring| ring.payload(payload.offset, payload.len, scratch)); sink(&builder(segment, tuple, payload), via); } } @@ -1020,22 +1014,41 @@ impl Tcp { result } - /// [`Self::recv`] for a reader that may take less than it is shown, such as a pipe that is - /// nearly full: `take` sees the oldest bytes held, as far as they lie in one piece, and answers - /// how many of them it took; no other byte leaves the buffer. With nothing held it is not - /// called, and the answer is [`Self::recv`]'s; `Data(0)` is a reader that took nothing. - pub fn recv_with(&mut self, now: Instant, id: ConnId, take: impl FnOnce(&[u8]) -> usize) -> Result { + /// The user's buffers from here on are `rx` and `tx`, which it reads and writes itself: what + /// the connection holds moves there at its offsets, out-of-order text included, and the + /// buffers grow no further than their lengths. Of a synchronized connection whose buffers + /// are its own and fit; otherwise [`Error::NotConnected`], and nothing is lent. + pub fn lend(&mut self, id: ConnId, rx: Box, tx: Box) -> Result<(), Error> { + match &mut self.conn(id)?.state { + Tcb::Sync(sync) => sync.lend(rx, tx).then_some(()).ok_or(Error::NotConnected), + Tcb::SynSent(_) | Tcb::SynRcvd(_) | Tcb::Ended(_) => Err(Error::NotConnected), + } + } + + /// The user read the `n` oldest bytes held out of its lent receive buffer: they leave it, + /// and the window opens by them. Answers how many were held. + pub fn consume(&mut self, now: Instant, id: ConnId, n: usize) -> Result { let conn = self.conn(id)?; - let result = match &mut conn.state { - Tcb::SynSent(_) | Tcb::SynRcvd(_) => Err(Error::WouldBlock), - Tcb::Sync(sync) => sync.recv_with(take, now), - Tcb::Ended(Ended { failure, rx }) => match rx.as_mut() { - Some(rx) if rx.unread() > 0 => Ok(Received::Data(rx.read_with(take))), - _ => failure.map_or(Ok(Received::End), |f| Err(Error::Failed(f))), - }, + let consumed = match &mut conn.state { + Tcb::SynSent(_) | Tcb::SynRcvd(_) => Err(Error::NotConnected), + Tcb::Sync(sync) => Ok(sync.consume(n, now)), + Tcb::Ended(Ended { rx, .. }) => Ok(rx.as_mut().map_or(0, |rx| rx.consume(n))), }; self.settle(id.index, now); - result + consumed + } + + /// The user placed `n` more bytes in its lent send buffer behind those queued: they are + /// queued, as [`Self::send`] queues its copy. Answers how many the buffer had room for. + pub fn queue(&mut self, now: Instant, id: ConnId, n: usize) -> Result { + let conn = self.conn(id)?; + let queued = match &mut conn.state { + Tcb::SynSent(_) | Tcb::SynRcvd(_) => Err(Error::NotConnected), + Tcb::Sync(sync) => sync.queue(n, now), + Tcb::Ended(ended) => Err(ended.failure.map_or(Error::Closing, Error::Failed)), + }; + self.settle(id.index, now); + queued } pub fn shutdown_write(&mut self, now: Instant, id: ConnId) -> Result<(), Error> { @@ -1123,19 +1136,20 @@ impl Tcp { pub fn status(&mut self, id: ConnId) -> Result { let in_time_wait = |demux: &BTreeMap, tuple| matches!(demux.get(tuple), Some(Entry::TimeWait(_))); let conn = slot(&mut self.conns, id.index, id.generation).filter(|c| c.user == User::Held).ok_or(Error::NoSuchSocket)?; - let (state, readable, writable, delivery_problem, failure) = match &conn.state { - Tcb::SynSent(sent) => (State::SynSent, 0, sent.buf.room(), sent.timer.stalled >= 3, None), - Tcb::SynRcvd(rcvd) => (State::SynReceived, 0, 0, rcvd.timer.stalled >= 3, None), + let (state, readable, writable, queued, starved, delivery_problem, failure) = match &conn.state { + Tcb::SynSent(sent) => (State::SynSent, 0, sent.buf.room(), sent.buf.len(), false, sent.timer.stalled >= 3, None), + Tcb::SynRcvd(rcvd) => (State::SynReceived, 0, 0, 0, false, rcvd.timer.stalled >= 3, None), Tcb::Sync(sync) => { let writable = if sync.tx.fin.is_some() { 0 } else { sync.tx.buf.room() }; - (sync.state(), sync.rx.unread(), writable, sync.delivery_problem, None) + let starved = sync.rx.starved(sync.smss()); + (sync.state(), sync.rx.unread(), writable, sync.tx.buf.len(), starved, sync.delivery_problem, None) } Tcb::Ended(ended) => { let state = if in_time_wait(&self.demux, &conn.tuple) { State::TimeWait } else { State::Closed }; - (state, ended.rx.as_ref().map_or(0, |rx| rx.unread()), 0, false, ended.failure) + (state, ended.rx.as_ref().map_or(0, |rx| rx.unread()), 0, 0, false, false, ended.failure) } }; - Ok(Status { state, readable, writable, failure, soft_error: conn.soft, delivery_problem }) + Ok(Status { state, readable, writable, queued, starved, failure, soft_error: conn.soft, delivery_problem }) } /// Any live connection, orphans included: what inspect lists. diff --git a/system/netstack/shard/tcp/tests/lent.rs b/system/netstack/shard/tcp/tests/lent.rs new file mode 100644 index 00000000000..bac42bfc257 --- /dev/null +++ b/system/netstack/shard/tcp/tests/lent.rs @@ -0,0 +1,146 @@ +//! Lent buffers: storage the user holds and reads and writes itself, which [tcp] copies into and +//! out of. No scenario ids: the specifications hold no such call. What lending must keep is +//! RFC 9293's: text past a hole is held for later processing and a FIN only once every byte before +//! it is in (§3.10.7.4), a read owes the window update a read owes (§3.8.6.2.2), and the text a +//! segment carries is the user's, in order (§3.9.1.2). + +mod common; + +use std::cell::RefCell; +use std::rc::Rc; + +use common::*; +use toyos_net_tcp::{Error, Lent, State}; + +/// A buffer the test holds beside [tcp]: the user's view of its bytes. +#[derive(Clone, Debug)] +struct Shared(Rc>>); + +impl Shared { + fn new(len: usize) -> Self { + Self(Rc::new(RefCell::new(vec![0; len]))) + } + + /// `len` bytes from `at`, wrapping at the end. + fn bytes(&self, at: usize, len: usize) -> Vec { + let bytes = self.0.borrow(); + (0..len).map(|i| bytes[(at + i) % bytes.len()]).collect() + } + + /// The user writes `data` at `at`, wrapping at the end. + fn write(&self, at: usize, data: &[u8]) { + let mut bytes = self.0.borrow_mut(); + let len = bytes.len(); + for (i, b) in data.iter().enumerate() { + bytes[(at + i) % len] = *b; + } + } +} + +impl Lent for Shared { + fn len(&self) -> usize { + self.0.borrow().len() + } + + fn put(&mut self, at: usize, data: &[u8]) { + self.0.borrow_mut()[at..at + data.len()].copy_from_slice(data); + } + + fn take(&self, at: usize, out: &mut [u8]) { + out.copy_from_slice(&self.0.borrow()[at..at + out.len()]); + } +} + +/// The connection's buffers lent: two of `len` bytes, the test's views of them. +fn lend(h: &mut H, len: usize) -> (Shared, Shared) { + let (rx, tx) = (Shared::new(len), Shared::new(len)); + let (lent, _) = h.call(h.t, |tcp, _, id| tcp.lend(id, Box::new(rx.clone()), Box::new(tx.clone()))); + assert_eq!(lent, Ok(())); + (rx, tx) +} + +/// The text a well-behaved peer sends in `[from, to)`. +fn text(from: u32, to: u32) -> Vec { + (from..to).map(pattern).collect() +} + +// A child holds text in order, text past a hole and a FIN behind that before it is lent (RFC 9293 +// §3.10.7.4: kept for later processing, the FIN not yet taken): lent, every byte is at its +// offset, and the hole filled after it makes the text whole and takes the FIN. +#[test] +fn what_a_connection_held_moves_to_its_lent_ring_at_its_offsets() { + let mut h = listening(); + h.input(0, seg(5000).syn().wnd(65_535).mss(1460)); + h.input(10, seg(5001).ack(1001).len(100)); + h.input(20, seg(5201).ack(1001).len(100).fin()); + h.conn = Some(h.tcp.accept(h.listener.unwrap()).unwrap().expect("a ready child")); + let (rx, _) = lend(&mut h, 65_535 * 2); + assert_eq!(h.status().readable, 100); + assert_eq!(rx.bytes(0, 100), text(5001, 5101), "the text in order"); + assert_eq!(rx.bytes(200, 100), text(5201, 5301), "the text past the hole, at its offset"); + let outs = h.input(30, seg(5101).ack(1001).len(100)); + assert_eq!((h.status().readable, h.status().state), (300, State::CloseWait), "the hole filled, and the FIN taken"); + expect(&outs, &["ACK=5302"]); + assert_eq!(rx.bytes(0, 300), text(5001, 5301)); +} + +// RX-13's window update, owed by the user's read out of the lent ring as by `recv`: 65,535 +// unread bytes shut the window, and a read that moves the edge by 1,500 from a window of 0 is +// acknowledged at once. +#[test] +fn a_read_out_of_the_lent_ring_owes_the_window_update_a_recv_does() { + let mut h = fixture_e(); + lend(&mut h, 65_535 * 2); + let mut seq = 5001u32; + while seq < 70_536 { + let n = (70_536 - seq).min(1460); + h.arrive(seg(seq).ack(1001).len(n as usize)); + seq += n; + } + h.at(1); + assert!(h.status().starved, "a shut window"); + let (r, outs) = h.call(1, |tcp, now, id| tcp.consume(now, id, 1000)); + assert_eq!(r, Ok(1000)); + nothing(&outs); + let (r, outs) = h.call(1, |tcp, now, id| tcp.consume(now, id, 500)); + assert_eq!(r, Ok(500)); + expect(&outs, &["ACK=70536 WND=1500"]); +} + +// What the user queued in the lent send ring is what the segments carry, in order (RFC 9293 +// §3.9.1.2), across the ring's end, and an acknowledgment lets go of what it covers. +#[test] +fn the_lent_send_ring_is_the_text_segments_carry_across_its_end() { + let mut h = fixture_e(); + let (_, tx) = lend(&mut h, 4000); + let data: Vec = (0..2920u32).map(|i| (i % 251) as u8).collect(); + tx.write(0, &data); + let (r, outs) = h.call(0, |tcp, now, id| tcp.queue(now, id, 2920)); + assert_eq!(r, Ok(2920)); + expect(&outs, &["SEQ=1001 LEN=1460", "SEQ=2461 LEN=1460"]); + assert_eq!([&outs[0].payload[..], &outs[1].payload[..]].concat(), data); + h.input(10, seg(5001).ack(3921)); + assert_eq!(h.status().queued, 0, "both acknowledged"); + let wrapped: Vec = (0..2920u32).map(|i| (i % 13) as u8).collect(); + tx.write(2920, &wrapped); + let (r, outs) = h.call(20, |tcp, now, id| tcp.queue(now, id, 2920)); + assert_eq!(r, Ok(2920)); + expect(&outs, &["SEQ=3921 LEN=1460", "SEQ=5381 LEN=1460"]); + assert_eq!([&outs[0].payload[..], &outs[1].payload[..]].concat(), wrapped, "the text across the ring's end"); + assert_eq!(h.status().queued, 2920); +} + +// A buffer is lent once, and only a synchronized connection's. +#[test] +fn a_connection_is_lent_once_and_only_once_synchronized() { + let mut h = H::new(65_535); + h.start(-10); + let id = h.tcp.connect(h.now(), A, Some(port(49152)), ep(B, 80)).unwrap(); + h.conn = Some(id); + let (r, _) = h.call(-10, |tcp, _, id| tcp.lend(id, Box::new(Shared::new(10)), Box::new(Shared::new(10)))); + assert_eq!(r, Err(Error::NotConnected), "a connection in SYN-SENT"); + h.input(0, seg(5000).ack(1001).syn().wnd(65_535).mss(1460)); + lend(&mut h, 65_535); + let (r, _) = h.call(0, |tcp, _, id| tcp.lend(id, Box::new(Shared::new(65_535)), Box::new(Shared::new(65_535)))); + assert_eq!(r, Err(Error::NotConnected), "lent already"); +} diff --git a/system/netstack/shard/tcp/tests/take.rs b/system/netstack/shard/tcp/tests/take.rs deleted file mode 100644 index 3e3567abdca..00000000000 --- a/system/netstack/shard/tcp/tests/take.rs +++ /dev/null @@ -1,130 +0,0 @@ -//! `recv_with`: the read of a reader that may take less than it is shown. No scenario ids: the -//! specifications hold no such call, which the track lists among stage 4's departures; their -//! receive scenarios are `receive.rs`'s, on `recv`, and these hold `recv_with` to the same buffer -//! and the same window update. From E. - -mod common; - -use common::*; -use toyos_net_tcp::{Error, Received, State}; - -/// A read whose reader answers `count`, without letting the clock move: the answer, what the -/// reader was shown, and what the read made A send. -fn take(h: &mut H, count: usize) -> (Result, Vec, Vec) { - let mut shown = Vec::new(); - let (r, outs) = h.call(h.t, |tcp, now, id| { - tcp.recv_with(now, id, |held| { - shown = held.to_vec(); - count - }) - }); - (r, shown, outs) -} - -/// The text a well-behaved peer sends in `[from, to)`. -fn text(from: u32, to: u32) -> Vec { - (from..to).map(pattern).collect() -} - -#[test] -fn what_the_reader_did_not_take_stays_held_in_order() { - let mut h = fixture_e(); - h.input(0, seg(5001).ack(1001).len(100)); - let (r, shown, _) = take(&mut h, 40); - assert_eq!((r, shown), (Ok(Received::Data(40)), text(5001, 5101))); - assert_eq!(h.status().readable, 60); - let (r, shown, _) = take(&mut h, 0); - assert_eq!((r, shown), (Ok(Received::Data(0)), text(5041, 5101)), "a reader that took nothing"); - assert_eq!(h.read(100), text(5041, 5101)); -} - -#[test] -fn with_nothing_held_the_reader_is_not_asked() { - let mut h = fixture_e(); - let mut asked = false; - let (r, _) = h.call(h.t, |tcp, now, id| { - tcp.recv_with(now, id, |_| { - asked = true; - 0 - }) - }); - assert_eq!((r, asked), (Err(Error::WouldBlock), false)); - h.input(0, seg(5001).ack(1001).fin()); - let (r, _) = h.call(h.t, |tcp, now, id| { - tcp.recv_with(now, id, |_| { - asked = true; - 0 - }) - }); - assert_eq!((r, asked), (Ok(Received::End), false)); -} - -// The window update RX-13 has a read owe: 65,535 unread bytes shut the window, and a take that -// moves the edge by 1,500 from a window of 0 is acknowledged at once. -#[test] -fn a_take_owes_the_window_update_a_read_does() { - let mut h = fixture_e(); - let mut seq = 5001u32; - while seq < 70_536 { - let n = (70_536 - seq).min(1460); - h.arrive(seg(seq).ack(1001).len(n as usize)); - seq += n; - } - h.at(1); - let (r, _, outs) = take(&mut h, 1000); - assert_eq!(r, Ok(Received::Data(1000))); - nothing(&outs); - let (r, _, outs) = take(&mut h, 500); - assert_eq!(r, Ok(Received::Data(500))); - expect(&outs, &["ACK=70536 WND=1500"]); -} - -// The buffer is a ring of 65,535 bytes: text that wraps it is shown in two pieces, and a reader -// that claims more than it was shown takes the piece and no byte of the next. -#[test] -fn text_across_the_rings_wrap_is_shown_in_two_pieces_and_none_is_lost() { - let mut h = fixture_e(); - let mut seq = 5001u32; - while seq < 70_536 { - let n = (70_536 - seq).min(1460); - h.arrive(seg(seq).ack(1001).len(n as usize)); - seq += n; - } - h.at(1); - let (r, _, _) = take(&mut h, 65_000); - assert_eq!(r, Ok(Received::Data(65_000))); - h.input(2, seg(70_536).ack(1001).len(1000)); - assert_eq!(h.status().readable, 1535); - - let mut pieces = Vec::new(); - loop { - match take(&mut h, usize::MAX) { - (Ok(Received::Data(n)), shown, _) => { - assert_eq!(n, shown.len(), "the count is what was shown"); - pieces.push(shown); - } - (Err(Error::WouldBlock), _, _) => break, - (other, _, _) => panic!("{other:?}"), - } - } - assert_eq!(pieces.iter().map(Vec::len).collect::>(), [535, 1000]); - assert_eq!(pieces.concat(), text(70_001, 71_536)); -} - -// Both FINs end the connection (RFC 9293 §3.6, case 1), and what the peer sent before its own is -// still the reader's: shown, taken, and only then the end. -#[test] -fn text_unread_when_both_fins_ended_the_connection_is_still_taken() { - let mut h = fixture_e(); - let (r, _) = h.call(0, |tcp, now, id| tcp.shutdown_write(now, id)); - r.unwrap(); - h.input(10, seg(5001).ack(1002)); - h.input(20, seg(5001).ack(1002).len(100).fin()); - assert_eq!(h.status().state, State::TimeWait); - let (r, shown, _) = take(&mut h, 40); - assert_eq!((r, shown), (Ok(Received::Data(40)), text(5001, 5101))); - let (r, shown, _) = take(&mut h, usize::MAX); - assert_eq!((r, shown), (Ok(Received::Data(60)), text(5041, 5101))); - let (r, shown, _) = take(&mut h, usize::MAX); - assert_eq!((r, shown), (Ok(Received::End), Vec::new())); -} diff --git a/system/netstack/src/client.rs b/system/netstack/src/client.rs index d421035c3a9..c8b1389e659 100644 --- a/system/netstack/src/client.rs +++ b/system/netstack/src/client.rs @@ -97,9 +97,9 @@ pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2); /// /// `MsgType::DnsLookup` is the only request carrying bytes rather than a struct, /// and `toyos::net::dns_lookup` frames a hostname into a 256-byte buffer; every -/// typed request is far smaller, `TcpConnectPipedRequest` at 32 bytes being the -/// widest. A client may declare anything up to `ipc::MAX_FRAME_LEN` — the excess -/// is counted down and discarded, never waited for. +/// typed request is far smaller. A client may declare anything up to +/// `ipc::MAX_FRAME_LEN` — the excess is counted down and discarded, never waited +/// for. pub const MAX_KEPT_REQUEST: usize = 256; /// One client's inbound framing. diff --git a/system/netstack/src/lent.rs b/system/netstack/src/lent.rs new file mode 100644 index 00000000000..b8419f9827d --- /dev/null +++ b/system/netstack/src/lent.rs @@ -0,0 +1,113 @@ +//! A region a client lent, as the node's [`Session`]: its header as words, its two rings as +//! [tcp]'s buffers, and the client's two doorbells, the connection the region came on and the +//! stream's bell. +//! +//! **A region is judged before it is mapped** ([`SharedMemory::lent`]): a handle that is no +//! region, one that is not memory the kernel allocated, and one that is not exactly +//! [`REGION_BYTES`] are refused by name, so netstack never maps a device's registers or a +//! region shorter than the layout. +//! +//! **The client writes the same memory**, so bytes are copied in and out once and no reference +//! is formed over it, as `toyos::fs`'s window is: a byte the client rewrites under a copy costs +//! only its own stream. The header is reached only as atomics. +//! +//! The mapping lives as long as anything holds it: the session, and [tcp]'s buffers, which +//! outlive the session where [tcp] finishes a connection its client left. + +use std::rc::Rc; + +use toyos::net::stream::{Words, RECEIVE_BYTES, RECEIVE_RING, REGION_BYTES, SEND_BYTES, SEND_RING}; +use toyos::shm::{self, SharedMemory}; +use toyos::{Connection, Pipe}; +use toyos_net_node::Session; +use toyos_net_tcp::Lent; +use toyos_transport::session::Rings; + +/// The region's mapping. +struct Mapping(SharedMemory); + +/// One of the region's rings, as [tcp]'s storage. +struct Ring { + mapping: Rc, + base: usize, + len: usize, +} + +impl std::fmt::Debug for Ring { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Ring({:#x}, {})", self.base, self.len) + } +} + +impl Ring { + /// The address of `len` bytes at `at`, which [`Lent`]'s contract keeps inside the ring. + fn at(&self, at: usize, len: usize) -> *mut u8 { + assert!(at.checked_add(len).is_some_and(|end| end <= self.len), "{len} bytes at {at} run past a ring of {}", self.len); + // SAFETY: the ring lies inside the mapping, which `mapping` keeps alive, and the + // assertion keeps the bytes inside the ring. + unsafe { self.mapping.0.as_ptr().add(self.base + at) } + } +} + +impl Lent for Ring { + fn len(&self) -> usize { + self.len + } + + fn put(&mut self, at: usize, data: &[u8]) { + let to = self.at(at, data.len()); + // SAFETY: `at` bounded the bytes inside the mapping; `data` is netstack's own memory + // and never the region. + unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), to, data.len()) } + } + + fn take(&self, at: usize, out: &mut [u8]) { + let from = self.at(at, out.len()); + // SAFETY: as in `put`, with the two sides swapped. + unsafe { core::ptr::copy_nonoverlapping(from, out.as_mut_ptr(), out.len()) } + } +} + +/// A stream's session as netstack holds it. +pub struct Region { + mapping: Rc, + connection: Rc, + /// The bell's write end: the client holds the read end. + bell: Pipe, +} + +impl Region { + /// The region `handle` names, judged and mapped, on `connection`, with `bell` netstack rings + /// a writer on. Refused, nothing is mapped and the handle is closed. + pub fn lent(handle: toyos::RawHandle, connection: Rc, bell: Pipe) -> Result { + let region = SharedMemory::lent(handle, REGION_BYTES)?; + Ok(Self { mapping: Rc::new(Mapping(region)), connection, bell }) + } +} + +impl Session for Region { + fn words(&self) -> &Words { + // SAFETY: the mapping is `REGION_BYTES` long, 2 MiB aligned, and lives as long as + // `self`; its header is `Words` and reached only as atomics. + unsafe { &*(self.mapping.0.as_ptr() as *const Words) } + } + + fn receive(&self) -> Box { + Box::new(Ring { mapping: self.mapping.clone(), base: RECEIVE_RING, len: RECEIVE_BYTES as usize }) + } + + fn send(&self) -> Box { + Box::new(Ring { mapping: self.mapping.clone(), base: SEND_RING, len: SEND_BYTES as usize }) + } + + /// A doorbell that is full is rung already, and one whose reader is gone has a client the + /// connection's hang-up reports. + fn ring(&self, rings: Rings) { + if rings.connection { + let _ = self.connection.write_nonblock(&[1]); + } + if rings.bell { + let _ = self.bell.write_nonblock(&[1]); + } + } +} diff --git a/system/netstack/src/main.rs b/system/netstack/src/main.rs index 05bbf0172a8..a2a55aaf382 100644 --- a/system/netstack/src/main.rs +++ b/system/netstack/src/main.rs @@ -1,19 +1,20 @@ //! The network server: one Ethernet card claimed from the kernel, ToyOS's own -//! stack over it (`toyos-net-node`), served to programs over the pipe ABI -//! (`toyos::net`). +//! stack over it (`toyos-net-node`), served to programs over `toyos::net`'s +//! protocol. //! //! **This program holds no protocol and reads no frame.** Every byte off the //! wire goes to the node as the card handed it over, and every decision about //! an address, a socket or a segment is the node's. What is here is what the //! node cannot do and stay pure: the card (`card`), the clock, the kernel's -//! random source, the kernel's pipes (`pipes`), the clients' connections -//! (`client`) and their requests (`serve`). +//! random source, the regions clients lend (`lent`), the kernel's pipes +//! (`pipes`), the clients' connections (`client`) and their requests +//! (`serve`). //! //! **One pass**: the card's link and its received frames, as one batch, go to the node, then //! every deadline that is due, then the node is offered the card's transmit //! room until it has no frame left or the card no room; what the node has to //! say is written to the log and to the clients that waited for it; and the -//! loop sleeps on the kernel until the card, a client, a watched pipe or the +//! loop sleeps on the kernel until the card, a client, a watched handle or the //! node's next deadline wakes it. A request is carried out in the pass that //! read it and its frames leave in the next, which follows at once. //! @@ -45,6 +46,7 @@ use toyos_net_wire::Instant; mod card; mod client; mod i219; +mod lent; mod pipes; mod serve; mod virtio_net; @@ -81,24 +83,24 @@ pub const HOSTNAME: &str = "toyos-t14"; /// boot and a lease that lands later is applied like any other. const LEASE_BOUND: Duration = Duration::from_millis(toyos_tco::LEASE_BOUND_MS); -/// The most payload each direction of a connection buffers in the stack, and -/// so the most a window offers: gigabit for a round trip up to 33 ms, at -/// window scale 7. Storage is taken as text is held, and the receive side -/// grows to this only as its reader keeps up (`toyos-net-tcp`'s `rx`). -const TCP_BUFFER: u32 = 4 << 20; +/// The most a connection's receive buffer grows to, and so the most a window +/// offers: gigabit for a round trip up to 33 ms, at window scale 7. It is the +/// region's receive ring, and the receive side grows to it only as its reader +/// keeps up (`toyos-net-tcp`'s `rx`); a send buffer is the region's send ring. +const TCP_BUFFER: u32 = toyos::net::stream::RECEIVE_BYTES; /// A kernel pipe is one 2 MiB page (`kernel/src/pipe.rs`). The client /// allocates it, and netstack holding its far end is what keeps it alive. const PIPE_BYTES: u64 = 2 * 1024 * 1024; /// What one place can make this machine hold, at the largest of the three -/// things a place is (`toyos-net-node`'s `places`): a stream, its two pipes -/// and two buffers; a listener, whose peers fill its queue of `LISTEN_READY` -/// finished connections with an initial receive buffer of text each, nobody -/// reading them to grow one, and its wake pipe; a datagram socket, its two -/// pipes and two queues. +/// things a place is (`toyos-net-node`'s `places`): a stream, its region and +/// the pipes of its connection and its bell; a listener, whose peers fill its +/// queue of `LISTEN_READY` finished connections with an initial receive buffer +/// of text each, nobody reading them to grow one, and its wake pipe; a +/// datagram socket, its two pipes and two queues. const PLACE_BYTES: u64 = { - let stream = 2 * PIPE_BYTES + 2 * TCP_BUFFER as u64; + let stream = toyos::net::stream::REGION_BYTES as u64 + 3 * PIPE_BYTES; let listener = toyos_net_tcp::limits::LISTEN_READY as u64 * toyos_net_tcp::limits::RECEIVE_BUFFER_INITIAL as u64 + PIPE_BYTES; let datagram = 2 * PIPE_BYTES + (toyos_net_udp::limits::RX_BYTES + toyos_net_udp::limits::TX_BYTES) as u64; assert!(stream >= listener && stream >= datagram); @@ -117,11 +119,11 @@ const PLACE_BUDGET_SHARE: u64 = 8; /// Watches that are no place's: the service's acceptor and the card's claim. const FIXED_WATCHES: u32 = 2; -/// The most places one poller can watch: every place's pipes are asked about -/// in the same batch as the fixed watches, the connections that have not said -/// what they want and the lookups' clients. -const MAX_PLACES: u32 = - (Poller::MAX_HANDLES - FIXED_WATCHES - MAX_PENDING_CONNS - serve::LOOKUP_WATCHES) / serve::WATCHES_PER_PLACE; +/// The most places one poller can watch: each place is one watched handle, a +/// stream's connection, a listener's wake pipe or a datagram socket's receive +/// pipe, asked about in the same batch as the fixed watches, the connections +/// that have not said what they want and the lookups' clients. +const MAX_PLACES: u32 = Poller::MAX_HANDLES - FIXED_WATCHES - MAX_PENDING_CONNS - serve::LOOKUP_WATCHES; /// How many places the node is given, of `total_mem` bytes of physical /// memory: its share by what a place costs, at least one, at most what the @@ -326,7 +328,7 @@ fn main() { let config = Config { mac: IndividualMac::new(MacAddr(mac)).unwrap_or_else(|| panic!("netstack: the card's address is a group's")), receive_buffer: TCP_BUFFER, - send_buffer: TCP_BUFFER, + send_buffer: toyos::net::stream::SEND_BYTES, secrets: secrets(), }; let host = HostName::new(HOSTNAME).unwrap_or_else(|| panic!("netstack: {HOSTNAME:?} is no host name")); @@ -358,7 +360,7 @@ fn main() { } let mut poller = watches::Watches::new( - FIXED_WATCHES + serve::WATCHES_PER_PLACE * MAX_PLACES + MAX_PENDING_CONNS + serve::LOOKUP_WATCHES, + FIXED_WATCHES + MAX_PLACES + MAX_PENDING_CONNS + serve::LOOKUP_WATCHES, ); const TOKEN_ACCEPTOR: u64 = 0; const TOKEN_NIC: u64 = 1; @@ -411,7 +413,7 @@ fn main() { poller.watch(&acceptor, READABLE, TOKEN_ACCEPTOR); poller.watch(card.claim(), READABLE, TOKEN_NIC); - sockets.watch(&node, &mut poller); + sockets.watch(&mut poller); for p in pending.iter() { poller.watch(&p.conn, READABLE, TOKEN_PENDING_BASE + p.conn.as_handle().0 as u64); } diff --git a/system/netstack/src/pipes.rs b/system/netstack/src/pipes.rs index 5aa1b4cc44a..187017385d9 100644 --- a/system/netstack/src/pipes.rs +++ b/system/netstack/src/pipes.rs @@ -1,23 +1,23 @@ -//! The kernel's pipe ends as the node's [`ToClient`], [`FromClient`] and -//! [`Wake`]: one non-blocking call each, and the kernel's refusal in the +//! The kernel's pipe end a listener's owner reads its wakes from, as the +//! node's [`Wake`]: one non-blocking write, and the kernel's refusal in the //! node's word for it. //! -//! **The node owns an end and netstack only watches it.** [`hold`] splits a +//! **The node owns the end and netstack only watches it.** [`hold`] splits a //! pipe end into the [`Held`] the node is handed and the [`Watched`] netstack -//! keeps: the handle closes when the node drops its half, which is how a -//! client reads the end of its stream or finds its writes refused, and from -//! then the watched half names nothing. +//! keeps: the handle closes when the node drops its half, which is how an +//! owner reads the end of its listener, and from then the watched half names +//! nothing. //! -//! **Untrusted input.** An end is whatever handle its client moved, and +//! **Untrusted input.** The end is whatever handle its owner moved, and //! nothing checks its kind at intake: one that is no pipe end, or the wrong //! end, answers a refusal here that is neither a full pipe nor a vanished -//! reader, and the node ends that client's stream or listener for it. +//! reader, and the node ends that listener for it. use std::rc::{Rc, Weak}; use toyos::Pipe; use toyos_abi::syscall::SyscallError; -use toyos_net_node::{FromClient, ReadRefusal, ToClient, Wake, WriteRefusal}; +use toyos_net_node::{Wake, WriteRefusal}; /// The half the node holds: the handle lives as long as this does. pub struct Held(Rc); @@ -46,21 +46,6 @@ fn write_refused(why: SyscallError) -> WriteRefusal { } } -impl ToClient for Held { - fn write(&mut self, bytes: &[u8]) -> Result { - self.0.write_nonblock(bytes).map_err(write_refused) - } -} - -impl FromClient for Held { - fn read(&mut self, out: &mut [u8]) -> Result { - self.0.read_nonblock(out).map_err(|why| match why { - SyscallError::WouldBlock => ReadRefusal::Empty, - _ => ReadRefusal::Broken, - }) - } -} - impl Wake for Held { fn wake(&mut self) -> Result<(), WriteRefusal> { match self.0.write_nonblock(&[1]) { diff --git a/system/netstack/src/serve.rs b/system/netstack/src/serve.rs index 5bc98909144..afe0ff86641 100644 --- a/system/netstack/src/serve.rs +++ b/system/netstack/src/serve.rs @@ -1,13 +1,18 @@ //! A client's requests onto the node's calls, and the node's answers back -//! onto the pipe ABI (`toyos::net`). +//! onto `toyos::net`'s protocol. //! //! **One request is one call of the node's**, and every decision about a -//! socket is the node's: what is here is the number a client names a socket -//! by, the clients that wait for an answer, the pipe ends netstack watches, -//! and the word each of the node's refusals is written in. Two requests reach -//! no call: a datagram's bytes travel in the socket's own pipes, which are -//! netstack's and not the node's, and a shutdown of the receiving half alone -//! asks nothing of the stack, std keeping that half's state itself. +//! socket is the node's: what is here is the number a client names a listener +//! or a datagram socket by, the clients that wait for an answer, the handles +//! netstack watches, and the word each of the node's refusals is written in. A +//! datagram's bytes travel in the socket's own pipes, which are netstack's and +//! not the node's, and reach no call. +//! +//! **A stream is its region and the connection its connect or accept came on** +//! (`toyos::net::stream`): netstack judges the region before it maps it +//! (`lent`), makes the stream's bell, answers once with the bell, and from then +//! watches the connection once, for the client's rings and its hang-up, which +//! is the stream's close. A client names no stream by a number. //! //! **Untrusted input.** A request's every field is its client's number: an id //! that names nothing, or a socket of another kind, is refused @@ -34,21 +39,24 @@ use std::collections::{BTreeMap, HashMap}; use std::net::Ipv4Addr; use std::time::Duration; +use std::rc::{Rc, Weak}; + use toyos::net::*; -use toyos::poller::{OTHER_END_GONE, READABLE, WRITABLE}; +use toyos::poller::{OTHER_END_GONE, READABLE}; use toyos::port::Acceptor; -use toyos::{ipc, say, AsHandle, Pipe}; +use toyos::{ipc, say, AsHandle, Connection, OwnedHandle, Pipe}; use toyos_abi::syscall::{SyscallError, MAX_BADGE}; use toyos_inspect::Snapshot; use toyos_net_node::{ AcceptRefused, ConnectRefused, DatagramId, Ended, Event, ListenRefused, ListenerId, LookupId, Node, NotStarted, - PipeEnd, Pipes, Refused, Resolving, StreamEvent, StreamId, WriteRefusal, + Refused, Resolving, StreamEvent, StreamId, WriteRefusal, }; use toyos_net_shard::ConnectError; use toyos_net_tcp::{Endpoint, Failure}; use toyos_net_wire::{Instant, Port}; use crate::client::{Client, Request}; +use crate::lent::Region; use crate::pipes::{self, Watched}; use crate::watches::Watches; use crate::HOSTNAME; @@ -56,8 +64,7 @@ use crate::HOSTNAME; /// A watch's token names the socket it was asked of by its id, in the low /// word, and never a place in a list: an answer can arrive after its socket /// is gone, and must then name nothing. -const TOKEN_FROM_CLIENT: u64 = 1 << 32; -const TOKEN_TO_CLIENT: u64 = 2 << 32; +const TOKEN_STREAM: u64 = 1 << 32; const TOKEN_LISTENER: u64 = 3 << 32; const TOKEN_DATAGRAM: u64 = 4 << 32; /// A client that waits for its lookup's answer, by its connection's handle. @@ -65,9 +72,6 @@ const TOKEN_DATAGRAM: u64 = 4 << 32; const TOKEN_LOOKUP: u64 = 5 << 32; const TOKEN_KIND: u64 = !(u32::MAX as u64); -/// Watches one place makes at most: a stream's two pipes. -pub const WATCHES_PER_PLACE: u32 = 2; - /// Watches the lookups make: each waiting client's connection. pub const LOOKUP_WATCHES: u32 = toyos_dns::MAX_LOOKUPS as u32; @@ -78,7 +82,6 @@ const MAX_ANSWERED: usize = (256 - 1) / 5; /// What a client's id names. enum Socket { - Stream(StreamId), Listener { id: ListenerId, wakes: Watched }, Datagram(Datagram), } @@ -91,13 +94,14 @@ struct Datagram { from_client: Pipe, } -/// The pipe ends of a stream the node holds, for as long as it holds either: -/// longer than its client's id names it, since a closed stream's pipe is -/// still read until it is empty. -struct Ends { - stream: StreamId, - to_client: Watched, - from_client: Watched, +/// A stream's connection as netstack watches it, for as long as the node +/// holds the stream's session. +struct Stream { + id: StreamId, + connection: Weak, + /// The connect not yet answered: its connection, and the bell's read end + /// its answer hands over. + answer: Option<(Rc, Pipe)>, } /// A client waiting for a datagram to reach its socket. @@ -109,35 +113,51 @@ struct Receiving { pub struct Sockets { ids: HashMap, next_id: u32, - ends: HashMap, - /// The id each stream's ends are kept under. + /// Each stream by the key its watch's token carries. + streams: HashMap, + /// The key each stream is kept under. by_stream: BTreeMap, - /// The client each unanswered connect is answered to, and the id it is - /// told. - connecting: BTreeMap, lookups: Vec<(LookupId, Client)>, /// At most one receive waits on a socket, so a client's requests alone /// hold no more of netstack's handles than its sockets do. receiving: BTreeMap, /// The places the node was given. places: usize, - /// A stream's pipe was answered ready in this wake: one pass over the + /// A stream's connection was answered in this wake: one pass over the /// streams follows the wake, however many answers it carried. bridge: bool, - /// The streams whose send pipe was answered readable in this wake. - readable: Vec, + /// The streams whose client rang in this wake. + rung: Vec, +} + +/// A stream's region off a request's connection, judged, and its bell made: +/// the session the node is lent and the bell's read end the answer hands +/// over. Refused by name, nothing is mapped. +fn lend(client: Client) -> Result<(Region, Pipe, Rc), (Client, u32)> { + let Some([region]) = client.conn.recv_handles_exact::<1>() else { return Err((client, ERR_INVALID_INPUT)) }; + let Ok((bell, ring)) = toyos::pipe_pair() else { + // The handle is consumed either way. + toyos_abi::syscall::close(region); + return Err((client, ERR_RESOURCE_EXHAUSTED)); + }; + let connection = Rc::new(client.conn); + match Region::lent(region, connection.clone(), ring) { + Ok(session) => Ok((session, bell, connection)), + Err(why) => { + say!("netstack: refusing a stream's region — {why}"); + let Ok(conn) = Rc::try_unwrap(connection) else { unreachable!("a refused region shares its connection") }; + Err((Client { conn }, ERR_INVALID_INPUT)) + } + } } -/// The two ends a connect or an accept moves, each split into the node's half -/// and the watched one. -fn data_pipes(client: &Client) -> Option<(Pipes, Watched, Watched)> { - let [to_client, from_client] = client.conn.recv_handles_exact::<{ DATA_HANDLES }>()?; - // SAFETY: the kernel moved both handles into this process with the frame - // just read, and nothing else answers for either. - let (to_client, from_client) = unsafe { (Pipe::from_raw(to_client), Pipe::from_raw(from_client)) }; - let (to_held, to_watched) = pipes::hold(to_client); - let (from_held, from_watched) = pipes::hold(from_client); - Some((Pipes { to_client: Box::new(to_held), from_client: Box::new(from_held) }, to_watched, from_watched)) +/// Answers a stream's request with its bell and `answer`, on the connection +/// that is the stream's from here on. +fn hand_over(connection: &Connection, bell: Pipe, answer: &T) { + let sent = connection.send_handles([OwnedHandle::from(bell)]).map_err(ipc::TrySendError::Syscall); + if let Err(why) = sent.and_then(|()| connection.try_send(RespType::Result as u32, answer)) { + say!("netstack: a stream's client left before its answer: {why:?}"); + } } /// The pipe ABI's word for each of the node's. @@ -198,7 +218,7 @@ fn listen_refused(refusal: ListenRefused) -> u32 { fn accept_refused(refusal: AcceptRefused) -> u32 { match refusal { AcceptRefused::NoListener | AcceptRefused::Nothing => ERR_NOT_CONNECTED, - AcceptRefused::NoPipes => ERR_INVALID_INPUT, + AcceptRefused::NoRegion => ERR_INVALID_INPUT, AcceptRefused::Full => ERR_RESOURCE_EXHAUSTED, } } @@ -244,18 +264,17 @@ impl Sockets { Self { ids: HashMap::new(), next_id: first_id.max(1), - ends: HashMap::new(), + streams: HashMap::new(), by_stream: BTreeMap::new(), - connecting: BTreeMap::new(), lookups: Vec::new(), receiving: BTreeMap::new(), places, bridge: false, - readable: Vec::new(), + rung: Vec::new(), } } - /// An id no socket and no stream's ends are kept under, and never 0. + /// An id no socket and no stream is kept under, and never 0. fn alloc_id(&mut self) -> u32 { loop { let id = self.next_id; @@ -263,38 +282,30 @@ impl Sockets { 0 => 1, next => next, }; - if !self.ids.contains_key(&id) && !self.ends.contains_key(&id) { + if !self.ids.contains_key(&id) && !self.streams.contains_key(&id) { return id; } } } - fn hold_stream(&mut self, stream: StreamId, to_client: Watched, from_client: Watched) -> u32 { - let id = self.alloc_id(); - self.ids.insert(id, Socket::Stream(stream)); - self.ends.insert(id, Ends { stream, to_client, from_client }); - self.by_stream.insert(stream, id); - id - } - - fn stream(&self, socket_id: u32) -> Option { - match self.ids.get(&socket_id) { - Some(Socket::Stream(id)) => Some(*id), - Some(Socket::Listener { .. } | Socket::Datagram(_)) | None => None, - } + fn hold_stream(&mut self, id: StreamId, connection: &Rc, answer: Option) { + let key = self.alloc_id(); + let answer = answer.map(|bell| (connection.clone(), bell)); + self.streams.insert(key, Stream { id, connection: Rc::downgrade(connection), answer }); + self.by_stream.insert(id, key); } fn listener(&self, socket_id: u32) -> Option { match self.ids.get(&socket_id) { Some(Socket::Listener { id, .. }) => Some(*id), - Some(Socket::Stream(_) | Socket::Datagram(_)) | None => None, + Some(Socket::Datagram(_)) | None => None, } } fn datagram(&self, socket_id: u32) -> Option<&Datagram> { match self.ids.get(&socket_id) { Some(Socket::Datagram(socket)) => Some(socket), - Some(Socket::Stream(_) | Socket::Listener { .. }) | None => None, + Some(Socket::Listener { .. }) | None => None, } } @@ -304,18 +315,16 @@ impl Sockets { pub fn request(&mut self, node: &mut Node, now: Instant, req: Request, acceptor: &Acceptor, draw: fn() -> u32) { match MsgType::from_u32(req.msg_type) { Some(MsgType::TcpClose) => self.close(node, now, &req), - Some(MsgType::TcpShutdown) => self.shutdown(node, now, &req), Some(MsgType::UdpBind) => self.udp_bind(node, &req, draw), Some(MsgType::UdpSendTo) => self.udp_send_to(node, now, &req), Some(MsgType::UdpRecvFrom) => self.udp_recv_from(node, now, req), Some(MsgType::UdpClose) => self.udp_close(node, now, &req), Some(MsgType::DnsLookup) => self.lookup(node, now, req, acceptor, draw), - Some(MsgType::TcpSetOption) => self.set_option(node, now, &req), Some(MsgType::TcpListenerSetOption) => self.listener_set_option(node, &req), Some(MsgType::UdpSetOption) => self.udp_set_option(node, &req), - Some(MsgType::TcpConnectPiped) => self.connect(node, now, req), + Some(MsgType::TcpConnect) => self.connect(node, now, req), Some(MsgType::TcpBindPiped) => self.listen(node, &req, draw), - Some(MsgType::TcpAcceptPiped) => self.accept(node, now, &req), + Some(MsgType::TcpAccept) => self.accept(node, now, req), None => { say!("netstack: unknown message type {}", req.msg_type); req.client.error(ERR_INVALID_INPUT); @@ -323,15 +332,14 @@ impl Sockets { } } - /// The client lets go of whatever its id names; an id that names nothing - /// is let go already. + /// The client lets go of the listener or datagram socket its id names; an + /// id that names nothing is let go already. fn close(&mut self, node: &mut Node, now: Instant, msg: &Request) { let Ok(req) = ipc::decode_payload::(msg.payload()) else { msg.client.error(ERR_INVALID_INPUT); return; }; match self.ids.remove(&req.socket_id) { - Some(Socket::Stream(id)) => node.close(now, id), Some(Socket::Listener { id, .. }) => { node.close_listener(now, id); } @@ -341,48 +349,6 @@ impl Sockets { msg.client.done(); } - fn shutdown(&mut self, node: &mut Node, now: Instant, msg: &Request) { - let Ok(req) = ipc::decode_payload::(msg.payload()) else { - msg.client.error(ERR_INVALID_INPUT); - return; - }; - let Some(id) = self.stream(req.socket_id) else { - msg.client.error(ERR_NOT_CONNECTED); - return; - }; - let known = match req.how { - // The receiving half alone: std keeps its state, and the id names - // a stream. - 0 => true, - 1 | 2 => node.shutdown_write(now, id), - _ => { - msg.client.error(ERR_INVALID_INPUT); - return; - } - }; - if known { - msg.client.done(); - } else { - msg.client.error(ERR_NOT_CONNECTED); - } - } - - fn set_option(&mut self, node: &mut Node, now: Instant, msg: &Request) { - let Ok(req) = ipc::decode_payload::(msg.payload()) else { - msg.client.error(ERR_INVALID_INPUT); - return; - }; - let Some(id) = self.stream(req.socket_id) else { - msg.client.error(ERR_NOT_CONNECTED); - return; - }; - match req.option { - OPT_NODELAY if node.set_nodelay(now, id, req.value != 0) => msg.client.done(), - OPT_NODELAY => msg.client.error(ERR_NOT_CONNECTED), - _ => msg.client.error(ERR_INVALID_INPUT), - } - } - fn listener_set_option(&mut self, node: &mut Node, msg: &Request) { let Ok(req) = ipc::decode_payload::(msg.payload()) else { msg.client.error(ERR_INVALID_INPUT); @@ -417,27 +383,31 @@ impl Sockets { } } + /// A stream's connect: its region lent, its connection the stream's from + /// here, and its answer once the node has one ([`Self::settle`]). fn connect(&mut self, node: &mut Node, now: Instant, msg: Request) { - let Ok(req) = ipc::decode_payload::(msg.payload()) else { + let Ok(req) = ipc::decode_payload::(msg.payload()) else { msg.client.error(ERR_INVALID_INPUT); return; }; - let (Some((pipes, to_client, from_client)), Some(port)) = (data_pipes(&msg.client), Port::new(req.port)) else { + let Some(port) = Port::new(req.port) else { msg.client.error(ERR_INVALID_INPUT); return; }; + let (session, bell, connection) = match lend(msg.client) { + Ok(lent) => lent, + Err((client, code)) => return client.error(code), + }; let remote = Endpoint { addr: Ipv4Addr::from(req.addr), port }; let timeout = (req.timeout_ms > 0).then(|| Duration::from_millis(u64::from(req.timeout_ms))); - match node.connect(now, remote, timeout, pipes) { - Ok(stream) => { - let id = self.hold_stream(stream, to_client, from_client); - self.connecting.insert(stream, (msg.client, id)); - } + match node.connect(now, remote, timeout, Box::new(session)) { + Ok(stream) => self.hold_stream(stream, &connection, Some(bell)), Err(refusal) => { if refusal == ConnectRefused::Full { say!("netstack: refusing connect, {} of {} places held", node.held(), self.places); } - msg.client.error(connect_refused(refusal)); + let Ok(conn) = Rc::try_unwrap(connection) else { unreachable!("a refused connect kept its session") }; + Client { conn }.error(connect_refused(refusal)); } } } @@ -464,37 +434,41 @@ impl Sockets { } } - fn accept(&mut self, node: &mut Node, now: Instant, msg: &Request) { - let Ok(req) = ipc::decode_payload::(msg.payload()) else { + /// A stream's accept: its region lent, and the oldest connection waiting + /// at the listener the stream on it, answered at once. + fn accept(&mut self, node: &mut Node, now: Instant, msg: Request) { + let Ok(req) = ipc::decode_payload::(msg.payload()) else { msg.client.error(ERR_INVALID_INPUT); return; }; - let moved = data_pipes(&msg.client); - let Some(listener) = self.listener(req.socket_id) else { + let Some(listener) = self.listener(req.listener) else { msg.client.error(accept_refused(AcceptRefused::NoListener)); return; }; - let (pipes, watched) = match moved { - Some((pipes, to_client, from_client)) => (Some(pipes), Some((to_client, from_client))), - None => (None, None), + let (session, bell, connection) = match lend(msg.client) { + Ok(lent) => lent, + Err((client, code)) => { + // Refused, the accept still spends its wake. + let _ = node.accept(now, listener, None); + return client.error(code); + } }; - match (node.accept(now, listener, pipes), watched) { - (Ok(accepted), Some((to_client, from_client))) => { - let socket_id = self.hold_stream(accepted.id, to_client, from_client); - msg.client.result(&TcpAcceptPipedResponse { - socket_id, + match node.accept(now, listener, Some(Box::new(session))) { + Ok(accepted) => { + hand_over(&connection, bell, &TcpAcceptResponse { remote_addr: accepted.remote.addr.octets(), remote_port: accepted.remote.port.get(), local_port: accepted.local.get(), - options: TcpOptions::new(accepted.nodelay), + nodelay: u32::from(accepted.nodelay), }); + self.hold_stream(accepted.id, &connection, None); } - (Ok(_), None) => unreachable!("the node made a stream of an accept that moved no pipes"), - (Err(refusal), _) => { + Err(refusal) => { if refusal == AcceptRefused::Full { say!("netstack: refusing accept, {} of {} places held", node.held(), self.places); } - msg.client.error(accept_refused(refusal)); + let Ok(conn) = Rc::try_unwrap(connection) else { unreachable!("a refused accept kept its session") }; + Client { conn }.error(accept_refused(refusal)); } } } @@ -685,19 +659,22 @@ impl Sockets { StreamEvent::Connected { id, local } => (id, Ok(local)), StreamEvent::Failed { id, failure } => (id, Err(connect_failed(failure))), StreamEvent::TimedOut { id } => (id, Err(ERR_TIMED_OUT)), - // Its client closed it from another connection. + // Its client left first, and nobody reads the answer. StreamEvent::Closed { id } => (id, Err(ERR_CONNECTION_REFUSED)), - StreamEvent::Cut { .. } => { - say!("netstack: resetting a connection — its client is gone and its peer took none of its bytes in 100 s"); + StreamEvent::Violated { violation, .. } => { + say!("netstack: resetting a stream — its client wrote what no client writes: {violation:?}"); continue; } }; - let Some((client, socket_id)) = self.connecting.remove(&id) else { - unreachable!("the node answered a connect nobody waits for") - }; + let answer_to = self.by_stream.get(&id).and_then(|key| self.streams.get_mut(key)?.answer.take()); + let Some((connection, bell)) = answer_to else { unreachable!("the node answered a connect nobody waits for") }; match answer { - Ok(local) => client.result(&TcpConnectResponse { socket_id, local_port: local.get(), _pad: 0 }), - Err(code) => client.error(code), + Ok(local) => hand_over(&connection, bell, &TcpConnectResponse { local_port: local.get(), _pad: 0 }), + Err(code) => { + if let Err(why) = connection.try_send(RespType::Error as u32, &ErrorResponse { code }) { + say!("netstack: a stream's client left before its answer: {why:?}"); + } + } } } @@ -713,7 +690,7 @@ impl Sockets { for (ended, refusal) in ended { let found = self.ids.iter().find_map(|(socket_id, socket)| match socket { Socket::Listener { id, .. } if *id == ended => Some(*socket_id), - Socket::Listener { .. } | Socket::Stream(_) | Socket::Datagram(_) => None, + Socket::Listener { .. } | Socket::Datagram(_) => None, }); // None where its owner's close arrived in the pass that ended it. let Some(socket_id) = found else { continue }; @@ -741,39 +718,28 @@ impl Sockets { } } - // A stream the node let go holds neither end, and its id names - // nothing from then. - let (ids, by_stream) = (&mut self.ids, &mut self.by_stream); - self.ends.retain(|socket_id, ends| { - let held = ends.to_client.held().is_some() || ends.from_client.held().is_some(); + // A stream the node let go holds no session, and its key names nothing + // from then. + let by_stream = &mut self.by_stream; + self.streams.retain(|_, stream| { + let held = stream.connection.strong_count() > 0; if !held { - by_stream.remove(&ends.stream); - if matches!(ids.get(socket_id), Some(Socket::Stream(_))) { - ids.remove(socket_id); - } + by_stream.remove(&stream.id); } held }); } - /// Asks the kernel about every pipe a pass could be owed for: a stream's - /// as the node's last pass left them, the wake pipe of each listener and - /// one pipe of each datagram socket for its owner's leaving, and the - /// connection of each client that waits for a lookup, which hangs up by - /// closing it. - pub fn watch(&self, node: &Node, poller: &mut Watches) { - for (stream, watch) in node.watches() { - let Some((socket_id, ends)) = self.by_stream.get(&stream).and_then(|id| Some((*id, self.ends.get(id)?))) else { - unreachable!("the node holds a stream whose ends netstack does not") - }; - let from = if watch.readable { READABLE } else { 0 } | if watch.writer { OTHER_END_GONE } else { 0 }; - if let (true, Some(pipe)) = (from != 0, ends.from_client.held()) { - poller.watch(&*pipe, from, TOKEN_FROM_CLIENT | u64::from(socket_id)); - } - let to = if watch.writable { WRITABLE } else { 0 } | if watch.reader { OTHER_END_GONE } else { 0 }; - if let (true, Some(pipe)) = (to != 0, ends.to_client.held()) { - poller.watch(&*pipe, to, TOKEN_TO_CLIENT | u64::from(socket_id)); - } + /// Asks the kernel about everything a pass could be owed for: each + /// stream's connection, for its client's rings once it is answered and its + /// hang-up always, the wake pipe of each listener and one pipe of each + /// datagram socket for its owner's leaving, and the connection of each + /// client that waits for a lookup, which hangs up by closing it. + pub fn watch(&self, poller: &mut Watches) { + for (key, stream) in &self.streams { + let Some(connection) = stream.connection.upgrade() else { continue }; + let flags = if stream.answer.is_some() { OTHER_END_GONE } else { READABLE | OTHER_END_GONE }; + poller.watch(&*connection, flags, TOKEN_STREAM | u64::from(*key)); } for (socket_id, socket) in &self.ids { match socket { @@ -783,7 +749,6 @@ impl Sockets { } } Socket::Datagram(socket) => poller.watch(&socket.to_client, OTHER_END_GONE, TOKEN_DATAGRAM | u64::from(*socket_id)), - Socket::Stream(_) => {} } } for (_, client) in &self.lookups { @@ -792,32 +757,28 @@ impl Sockets { } /// What the kernel answered one of [`Self::watch`]'s watches, or `false` - /// for a token that is none of them. An answer about a socket that is - /// gone since, or a pipe end the node let go, says nothing: closing an - /// end ends its watch, and that end is an answer too. + /// for a token that is none of them. An answer about a socket or a stream + /// that is gone since says nothing. pub fn answered(&mut self, node: &mut Node, now: Instant, token: u64, answer: Result) -> bool { let socket_id = token as u32; match token & TOKEN_KIND { - kind @ (TOKEN_FROM_CLIENT | TOKEN_TO_CLIENT) => { - let Some(ends) = self.ends.get(&socket_id) else { return true }; - let (end, held) = if kind == TOKEN_FROM_CLIENT { - (PipeEnd::FromClient, ends.from_client.held().is_some()) - } else { - (PipeEnd::ToClient, ends.to_client.held().is_some()) - }; - match answer { - _ if !held => {} + TOKEN_STREAM => { + let Some(stream) = self.streams.get(&socket_id) else { return true }; + let Some(connection) = stream.connection.upgrade() else { return true }; + let left = match answer { Err(why) => { - say!("netstack: resetting a connection — the kernel refused the watch of its {end:?} pipe: {why:?}"); - node.pipe_broken(now, ends.stream, end); - } - Ok(met) if met & OTHER_END_GONE != 0 => node.pipe_gone(now, ends.stream, end), - Ok(_) => { - if end == PipeEnd::FromClient { - self.readable.push(ends.stream); - } - self.bridge = true; + say!("netstack: closing a stream — the kernel refused the watch of its connection: {why:?}"); + true } + Ok(met) if met & OTHER_END_GONE != 0 => true, + // Doorbells, all taken: what they rang for is the pass's to find. + Ok(_) => drained(&connection), + }; + if left { + node.left(now, stream.id); + } else { + self.rung.push(stream.id); + self.bridge = true; } } TOKEN_LISTENER => { @@ -861,26 +822,39 @@ impl Sockets { /// The pass over the streams the last wake's answers asked for, once. pub fn bridge(&mut self, node: &mut Node, now: Instant) { if std::mem::take(&mut self.bridge) { - node.bridge(now, self.readable.drain(..)); + node.bridge(now, self.rung.drain(..)); } } /// The socket table as `inspect` reads it: counts, and no endpoint, /// because every client holding `netstack` can ask. pub fn inspect(&self, node: &Node, snap: &mut Snapshot) { - let (mut streams, mut listeners, mut udp) = (0u32, 0u32, 0u32); + let (mut listeners, mut udp) = (0u32, 0u32); for socket in self.ids.values() { match socket { - Socket::Stream(_) => streams += 1, Socket::Listener { .. } => listeners += 1, Socket::Datagram(_) => udp += 1, } } - snap.put("sockets.tcp", streams); + snap.put("sockets.tcp", self.streams.len() as u32); snap.put("sockets.listeners", listeners); snap.put("sockets.udp", udp); - snap.put("piped.live", node.streams()); + snap.put("streams.live", node.streams()); snap.put("places.held", node.held()); snap.put("places.max", self.places); } } + +/// Takes every ring a stream's client wrote on its connection: `true` where +/// the client hung up instead. +fn drained(connection: &Connection) -> bool { + let mut bytes = [0u8; 64]; + loop { + match connection.read_nonblock(&mut bytes) { + Ok(0) => return true, + Ok(_) => {} + Err(SyscallError::WouldBlock) => return false, + Err(_) => return true, + } + } +} diff --git a/tests/guest/rust-tests/Cargo.lock b/tests/guest/rust-tests/Cargo.lock index bd36d4c8f7a..6d6c2ef4c3b 100644 --- a/tests/guest/rust-tests/Cargo.lock +++ b/tests/guest/rust-tests/Cargo.lock @@ -724,6 +724,7 @@ name = "toyos" version = "0.18.0" dependencies = [ "toyos-abi", + "toyos-transport", ] [[package]] @@ -782,6 +783,17 @@ dependencies = [ "toyos-abi", ] +[[package]] +name = "toyos-transport" +version = "0.1.0" +dependencies = [ + "toyos-untrusted", +] + +[[package]] +name = "toyos-untrusted" +version = "0.1.0" + [[package]] name = "toyos-window" version = "0.20.0" diff --git a/tests/guest/rust-tests/src/bin/abuse_pipe_owner.rs b/tests/guest/rust-tests/src/bin/abuse_pipe_owner.rs index dfcb08bec5d..75ac9322411 100644 --- a/tests/guest/rust-tests/src/bin/abuse_pipe_owner.rs +++ b/tests/guest/rust-tests/src/bin/abuse_pipe_owner.rs @@ -53,7 +53,6 @@ const REACHES: &[(&str, &str)] = &[ ("read", "read as if it named something"), ("write", "took a write"), ("map", "handed over a ring page"), - ("join", "a connection was joined out of a sibling's ends"), ("control", "a slot nobody ever published answered"), ]; @@ -66,13 +65,8 @@ fn main() { } fn test() { - // Pipes this process created are its own business, and joining two of them - // into one duplex object is too: `SYS_CONNECTION_JOIN` grants nothing, - // because everything it reaches is already the caller's. + // Pipes this process created are its own business. let (own_read, own_write) = toyos::pipe_pair().expect("a pipe of our own"); - let joined = syscall::connection_join(own_read.as_handle(), own_write.as_handle()) - .expect("two ends this process holds must join"); - syscall::close(joined); own_write.write(b"round trip").expect("write our own pipe"); let mut buf = [0u8; 16]; let n = own_read.read(&mut buf).expect("read our own pipe"); @@ -146,7 +140,6 @@ fn attacker(role: &str) -> ! { "read" => format!("{:?}", syscall::read_nonblock(read, &mut buf)), "write" => format!("{:?}", syscall::write_nonblock(write, b"injected")), "map" => format!("{:?}", syscall::pipe_map(read).map(|_| ())), - "join" => format!("{:?}", syscall::connection_join(read, write)), "control" => { format!("{:?}", syscall::read_nonblock(RawHandle(UNPUBLISHED_SLOT), &mut buf)) } diff --git a/tests/guest/rust-tests/src/bin/stream_handles.rs b/tests/guest/rust-tests/src/bin/stream_handles.rs new file mode 100644 index 00000000000..96c78c5dbc3 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/stream_handles.rs @@ -0,0 +1,255 @@ +//! A stream belongs to the process that connected or accepted it, and std +//! keeps it so on every path a raw handle takes. +//! +//! - **Taken back, a handle shares its one end.** A stream's handle given out +//! by `into_raw_fd` or `From for OwnedFd`, or made by +//! `try_clone_to_owned`, and taken back reads on where the last read +//! stopped: one end, one set of cursors. +//! - **A handle the process holds no end for is refused by name**, and nothing +//! is written on its connection: `TcpStream::from_raw_fd` of a bare `dup` +//! panics with `toyos::net`'s refusal. +//! - **A stream's handle is no file**: `File::from_raw_fd` of one panics with +//! that refusal. +//! - **No spawn moves one**: a stdio slot and a slot past them by +//! `inherit_handle`, `endow` and `provide` each answer the refusal, start no +//! child, and leave the stream reading and writing through the handle. +//! +//! **This test is netstack for its children**: a port of its own served to each +//! as `netstack`, as `netstack_gone_mid_bind` does, which answers a connect +//! with the region judged, [`PEER`]'s bytes published into it as netstack +//! publishes what arrived, and a bell, so no NIC and no peer is needed. It +//! runs in a guest because std's ToyOS arms build only for `toyos` and stand +//! on the kernel's handle table. + +use std::fs::File; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::os::fd::{AsFd, AsRawFd, FromRawFd, IntoRawFd, OwnedFd}; +use std::os::toyos::process::CommandExt; +use std::process::{Child, Command, Stdio}; + +use toyos::net::stream::{Misuse, Server, Words, LAYOUT, RECEIVE_RING, REGION_BYTES}; +use toyos::net::{MsgType, RespType, TcpConnectRequest, TcpConnectResponse}; +use toyos::shm::SharedMemory; +use toyos::{namespace, port, AsHandle, Connection, OwnedHandle, Pipe}; +use toyos_abi::syscall::{self, SVC_LABEL}; +use toyos_abi::RawHandle; + +const SELF_PATH: &str = "/system/bin/test_rs_stream_handles"; + +/// What `toyos::net`'s connect resolves. +const SERVICE: &str = "netstack"; + +/// Where the children connect: nothing answers there but this test. +const ADDR: &str = "192.0.2.2:80"; + +/// What the peer sent before the client read anything. +const PEER: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; + +fn main() { + match std::env::args().nth(1).as_deref() { + Some("client") => client(), + Some("foreign") => foreign(), + Some("as-file") => as_file(), + Some("child") => { + say("started"); + std::process::exit(0) + } + Some(other) => panic!("unknown role {other:?}"), + None => test(), + } +} + +fn test() { + shares_and_refuses_moves(); + refuses_a_foreign_handle_writing_nothing(); + refuses_a_stream_as_a_file(); + println!("a stream's handle reaches its one end in its own process and goes nowhere else"); +} + +/// The handles given out and taken back share one end, and no spawn moves one. +fn shares_and_refuses_moves() { + let (mut child, _netstack) = run("client"); + let out = BufReader::new(child.stdout.take().expect("the child's stdout")); + let said: Vec = out.lines().map(|line| line.expect("a line")).collect(); + let status = child.wait().expect("wait the client"); + assert_eq!( + said, + ["shared", "stdio slot refused", "slot past stdio refused", "endow refused", "provide refused"], + "the client said otherwise", + ); + assert!(status.success(), "the client exited {status:?}"); + println!(" given out and taken back: one end; every spawn path refused, and the stream still serves"); +} + +/// A bare `dup` of a stream's handle is no stream of the process: refused by +/// name, with nothing written on its connection before the process ends. +fn refuses_a_foreign_handle_writing_nothing() { + let (mut child, netstack) = run("foreign"); + let mut out = BufReader::new(child.stdout.take().expect("the child's stdout")); + assert_eq!(line(&mut out), "duplicated"); + // What the connect may have rung, taken; from here the connection carries nothing but its end. + let mut bytes = [0u8; 64]; + while syscall::read_nonblock(netstack.conn.as_handle(), &mut bytes).is_ok_and(|n| n > 0) {} + child.stdin.take().expect("the child's stdin").write_all(b"go\n").expect("tell the child"); + let said = child.wait_with_output().expect("wait the child"); + let stderr = String::from_utf8_lossy(&said.stderr); + assert!(!said.status.success(), "the foreign handle was taken: {}", String::from_utf8_lossy(&said.stdout)); + assert!( + stderr.contains("is no stream of this process's: a stream belongs to the process that connected or accepted it"), + "refused, but not by name: {stderr}", + ); + let read = syscall::read(netstack.conn.as_handle(), &mut bytes); + assert_eq!(read, Ok(0), "the refusal wrote on the stream's connection before its end"); + println!(" a foreign handle: refused by name, nothing written"); +} + +/// `File::from_raw_fd` of a stream's handle panics with the refusal. +fn refuses_a_stream_as_a_file() { + let (child, _netstack) = run("as-file"); + let said = child.wait_with_output().expect("wait the child"); + let stderr = String::from_utf8_lossy(&said.stderr); + assert!(!said.status.success(), "the stream was taken as a file: {}", String::from_utf8_lossy(&said.stdout)); + assert!(stderr.contains("is a stream's, which is read and written through its region, not as a file"), "refused, but not by name: {stderr}"); + println!(" a stream as a file: refused by name"); +} + +/// The stream's server end as this test plays netstack, kept for the test's run. +struct Netstack { + conn: Connection, + _region: SharedMemory, + _bell: Pipe, + _server: Server, +} + +/// Spawns `role` with this test as its netstack, and serves its connect. +fn run(role: &str) -> (Child, Netstack) { + let (acceptor, connector) = port::create().expect("a port"); + let ns = namespace::build().add(SERVICE, &connector).finish().expect("a namespace"); + let child = Command::new(SELF_PATH) + .arg(role) + .endow(SVC_LABEL, ns.into_raw().0) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap_or_else(|e| panic!("spawn {role}: {e}")); + let conn = acceptor.accept().expect("the child's connect"); + let header = conn.recv_header().expect("a request"); + assert_eq!(header.msg_type, MsgType::TcpConnect as u32, "a stream's connect"); + let _: TcpConnectRequest = conn.recv_payload(&header).expect("the connect's request"); + let [region] = conn.recv_handles_exact::<1>().expect("the stream's region"); + let region = SharedMemory::lent(region, REGION_BYTES).expect("a region netstack takes"); + // SAFETY: the mapping is `REGION_BYTES`, 2 MiB aligned, and its header is reached only as atomics. + let words = unsafe { &*(region.as_ptr() as *const Words) }; + let mut server = Server::new(words, LAYOUT); + // SAFETY: the receive ring is inside the mapping, and the client reads nothing yet. + unsafe { std::ptr::copy_nonoverlapping(PEER.as_ptr(), region.as_ptr().add(RECEIVE_RING), PEER.len()) }; + let _ = server.publish(words, PEER.len() as u32).expect("the client keeps the protocol"); + let (bell, ring) = toyos::pipe_pair().expect("the bell"); + conn.send_handles([OwnedHandle::from(bell)]).expect("hand over the bell"); + conn.send(RespType::Result as u32, &TcpConnectResponse { local_port: 49_152, _pad: 0 }).expect("answer the connect"); + (child, Netstack { conn, _region: region, _bell: ring, _server: server }) +} + +fn line(out: &mut impl BufRead) -> String { + let mut line = String::new(); + out.read_line(&mut line).expect("a line from a child"); + line.trim().to_string() +} + +fn say(what: &str) { + println!("{what}"); + std::io::stdout().flush().expect("flush"); +} + +fn connect() -> TcpStream { + TcpStream::connect(ADDR.parse::().expect("an address")).expect("this test's netstack answers") +} + +/// The next `n` bytes the stream reads, which must be the peer's from `at`. +fn reads(stream: &mut TcpStream, at: usize, n: usize) { + let mut got = vec![0u8; n]; + stream.read_exact(&mut got).expect("the peer's bytes"); + assert_eq!(got, &PEER[at..at + n], "read from where the last read stopped"); +} + +/// A client whose stream's handle is given out three ways and taken back, then +/// handed to every spawn path. +fn client() -> ! { + let stream = connect(); + let raw = stream.into_raw_fd(); + // SAFETY: the handle `into_raw_fd` just gave out, taken back once. + let mut given = unsafe { TcpStream::from_raw_fd(raw) }; + reads(&mut given, 0, 4); + let owned: OwnedFd = given.into(); + let mut taken = TcpStream::from(owned); + reads(&mut taken, 4, 4); + let mut cloned = TcpStream::from(taken.as_fd().try_clone_to_owned().expect("a second handle")); + reads(&mut cloned, 8, 4); + drop(cloned); + reads(&mut taken, 12, 4); + say("shared"); + + let handle = taken.as_raw_fd() as u32; + let paths: [(&str, fn(&mut Command, u32)); 4] = [ + ("stdio slot", |command, handle| { + command.inherit_handle(1, handle); + }), + ("slot past stdio", |command, handle| { + command.inherit_handle(5, handle); + }), + ("endow", |command, handle| { + command.endow("stream", handle); + }), + ("provide", |command, handle| { + command.provide("stream", handle); + }), + ]; + let mut at = 16; + for (path, moving) in paths { + let mut command = Command::new(SELF_PATH); + command.arg("child").stdout(Stdio::null()); + moving(&mut command, handle); + match command.spawn() { + Err(refused) + if refused.kind() == std::io::ErrorKind::InvalidInput + && refused.to_string() == Misuse::Moved(RawHandle(handle)).to_string() => + { + taken.write_all(b"x").unwrap_or_else(|e| panic!("{path}: the stream writes no more: {e}")); + reads(&mut taken, at, 1); + at += 1; + say(&format!("{path} refused")); + } + Err(other) => say(&format!("{path} answered {other}")), + Ok(mut child) => { + let _ = child.wait(); + say(&format!("{path} took the stream's handle")); + } + } + } + std::process::exit(0) +} + +/// A client that takes a bare `dup` of its stream's handle as a stream. +fn foreign() -> ! { + let stream = connect(); + let copy = syscall::dup(RawHandle(stream.as_raw_fd() as u32)).expect("a bare duplicate"); + say("duplicated"); + let mut go = String::new(); + std::io::stdin().read_line(&mut go).expect("the test's go"); + // SAFETY: a handle this process holds and nothing else answers for. + let _adopted = unsafe { TcpStream::from_raw_fd(copy.0 as i32) }; + say("adopted"); + std::process::exit(0) +} + +/// A client that takes its stream's handle as a file. +fn as_file() -> ! { + let stream = connect(); + // SAFETY: a handle this process holds; the panic is the claim. + let file = unsafe { File::from_raw_fd(stream.as_raw_fd()) }; + say("taken as a file"); + let _ = file.into_raw_fd(); + std::process::exit(0) +} From 8b1e167fd90f3c9643f3f56eee314baf379b932b Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 02:25:44 +0200 Subject: [PATCH 2/6] netstack watches a stream's connection READABLE alone, which the kernel takes The T14's internet_download boot refused every stream at its first pass: "the kernel refused the watch of its connection: NotSupported". netstack watched a stream's connection `READABLE | OTHER_END_GONE`, and the kernel refuses `OTHER_END_GONE` on a Connection (`ops::pipe_end_watch` answers a pipe end alone, `inbox::arm`). A pipe read end with no writer is readable (`kernel/pure/pipe.rs`), so `READABLE` answers both the client's rings and its hang-up, which the drain reads as 0. QEMU never caught it: no QEMU test runs a stream through netstack, and `stream_handles`' fake netstack watched nothing. The watch is now one declaration, `toyos::net::stream::SERVER_WATCH`, which netstack registers and `stream_handles` holds to the kernel: the watch is taken and answered by the client's hang-up. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- sdk/toyos/src/net/stream.rs | 5 ++++ system/netstack/src/serve.rs | 10 ++++---- .../rust-tests/src/bin/stream_handles.rs | 24 +++++++++++++++++-- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index 81cb627e8a1..2d1996b7e24 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -74,6 +74,11 @@ pub type Words = [AtomicU32; WORDS]; /// netstack's end of a stream. pub type Server = session::Server; +/// What netstack watches a stream's connection for: the client's rings, and +/// its hang-up, which the connection reads as its end. A connection is no pipe +/// end, so the kernel refuses it `OTHER_END_GONE`. +pub const SERVER_WATCH: u32 = crate::poller::READABLE; + /// The receive ring's `end`: the peer's reset, or netstack's own. pub const RESET: u32 = 2; /// The receive ring's `end`: [tcp] gave the connection up, by its user timeout diff --git a/system/netstack/src/serve.rs b/system/netstack/src/serve.rs index afe0ff86641..8a3e1edca51 100644 --- a/system/netstack/src/serve.rs +++ b/system/netstack/src/serve.rs @@ -42,6 +42,7 @@ use std::time::Duration; use std::rc::{Rc, Weak}; use toyos::net::*; +use toyos::net::stream; use toyos::poller::{OTHER_END_GONE, READABLE}; use toyos::port::Acceptor; use toyos::{ipc, say, AsHandle, Connection, OwnedHandle, Pipe}; @@ -731,15 +732,13 @@ impl Sockets { } /// Asks the kernel about everything a pass could be owed for: each - /// stream's connection, for its client's rings once it is answered and its - /// hang-up always, the wake pipe of each listener and one pipe of each + /// stream's connection, for its client's rings and its hang-up, the wake pipe of each listener and one pipe of each /// datagram socket for its owner's leaving, and the connection of each /// client that waits for a lookup, which hangs up by closing it. pub fn watch(&self, poller: &mut Watches) { for (key, stream) in &self.streams { let Some(connection) = stream.connection.upgrade() else { continue }; - let flags = if stream.answer.is_some() { OTHER_END_GONE } else { READABLE | OTHER_END_GONE }; - poller.watch(&*connection, flags, TOKEN_STREAM | u64::from(*key)); + poller.watch(&*connection, stream::SERVER_WATCH, TOKEN_STREAM | u64::from(*key)); } for (socket_id, socket) in &self.ids { match socket { @@ -770,8 +769,7 @@ impl Sockets { say!("netstack: closing a stream — the kernel refused the watch of its connection: {why:?}"); true } - Ok(met) if met & OTHER_END_GONE != 0 => true, - // Doorbells, all taken: what they rang for is the pass's to find. + // Doorbells, all taken, or the end: a ring is the pass's to find. Ok(_) => drained(&connection), }; if left { diff --git a/tests/guest/rust-tests/src/bin/stream_handles.rs b/tests/guest/rust-tests/src/bin/stream_handles.rs index 96c78c5dbc3..9fc578ab9ff 100644 --- a/tests/guest/rust-tests/src/bin/stream_handles.rs +++ b/tests/guest/rust-tests/src/bin/stream_handles.rs @@ -14,6 +14,9 @@ //! `inherit_handle`, `endow` and `provide` each answer the refusal, start no //! child, and leave the stream reading and writing through the handle. //! +//! - **netstack's watch of a stream's connection** (`SERVER_WATCH`) is one +//! the kernel takes, and is answered by the client's hang-up. +//! //! **This test is netstack for its children**: a port of its own served to each //! as `netstack`, as `netstack_gone_mid_bind` does, which answers a connect //! with the region judged, [`PEER`]'s bytes published into it as netstack @@ -28,7 +31,8 @@ use std::os::fd::{AsFd, AsRawFd, FromRawFd, IntoRawFd, OwnedFd}; use std::os::toyos::process::CommandExt; use std::process::{Child, Command, Stdio}; -use toyos::net::stream::{Misuse, Server, Words, LAYOUT, RECEIVE_RING, REGION_BYTES}; +use toyos::net::stream::{Misuse, Server, Words, LAYOUT, RECEIVE_RING, REGION_BYTES, SERVER_WATCH}; +use toyos::poller::Poller; use toyos::net::{MsgType, RespType, TcpConnectRequest, TcpConnectResponse}; use toyos::shm::SharedMemory; use toyos::{namespace, port, AsHandle, Connection, OwnedHandle, Pipe}; @@ -69,7 +73,7 @@ fn test() { /// The handles given out and taken back share one end, and no spawn moves one. fn shares_and_refuses_moves() { - let (mut child, _netstack) = run("client"); + let (mut child, netstack) = run("client"); let out = BufReader::new(child.stdout.take().expect("the child's stdout")); let said: Vec = out.lines().map(|line| line.expect("a line")).collect(); let status = child.wait().expect("wait the client"); @@ -80,6 +84,22 @@ fn shares_and_refuses_moves() { ); assert!(status.success(), "the client exited {status:?}"); println!(" given out and taken back: one end; every spawn path refused, and the stream still serves"); + hang_up_answers_the_server_watch(&netstack.conn); +} + +/// netstack's watch of a stream's connection, made after its client left: the +/// kernel takes it and answers it, and the connection then reads its rings and +/// its end. +fn hang_up_answers_the_server_watch(conn: &Connection) { + let poller = Poller::new(1); + poller.watch_raw(conn.as_handle(), SERVER_WATCH, 0); + let mut answers = Vec::new(); + poller.wait_answers(1, 10_000_000_000, |_, answer| answers.push(answer)); + assert!(matches!(answers[..], [Ok(_)]), "the kernel answered netstack's watch {answers:?}"); + let mut bytes = [0u8; 64]; + while syscall::read_nonblock(conn.as_handle(), &mut bytes).is_ok_and(|n| n > 0) {} + assert_eq!(syscall::read_nonblock(conn.as_handle(), &mut bytes), Ok(0), "the hang-up is the connection's end"); + println!(" netstack's watch of a stream's connection: taken, and answered by the hang-up"); } /// A bare `dup` of a stream's handle is no stream of the process: refused by From c0e87be2e335bc87bb99835dc13d422f60bddfc7 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 02:53:34 +0200 Subject: [PATCH 3/6] review round 1: a freed stream slot stays reserved until its halves are cleared; records the change made false updated - toyos::net::stream: an end freed by its last handle or call keeps its slot reserved until `free` has cleared its halves and closed its bell and region, so a concurrent connect cannot take the slot and have its halves cleared under it (the review's blocker 2). - [tcp]'s `Outgoing` says the payload is copied out of a lent buffer, and the track records the send path's second copy (blocker 3). - issues: toyos-has-its-own-network-stack.md loses the ownerless clock and the reset-stream option bullets and names lend/consume/queue for recv_with; a-streams-failure-reaches-its-client-as-a-reset-whatever-it-was.md is narrowed to what is owed, the timeout and unreachable words' tests; a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md closes, since no option request is left to refuse (`set_nodelay` writes the region's word, which stays writable after a reset); and nothing-reads-whether-a-streams-tcp-nodelay-reached-netstack.md closes, since the word is what netstack reads and the node's `nodelay_reaches_the_stack` and `nodelay_set_before_the_connect_holds_from_the_first_segment` read its effect on the wire (blocker 4). - issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md files the region's commitment and the end table's bound, with exits (blocker 5). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...eight-mebibytes-and-a-process-holds-128.md | 27 +++++++++++ ...uses-the-option-requests-a-host-answers.md | 46 ------------------- ...s-its-client-as-a-reset-whatever-it-was.md | 38 +++++---------- ...ves-every-byte-through-two-kernel-pipes.md | 6 ++- ...-a-streams-tcp-nodelay-reached-netstack.md | 34 -------------- issues/toyos-has-its-own-network-stack.md | 10 ++-- sdk/toyos/src/net/stream.rs | 8 +++- system/netstack/shard/tcp/src/stack.rs | 5 +- 8 files changed, 58 insertions(+), 116 deletions(-) create mode 100644 issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md delete mode 100644 issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md delete mode 100644 issues/nothing-reads-whether-a-streams-tcp-nodelay-reached-netstack.md diff --git a/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md b/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md new file mode 100644 index 00000000000..5840bb152e0 --- /dev/null +++ b/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md @@ -0,0 +1,27 @@ +--- +status: open +kind: defect +opened: 2026-10-14 +--- + +# A stream commits 8 MiB at its connect, and a process holds at most 128 + +Every connect and accept makes its region whole before a byte moves: +`toyos::net::stream::REGION_BYTES`, four 2 MiB pages from `SYS_SHM_CREATE`, +physically contiguous, where the pipes it replaced took a page on first write +and [tcp]'s rings grew as text was held. A program that opens many short +connections pays 8 MiB for each, and a fragmented machine can refuse a +connect a pipe would have taken. #863 accepted the cost with the design. + +The process's ends are a fixed table, `MAX_STREAMS` = 128 streams and 256 +handles of their connections (`sdk/toyos/src/net/stream.rs`), since `toyos` +allocates nothing: past it a connect or an accept is refused +`ResourceExhausted` before anything is made. 128 is libc's `MAX_SOCKETS`, an +estimate no measurement set, and a new limit for a std program. + +**Exit condition**: regions whose pages are allocated as the rings fill, so a +stream commits what it holds; and the table grown on demand, or its bound +read off a measured program's need. + +**Owner**: the orchestrator, who stages +`issues/netstack-moves-every-byte-through-two-kernel-pipes.md`. diff --git a/issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md b/issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md deleted file mode 100644 index 1173cc3d7d8..00000000000 --- a/issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-08 ---- - -# A stream its peer reset refuses the option requests a host answers - -The node lets a stream go once it holds neither of its pipe ends -(`system/netstack/node/src/streams.rs`), and netstack's id for it goes with -it (`system/netstack/src/serve.rs`, `Sockets::settle`). A peer's reset ends -both pipes once the client's pipe has taken the text received before it, -so a client that still holds the stream names an id -netstack no longer has, and `Sockets::set_option` answers it -`ERR_NOT_CONNECTED`. - -What a host answers `std::net::TcpStream` on a stream whose peer reset it, -after the read that reported the reset: - -| | `set_nodelay(true)` | `nodelay()` | -|---|---|---| -| macOS (Darwin 27.0.0), measured | `Err(InvalidInput)`, `EINVAL` | `Ok(false)` | -| Linux, by the review of the change that made this, not measured | `Ok(())` | `Ok` | -| ToyOS, read from the code | `Err(NotConnected)` | `Ok`, the value last set | - -So the write is refused as macOS refuses it, under another kind. The read asks -netstack nothing: std answers from the value its `TcpStream` holds -(`sdk/std/sys/net/connection.rs`) and libc's `getsockopt` from its socket's -entry (`sdk/libc/src/socket.rs`). Keeping the id until the client's close -request is not the fix: a client that dies sends none, and netstack, having -closed its pipe ends, has nothing left that says the client is gone. - -libc's `connect` asks the same request of a connection it has just been -answered, to hand over a `TCP_NODELAY` set before it: a peer that resets -between netstack's answer and that request makes it fail `ENOTCONN`, with the -connection closed, which no host's `connect` answers. Its `accept` asks -nothing after its answer, which carries the connection's option. - -**Exit condition**: a guest test on `tests/images/netcase.toml` whose peer resets a stream -the client still holds, and `nodelay()` on it answers `Ok`, with -`netstack_socket_churn` still green; and the two compromises above ended: the -option request on a stream its peer reset is answered by netstack, and a libc -`connect` whose peer resets before the hand-over answers as the host does, -held by a test that can be red. - -**Owner**: whoever holds `issues/toyos-has-its-own-network-stack.md`. diff --git a/issues/a-streams-failure-reaches-its-client-as-a-reset-whatever-it-was.md b/issues/a-streams-failure-reaches-its-client-as-a-reset-whatever-it-was.md index 22afb374cf4..994969a59ba 100644 --- a/issues/a-streams-failure-reaches-its-client-as-a-reset-whatever-it-was.md +++ b/issues/a-streams-failure-reaches-its-client-as-a-reset-whatever-it-was.md @@ -4,33 +4,19 @@ kind: defect opened: 2026-10-09 --- -# A stream's failure reaches its client as a reset, whatever it was +# A stream's failure is told its client by name, and only a reset is tested -The pipe ABI tells a stream's client how the stream ended by the order its two -pipes end in (`system/netstack/node/src/streams.rs`'s header), which carries -one bit: the peer's FIN, or a failure. The node knows which failure -(`toyos_net_tcp::Failure`: a reset, R2's timeout, an ICMP error), and std -answers every one `ConnectionReset`, where Linux answers a connection [tcp] -gave up on `ETIMEDOUT` and one an ICMP error ended `EHOSTUNREACH` or -`ENETUNREACH`. libc answers every one `ECONNRESET` -(`sdk/libc/src/streamend.rs`). +The receive ring's `end` word carries which failure ended a stream: +`RESET`, `TIMED_OUT` or `UNREACHABLE` (`sdk/toyos/src/net/stream.rs`), which +the node stores from [tcp]'s `Failure` (`system/netstack/node/src/streams.rs`), +std answers `ConnectionReset`, `TimedOut` or `HostUnreachable` +(`sdk/std/sys/net/connection.rs`) and libc `ECONNRESET`, `ETIMEDOUT` or +`EHOSTUNREACH` (`sdk/libc/src/socket.rs`). The node's host tests read a +peer's reset as `RESET`; nothing reads the other two words, and no guest test +reads any of them through std or libc. -A pipe's end carries no word, and every other carrier costs what this one does -not: a third pipe or a kept request connection is a 2 MiB page each -(`kernel/src/pipe.rs`, `PIPE_SIZE`), and a reason the node keeps for a client to -ask about after both pipes are gone is kept for ever for a client that died, -since nothing then tells the node it left. The clean carrier is a word the -writer of a pipe sets as it lets its end go and the reader reads at the end, -which is the kernel's to add. - -The same word set would answer a connect that an ICMP error ended, which the -move's netstack answers `ERR_OTHER` for want of a word. - -Read from the code, not measured. - -**Exit condition**: a pipe's end carries its reason, the node writes -`Failure`'s through it, and std and libc answer each as Linux does; a node -host test reads each failure's word, and a guest test whose peer is -unreachable reads it as unreachable. +**Exit condition**: a node host test reads a connection [tcp] gave up on as +`TIMED_OUT` and one an ICMP error ended as `UNREACHABLE`, and a guest test +whose peer is unreachable reads it as unreachable through std. **Owner**: whoever holds `issues/toyos-has-its-own-network-stack.md`. diff --git a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md index d26692a9e21..1c08c57f899 100644 --- a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md +++ b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md @@ -144,7 +144,11 @@ The design and its roast are PR #863's body. MB posted beside the scout's rows. No tree that lands has that row or that counter: the boot carries the scout's instrument, `git diff 777c96b2b 2f9e090b8` (`wt/toyos-netring-scout` over its base), onto stage 3's head, - less the netstack stamps on the pipe writes stage 3 deletes. + less the netstack stamps on the pipe writes stage 3 deletes. As built, a + sent byte is copied twice by netstack, out of the lent send ring into a + scratch buffer and from it into the frame, since the frame builder takes + one slice; a builder that copies each span of the ring into the frame + itself would remove the first. - Stage 4: a listener's connection carries its wakes in place of its pipe, a datagram socket is a region as a stream is, and the pipe ABI's messages and `toyos::net`'s pipe handles go. netstack copies each datagram record's diff --git a/issues/nothing-reads-whether-a-streams-tcp-nodelay-reached-netstack.md b/issues/nothing-reads-whether-a-streams-tcp-nodelay-reached-netstack.md deleted file mode 100644 index 1db0339feac..00000000000 --- a/issues/nothing-reads-whether-a-streams-tcp-nodelay-reached-netstack.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-08 ---- - -# Nothing reads whether a stream's TCP_NODELAY reached netstack - -netstack answers `MsgType::TcpSetOption` done or with an error, and nothing -answers what a stream holds once it is a client's: the pipe ABI says it once, -in the accept's answer (`TcpOptions`, `sdk/toyos/src/net.rs`), and has no option -read, and std's `nodelay()` and libc's `getsockopt` answer the value their own -side stored. So a setter of a stream's option that stores the value and sends -nothing is seen by no test. Measured on libc's `connect`, which hands over a -`TCP_NODELAY` set before the connection existed: with the hand-over deleted, -the guest case that sets the option before `connect` and reads it after -(`tests/guest/c/nodelay_kept.c`, the `libc_sockets` test) stays green. The -only effect of the option is when a small write leaves the machine, which a -QEMU test may not time. - -A listener's is read: what an accepted socket answers is what netstack said of -its connection, so libc's `bind` naming no option in its request, or its -`setsockopt` on a listener storing the value and sending nothing, turns that -case red. - -**Exit**: a test reads what the stack holds for a stream's Nagle switch, on -the node under a host test of its option call or through a guest's wire, and -turns red when libc's `connect` or `setsockopt` on a connection, or std's -`set_nodelay`, stores the value and sends no request. - -**Owner**: whoever holds `issues/toyos-has-its-own-network-stack.md`, with -the reset-stream option test of -`issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md`, -which asks the same request. diff --git a/issues/toyos-has-its-own-network-stack.md b/issues/toyos-has-its-own-network-stack.md index bf82c074606..b1e3334a740 100644 --- a/issues/toyos-has-its-own-network-stack.md +++ b/issues/toyos-has-its-own-network-stack.md @@ -29,10 +29,8 @@ What the node does not yet meet: - A port is one listener's, whatever address either listen named: a listen on a port a listener holds is refused `ListenRefused::InUse` (`system/netstack/node/src/listeners.rs`), so [tcp]'s rule that a SYN goes to the listener that named its address before one at every address (LS-09) is not reachable from the node, and one program's two listeners on one port at two addresses, which a host allows one user, are refused too. The node has no word for the program behind a listen: with LS-09 reachable, any program that reaches netstack would take every later connection of another program's listener at every address by naming the machine's address on its port. Exit: a listen carries its program, a second listener on a held port is allowed only to the program that holds every listener on it, and a test of the node finds one program's pair standing, a SYN handed to the one that named its address, and another program's listen on that port refused. - A listener's handshakes in progress are [tcp]'s `limits::LISTEN_PENDING`, each held until `limits::SYNACK_GIVE_UP`, and a SYN past them is dropped: 256 SYNs a minute from addresses that never answer shut a port to every other peer for as long as they keep coming. [tcp] has no SYN cookie and lets no older handshake go for a newer one. Exit: a test in `system/netstack/shard/tcp/tests/` in which a peer's handshake completes while LS-12's flood runs. - The node counts a wake unspent until an accept arrives, and cannot tell an accept on its way from one that will never be sent: an owner that reads a wake and sends no accept is owed one wake fewer from then on, and the last connection waiting is not announced. That is the node's half of `issues/an-accept-that-never-reaches-netstack-strands-its-listener.md`. Exit: that issue's, by a client that cannot spend a wake without its accept arriving, or by an accept that waits in netd and needs no wake. -- A stream its client can see no more, its reader gone or the peer's FIN read through and its client writing no more, is reset once its pipe has given up no byte for 100 s, and a peer address restarts that clock for at most 16 such streams at once (`OWNERLESS_PER_PEER`, `system/netstack/node/src/streams.rs`); one past them has 100 s from the pass that found it so, until one of the 16 is done and it takes its place. What holds in this stage, from the code: per address, 16, an estimate no measurement set. There is no floor on the rate: each of the 16 is kept for as long as its pipe gives up one byte per 100 s, which for a full 2 MiB pipe is 2,097,152 x 100 s, over six years, and then for its tail in [tcp] (the line below on a client gone with nothing left in its pipe). The 16 is no bound across addresses: addresses are not counted, and where a connect's address is its client's choice, an accepted stream's is the peer's, which on the link answers for as many addresses as it likes. The bound across them is the node's places (`system/netstack/node/src/places.rs`), of which every stream holds one until the node lets it go, and there is no second one: in `peers_at_more_addresses_than_there_are_places_hold_no_stream_past_them` (`system/netstack/node/tests/listeners.rs`) peers at four addresses bring sixteen connections each to a node with three places, which holds two such streams at a time, each kept alive past its 100 s, and refuses every accept and connect past them. Exit: the T14 or a guest shows a peer holding one at a byte per 100 s, and the cut takes a floor on the rate, or the owner rules it needs none. -- A stream its peer reset is gone from the node once its client's pipe has taken the text received before the reset, so for a client that still holds it `Node::set_nodelay` answers `false`: `issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md`, carried by the node as built. std's `nodelay()` asks netd nothing: it answers from the value its `TcpStream` holds (`sdk/std/sys/net/connection.rs`), and libc's `getsockopt` of `TCP_NODELAY` from its socket's entry (`sdk/libc/src/socket.rs`). What the node carries is `set_nodelay` on a reset stream, refused where macOS refuses it under another kind and Linux answers. The node cannot hold the stream for the client: its two pipe ends are all it has of the client, letting both go is how the client learns of the reset, and with both gone nothing tells it when the client leaves, so a kept stream would be kept for ever. Exit, outside the node: that issue's guest test is green, and a `getsockopt` of `TCP_NODELAY` on a stream its peer reset is answered, by libc from a value its socket holds as std's does or by a pipe ABI that gives netd a handle whose other end's leaving the kernel reports. -- How a stream ended reaches its client as the order its two pipes end in, which is one bit: the peer's FIN, or a failure. The node keeps a send pipe whose FIN is queued until its writer leaves and lets a failed stream's send pipe go first (`system/netstack/node/src/streams.rs`), so std and libc read a FIN as a FIN and a failure as `ConnectionReset` or `ECONNRESET`, whichever failure it was: `issues/a-streams-failure-reaches-its-client-as-a-reset-whatever-it-was.md`. Exit: that issue's. -- Every batch of received frames and every deadline ends in a pass over every stream, every transmit opportunity walks every stream twice (once for the count each peer address keeps alive, once to pass those whose connect is not answered), and `Node::next_deadline` reads every stream: linear in streams per batch, as netstack's bridge on smoltcp was per frame. For an idle stream a pass is one `recv_with`, two `status` and one read of its client's pipe, which in netstack is a system call: 1,000 system calls a batch with 1,000 idle streams. A pass's `recv_with` also re-files its connection's deadline and offers it to the transmit round, so one batch makes every idle stream eligible and the next opportunity serves each to learn it has nothing. [tcp] cannot say which connections a segment, a timer, an ICMP error or a failed next hop moved: it reports `drain_eligible` and `drain_gone`, which are the round's. The change: [tcp] marks a held connection in `for_conn`, `tick`, `icmp` and `end` and hands the marks out once (`drain_moved`); the node maps a connection to its stream, keeps its deadlines ordered, and passes only the streams [tcp] moved, the one a pipe's wake names, and those due. A pipe's end-gone answer (`Node::pipe_gone`) runs a pass of its own as well, beside the one `Sockets::bridge` runs for a wake's readiness answers, so k clients leaving in one wake cost k + 1 passes. Not measured at the move. The count is what a guest can read and a time is not: netstack's places end at 103, what one poller watches (`MAX_PLACES`, `system/netstack/src/main.rs`), so 1,000 idle streams cannot exist, the T14's bench opens no peer that holds streams, and QEMU gives no verdict on time. Exit: a guest test counts netstack's reads of its clients' pipes per batch received beside one bulk stream, with 1 idle stream and with 100, and the line closes when the two counts are equal. +- How a stream ended reaches its client by the receive ring's `end` word, which names each failure; only a reset is tested: `issues/a-streams-failure-reaches-its-client-as-a-reset-whatever-it-was.md`. Exit: that issue's. +- Every batch of received frames and every deadline ends in a pass over every stream, every transmit opportunity walks every stream twice (once for the count each peer address keeps alive, once to pass those whose connect is not answered), and `Node::next_deadline` reads every stream: linear in streams per batch, as netstack's bridge on smoltcp was per frame. For an idle stream a pass is a `status` and loads of its region's words, and no system call. [tcp] cannot say which connections a segment, a timer, an ICMP error or a failed next hop moved: it reports `drain_eligible` and `drain_gone`, which are the round's. The change: [tcp] marks a held connection in `for_conn`, `tick`, `icmp` and `end` and hands the marks out once (`drain_moved`); the node maps a connection to its stream, keeps its deadlines ordered, and passes only the streams [tcp] moved, those a client's ring names, and those due. A client's hang-up (`Node::left`) runs a pass of its own as well, beside the one `Sockets::bridge` runs for a wake's answers, so k clients leaving in one wake cost k + 1 passes. Not measured at the move. The count is what a guest can read and a time is not: netstack's places end at what one poller watches (`MAX_PLACES`, `system/netstack/src/main.rs`), so 1,000 idle streams cannot exist, the T14's bench opens no peer that holds streams, and QEMU gives no verdict on time. Exit: a guest test counts the streams a pass moves per batch received beside one bulk stream, with 1 idle stream and with 100, and the line closes when the two counts are equal. - A client that is gone with nothing left in its pipe is finished by [tcp]'s rules for a user who let go: a reset once it has been idle 60 s, where netd on smoltcp reset it 100 s after the client left whatever the peer said. A peer that acknowledges a byte a minute holds such a connection for as long as it has bytes to acknowledge. Exit: the T14 or a guest shows such a peer holding one, and [tcp]'s rule takes a bound from the client's leaving; or the owner rules the idle bound is the one. - `Tcp::ready`, `Tcp::orphans` and `Tcp::options` are additions to [tcp] the specifications do not hold, made for the node's wakes, its places and the copy a stream keeps of its connection's options; their tests (`system/netstack/shard/tcp/tests/held.rs`) name no scenario id. `Shard::listen` refuses an address [ip] does not hold usable, by the rule [udp]'s bind refuses by (`udp.bind-address-not-local`), and counts nothing for it; the node's test of it (`a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds`) names no scenario either. A listener whose address the machine loses stands as a datagram socket bound to it does, with its port and its place and its owner told nothing, and answers again once the address is back (`a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back`, `system/netstack/node/tests/lease.rs`); no host was read for it. Exit: the specifications hold the three calls and the refusal, and the tests name their scenarios. - The word each of [udp]'s refusals is answered in on the pipe ABI is `Refused` in `system/netstack/node/src/datagram.rs`, tested by `each_refusal_is_answered_in_the_pipes_word_for_it` against no reader's scenario; `udp.no-ephemeral-port` has a word and no test that reaches it. Exit: the scenario owed below exists, and the test names its id. @@ -72,8 +70,8 @@ What stage 4 departs from its specifications: - [ip]'s `Event::Cleared` and `Event::Room` are not in the specifications. Exit: they name them. - The readers' stage 4 design has a flow waiting for its next hop keep its place in the round. [shard]'s round lets it go, deficit and all, and [tcp] offers it again when woken, at the round's tail, so a waiting flow costs the round nothing. Exit: a `system/netstack/shard/tests/drr.rs` test finds a woken flow served in the place it held when it began to wait. - The readers' stage 4 design has a flow leave the round once it has nothing eligible, its deficit reset. [udp] says so as it hands out a sender's last datagram; [tcp] learns it only when it serves the connection, since only its `next_segment` decides what is due, so a connection whose turn ended on its last segment stays in the round with its deficit until its next turn, and what it queues before then is sent against that deficit, debt included, as RFC 8290 §4.2 keeps an emptied queue until it is next selected. Exit: [tcp] reports at hand-off that nothing is left, and a `system/netstack/shard/tests/drr.rs` test finds such a connection out of the round, its deficit reset. -- `Tcp::recv_with`, the read of a reader that may take less than it is shown, is not in the specifications: the node's bridge to a client's pipe needs it, since a byte `recv` copied out has opened the window and cannot be put back. Its tests (`system/netstack/shard/tcp/tests/take.rs`) name no scenario. Exit: the specification names the call and its scenarios, and the tests take their ids. -- RFC 9293 §3.10.7.4 says a reset in a synchronized state SHOULD flush every queue. [tcp] keeps the in-order text received before it, which `recv` and `recv_with` give ahead of `Err(Failed(Reset))` (`Ended`, `system/netstack/shard/tcp/src/stack.rs`), as Linux and the BSDs read buffered data ahead of `ECONNRESET`: the orchestrator ruled to follow them, since ToyOS's clients are written against those stacks, and since text a pass has already written to a client's pipe cannot be flushed, so only keeping it makes what a client reads independent of how the card batched frames. `s_rx_023_a_reset_comes_after_the_text_before_it` holds it. Exit: the specification names the departure, or the owner rules for the flush. +- `Tcp::lend`, `Tcp::consume` and `Tcp::queue`, the calls that make a connection's buffers a region its user reads and writes itself, are not in the specifications. Their tests (`system/netstack/shard/tcp/tests/lent.rs`) name no scenario. Exit: the specification names the calls and their scenarios, and the tests take their ids. +- RFC 9293 §3.10.7.4 says a reset in a synchronized state SHOULD flush every queue. [tcp] keeps the in-order text received before it, which `recv` and a lent buffer give ahead of `Err(Failed(Reset))` (`Ended`, `system/netstack/shard/tcp/src/stack.rs`), as Linux and the BSDs read buffered data ahead of `ECONNRESET`: the orchestrator ruled to follow them, since ToyOS's clients are written against those stacks, and since text a pass has already written to a client's pipe cannot be flushed, so only keeping it makes what a client reads independent of how the card batched frames. `s_rx_023_a_reset_comes_after_the_text_before_it` holds it. Exit: the specification names the departure, or the owner rules for the flush. - The readers' TCP specification disagrees with itself, and [tcp] follows its user timeout: one replaces R2 and runs only while sent data is unacknowledged or a zero window holds data back, so with one set a connection whose data never left, for want of credit or of a next hop, has no give-up, where its rule that give-up clocks run on wall time has a device that never offers credit end its connections. Exit: RFC 9293 §3.10.8 ends a connection in any state once its user timeout expires: a test that sets one and withholds credit, as `s_pl_009_give_up_without_credit` does without one, finds the connection ended. Open for the owner. On 2026-10-03 he chose "Fair scheduler now": "Build the byte-fair scheduler as the network spec describes, about 280 lines, and decide later on the laptop whether new connections should jump the queue." It is built as one round, in the order flows became eligible, and nobody jumps it. The track reads a queue that new connections jump as RFC 8290's new-flows list, and offers him this evidence: a sparse flow such as a DNS query waits one turn of every flow ahead of it in the round, up to a quantum of 1,514 bytes each, where [udp]'s datagrams and [tcp]'s segments alternated frame by frame before, so a query waited at most one TCP frame: in DRR-02 a 100-byte segment that becomes eligible leaves behind a full frame of each of the three bulk flows ahead of it (`s_shard_drr_002_a_bulk_flow_starves_no_other_beyond_its_quantum`); a T14 measurement at stage 5 of a sparse flow's wait behind bulk flows would add to it. Exit: the owner rules on whether new connections jump the queue. diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index 2d1996b7e24..0521c1a45f3 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -187,10 +187,13 @@ impl Table { self.free_unused(slot) } + /// The end, where nothing reaches it any more. Its slot stays reserved + /// until [`free`] has cleared its halves, so no new stream takes the slot + /// and has its halves cleared under it. fn free_unused(&mut self, slot: usize) -> Option { let end = *self.end(slot); (end.handles == 0 && end.calls == 0).then(|| { - self.slots[slot] = Slot::Free; + self.slots[slot] = Slot::Making; end }) } @@ -221,6 +224,7 @@ fn free(slot: usize, end: End) { *HALVES[slot].tx.lock() = None; syscall::close(end.bell); syscall::close(end.region); + TABLE.lock().unreserve(slot); } /// Why a stream call refused, in the protocol's words. @@ -750,6 +754,8 @@ mod tests { assert_eq!(table.forget(h(11)), Some(None), "a call is left"); assert_eq!(table.find(h(11)), None, "the handle reaches nothing"); assert_eq!(table.returned(called), Some(End { handles: 0, calls: 0, ..end() }), "the last call frees it"); + assert_eq!(table.reserve(), Some(slot + 1), "a freed slot is not taken before its halves are cleared"); + table.unreserve(slot); assert_eq!(table.slots[slot], Slot::Free); } diff --git a/system/netstack/shard/tcp/src/stack.rs b/system/netstack/shard/tcp/src/stack.rs index b7cdc101d88..d29f07a61bd 100644 --- a/system/netstack/shard/tcp/src/stack.rs +++ b/system/netstack/shard/tcp/src/stack.rs @@ -49,8 +49,9 @@ pub enum Served { Done, } -/// A segment at the moment it leaves: headers from the state of now, payload borrowed from the -/// send buffer. +/// A segment at the moment it leaves: headers from the state of now, and its payload, borrowed +/// from the send buffer where it lies in one piece of [tcp]'s own storage and otherwise copied out +/// of it first, as a lent buffer's always is. #[derive(Clone, Copy, Debug)] pub struct Outgoing<'a> { pub source: Ipv4Addr, From 1920ac48d43730451c1f39734d7bfb16fe685345 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 11:28:37 +0200 Subject: [PATCH 4/6] review round 2: a lent receive ring's window grows as a read does, shown; two records made true MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [tcp]'s net tests: an application can read its connection's lent receive ring itself and give the text back with `consume` (`Net::lend`), and `a_lent_receive_ring_grows_the_window_as_a_read_does` runs the T14-shaped download (gigabit, 15 ms, 32 MiB) that way: the window passes 65,535 and reaches the 4 MiB buffer. Red, at 65,535, when `consume` grows nothing. The T14's 23.7 Mb/s is therefore not [tcp]'s window on a lent ring; the baseline boot of main with the scout's instrument attributes it. - issues/toyos-has-its-own-network-stack.md: the §3.10.7.4 bullet says text published into a client's receive ring, not written to its pipe. - toyos::net::stream: `Slot::Making` is also an end `free` lets go of. - serve.rs: `watch`'s comment rewrapped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- issues/toyos-has-its-own-network-stack.md | 2 +- sdk/toyos/src/net/stream.rs | 3 +- system/netstack/shard/tcp/tests/common/net.rs | 56 +++++++++++++++++++ system/netstack/shard/tcp/tests/net.rs | 18 ++++++ system/netstack/src/serve.rs | 7 ++- 5 files changed, 81 insertions(+), 5 deletions(-) diff --git a/issues/toyos-has-its-own-network-stack.md b/issues/toyos-has-its-own-network-stack.md index b1e3334a740..67a85773592 100644 --- a/issues/toyos-has-its-own-network-stack.md +++ b/issues/toyos-has-its-own-network-stack.md @@ -71,7 +71,7 @@ What stage 4 departs from its specifications: - The readers' stage 4 design has a flow waiting for its next hop keep its place in the round. [shard]'s round lets it go, deficit and all, and [tcp] offers it again when woken, at the round's tail, so a waiting flow costs the round nothing. Exit: a `system/netstack/shard/tests/drr.rs` test finds a woken flow served in the place it held when it began to wait. - The readers' stage 4 design has a flow leave the round once it has nothing eligible, its deficit reset. [udp] says so as it hands out a sender's last datagram; [tcp] learns it only when it serves the connection, since only its `next_segment` decides what is due, so a connection whose turn ended on its last segment stays in the round with its deficit until its next turn, and what it queues before then is sent against that deficit, debt included, as RFC 8290 §4.2 keeps an emptied queue until it is next selected. Exit: [tcp] reports at hand-off that nothing is left, and a `system/netstack/shard/tests/drr.rs` test finds such a connection out of the round, its deficit reset. - `Tcp::lend`, `Tcp::consume` and `Tcp::queue`, the calls that make a connection's buffers a region its user reads and writes itself, are not in the specifications. Their tests (`system/netstack/shard/tcp/tests/lent.rs`) name no scenario. Exit: the specification names the calls and their scenarios, and the tests take their ids. -- RFC 9293 §3.10.7.4 says a reset in a synchronized state SHOULD flush every queue. [tcp] keeps the in-order text received before it, which `recv` and a lent buffer give ahead of `Err(Failed(Reset))` (`Ended`, `system/netstack/shard/tcp/src/stack.rs`), as Linux and the BSDs read buffered data ahead of `ECONNRESET`: the orchestrator ruled to follow them, since ToyOS's clients are written against those stacks, and since text a pass has already written to a client's pipe cannot be flushed, so only keeping it makes what a client reads independent of how the card batched frames. `s_rx_023_a_reset_comes_after_the_text_before_it` holds it. Exit: the specification names the departure, or the owner rules for the flush. +- RFC 9293 §3.10.7.4 says a reset in a synchronized state SHOULD flush every queue. [tcp] keeps the in-order text received before it, which `recv` and a lent buffer give ahead of `Err(Failed(Reset))` (`Ended`, `system/netstack/shard/tcp/src/stack.rs`), as Linux and the BSDs read buffered data ahead of `ECONNRESET`: the orchestrator ruled to follow them, since ToyOS's clients are written against those stacks, and since text a pass has already published into a client's receive ring cannot be flushed, so only keeping it makes what a client reads independent of how the card batched frames. `s_rx_023_a_reset_comes_after_the_text_before_it` holds it. Exit: the specification names the departure, or the owner rules for the flush. - The readers' TCP specification disagrees with itself, and [tcp] follows its user timeout: one replaces R2 and runs only while sent data is unacknowledged or a zero window holds data back, so with one set a connection whose data never left, for want of credit or of a next hop, has no give-up, where its rule that give-up clocks run on wall time has a device that never offers credit end its connections. Exit: RFC 9293 §3.10.8 ends a connection in any state once its user timeout expires: a test that sets one and withholds credit, as `s_pl_009_give_up_without_credit` does without one, finds the connection ended. Open for the owner. On 2026-10-03 he chose "Fair scheduler now": "Build the byte-fair scheduler as the network spec describes, about 280 lines, and decide later on the laptop whether new connections should jump the queue." It is built as one round, in the order flows became eligible, and nobody jumps it. The track reads a queue that new connections jump as RFC 8290's new-flows list, and offers him this evidence: a sparse flow such as a DNS query waits one turn of every flow ahead of it in the round, up to a quantum of 1,514 bytes each, where [udp]'s datagrams and [tcp]'s segments alternated frame by frame before, so a query waited at most one TCP frame: in DRR-02 a 100-byte segment that becomes eligible leaves behind a full frame of each of the three bulk flows ahead of it (`s_shard_drr_002_a_bulk_flow_starves_no_other_beyond_its_quantum`); a T14 measurement at stage 5 of a sparse flow's wait behind bulk flows would add to it. Exit: the owner rules on whether new connections jump the queue. diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index 0521c1a45f3..17dfc9c017c 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -126,7 +126,8 @@ struct End { #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Slot { Free, - /// Taken by a connect or an accept that has not answered. + /// Reserved: by a connect or an accept that has not answered, or by an end + /// [`free`] is letting go of. Making, Live(End), } diff --git a/system/netstack/shard/tcp/tests/common/net.rs b/system/netstack/shard/tcp/tests/common/net.rs index 20b10075b58..44d40a72ff7 100644 --- a/system/netstack/shard/tcp/tests/common/net.rs +++ b/system/netstack/shard/tcp/tests/common/net.rs @@ -76,6 +76,33 @@ pub struct App { pub reading: bool, /// Write at most this many bytes per pass; `None` writes all the stack takes. pub write_limit: Option, + /// The receive buffer lent ([`Net::lend`]): read out of it and given back with `consume`. + pub lent: Option, +} + +/// A lent buffer the test holds beside [tcp]: a ring of bytes [tcp] copies into. +#[derive(Clone, Debug)] +pub struct Shared(pub std::rc::Rc>>); + +impl toyos_net_tcp::Lent for Shared { + fn len(&self) -> usize { + self.0.borrow().len() + } + + fn put(&mut self, at: usize, data: &[u8]) { + self.0.borrow_mut()[at..at + data.len()].copy_from_slice(data); + } + + fn take(&self, at: usize, out: &mut [u8]) { + out.copy_from_slice(&self.0.borrow()[at..at + out.len()]); + } +} + +/// An application's lent receive ring and where its next unread byte lies. +#[derive(Debug)] +pub struct Lending { + rx: Shared, + at: usize, } /// What the link does with one datagram. @@ -240,6 +267,7 @@ impl Net { read_limit: None, reading: true, write_limit: None, + lent: None, }); } @@ -291,6 +319,24 @@ impl Net { app.shut = true; } let mut budget = app.read_limit.unwrap_or(usize::MAX); + if let Some(lending) = app.lent.as_mut().filter(|_| app.reading && app.end.is_none()) { + let readable = node.tcp.status(app.id).map_or(0, |s| s.readable).min(budget); + let ring = lending.rx.0.borrow(); + let bytes: Vec = (0..readable).map(|i| ring[(lending.at + i) % ring.len()]).collect(); + lending.at = (lending.at + readable) % ring.len(); + drop(ring); + app.received.feed(&bytes); + assert_eq!(node.tcp.consume(now, app.id, readable), Ok(readable)); + if readable == 0 { + match node.tcp.recv(now, app.id, &mut []) { + Ok(Received::End) => app.end = Some(End::Fin), + Ok(Received::Data(_)) | Err(Error::WouldBlock) => {} + Err(Error::Failed(f)) => app.end = Some(End::Failed(f)), + Err(e) => panic!("recv: {e:?}"), + } + } + budget = 0; + } while app.reading && app.end.is_none() && budget > 0 { let mut buf = vec![0u8; budget.min(65_536)]; match node.tcp.recv(now, app.id, &mut buf) { @@ -447,6 +493,16 @@ impl Net { } /// Every application finished: both FINs in, or a failure. + /// App `app`'s connection lent two buffers of `len` bytes; it reads the receive ring itself + /// from here on. + pub fn lend(&mut self, app: usize, len: usize) { + let ring = || Shared(std::rc::Rc::new(std::cell::RefCell::new(vec![0; len]))); + let (rx, tx) = (ring(), ring()); + let App { node, id, .. } = self.apps[app]; + assert_eq!(self.nodes[node].tcp.lend(id, Box::new(rx.clone()), Box::new(tx)), Ok(())); + self.apps[app].lent = Some(Lending { rx, at: 0 }); + } + pub fn finished(&self) -> bool { self.apps.iter().all(|a| a.end.is_some() && (a.shut || matches!(a.end, Some(End::Failed(_))))) && !self.apps.is_empty() } diff --git a/system/netstack/shard/tcp/tests/net.rs b/system/netstack/shard/tcp/tests/net.rs index 91b753e36f9..61e5ff4a529 100644 --- a/system/netstack/shard/tcp/tests/net.rs +++ b/system/netstack/shard/tcp/tests/net.rs @@ -498,6 +498,24 @@ fn a_peer_sending_segments_shorter_than_our_send_mss_still_grows_the_window() { assert_eq!(peaks[0].window, BUFFER, "{:?}", peaks[0]); } +/// The same download into a lent receive ring, as a stream's region is: the reader takes text out +/// of the ring itself and gives it back with `consume`, which grows the window as a read does, so +/// the window passes 65,535 and reaches the whole buffer. +#[test] +fn a_lent_receive_ring_grows_the_window_as_a_read_does() { + let mut net = Net::buffered(15, BUFFER); + net.nodes[1].credit_per_ms = Some(125_000); + net.connections(1, 443, [0, 32 * MIB]); + assert!(net.run(100, |n| n.apps.len() == 2)); + let reader = net.apps.iter().position(|a| a.node == 0).unwrap(); + net.lend(reader, BUFFER as usize); + let peaks = watch(&mut net, 600_000); + assert!(net.finished(), "the transfer did not finish:\n{}", net.dump()); + net.assert_exact(); + assert!(peaks[0].window > 65_535, "{:?}", peaks[0]); + assert_eq!(peaks[0].window, BUFFER, "{:?}", peaks[0]); +} + /// RFC 8985 §7.3: no probe leaves unless an RTT sample came since the last. Without timestamps a /// retransmission voids the round's timing, so once the path's round trip passes twice the SRTT /// a probe every round would leave before every ACK and SRTT would never move again. Here, after diff --git a/system/netstack/src/serve.rs b/system/netstack/src/serve.rs index 8a3e1edca51..036855e1a43 100644 --- a/system/netstack/src/serve.rs +++ b/system/netstack/src/serve.rs @@ -732,9 +732,10 @@ impl Sockets { } /// Asks the kernel about everything a pass could be owed for: each - /// stream's connection, for its client's rings and its hang-up, the wake pipe of each listener and one pipe of each - /// datagram socket for its owner's leaving, and the connection of each - /// client that waits for a lookup, which hangs up by closing it. + /// stream's connection, for its client's rings and its hang-up, the wake + /// pipe of each listener and one pipe of each datagram socket for its + /// owner's leaving, and the connection of each client that waits for a + /// lookup, which hangs up by closing it. pub fn watch(&self, poller: &mut Watches) { for (key, stream) in &self.streams { let Some(connection) = stream.connection.upgrade() else { continue }; From 3375d0b6e820bc75405e72f0e2634bc1de1181ec Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 11:55:30 +0200 Subject: [PATCH 5/6] issues: the T14's first download after boot runs slower than the rest Round 3's T14 boots of PR #905 read run 1 at 107 Mb/s on main's pipe path and 470 Mb/s on stage 3, against 722 to 878 Mb/s for runs 2 to 6 on both, with the CPU idle: a first-run stall independent of stage 3, filed with its candidates and exit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...ad-after-boot-runs-slower-than-the-rest.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 issues/the-t14s-first-download-after-boot-runs-slower-than-the-rest.md diff --git a/issues/the-t14s-first-download-after-boot-runs-slower-than-the-rest.md b/issues/the-t14s-first-download-after-boot-runs-slower-than-the-rest.md new file mode 100644 index 00000000000..745907328eb --- /dev/null +++ b/issues/the-t14s-first-download-after-boot-runs-slower-than-the-rest.md @@ -0,0 +1,33 @@ +--- +status: open +kind: defect +opened: 2026-10-14 +--- + +# The T14's first download after boot runs slower than the rest + +On the T14, `internet_download`'s first run of a boot is far slower than its +later runs, with the CPU idle, through main's pipe path and through stage 3's +region alike. Round 3 of PR #905, one boot per image, the same 170,439,044-byte +file each run, at a 16 ms round trip: + +| image | run 1 | runs 2 to 6 | +|---|---|---| +| main `f0a987bc6` with the instrument (pipes) | 12.747 s, 107.0 Mb/s, busy 0.016 | 778 to 868 Mb/s | +| stage 3 (`toyos::net::stream`'s region) | 2.900 s, 470.1 Mb/s, busy 0.057 | 722 to 878 Mb/s | + +The round before, stage 3's image read 101,985,020 bytes in 34.4 s (23.7 Mb/s, +busy 0.007) before the job's window cut its first run, which had no later runs +to compare. The readings are on `ToyOSOrg/evidence` branch `pr-905`, +`round2/metal/` and `round3/metal/`. Nothing on the host or in QEMU has shown it. + +Candidates, none measured: the I219 driven in MSI mode since #917, which came +into main between the first boot that downloaded fast (the scout's, all six +runs whole) and these; the client's wake to read; or a connection's warm-up in +netstack ([tcp]'s receive window grows by what is read per round trip, and a +lent ring's growth is shown by `a_lent_receive_ring_grows_the_window_as_a_read_does`). + +**Exit condition**: on the T14, `internet_download`'s first run falls within +the band of the boot's other runs. + +**Owner**: netstack. From 4a9492a104ab5a11b736edf7fa9c94a541ad34c1 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 12:17:40 +0200 Subject: [PATCH 6/6] review round 3: stage 3 marked done in its track, with its T14 exit readings; `finished` keeps its doc - issues/netstack-moves-every-byte-through-two-kernel-pipes.md: stage 3 is done, as stages 1 and 2 are: what it built, the regression and the second send copy it leaves, and the T14 exit as read against main's baseline under the same instrument (pipe_copy_bytes 0.000135 per byte against 2.0029; hold, spin, C0 and rate), linked to the evidence commit. - [tcp]'s net harness: `finished`'s doc comment is back on `finished`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...ves-every-byte-through-two-kernel-pipes.md | 110 +++--------------- system/netstack/shard/tcp/tests/common/net.rs | 2 +- 2 files changed, 16 insertions(+), 96 deletions(-) diff --git a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md index 1c08c57f899..a6e43f0693c 100644 --- a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md +++ b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md @@ -54,101 +54,21 @@ The design and its roast are PR #863's body. - Stage 2, done (PR #878): a byte ring with untrusted cursors and want words in `toyos-transport` (`lib/blockring/transport`), host-tested and modelled in loom. -- Stage 3 (PR #905): a TCP stream is one region and one connection, a - connected one and an accepted one alike, as the orchestrator ruled on #905 - (R1): netstack's receive and send buffers are the region's two rings, and - [tcp]'s heap storage stays for a passive child before its accept, at most - `limits::RECEIVE_BUFFER_INITIAL` a child and `limits::LISTEN_READY` - children a listener. An accept takes one of the wakes netstack writes on the - listener's pipe for each child that waits before it makes a region, so a - nonblocking accept with none answers `WouldBlock` having made nothing; it - lends the region, netstack copies what the child held into the receive ring - at its offsets, text past a hole included and the FIN [tcp]'s state, before - it answers, and the answer carries the peer's address and the `nodelay` the - child inherited from its listener. `SYS_CONNECTION_JOIN` goes. The - connection carries the connect or the accept and its answer, then only - doorbells and the hang-up. A stream belongs to the process that connected - or accepted it: `toyos::net` refuses by name a handle it holds no end for. - netstack rings a reader on the connection and a writer waiting for send room - on a pipe of the stream's own, so neither drains a wake rung for the other. - Within a direction no thread holds the end across its wait, and a thread - that took the doorbell while another waits on it passes the take on through - netstack. A registration arms the want of each direction it names and rings - netstack if that direction is already ready, and netstack rings back. A - fatal `end`, any but a FIN, makes the send direction ready to every look, as - POSIX's `select` counts a descriptor ready for writing when a write "would - not block, whether or not the function would transfer data successfully", - and a `write` reads it before it copies, answering its error and copying - nothing, as RFC 9293 flushes every segment queue at a reset or a user - timeout (§3.10.7.4, §3.10.8) and answers a SEND that finds no connection an - error (§3.10.2). std and libc read and write the rings through - `toyos::net`, whose table of the process's ends is the instance std links - (R2): the mio and socket2 forks reach a stream through std, and - `std::os::toyos::net` gives them what std has no word for, the bell to watch - and the registration. `toyos-transport` and `toyos-untrusted`, whose code - the client end is, are published SDK crates in std's graph (R3). std gains - `FromRawFd`, `IntoRawFd` and the `OwnedFd` conversions for `TcpStream`, - which take only the process's own ends, and the `rust/` fork's ToyOS arms of - `OwnedFd`'s drop and `BorrowedFd::try_clone_to_owned` let go of and enter a - stream's handle through `toyos::net`. A stream's handle reads doorbells, so - `File::from_raw_fd` of one panics with `toyos::net`'s refusal, and every - path std offers to move a raw handle into another process asks - `toyos::net` of each handle before anything moves and answers its refusal: - a spawn's stdio slots, `CommandExt::inherit_handle` into any slot, - `CommandExt::endow` and `Command::provide`, each with no child started and - the handle still the caller's, so no refused move leaves an entry that - outlives its handle's number. `toyos::ipc`'s `send_handles` is refused by - its type: it takes `OwnedHandle`s, and a stream's handle becomes one only - through `OwnedHandle::from_raw`, whose `# Safety` excludes a handle - `toyos::net` answers for. The bare `dup`, `SYS_HANDLE_SEND` and `SYS_SPAWN` - syscalls stay unchecked. A regression, - `issues/a-stream-cannot-move-between-processes.md`. Exit: three loom models - that drive the client end and netstack's ringing code as stage 3 lands - them, each red under a control that reverts its decision. In one, a - stream's reader and writer, each asleep on its own want while netstack's - thread releases room and publishes data in either order, end with neither - waiting, and the control rings both wants on one doorbell. In another, a - blocking call and a `poll` of one direction, the poll's watch answered by - the doorbell's level at its look as the kernel's `deliver` answers it and - the blocking call leaving bytes, the receive ring's `end` or room behind, - end with the poll answered, and the control does not pass the take on. In - the third, a source registered while its direction is ready, in both - directions, with the receive ring's `end`, and with a fatal `end` while the - send ring is full, made ready before the registration or racing it, gets - its first event; it is red without the registration's arm and ring, red - with either of the two removed, red under stage 2's `want-relaxed`, and red - with the fatal `end` where a look reads only the send ring's head. Over the - region, `system/netstack/node/tests/listeners.rs`'s - `a_listener_answers_a_syn_and_wakes_its_owner_when_the_handshake_ends` - passes, and so does a case in which a child holds, before its accept, text - in order, text past a hole and its FIN behind that, the hole is filled after - the accept, and the region reads all of it in order, then its `end`. That - case is red under each of three controls that drop at the accept one thing - the child held: its in-order bytes, which also reds the first, its - out-of-order range, its FIN. A test: `toyos::net`, handed a stream handle - the process holds no end for, refuses it by name and writes nothing on its - connection, and a stream's own handle, given out by `into_raw_fd` or - `From for OwnedFd` or made by `try_clone_to_owned`, and taken - back, shares the one end and its cursors, red when giving the handle out - lets its entry go. A test: `File::from_raw_fd` of a stream's handle panics - with that refusal, red when it takes the handle silently. A test per moving - path, each handed a stream's handle: a spawn's stdio slot, `inherit_handle` - into a slot past the stdio slots, `endow` and `provide` each answer the - refusal and start no child, and the stream then reads and writes through - that handle; each is red when its path takes the handle silently, and - `endow`'s and `provide`'s are red when the handle moves and its entry stays. - A test: a `write` with room after a reset answers `ConnectionReset` and - copies nothing, red when `write` copies before it reads the `end`. And a T14 - boot of round 3's `internet_download` on stage 3's head reads, in every run, - `pipe_copy_bytes` under 0.01 per byte downloaded, its C0, hold and spin per - MB posted beside the scout's rows. No tree that lands has that row or that - counter: the boot carries the scout's instrument, `git diff 777c96b2b - 2f9e090b8` (`wt/toyos-netring-scout` over its base), onto stage 3's head, - less the netstack stamps on the pipe writes stage 3 deletes. As built, a - sent byte is copied twice by netstack, out of the lent send ring into a - scratch buffer and from it into the frame, since the frame builder takes - one slice; a builder that copies each span of the ring into the frame - itself would remove the first. +- Stage 3, done (PR #905): a TCP stream, connected or accepted, is one + region whose two rings are [tcp]'s buffers and the connection it was opened + on, which then carries only doorbells and the hang-up; std, libc, mio and + socket2 reach it through `toyos::net::stream`, and `SYS_CONNECTION_JOIN` is + gone. A stream's handle cannot move to another process + (`issues/a-stream-cannot-move-between-processes.md`). A sent byte is copied + twice by netstack, out of the lent send ring into a scratch buffer and from + it into the frame, since the frame builder takes one slice; a builder that + copies each span of the ring into the frame itself would remove the first. + The exit read on the T14, `internet_download`, six whole runs per image, + against main `f0a987bc6` under the same instrument, medians of runs 2 to 6 + per MB: `pipe_copy_bytes` 0.000135 per byte downloaded (every run under + 0.00015) against 2.0029, the pipe lock held 0.179 ms against 0.587 and spun + 0.092 against 0.108, C0 7.735 ms against 7.770, 812.7 Mb/s against 824.1 + (https://github.com/ToyOSOrg/evidence/tree/7cc1bda0dcb038084de60d65c64f30a3956a13a3/round3). - Stage 4: a listener's connection carries its wakes in place of its pipe, a datagram socket is a region as a stream is, and the pipe ABI's messages and `toyos::net`'s pipe handles go. netstack copies each datagram record's diff --git a/system/netstack/shard/tcp/tests/common/net.rs b/system/netstack/shard/tcp/tests/common/net.rs index 44d40a72ff7..119340ea15b 100644 --- a/system/netstack/shard/tcp/tests/common/net.rs +++ b/system/netstack/shard/tcp/tests/common/net.rs @@ -492,7 +492,6 @@ impl Net { true } - /// Every application finished: both FINs in, or a failure. /// App `app`'s connection lent two buffers of `len` bytes; it reads the receive ring itself /// from here on. pub fn lend(&mut self, app: usize, len: usize) { @@ -503,6 +502,7 @@ impl Net { self.apps[app].lent = Some(Lending { rx, at: 0 }); } + /// Every application finished: both FINs in, or a failure. pub fn finished(&self) -> bool { self.apps.iter().all(|a| a.end.is_some() && (a.shut || matches!(a.end, Some(End::Failed(_))))) && !self.apps.is_empty() }