From c6a2861e8f8a587c1083ad8dfc83fec5a3beffd8 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 12 Oct 2026 21:21:44 +0200 Subject: [PATCH 1/3] tests: cut netstack_socket_churn, an integration test of a userland program netstack_socket_churn starts netstack in a guest and drives it through a sequence of steps against the harness's host servers, which reviewer.md's Tests rule (#890) refuses outside the kernel: it is cut, and what it checked is held by netstack's own tests or recorded as owed. What it reached of the node is held by system/netstack/node/tests: listeners.rs's a_connect_past_the_places_is_refused_and_sends_nothing, a_departed_clients_connection_holds_its_place_until_it_is_cut and a_wake_the_owners_pipe_refuses_ends_the_listener, and streams.rs's a_client_that_left_is_finished_by_the_stack, a_writer_that_leaves_after_the_shutdown_leaves_the_stream_to_its_reader, a_pipe_that_is_no_pipe_resets_its_connection and a_refused_watch_resets_only_a_stream_that_holds_the_end. What only system/netstack/src/serve.rs does has no test since this cut: a second receive refused ERR_RESOURCE_EXHAUSTED, a waiting receive answered by its datagram (settle), and the kernel's watch answers that end a stream, a listener or a datagram socket (Sockets::answered). issues/servers-let-a-client-go-on-the-handshake-limit-with-its-request-unread.md records them as owed. It was also red once, in a whole cargo test at be929d941: netstack let client 86031 go on its handshake limit, and the connect loop read NetstackNotFound where it asserts ResourceExhausted. That defect is the record above. The cut takes the guest binary, its RUST_SKIP and MACHINE_TESTS entries, its harness function and dispatch arm, and the netcase image's comment that named it as a reason test-runner receives netstack; netstack_streams, libc_sockets and https_fetch still need that grant. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- .../src/bin/netstack_socket_churn.rs | 209 ------------------ tests/images/netcase.toml | 2 - tests/toyos.rs | 40 ---- 3 files changed, 251 deletions(-) delete mode 100644 tests/guest/rust-tests/src/bin/netstack_socket_churn.rs diff --git a/tests/guest/rust-tests/src/bin/netstack_socket_churn.rs b/tests/guest/rust-tests/src/bin/netstack_socket_churn.rs deleted file mode 100644 index 0534d66d57a..00000000000 --- a/tests/guest/rust-tests/src/bin/netstack_socket_churn.rs +++ /dev/null @@ -1,209 +0,0 @@ -//! Piped TCP connections that end with no close request from their client, as -//! a client that died leaves them: once netstack has let each connection go, -//! its stream count is what it was before the first, and so is its count of -//! places held, once the stack has finished each connection alone. -//! -//! argv[1] is the port of the harness's host server, which ends every -//! connection it accepts at once. Each round reads the peer's end of stream, -//! drops both pipe ends, and waits for netstack's own count of live piped -//! connections to return. -//! -//! argv[2] is the port of a second, which holds every connection it accepts -//! and reads nothing. On it, a client that moves netstack a handle that is no -//! pipe end where its receive end belongs, and keeps its send end: the kernel -//! refuses netstack's watch of it, and that refusal is all that ends the -//! connection. -//! -//! On the same server, a client that shut its sending half down and left: -//! netstack let its send pipe go at the shutdown and its peer sends nothing, -//! so nothing but the kernel's word on its receive pipe's other end can tell -//! netstack it is gone. -//! -//! A listener and a datagram socket whose owner drops its pipe ends with no -//! close request, and no peer near either: the kernel's word on the pipe's -//! other end is all that ends them, and netstack's count of each returns. -//! -//! argv[3] is the port of a third, which answers each datagram with itself. A -//! receive is asked of netstack before any datagram has been sent, so it waits -//! there, and is answered by the datagram the server sends back; a second -//! receive on that socket while the first waits is refused. -//! -//! Last, connects to the holding server until netstack refuses one: it holds -//! as many places as it said it has, and the connect past them is answered -//! `ResourceExhausted`. Last because the stack keeps each of those -//! connections' places until it has finished them with a peer that never -//! ends its half. - -use std::time::{Duration, Instant}; - -use toyos::net::{ - MsgType, NetError, NetstackConn, TcpConnectPipedRequest, TcpConnectResponse, UdpRecvFromRequest, UdpRecvResponse, -}; -use toyos::OwnedHandle; -use toyos_inspect::{Value, NET}; - -/// The host, as QEMU's user network names it to a guest. -const HOST: [u8; 4] = [10, 0, 2, 2]; - -const ROUNDS: usize = 4; - -/// A hang ceiling on netstack letting one ended connection go: it does so on -/// the pass the client's leaving wakes, and the stack finishes the connection -/// when the peer acknowledges its FIN. -const LET_GO: Duration = Duration::from_secs(20); - -fn count(key: &str) -> u64 { - let snapshot = inspect::ask(NET).unwrap_or_else(|why| panic!("netstack's inspect answer: {why}")); - match snapshot.get(key) { - Some(Value::U64(n)) => *n, - other => panic!("netstack's snapshot carries {key} as {other:?}"), - } -} - -/// Asks netstack for `key` until it is `wanted`: each question is a round -/// trip netstack answers on a pass of its own. -fn returns(key: &str, wanted: u64, what: &str) { - let asked = Instant::now(); - loop { - let found = count(key); - if found == wanted { - return; - } - assert!(asked.elapsed() < LET_GO, "netstack's {key} is {found} and not {wanted} {LET_GO:?} after {what}"); - } -} - -/// A connection the peer holds open, whose client shuts its sending half down -/// and is gone. -fn left_after_a_shutdown(port: u16) { - let live = count("net.piped.live"); - let conn = toyos::net::tcp_connect(HOST, port, 0).unwrap_or_else(|e| panic!("the holding server: {e:?}")); - assert_eq!(count("net.piped.live"), live + 1, "netstack counts the connection it just answered"); - toyos::net::tcp_shutdown(conn.socket_id, 1).unwrap_or_else(|e| panic!("shutting the sending half: {e:?}")); - drop(conn); - returns("net.piped.live", live, "a client that shut its sending half down dropped both its ends"); -} - -fn owners_that_left_are_let_go() { - let (listeners, udp) = (count("net.sockets.listeners"), count("net.sockets.udp")); - let listener = toyos::net::tcp_bind([0, 0, 0, 0], 0).unwrap_or_else(|e| panic!("a listener: {e:?}")); - let socket = toyos::net::udp_bind([0, 0, 0, 0], 0).unwrap_or_else(|e| panic!("a datagram socket: {e:?}")); - assert_eq!(count("net.sockets.listeners"), listeners + 1, "netstack counts the listener it just answered"); - assert_eq!(count("net.sockets.udp"), udp + 1, "netstack counts the datagram socket it just answered"); - // Their pipe ends, and no close request. - drop(listener); - drop(socket); - returns("net.sockets.listeners", listeners, "a listener's owner dropped its wake pipe"); - returns("net.sockets.udp", udp, "a datagram socket's owner dropped its pipes"); -} - -fn a_receive_that_waits_is_answered(port: u16) { - const SAID: &[u8] = b"a datagram for a receive that waits"; - let socket = toyos::net::udp_bind([0, 0, 0, 0], 0).unwrap_or_else(|e| panic!("a datagram socket: {e:?}")); - let waiting = NetstackConn::connect() - .and_then(|netstack| { - netstack.request(MsgType::UdpRecvFrom, &UdpRecvFromRequest { socket_id: socket.socket_id.0, max_len: 64 }) - }) - .unwrap_or_else(|e| panic!("asking for a datagram: {e:?}")); - // netstack reads its clients' requests in the order they connected, so - // this answer says the receive above is waiting there. - count("net.sockets.udp"); - let second = toyos::net::udp_recv_from(socket.socket_id, 64); - assert_eq!(second.err(), Some(NetError::ResourceExhausted), "a second receive on a socket whose first waits"); - assert_eq!(socket.tx.write_nonblock(SAID), Ok(SAID.len()), "a datagram into the socket's send pipe"); - toyos::net::udp_send_to(socket.socket_id, HOST, port, SAID.len() as u16) - .unwrap_or_else(|e| panic!("sending to the answering server: {e:?}")); - let answer: UdpRecvResponse = waiting.response().unwrap_or_else(|e| panic!("the receive that waited: {e:?}")); - assert_eq!((answer.addr, answer.port, usize::from(answer.len)), (HOST, port, SAID.len()), "the answer's source and length"); - let mut back = [0u8; 64]; - assert_eq!(socket.rx.read(&mut back), Ok(SAID.len()), "the answer's bytes in the socket's receive pipe"); - assert_eq!(&back[..SAID.len()], SAID); - toyos::net::udp_close(socket.socket_id).unwrap_or_else(|e| panic!("closing the datagram socket: {e:?}")); -} - -fn a_connect_past_the_places_is_refused(port: u16) { - let (max, held) = (count("net.places.max"), count("net.places.held")); - let mut kept = Vec::new(); - let refused = loop { - match toyos::net::tcp_connect(HOST, port, 0) { - Ok(conn) => kept.push(conn), - Err(refusal) => break refusal, - } - assert!(kept.len() as u64 <= max, "netstack holds more connections than the {max} places it said it has"); - }; - assert_eq!(refused, NetError::ResourceExhausted, "the connect past netstack's places"); - assert_eq!( - kept.len() as u64, - max - held, - "netstack said it has {max} places with {held} held, and refused the connect after {} more", - kept.len() - ); - assert_eq!(count("net.places.held"), max, "every place is held where the connect was refused"); -} - -/// A connection the peer holds open, whose client keeps its send end and moved -/// netstack an acceptor for the end netstack writes the peer's bytes into. The -/// peer sends none, so netstack never writes it: its watch is refused, and the -/// connection is reset and let go. -fn a_receive_end_that_is_no_pipe_end_is_refused(port: u16) { - let live = count("net.piped.live"); - let (acceptor, _connector) = toyos::port::create().expect("a port"); - // SAFETY: the acceptor's one handle, which nothing else answers for. - let no_pipe_end = unsafe { OwnedHandle::from_raw(acceptor.into_raw()) }; - let (from_client, kept) = toyos::pipe_pair().expect("a pipe"); - let connected: TcpConnectResponse = NetstackConn::connect() - .and_then(|netstack| { - netstack.request_with_handles( - [no_pipe_end, OwnedHandle::from(from_client)], - MsgType::TcpConnectPiped, - &TcpConnectPipedRequest { addr: HOST, port, _pad: 0, timeout_ms: 0 }, - ) - }) - .and_then(|pending| pending.response()) - .unwrap_or_else(|e| panic!("the holding server, with an acceptor for a receive end: {e:?}")); - returns( - "net.piped.live", - live, - &format!("the kernel refused the watch of connection {}'s receive end, which is no pipe end", connected.socket_id), - ); - drop(kept); -} - -fn main() { - let port = |at: usize| -> u16 { - std::env::args() - .nth(at) - .and_then(|p| p.parse().ok()) - .expect("usage: netstack_socket_churn ") - }; - let (port, holding, answering) = (port(1), port(2), port(3)); - let (streams, live) = (count("net.sockets.tcp"), count("net.piped.live")); - let held = count("net.places.held"); - for round in 1..=ROUNDS { - let conn = toyos::net::tcp_connect(HOST, port, 0) - .unwrap_or_else(|e| panic!("connection {round} to the host server: {e:?}")); - let mut byte = [0u8; 1]; - assert_eq!(conn.rx.read(&mut byte), Ok(0), "connection {round}: the host ends it without a byte"); - // Both pipe ends, and no close request. - drop(conn); - returns("net.piped.live", live, &format!("both ends of connection {round} closed")); - } - let left = count("net.sockets.tcp"); - println!( - "netstack_socket_churn: {ROUNDS} connections ended and let go; netstack held {streams} stream(s) before \ - them and holds {left} after" - ); - assert_eq!(left, streams, "netstack keeps a stream for a connection it let go"); - returns("net.places.held", held, "the last of the connections it let go ended on the wire"); - a_receive_end_that_is_no_pipe_end_is_refused(holding); - println!("netstack_socket_churn: a connection whose receive end is no pipe end was reset"); - left_after_a_shutdown(holding); - println!("netstack_socket_churn: a client that shut down and left was let go"); - owners_that_left_are_let_go(); - println!("netstack_socket_churn: a listener and a datagram socket whose owner left were let go"); - a_receive_that_waits_is_answered(answering); - println!("netstack_socket_churn: a receive that waited was answered by its datagram"); - a_connect_past_the_places_is_refused(holding); - println!("netstack_socket_churn: the connect past netstack's places was refused"); - println!("netstack_socket_churn: ok"); -} diff --git a/tests/images/netcase.toml b/tests/images/netcase.toml index e956a131d72..a6c56e547cf 100644 --- a/tests/images/netcase.toml +++ b/tests/images/netcase.toml @@ -33,8 +33,6 @@ serves = ["soundserver"] devices = ["pci:1af4:1059"] syscap = ["rt"] -# `netstack` because `netstack_socket_churn` connects through it and reads its -# counts. [programs.test-runner] receives = ["netstack"] diff --git a/tests/toyos.rs b/tests/toyos.rs index c5b2f432033..f95ce314318 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -138,9 +138,6 @@ const RUST_SKIP: &[&str] = &[ // launches ask DATA's server while this job's share holds all it may: the // `fs_share` metal row runs it on tests/images/proctreecase.toml. "fs_share", - // It needs a NIC in front of netstack and a host server behind it: - // `netstack_socket_churn` runs it on `tests/images/netcase.toml`. - "netstack_socket_churn", // It needs a NIC in front of netstack, a host server behind it and a host // that dials its listener: `netstack_streams` runs it on `tests/images/netcase.toml`. "netstack_streams", @@ -337,10 +334,6 @@ const MACHINE_TESTS: &[&str] = &[ // in reach, and the register sequence and the DMA it frees are no host // test's. "amdvi_firmware_left", - // netstack's own state behind a real stack and a peer that ends its - // connections: netstack is one binary that owns its NIC, with no host - // build, and the T14's peer is the bench's network. - "netstack_socket_churn", // The stack against a TCP its writers did not write, the host kernel's // behind QEMU's user network, through the kernel's own pipes and each of // netstack's two drivers: netstack has no host build, the stack's host @@ -3447,38 +3440,6 @@ fn scanout_wc(console: &str) -> Result<(), String> { Ok(()) } -/// netstack's stream count returns once connections that ended without their -/// client's close request are let go, a client that left a connection its -/// peer holds is counted gone, and a connection whose receive end the kernel -/// refuses netstack's watch of is reset; a listener and a datagram socket -/// whose owner left are let go, a receive that waits is answered by its -/// datagram, and the connect past netstack's places is refused. One host -/// server here ends each connection it accepts at once, one holds each and -/// one answers each datagram; the guest's comparisons are the verdict. -fn netstack_socket_churn() -> Result<(), String> { - const JOB: &str = "netstack_socket_churn"; - let server = std::net::TcpListener::bind(("127.0.0.1", 0)).map_err(|e| format!("the host server: {e}"))?; - let port = server.local_addr().map_err(|e| format!("the host server's port: {e}"))?.port(); - // Ends with the process: a guest that never dials leaves it in `accept`. - thread::spawn(move || server.incoming().for_each(drop)); - let (holding, answering) = (holding_server()?, answering_server()?); - - let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); - let mut qemu = boot_netcase(&[], &[(JOB.to_string(), bin)], BootOptions::default())?; - let result = qemu - .run_test(&format!("test_rs_netstack_socket_churn {port} {holding} {answering}"), Duration::from_secs(120)); - if let Some(why) = &result.error { - return Err(format!("{why}\nthe job said:\n{}", result.stdout)); - } - if result.exit_code != Some(0) { - return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); - } - if !result.stdout.lines().any(|l| l.trim_end().ends_with("netstack_socket_churn: ok")) { - return Err(format!("the guest never said it was done:\n{}", result.stdout)); - } - Ok(()) -} - /// What one job of netstack's said, as its verdict: it ended by itself, with /// 0, and with its last line. fn job_ok(job: &str, result: &qemu::TestResult) -> Result<(), String> { @@ -3902,7 +3863,6 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { match name { "iommu_virtio_platform" => common::iommu::iommu_virtio_platform(test_config), "amdvi_firmware_left" => common::iommu::amdvi_firmware_left(test_config), - "netstack_socket_churn" => netstack_socket_churn(), "netstack_streams" => netstack_streams(qemu::Profile::Headless), "netstack_streams_e1000e" => netstack_streams(qemu::Profile::HeadlessE1000e), "libc_sockets" => libc_sockets(), From 042485e0ee363d93089019ec13273dcb54d712b7 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 12 Oct 2026 22:31:43 +0200 Subject: [PATCH 2/3] issues: servers let a client go on the handshake limit with its request unread Eight servers carry a private two-second HANDSHAKE_TIMEOUT, and six of them let a connection go on it before reading what the same wait reported, so a request already whole is dropped when the pass that would read it comes late. The supervisor and fileserver serve first, as 5a68db78b made init do after a stop asked at 2.196 s was dropped unread at 4.753 s; that commit is the recorded real failure of the same order. netstack_socket_churn went red that way once, at be929d941. The record covers soundserver's control slots and the surface hosts, servers under the same doctrine, which hold a silent connection with no limit and accept and close one past a full table. Its exit, under the owner's rulings on PR #897: every server reads before it lets go; the limit stays, logged by name, as one declaration beside toyos::ipc::FrameRx that takes its clock, with a host test of the order, and a length taken from nginx's and Apache httpd's published defaults; no fresh whole-limit wait; a full table blocks, as RFC 9112 section 9.5 asks; the supervisor's power port gets a table of its own; a connection let go is closed with no answer, as nginx does, and its client names it as the server letting it go, never NetstackNotFound; and system/netstack/src/serve.rs gets tests for what the cut churn test alone reached. The tests that hold the rest of what it reached are named. The owner is the orchestrator: nobody holds the network track. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...handshake-limit-with-its-request-unread.md | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 issues/servers-let-a-client-go-on-the-handshake-limit-with-its-request-unread.md diff --git a/issues/servers-let-a-client-go-on-the-handshake-limit-with-its-request-unread.md b/issues/servers-let-a-client-go-on-the-handshake-limit-with-its-request-unread.md new file mode 100644 index 00000000000..9dcfe529ded --- /dev/null +++ b/issues/servers-let-a-client-go-on-the-handshake-limit-with-its-request-unread.md @@ -0,0 +1,143 @@ +--- +status: open +kind: defect +opened: 2026-10-12 +--- + +# Servers let a client go on the handshake limit with its request unread + +Eight servers hold each connection they accept in a table until its first request is whole, and +let it go `HANDSHAKE_TIMEOUT`, 2 s, after its accept. Each declares its own copy: +`system/compositor/src/client.rs`, `system/diskserver/src/main.rs`, +`system/filepicker/src/main.rs`, `system/fileserver/src/main.rs`, +`system/logkeeper/src/inspect.rs`, `system/netstack/src/client.rs`, +`system/supervisor/src/main.rs` and `system/usbd/src/main.rs`. The limit is what drains the table, +so that a client that connects and says nothing cannot hold it; the defect is when a pass applies +it. Two more servers hold such a connection with no limit at all: soundserver's control plane, 63 +slots (`MAX_CONTROL_CLIENTS`, `system/soundserver/src/control.rs`), and every surface host, +console's and terminal's, 4 (`toyos::surface::MAX_CLIENTS`, `sdk/toyos/src/surface.rs`). Lines are +at `1a7464692`. + +**The order.** Six let a connection go on the limit before they read what the same wait reported: +netstack (`main.rs:449-456`, ahead of its read at `:498`), the compositor (`session.rs:353-360`, +ahead of `take_frames` at `:379`), diskserver (`main.rs:687-688`), filepicker (`main.rs:537-544`), +logkeeper (`inspect.rs:153-154`) and usbd (`main.rs:136-139`, each connection's limit checked +before it is read). A request already whole in its connection is dropped when the pass that would +read it comes late, so the limit times the server's passes as well as its client, and a machine +whose CPUs are busy elsewhere is late with nothing done wrong. Filepicker's picker session holds +its loop (`main.rs:610-613`): a connection it accepted before a session and had not read is let go +on the pass after it, once the session outlasts the limit. The supervisor (`main.rs:1299-1311`) +and fileserver (`main.rs:411-429`) serve first, as `5a68db78b` made init do after a stop asked +through its power port at 2.196 s was dropped unread at 4.753 s. They still read only the +connections the wait reported, so a request made whole after the wait returned and before the +limit's clock is read is let go whole as well. The compositor's table also holds a connection +whose copy it began, until its commit, timed by the same limit from the copy's beginning +(`client.rs:124-126`, `session.rs:1091`). + +**What a client reads.** A connection let go on the limit is closed with nothing said to it. +netstack's client reads that as `NetError::NetstackNotFound` (`hangup`, `sdk/toyos/src/net.rs`), +the word for a machine with no netstack. + +**The log.** netstack, the compositor, filepicker, fileserver, the supervisor and usbd name the +client they let go on the limit. diskserver's drop says nothing (`main.rs:688`), and logkeeper's +says nothing by choice (`inspect.rs:149-152`): a line per drop, it says, would let every holder of +`log` write into the log at its own rate. A drop on the limit comes at the server's rate, not the +client's: every entry a server drops in a span as long as the limit was in its table when the span +began, so it drops at most its table's size per limit. Today a drop's line cannot say whether the +client or the server's pass was late; once a connection is read before it is let go, a drop says +the client had not sent its request. + +**The wait.** netstack (`main.rs:435-437`) and logkeeper (`inspect.rs:144-145`) wait a fresh whole +limit on each pass rather than what is left of the oldest entry's, so an entry can wait nearly +twice the limit: what the supervisor calls "a bound that is not the bound" (`main.rs:1198-1203`). +diskserver, filepicker, fileserver, the supervisor and usbd wait what is left; the compositor +wakes once a frame. + +**A full table.** netstack (`main.rs:481-488`), the compositor (`session.rs:694-709`), filepicker +(`main.rs:554-558`), the supervisor (`main.rs:1232-1238`), usbd (`main.rs:127-129`) and logkeeper +(`inspect.rs:156-161`) accept the next connection and close it, naming it only in their own log +and logkeeper not even there; netstack's client reads that as `NetstackNotFound` too. soundserver +accepts it and signals `MSG_STREAM_ERROR` (`control.rs:185-188`), and a surface host accepts it and +closes it, which its client reads as `GrabError::HostGone` (`surface.rs:181-196`). diskserver +(`main.rs:650-651`) and fileserver (`main.rs:390-391`) stop accepting until an entry leaves, and +the connection waits in its port's queue, past whose 32 the kernel answers the connect +`ResourceExhausted` (`MAX_PENDING_CONNECTIONS`, `kernel/src/object/port.rs`). fileserver's +per-share part (`HANDSHAKE_SHARE`, `main.rs:455-462`) is a bound of another kind: one share's, +which stopping the acceptor would put on every share, and it answers `ResourceExhausted` in a frame +before it lets go. + +**The supervisor's one table.** Its launcher, grants, swap and power connections share one table +of `MAX_PENDING_LAUNCHES` (`main.rs:1213-1248`), so once a full table blocks, a stop asked through +the power port waits behind silent launcher connections: the lateness `5a68db78b` recorded. + +**What is published.** RFC 9110 §15.5.9 gives an HTTP server a limit on a request it has not +received whole, and a word for it, 408 (Request Timeout) +([RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-15.5.9)). RFC 9112 §9.5 sets no length +for a server's timeout, and asks a server to let the transport's flow control resolve a temporary +overload rather than close connections for their clients to retry +([RFC 9112](https://www.rfc-editor.org/rfc/rfc9112#section-9.5)). The published defaults for a +request's header are 60 s in nginx, `client_header_timeout` +([source](https://github.com/nginx/nginx/blob/0a5db852e18d1c22c62a5ede1412ac3f416111a0/src/http/ngx_http_core_module.c#L3620-L3621), +[documentation](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_timeout)), +and 20 s, stretched to 40 s while it arrives at 500 bytes/s, in Apache httpd, +`RequestReadTimeout header=20-40,MinRate=500` +([source](https://github.com/apache/httpd/blob/1e06e9860794ce39605b79a0a787abfe084e5677/modules/filters/mod_reqtimeout.c#L35-L37), +[documentation](https://httpd.apache.org/docs/2.4/mod/mod_reqtimeout.html#requestreadtimeout)). +2 s is a tenth of the shorter. On the limit, nginx closes the connection with no answer +([source](https://github.com/nginx/nginx/blob/0a5db852e18d1c22c62a5ede1412ac3f416111a0/src/http/ngx_http_request.c#L1130-L1134)). + +**Seen** once, in a whole `cargo test` at `be929d941`, then the head of #886, which changes +nothing in `system/netstack`, `sdk/toyos/src/net.rs` or the test against its base `ecd6d1cc5`: +`netstack_socket_churn` red after 14 s, netstack's `dropping client 86031 — it never finished its +request` and then the test's loop of connects answered `NetstackNotFound` where it asserts +`ResourceExhausted` +([log](https://github.com/ToyOSOrg/evidence/blob/4dab62b2bd351b7467f238b7e8f7452237db7315/build/gates-be929d941/guest.log#L647-L682), +[summary](https://github.com/ToyOSOrg/evidence/blob/4dab62b2bd351b7467f238b7e8f7452237db7315/build/gates-be929d941/summary.txt)). +The harness priced its liveness ceilings at 1.00x on 14 cores +([log](https://github.com/ToyOSOrg/evidence/blob/4dab62b2bd351b7467f238b7e8f7452237db7315/build/gates-be929d941/guest.log#L1087-L1088)). +That run's host load was not recorded. The same gates at `f126fb514` were green, the test in 15 s +([log](https://github.com/ToyOSOrg/evidence/blob/4dab62b2bd351b7467f238b7e8f7452237db7315/build/gates/guest.log#L680)), +at one-minute load averages of 36.22 as the suite began and 36.73 as it ended +([guest.load](https://github.com/ToyOSOrg/evidence/blob/4dab62b2bd351b7467f238b7e8f7452237db7315/build/gates/guest.load)). +The log does not say whether the client sent its request late or netstack read it late. + +**The cut test.** `netstack_socket_churn` is cut, an integration test of a userland program that +`.claude/agents/reviewer.md`'s **Tests** refuses; it does not come back. What it reached of the +node is held by `system/netstack/node/tests/listeners.rs`'s +`a_connect_past_the_places_is_refused_and_sends_nothing`, +`a_departed_clients_connection_holds_its_place_until_it_is_cut` and +`a_wake_the_owners_pipe_refuses_ends_the_listener`, and by +`system/netstack/node/tests/streams.rs`'s `a_client_that_left_is_finished_by_the_stack`, +`a_writer_that_leaves_after_the_shutdown_leaves_the_stream_to_its_reader`, +`a_pipe_that_is_no_pipe_resets_its_connection` and +`a_refused_watch_resets_only_a_stream_that_holds_the_end`. What only `system/netstack/src/serve.rs` +does, which carries no test, no test reaches since the cut. + +**Owner**: the orchestrator, which dispatches the fix. + +**Exit condition**: + +- Each of the eight reads a connection before it lets it go on the limit, and serves the request + it finds whole there however late its pass came; soundserver and every surface host read a + connection before they let it go. +- The limit stays, on a connection that has sent no whole request, and every server, diskserver + and logkeeper among them, logs that drop by name. +- The eight copies are one declaration every server reads, beside `FrameRx` in `toyos::ipc`, that + takes its clock as an argument; a host test there goes red when a connection whose request is + whole is let go on a pass past its limit. +- Its length is taken from nginx's and Apache httpd's published defaults and cites them, or + another is defended against them at the declaration. +- No server waits a fresh whole limit: each wakes by what is left of its oldest entry's. +- A server whose table is full, soundserver and every surface host among them, stops accepting + until an entry leaves it, as diskserver and fileserver do and RFC 9112 §9.5 asks, and none + accepts a connection to close it because the table is full: the port's queue behind it holds the + connection, and the kernel refuses the one past that queue by name. +- The supervisor's power connections wait in a table of their own, so a stop never waits behind + launcher, grants or swap connections. +- A connection let go on the limit is closed with no answer, as nginx does. Its client names that + as the server letting it go, never as no server, `NetError::NetstackNotFound` among them, and + every client library of these servers does the same. +- `system/netstack/src/serve.rs` has its own tests of a second receive refused + `ERR_RESOURCE_EXHAUSTED` (`serve.rs:563-567`), a waiting receive answered by its datagram + (`settle`, `:676-680`), and the kernel's watch answers that end a stream, a listener or a datagram + socket whose owner left or whose end is no pipe (`Sockets::answered`, `:798-843`). From f81eea55c0b0543aac16db73dfe999fda60bafd9 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 12 Oct 2026 22:31:43 +0200 Subject: [PATCH 3/3] issues: one program that fills a server's table holds every other client out Found while recording the handshake limit's order, and filed under the owner's ruling on PR #897. A server's table of connections it has not yet served is one number for every client: one program that reconnects each time it is let go keeps the eight limited tables full for as long as it runs, and holds soundserver's 63 control slots or a surface's 4, which have no limit, until it hangs up. Once full tables block and the limit takes a length of 20 s to 60 s, every other client waits behind it. Only fileserver shares its table out, per session's share. The exit is a per-client share of each table, keyed by a badge that whoever mints the connector writes: the supervisor for the servers it spawns, console and terminal for the ports they create for their own shells. A client past its share is refused by name, with a host test of each table. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...vers-table-holds-every-other-client-out.md | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 issues/one-program-that-fills-a-servers-table-holds-every-other-client-out.md diff --git a/issues/one-program-that-fills-a-servers-table-holds-every-other-client-out.md b/issues/one-program-that-fills-a-servers-table-holds-every-other-client-out.md new file mode 100644 index 00000000000..3357217c655 --- /dev/null +++ b/issues/one-program-that-fills-a-servers-table-holds-every-other-client-out.md @@ -0,0 +1,44 @@ +--- +status: open +kind: defect +opened: 2026-10-12 +--- + +# One program that fills a server's table holds every other client out + +A server holds the connections it has accepted and not yet served in a table of a fixed size, and +that size is one number for every client. The eight servers whose `HANDSHAKE_TIMEOUT` empties +their table (`issues/servers-let-a-client-go-on-the-handshake-limit-with-its-request-unread.md`) +let a silent connection go on that limit, so one program that connects again each time one is let +go keeps the table full for as long as it runs. soundserver's control plane, 63 slots +(`MAX_CONTROL_CLIENTS`, `system/soundserver/src/control.rs`), and a surface host's clients, 4 +(`toyos::surface::MAX_CLIENTS`, `sdk/toyos/src/surface.rs`), have no limit at all: a program that +connects that many times and says nothing holds them until it hangs up. + +Meanwhile every other client is let go at the full table, or, where the server stops accepting, +waits in the port's queue behind it. That record's exit has every full table stop accepting and +takes the limit's length from published defaults of 20 s to 60 s, so each other client waits up to +a whole limit behind the one program, and behind soundserver's or a surface's for as long as that +program runs. + +Only fileserver shares its table out: one share holds at most `HANDSHAKE_SHARE`, a quarter of +`MAX_HANDSHAKES`, and its next connection is answered `ResourceExhausted` and let go +(`system/fileserver/src/main.rs:80-86`, `:455-462`). Its share is the one the grant on the +connection's badge names, which the supervisor mints per session, not per program +(`system/supervisor/src/main.rs`, `mint`). A badge naming the program, its row and session, is on +every connection to netstack and to the supervisor's launcher and grants ports +(`Grants::netstack`, `launcher_namespace`, `grants_namespace`); diskserver's names a partition, and +the others' connections carry none. `issues/netstacks-places-are-one-number-for-every-client.md` and +`issues/netstack-lookup-slots-have-no-per-client-share.md` record the same hole on netstack's places +and lookups. The supervisor mints the connector of every server it spawns, but not a surface +host's: console and terminal each create their own port and move its connector only into the shell +they spawn (`system/console/src/main.rs:154-156`, `system/terminal/src/main.rs:55-56`). Lines are at +`1a7464692`. + +**Owner**: the orchestrator, which dispatches the fix. + +**Exit condition**: every table named above holds at most a per-client share of itself, keyed by a +badge that names the client, written by whoever mints the connector: the supervisor for the servers +it spawns, console and terminal for their own ports; a client past its share has its next connection +refused by name while another client's is taken, and a host test of each table goes red when one +client holds more than its share.