From a7393e6135bc2367830928541ee024548854766f Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 00:12:55 +0800 Subject: [PATCH 01/26] fix: close device, stream, and preview lifecycle gaps --- boot/init/src/boot.rs | 41 ++++++++++++-- docs/deploy.md | 8 +-- docs/sandboxd-api.md | 22 ++++++-- docs/sdk-python.md | 3 +- docs/sdk.md | 3 +- docs/silkd.md | 2 +- sandboxd/config/config.go | 5 +- sandboxd/main.go | 2 +- sandboxd/server/preview.go | 58 +++++++------------ sandboxd/server/preview_test.go | 99 +++++++++++++++++++++++++-------- sandboxd/server/server.go | 3 + sandboxd/server/server_test.go | 2 +- sdk/go/pty.go | 26 ++++----- sdk/go/pty_test.go | 31 +++++++++-- silkd/src/watch.rs | 56 +++++++++++++------ 15 files changed, 243 insertions(+), 118 deletions(-) diff --git a/boot/init/src/boot.rs b/boot/init/src/boot.rs index ec520295..1034af29 100644 --- a/boot/init/src/boot.rs +++ b/boot/init/src/boot.rs @@ -2,6 +2,7 @@ //! switch_root → exec. use std::fs; +use std::path::Path; use std::time::{Duration, Instant}; use crate::cfg::{self, BootCfg}; @@ -235,12 +236,18 @@ fn scan_serials(ids: &[&str], found: &mut [Option]) { let Ok(serial) = fs::read_to_string(&path) else { continue; }; - let serial = serial.trim_end(); - for (i, id) in ids.iter().enumerate() { - if found[i].is_none() && *id == serial { - found[i] = Some(format!("/dev/{name}")); - } - } + record_serial(ids, found, serial.trim_end(), &format!("/dev/{name}")); + } + } +} + +fn record_serial(ids: &[&str], found: &mut [Option], serial: &str, device: &str) { + if !Path::new(device).exists() { + return; + } + for (i, id) in ids.iter().enumerate() { + if found[i].is_none() && *id == serial { + found[i] = Some(device.into()); } } } @@ -251,3 +258,25 @@ fn uptime() -> String { .and_then(|s| s.split_ascii_whitespace().next().map(String::from)) .unwrap_or_else(|| "?".into()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn record_serial_waits_for_device_node() { + let ids = ["layer"]; + let mut found = [None]; + + record_serial( + &ids, + &mut found, + "layer", + "/dev/sandbox-init-missing-device", + ); + assert!(found[0].is_none()); + + record_serial(&ids, &mut found, "layer", "/dev/null"); + assert_eq!(found[0].as_deref(), Some("/dev/null")); + } +} diff --git a/docs/deploy.md b/docs/deploy.md index 24048d6a..dbb26b2a 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -96,7 +96,7 @@ sandboxd reads one JSON file (`-config`, default | `refill_concurrency` | 0 (auto) | concurrent VM provisioning budget, shared by warm-pool refills, fork clones, and the reap/hibernate/reconcile engine batches. 0 sizes it from the node: `NumCPU*2/3` clamped to [4, 256] — a 384-core node gets 256; small nodes keep a floor of 4 | | `preview_listen` | (off) | address for a preview HTTP server that serves guest ports under signed URLs; needs `preview_secret` | | `preview_secret` | — | cluster-shared HMAC secret signing preview tokens (all nodes share one) | -| `preview_advertise` | = `preview_listen` | the base URL a browser/proxy reaches this node's preview server at | +| `preview_advertise` | = `preview_listen` | the browser-facing preview base URL; nodes behind one TLS proxy may share it, while signed tokens route internally through each owner's `advertise_addr` | | `checkpoint_dir` | `/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — records are generation-addressed with `meta.json` as the atomic commit pointer, so no cross-node locking is required of the filesystem. A re-publish retains its superseded export generation for at least ~1h and until a following hourly sweep so an in-flight clone that resolved the old metadata can finish; budget the current generation plus every generation retained across that grace-and-sweep window. An explicit delete can make a concurrent clone fail visibly. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide) | | `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. Re-publish retains prior export generations until Delete so an in-flight fetch that selected old metadata can finish; budget storage for those generations. An explicit S3 Delete can still make a concurrent fetch that has not finished materializing fail visibly. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` | | `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the expiry eligibility point for a healed replica a delete broadcast missed, after which its next successful hourly sweep removes it; persistent sweep failure extends retention until one succeeds, so it is not a hard ceiling | @@ -452,9 +452,9 @@ sandboxd: life is clamped to the claim's lease. - **Serving**: any node's preview listener verifies the token (no shared state), then reverse-proxies to the guest port over the relay if it owns - the sandbox, or forwards to the owner node otherwise. A released sandbox - is gone from the claim map, so its URL stops resolving — revocation is - the liveness lookup, not a list. + the sandbox, or forwards over HTTP to the owner node's `advertise_addr` + otherwise. A released sandbox is gone from the claim map, so its URL stops + resolving — revocation is the liveness lookup, not a list. - **The public entry point is a commodity dumb proxy.** Because any node can accept and forward, front the nodes with whatever terminates TLS and round-robins: a cloud HTTPS load balancer with a managed wildcard cert diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 0095ac8b..d268fbec 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -99,9 +99,21 @@ can lag the serial match the same way it does for an eager mount's device settle. Confirm both before mounting: ```sh -for dev in /sys/block/*/serial; do - [ "$(cat "$dev")" = scratch-db ] && echo "/dev/$(basename "$(dirname "$dev")")" +device= +tries=0 +while [ "$tries" -lt 200 ]; do + for serial in /sys/block/*/serial /sys/block/*/device/serial; do + [ -r "$serial" ] || continue + [ "$(cat "$serial")" = scratch-db ] || continue + block=${serial#/sys/block/} + candidate="/dev/${block%%/*}" + [ -b "$candidate" ] && { device="$candidate"; break 2; } + done + tries=$((tries + 1)) + sleep 0.01 done +[ -n "$device" ] || { echo "scratch-db device not ready" >&2; exit 1; } +printf '%s\n' "$device" ``` Then mount it however the workload needs. A `ro` entry is attached @@ -333,9 +345,9 @@ guest HTTP port from a browser: body `{"token": "...", "port": 8080, "ttl_seconds": 0}` → `{"url": "http:///p//"}`. The URL's life is clamped to the claim's remaining lease. 501 when the node has no `preview_listen`. The signed token embeds the sandbox id, port, and -owner node, so any node's preview listener can serve it (forwarding to the -owner) and a released sandbox's URL simply stops resolving — no revocation -list. See [deploy](deploy.md#preview-urls). +owner `advertise_addr`, so any node's preview listener can serve it (forwarding +to the owner's main listener) and a released sandbox's URL simply stops +resolving — no revocation list. See [deploy](deploy.md#preview-urls). ## POST /v1/sandboxes/{id}/checkpoint diff --git a/docs/sdk-python.md b/docs/sdk-python.md index f6864e86..e7508bf4 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -362,7 +362,8 @@ w.close() `watch` returns once the guest acknowledges the watch is armed — events caused after it returns are guaranteed captured. A bad path fails -synchronously. +synchronously; if the consumer falls too far behind, iteration raises the +terminal overflow instead of silently dropping events. ## Git diff --git a/docs/sdk.md b/docs/sdk.md index ba49dad4..262a45fb 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -443,7 +443,8 @@ err = w.Err() // why the stream ended; nil after Close `Watch` returns once the guest acknowledges the watch is armed — events caused after it returns are guaranteed captured. A bad path fails -synchronously. +synchronously; if the consumer falls too far behind, `Err` reports a terminal +overflow instead of the stream silently dropping events. ## Git diff --git a/docs/silkd.md b/docs/silkd.md index 51b17057..3e36e9f0 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -36,7 +36,7 @@ a frame only one side can parse fails CI. | fs | `fs_write {path, mode?}` (+`data`/`data_end` frames) / `fs_read` / `fs_list` / `fs_stat` / `fs_mkdir {parents?}` / `fs_rm {recursive?}` / `fs_rename {from, to}` | streaming both directions; write commits atomically via temp+rename and inherits an overwritten file's mode; `fs_list` streams 4096-entry batches | | tree | `fs_push {dest}` (+tar as `data` frames) / `fs_pull {path}` | whole trees as tar streams through the guest tar | | search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files | -| watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher errors arrive as a terminal `error` | +| watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | | pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | a shell under a pseudo-terminal; output as `stdout` frames, input as `stdin` frames, exit terminal. PTYs register in the proc table like any exec | | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | structured results (porcelain-v2 status, commit hash, branch list). `auth` is injected as an in-memory header, never written to guest disk | | port | `port_forward {port}` | relays guest TCP 127.0.0.1:port over this connection: `ready` once connected, then `data` both ways (`data_end` half-closes the guest socket); the server closing ends the stream with `done`. Works on both lanes — the no-network lane's only way in | diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 16c4367e..7c2a6082 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -221,10 +221,7 @@ type Config struct { ArchiveAfterSeconds int `json:"archive_after_seconds,omitempty"` ArchiveDeleteAfterSeconds int `json:"archive_delete_after_seconds,omitempty"` - // PreviewListen, when set, starts a preview HTTP server on that address - // serving guest ports under signed URLs. PreviewSecret (cluster-shared) - // signs the tokens; PreviewAdvertise is the base a browser/proxy reaches - // this node's preview server at, defaulting to PreviewListen. + // PreviewAdvertise is the browser-facing base URL and may be shared behind one proxy. PreviewListen string `json:"preview_listen,omitempty"` PreviewSecret string `json:"preview_secret,omitempty"` //nolint:gosec // config field, not a hardcoded credential PreviewAdvertise string `json:"preview_advertise,omitempty"` diff --git a/sandboxd/main.go b/sandboxd/main.go index 904b3fbb..14a58c92 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -132,7 +132,7 @@ func main() { var preview *server.PreviewServer if cfg.PreviewListen != "" { - preview = server.NewPreviewServer(cfg.PreviewSecret, cfg.PreviewAdvertise, mgr) + preview = server.NewPreviewServer(cfg.PreviewSecret, cfg.PreviewAdvertise, cfg.AdvertiseAddr, mgr) } srv := server.New(cfg.APIToken, cfg.Tenants, cfg.AdvertiseAddr, mgr, eng, placer, prober, probeKey, preview) httpSrv := &http.Server{ diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 1c34d7d7..31967d06 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -20,9 +20,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// previewClaims is the signed payload of a preview URL: it authorizes serving -// guest `Port` of sandbox `ID`, owned by the node reachable at preview base -// `Owner`, until `Exp`. Signed so any node can verify without shared state. +// previewClaims is the signed guest target and owner route carried by a preview URL. type previewClaims struct { ID string `json:"id"` Port uint16 `json:"port"` @@ -35,31 +33,25 @@ type PreviewManager interface { PreviewDial(ctx context.Context, id string, port uint16) (net.Conn, error) } -// PreviewServer serves guest HTTP apps under signed, expiring URLs, built on -// the port relay. The whole mechanism lives here: a signed token needs no -// revocation list (a released sandbox simply isn't in the claim map), and -// because the token carries its owner's preview address, any node can accept -// a request and proxy it to the owner — so the public entry point is a dumb -// TLS proxy, not a stateful gateway. +// PreviewServer serves signed guest HTTP URLs and forwards requests to their owner node. type PreviewServer struct { secret []byte - advertise string + base string + owner string mgr PreviewManager transport *http.Transport } -// NewPreviewServer returns nil when preview is not configured (empty secret). -// advertise is this node's preview base (host:port a browser/proxy reaches). -func NewPreviewServer(secret, advertise string, mgr PreviewManager) *PreviewServer { +// NewPreviewServer returns nil when preview is not configured. +func NewPreviewServer(secret, base, owner string, mgr PreviewManager) *PreviewServer { if secret == "" { return nil } - p := &PreviewServer{secret: []byte(secret), advertise: advertise, mgr: mgr} - // One shared transport so a page's sub-resource fan-out reuses kept-alive - // guest conns instead of re-dialing per request; the Director keys each - // request's host to the sandbox+port so the idle pool never mixes claims. + p := &PreviewServer{secret: []byte(secret), base: base, owner: owner, mgr: mgr} + // Every request dials through PreviewDial so release revokes a URL even while + // VM removal is waiting for its eventual retry. p.transport = &http.Transport{ - IdleConnTimeout: 90 * time.Second, + DisableKeepAlives: true, DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { id, portStr, err := net.SplitHostPort(addr) if err != nil { @@ -75,23 +67,20 @@ func NewPreviewServer(secret, advertise string, mgr PreviewManager) *PreviewServ return p } -// Mint returns a preview URL for a guest port, valid for ttl (bounded by the -// caller to the claim's deadline). Called on the owner node with the sandbox -// already authorized, so the token names this node as owner. +// Mint returns a preview URL for a guest port, valid for ttl. func (p *PreviewServer) Mint(id string, port uint16, ttl time.Duration) string { - claims := previewClaims{ID: id, Port: port, Owner: p.advertise, Exp: time.Now().Add(ttl).Unix()} + claims := previewClaims{ID: id, Port: port, Owner: p.owner, Exp: time.Now().Add(ttl).Unix()} payload, _ := json.Marshal(claims) enc := base64.RawURLEncoding.EncodeToString(payload) token := enc + "." + p.sign(enc) - base := p.advertise + base := p.base if !strings.Contains(base, "://") { base = "http://" + base } return fmt.Sprintf("%s/p/%s/", strings.TrimRight(base, "/"), token) } -// Handler serves the preview_listen address: verify the token, then either -// reverse-proxy to the guest port locally or forward to the owner node. +// Handler serves preview requests and health checks on preview_listen. func (p *PreviewServer) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/p/{token}/", p.serve) @@ -105,27 +94,22 @@ func (p *PreviewServer) serve(w http.ResponseWriter, r *http.Request) { http.Error(w, "invalid or expired preview token", http.StatusForbidden) return } - if claims.Owner != p.advertise { + if claims.Owner != p.owner { p.forward(w, r, claims.Owner) return } p.proxyLocal(w, r, claims) } -// proxyLocal reverse-proxies to the guest port over a relay connection whose -// liveness lookup is the revocation check: a released sandbox is gone from -// the claim map, so PreviewDial fails and the URL 404s. +// proxyLocal uses the live claim lookup as stateless revocation. func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claims previewClaims) { rp := &httputil.ReverseProxy{ Director: func(req *http.Request) { req.URL.Scheme = "http" - // Host = sandbox:port so the shared transport's idle pool keys per - // claim; DialContext parses it back to PreviewDial. + // The synthetic host carries PreviewDial's target. req.URL.Host = fmt.Sprintf("%s:%d", claims.ID, claims.Port) req.URL.Path = "/" + strings.TrimPrefix(req.URL.Path, "/p/"+r.PathValue("token")+"/") - // The preview URL is bearer-authorized by its token; never hand - // the browser's ambient credentials for the preview domain to - // untrusted guest code. + // Browser credentials for the preview domain must not reach guest code. req.Header.Del("Cookie") req.Header.Del("Authorization") }, @@ -138,8 +122,7 @@ func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claim rp.ServeHTTP(w, r) //nolint:gosec // target derived from an HMAC-signed token, not client input } -// forward relays the request to the owner node's preview_listen — the token -// self-authorizes, so this node is a dumb hop. +// forward relays the signed request to the owner node's main listener. func (p *PreviewServer) forward(w http.ResponseWriter, r *http.Request, owner string) { target := &url.URL{Scheme: "http", Host: owner} rp := httputil.NewSingleHostReverseProxy(target) @@ -172,8 +155,7 @@ func (p *PreviewServer) verify(token string) (previewClaims, bool) { return claims, true } -// handlePreview mints a preview URL for a claimed sandbox's port. The sandbox -// token authorizes it; the TTL is clamped to the claim's remaining lease. +// handlePreview mints a preview URL bounded by the claim's remaining lease. func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) { if s.preview == nil { writeErr(w, http.StatusNotImplemented, "preview not configured") diff --git a/sandboxd/server/preview_test.go b/sandboxd/server/preview_test.go index 6a7691fd..a3c5988a 100644 --- a/sandboxd/server/preview_test.go +++ b/sandboxd/server/preview_test.go @@ -7,28 +7,32 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" ) func TestPreviewTokenRoundTrip(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", &fakePreviewMgr{}) + ps := NewPreviewServer("secret", "https://preview.example.com", "node:7777", &fakePreviewMgr{}) token := mintToken(ps, "sb_1", 8080, time.Hour) claims, ok := ps.verify(token) - if !ok || claims.ID != "sb_1" || claims.Port != 8080 || claims.Owner != "node:9000" { + if !ok || claims.ID != "sb_1" || claims.Port != 8080 || claims.Owner != "node:7777" { t.Fatalf("verify %+v ok=%v", claims, ok) } + if url := ps.Mint("sb_1", 8080, time.Hour); !strings.HasPrefix(url, "https://preview.example.com/p/") { + t.Errorf("url %q, want public preview base", url) + } if _, ok := ps.verify(token + "x"); ok { t.Error("tampered token verified") } - if _, ok := NewPreviewServer("other-secret", "node:9000", &fakePreviewMgr{}).verify(token); ok { + if _, ok := NewPreviewServer("other-secret", "https://preview.example.com", "node:7777", &fakePreviewMgr{}).verify(token); ok { t.Error("token verified under the wrong secret") } } func TestPreviewRejectsExpired(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", &fakePreviewMgr{}) + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{}) token := mintToken(ps, "sb_1", 8080, -time.Second) // already expired if _, ok := ps.verify(token); ok { t.Error("expired token verified") @@ -36,8 +40,6 @@ func TestPreviewRejectsExpired(t *testing.T) { } func TestPreviewProxiesToGuest(t *testing.T) { - // A real HTTP server stands in for the guest app; PreviewDial hands the - // proxy a raw conn to it. guest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, "guest saw "+r.URL.Path) })) @@ -45,7 +47,7 @@ func TestPreviewProxiesToGuest(t *testing.T) { guestAddr := strings.TrimPrefix(guest.URL, "http://") dialed := false - ps := NewPreviewServer("secret", "node:9000", &fakePreviewMgr{ + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ dial: func(id string, port uint16) (net.Conn, error) { dialed = true if id != "sb_1" || port != 8080 { @@ -71,8 +73,7 @@ func TestPreviewProxiesToGuest(t *testing.T) { } func TestPreviewRevokedWhenDialFails(t *testing.T) { - // A released sandbox: PreviewDial errors, so the URL 502s — statelessly. - ps := NewPreviewServer("secret", "node:9000", &fakePreviewMgr{ + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ dial: func(string, uint16) (net.Conn, error) { return nil, net.ErrClosed }, }) ts := httptest.NewServer(ps.Handler()) @@ -89,31 +90,83 @@ func TestPreviewRevokedWhenDialFails(t *testing.T) { } } +func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { + guest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "guest") + })) + t.Cleanup(guest.Close) + guestAddr := strings.TrimPrefix(guest.URL, "http://") + + var live atomic.Bool + var dials atomic.Int32 + live.Store(true) + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ + dial: func(string, uint16) (net.Conn, error) { + dials.Add(1) + if !live.Load() { + return nil, net.ErrClosed + } + return net.Dial("tcp", guestAddr) + }, + }) + ts := httptest.NewServer(ps.Handler()) + t.Cleanup(ts.Close) + url := ts.URL + "/p/" + mintToken(ps, "sb_1", 8080, time.Hour) + "/" + + resp, err := http.Get(url) + if err != nil { + t.Fatalf("first get: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("first status = %d, want 200", resp.StatusCode) + } + + live.Store(false) + resp, err = http.Get(url) + if err != nil { + t.Fatalf("get after release: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadGateway { + t.Errorf("status after release = %d, want 502", resp.StatusCode) + } + if got := dials.Load(); got != 2 { + t.Errorf("PreviewDial calls = %d, want one per request", got) + } +} + func TestPreviewForwardsToOwner(t *testing.T) { - // A token owned by a different node is proxied to that node verbatim. - var forwarded bool - owner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - forwarded = true - _, _ = io.WriteString(w, "owner served") + guest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "guest saw "+r.URL.Path) })) - t.Cleanup(owner.Close) - ownerAddr := strings.TrimPrefix(owner.URL, "http://") + t.Cleanup(guest.Close) + guestAddr := strings.TrimPrefix(guest.URL, "http://") - // Mint on the owner, serve on a different node. - ownerPS := NewPreviewServer("secret", ownerAddr, &fakePreviewMgr{}) - token := mintToken(ownerPS, "sb_1", 8080, time.Hour) + owner := httptest.NewUnstartedServer(nil) + ownerAddr := owner.Listener.Addr().String() + ownerPS := NewPreviewServer("secret", "https://preview.example.com", ownerAddr, &fakePreviewMgr{ + dial: func(string, uint16) (net.Conn, error) { return net.Dial("tcp", guestAddr) }, + }) + ownerSrv := New("", nil, ownerAddr, &fakeManager{}, &fakeDialer{}, nil, nil, nil, ownerPS) + owner.Config.Handler = ownerSrv.Handler() + owner.Start() + t.Cleanup(func() { owner.Close(); ownerSrv.CloseRelays() }) - entry := NewPreviewServer("secret", "entry:9000", &fakePreviewMgr{}) + token := mintToken(ownerPS, "sb_1", 8080, time.Hour) + entry := NewPreviewServer("secret", "https://preview.example.com", "entry:7777", &fakePreviewMgr{}) ts := httptest.NewServer(entry.Handler()) t.Cleanup(ts.Close) - resp, err := http.Get(ts.URL + "/p/" + token + "/") + resp, err := http.Get(ts.URL + "/p/" + token + "/via-owner") if err != nil { t.Fatalf("get: %v", err) } defer resp.Body.Close() - if !forwarded { - t.Error("request not forwarded to the owner node") + body, _ := io.ReadAll(resp.Body) + if string(body) != "guest saw /via-owner" { + t.Errorf("body %q, want request forwarded through owner to guest", body) } } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index ad7d0d5a..3da4416d 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -227,6 +227,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /v1/sandboxes", s.requireToken(s.handleSandboxes)) mux.HandleFunc("GET /metrics", s.requireRoot(s.handleMetrics)) mux.HandleFunc("GET /healthz", s.handleHealthz) + if s.preview != nil { + mux.HandleFunc("/p/{token}/", s.preview.serve) + } return mux } diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index e39c7637..f9cb721b 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -1472,7 +1472,7 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { mgr := &fakeManager{claimDeadline: func(string, string) (time.Time, error) { return time.Time{}, nil }} - ps := NewPreviewServer("secret", "node:7777", &fakePreviewMgr{}) + ps := NewPreviewServer("secret", "node:7777", "node:7777", &fakePreviewMgr{}) srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, nil, nil, ps) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) diff --git a/sdk/go/pty.go b/sdk/go/pty.go index c340ce62..f9571e8d 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -3,6 +3,7 @@ package sandbox import ( "context" "io" + "net" "sync" "github.com/cocoonstack/sandbox/protocol/wire" @@ -27,13 +28,13 @@ type Pty struct { stop func() out *io.PipeReader - mu sync.Mutex - exitCode int - exited bool + mu sync.Mutex + exitCode int + exited bool + closeOnce sync.Once } -// Read returns terminal output; io.EOF signals the shell has exited (check -// ExitCode after). +// Read returns terminal output; io.EOF means ExitCode is ready. func (p *Pty) Read(b []byte) (int, error) { return p.out.Read(b) } @@ -61,21 +62,21 @@ func (p *Pty) Resize(ctx context.Context, cols, rows uint16) error { // Close ends the pty session (silkd sees the disconnect and kills the shell). func (p *Pty) Close() error { - p.stop() + p.closeOnce.Do(func() { + p.stop() + _ = p.out.CloseWithError(net.ErrClosed) + }) return nil } -// ExitCode reports the shell's exit code once Read has returned io.EOF; ok is -// false while the shell is still running. +// ExitCode reports the shell's exit code after Read returns io.EOF. func (p *Pty) ExitCode() (code int, ok bool) { p.mu.Lock() defer p.mu.Unlock() return p.exitCode, p.exited } -// drain relays response frames: output into the pipe, the exit code into -// state, a terminal error to unblock Read; an OpenPty-ctx cancel surfaces as -// ctx.Err(), matching Watcher and PortConn. +// drain relays response frames into the output pipe and terminal state. func (p *Pty) drain(ctx context.Context, pw *io.PipeWriter) { for { resp, err := recv(ctx, p.conn) @@ -105,8 +106,7 @@ func (p *Pty) drain(ctx context.Context, pw *io.PipeWriter) { } } -// OpenPty starts a shell under a pty. The ctx governs the pty's lifetime: -// canceling it (or calling Close) tears the session down. +// OpenPty starts a shell whose lifetime is governed by ctx or Close. func (s *Sandbox) OpenPty(ctx context.Context, opts PtyOpts) (*Pty, error) { req := &wire.PtyOpen{Cols: opts.Cols, Rows: opts.Rows, Cwd: opts.Cwd, Env: opts.Env, User: opts.User} conn, done, err := s.call(ctx, req) diff --git a/sdk/go/pty_test.go b/sdk/go/pty_test.go index 389b6fe6..b65e19b2 100644 --- a/sdk/go/pty_test.go +++ b/sdk/go/pty_test.go @@ -19,7 +19,6 @@ func TestPtyEchoAndExit(t *testing.T) { } defer pty.Close() - // The fake echoes input; write a line and read it back. if _, err = pty.Write([]byte("ping\n")); err != nil { t.Fatalf("write: %v", err) } @@ -35,7 +34,6 @@ func TestPtyEchoAndExit(t *testing.T) { t.Fatalf("resize: %v", err) } - // "exit" ends the shell → Read returns EOF and ExitCode is set. if _, err := pty.Write([]byte("exit\n")); err != nil { t.Fatalf("write exit: %v", err) } @@ -48,8 +46,6 @@ func TestPtyEchoAndExit(t *testing.T) { } } -// TestPtyCtxCancelIsTyped: canceling the OpenPty ctx must surface as a typed -// context error through Read, matching Watcher and PortConn. func TestPtyCtxCancelIsTyped(t *testing.T) { sb := fakeSandbox(t) ctx, cancel := context.WithCancel(t.Context()) @@ -65,3 +61,30 @@ func TestPtyCtxCancelIsTyped(t *testing.T) { t.Errorf("Read after ctx cancel = %v, want context.Canceled", err) } } + +func TestPtyCloseUnblocksUnreadOutput(t *testing.T) { + sb := fakeSandbox(t) + pty, err := sb.OpenPty(t.Context(), PtyOpts{Cols: 80, Rows: 24}) + if err != nil { + t.Fatalf("OpenPty: %v", err) + } + if _, err = pty.Write([]byte("unread\n")); err != nil { + t.Fatalf("Write: %v", err) + } + stops := 0 + stop := pty.stop + pty.stop = func() { stops++; stop() } + + if err = pty.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if err = pty.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } + if _, err = pty.Read(make([]byte, 1)); !errors.Is(err, io.ErrClosedPipe) { + t.Errorf("Read after Close = %v, want io.ErrClosedPipe", err) + } + if stops != 1 { + t.Errorf("stop called %d times, want 1", stops) + } +} diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index 353126d3..a8b34e3a 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -2,21 +2,18 @@ //! disconnects. Watch is the one connection-bound verb — an event feed has no //! meaningful detached state, so it lives only as long as its connection. +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + use notify::{RecursiveMode, Watcher}; use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncWrite}; use tokio::sync::mpsc; use crate::proto::{self, ErrorKind, EventKind, Response}; -/// Watches `path` (recursively when set): a `ready` frame once the watch is -/// armed (events after it are guaranteed captured), then `event` frames until -/// the client disconnects or the watcher dies. The blocking notify watcher -/// runs on its own thread and forwards frames into an async channel; the -/// client half is polled concurrently so a disconnect ends the watch even -/// when no event is pending (otherwise an abandoned quiet watch would leak -/// the task and the notify thread). A watcher error (inotify overflow, -/// watcher death) arrives as an `error` frame, which is terminal. Dropping -/// the watcher on return stops it. +const OVERFLOW_MESSAGE: &str = "watch event queue overflow"; + +/// Watches `path`, writing `ready`, ordered events, or a terminal error until disconnect. pub async fn watch( reader: &mut R, w: &mut W, @@ -28,12 +25,10 @@ where W: AsyncWrite + Unpin, { let (tx, mut rx) = mpsc::channel::(256); + let overflowed = Arc::new(AtomicBool::new(false)); + let callback_overflowed = Arc::clone(&overflowed); let mut watcher = match notify::recommended_watcher(move |res| { - for frame in to_frames(res) { - // Best-effort: a full channel means the client is slower than the - // filesystem; drop rather than block the notify thread. - let _ = tx.try_send(frame); - } + enqueue_frames(&tx, &callback_overflowed, to_frames(res)); }) { Ok(watcher) => watcher, Err(e) => return proto::error_frame(w, ErrorKind::Internal, e.to_string()).await, @@ -48,6 +43,9 @@ where } proto::write_frame(w, &Response::Ready).await?; loop { + if overflowed.load(Ordering::Acquire) { + return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await; + } tokio::select! { frame = rx.recv() => match frame { Some(frame) => { @@ -66,8 +64,22 @@ where } } -/// Maps one notify result to the frames to stream: `event` frames for a good -/// event, a single terminal `error` frame for a watcher error. +fn enqueue_frames(tx: &mpsc::Sender, overflowed: &AtomicBool, frames: Vec) { + if overflowed.load(Ordering::Acquire) { + return; + } + for frame in frames { + match tx.try_send(frame) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(_)) => { + overflowed.store(true, Ordering::Release); + return; + } + Err(mpsc::error::TrySendError::Closed(_)) => return, + } + } +} + fn to_frames(res: notify::Result) -> Vec { use notify::EventKind as N; let event = match res { @@ -126,4 +138,16 @@ mod tests { other => panic!("expected event frame, got {other:?}"), } } + + #[test] + fn full_channel_sets_overflow() { + let (tx, mut rx) = mpsc::channel(1); + let overflowed = AtomicBool::new(false); + + enqueue_frames(&tx, &overflowed, vec![Response::Ready, Response::Ready]); + + assert!(overflowed.load(Ordering::Acquire)); + assert!(matches!(rx.try_recv(), Ok(Response::Ready))); + assert!(rx.try_recv().is_err()); + } } From 47d55e8ffefea41d63ef703a99409998460c564d Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 01:50:29 +0800 Subject: [PATCH 02/26] fix: watch overflow signals by dropping the sender, not a flag The overflow flag was only checked at the loop top, so a callback preempted between a full try_send and the store could leave the loop parked on an empty channel forever - a silent quiet watch, the exact loss the terminal error exists to prevent. Dropping the sender makes the closed channel the wakeup itself: the buffered ordered prefix drains first, then recv() returns None and the terminal overflow error goes out. Deletes the Arc/atomic machinery outright. The pty close error never reached readers (pipe readers always see ErrClosedPipe), so plain Close replaces CloseWithError. --- sdk/go/pty.go | 3 +-- silkd/src/watch.rs | 47 +++++++++++++++++++++------------------------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/sdk/go/pty.go b/sdk/go/pty.go index f9571e8d..a3a31624 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -3,7 +3,6 @@ package sandbox import ( "context" "io" - "net" "sync" "github.com/cocoonstack/sandbox/protocol/wire" @@ -64,7 +63,7 @@ func (p *Pty) Resize(ctx context.Context, cols, rows uint16) error { func (p *Pty) Close() error { p.closeOnce.Do(func() { p.stop() - _ = p.out.CloseWithError(net.ErrClosed) + _ = p.out.Close() }) return nil } diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index a8b34e3a..0e45b653 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -2,9 +2,6 @@ //! disconnects. Watch is the one connection-bound verb — an event feed has no //! meaningful detached state, so it lives only as long as its connection. -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; - use notify::{RecursiveMode, Watcher}; use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncWrite}; use tokio::sync::mpsc; @@ -25,10 +22,9 @@ where W: AsyncWrite + Unpin, { let (tx, mut rx) = mpsc::channel::(256); - let overflowed = Arc::new(AtomicBool::new(false)); - let callback_overflowed = Arc::clone(&overflowed); + let mut tx = Some(tx); let mut watcher = match notify::recommended_watcher(move |res| { - enqueue_frames(&tx, &callback_overflowed, to_frames(res)); + forward_frames(&mut tx, to_frames(res)); }) { Ok(watcher) => watcher, Err(e) => return proto::error_frame(w, ErrorKind::Internal, e.to_string()).await, @@ -43,9 +39,6 @@ where } proto::write_frame(w, &Response::Ready).await?; loop { - if overflowed.load(Ordering::Acquire) { - return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await; - } tokio::select! { frame = rx.recv() => match frame { Some(frame) => { @@ -55,7 +48,9 @@ where return Ok(()); } } - None => return Ok(()), + // While the watcher lives, only overflow drops the sender: the + // buffered prefix has all been delivered, then the terminal error. + None => return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await, }, // The client sends nothing during a watch, so any readable state — // EOF (disconnect), a stray frame, or an error — ends the watch. @@ -64,18 +59,15 @@ where } } -fn enqueue_frames(tx: &mpsc::Sender, overflowed: &AtomicBool, frames: Vec) { - if overflowed.load(Ordering::Acquire) { - return; - } +/// Forwards frames into the bounded channel; a full channel drops the sender, +/// so the closed channel itself is the overflow signal — a wakeup the async +/// loop cannot miss even while parked on an empty queue. +fn forward_frames(tx: &mut Option>, frames: Vec) { + let Some(sender) = tx.as_ref() else { return }; for frame in frames { - match tx.try_send(frame) { - Ok(()) => {} - Err(mpsc::error::TrySendError::Full(_)) => { - overflowed.store(true, Ordering::Release); - return; - } - Err(mpsc::error::TrySendError::Closed(_)) => return, + if sender.try_send(frame).is_err() { + *tx = None; + return; } } } @@ -140,14 +132,17 @@ mod tests { } #[test] - fn full_channel_sets_overflow() { + fn full_channel_drops_sender_after_the_delivered_prefix() { let (tx, mut rx) = mpsc::channel(1); - let overflowed = AtomicBool::new(false); + let mut tx = Some(tx); - enqueue_frames(&tx, &overflowed, vec![Response::Ready, Response::Ready]); + forward_frames(&mut tx, vec![Response::Ready, Response::Ready]); - assert!(overflowed.load(Ordering::Acquire)); + assert!(tx.is_none()); assert!(matches!(rx.try_recv(), Ok(Response::Ready))); - assert!(rx.try_recv().is_err()); + assert!(matches!( + rx.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + )); } } From 082a3c1db31977a19501a72bc2a00eed9132b9bb Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 02:12:54 +0800 Subject: [PATCH 03/26] perf(preview): pooled guest conns with per-request touch authorization DisableKeepAlives paid a measured +0.23ms relay dial per request to get per-request revocation, activity, and audit. PreviewTouch does the same three against the claim map for microseconds, so the kept-alive pool returns for sub-resource fan-out; the idle pool stays keyed per claim. Measured on .79: preview seq p50 back to the pre-change band (0.17-0.21ms vs 0.44ms), warm claims unchanged. deploy.md documents the bearer-token payload and the shared browser origin under one preview_advertise. --- docs/deploy.md | 6 ++++ sandboxd/pool/claim.go | 28 +++++++++++----- sandboxd/pool/telemetry_test.go | 10 +++--- sandboxd/server/preview.go | 16 ++++++--- sandboxd/server/preview_test.go | 58 ++++++++++++++++++++------------- 5 files changed, 76 insertions(+), 42 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index dbb26b2a..90bd8b15 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -461,3 +461,9 @@ sandboxd: (GCP/AWS) in production, or a plain nginx/Caddy for self-hosting. It understands nothing about tokens — not sandboxd's code. Dev and e2e hit `preview_listen` directly over HTTP. +- **Sharing scope**: a preview URL is a bearer link, and its token payload + (sandbox id, port, owner `advertise_addr`) is readable by whoever holds + it — keep `advertise_addr` off browser-routable networks. All URLs under + one `preview_advertise` share one browser origin, so different claims' + apps are same-origin to the browser; workloads needing browser-side + isolation need per-sandbox subdomains on the fronting proxy. diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index bb9f9195..5a4c1366 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -103,10 +103,25 @@ func (m *Manager) ClaimDeadline(id, token string) (time.Time, error) { return sb.Deadline, nil } -// PreviewDial opens a byte stream to a guest port for the preview server. The -// caller has already verified the signed preview token, so no sandbox token -// is needed; the live-claim lookup is the revocation check — a released or -// reaped sandbox is absent and this fails. A hibernated sandbox wakes. +// PreviewTouch authorizes one preview request: the token is already verified, +// so the live-claim lookup is the whole check — a released sandbox is absent +// and its URL stops resolving. Stamps data-plane activity and writes the +// audit record (preview bypasses the relay's tap; this is the only trace). +func (m *Manager) PreviewTouch(ctx context.Context, id string, port uint16) error { + m.mu.Lock() + sb, ok := m.claimed[id] + m.mu.Unlock() + if !ok { + return ErrUnknownSandbox + } + sb.Touch() + m.recordAudit(ctx, id, auditFrame{Op: "preview", Port: port}) + return nil +} + +// PreviewDial opens a byte stream to a guest port for the preview proxy's +// connection pool; PreviewTouch authorizes each request separately. A +// hibernated sandbox wakes. func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net.Conn, error) { m.mu.Lock() sb, ok := m.claimed[id] @@ -114,11 +129,6 @@ func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net. if !ok { return nil, ErrUnknownSandbox } - sb.Touch() // a live preview stream is data-plane activity - // Preview bypasses the relay's audit tap (it dials the engine directly), - // so record the access here — the only data-plane entry that would - // otherwise leave no audit trace. - m.recordAudit(ctx, id, auditFrame{Op: "preview_dial", Port: port}) sock, err := m.wakeResolved(ctx, sb) if err != nil { return nil, err diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index d573cf32..d8b5d876 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -189,7 +189,7 @@ func TestSandboxesIndexOmitsTokens(t *testing.T) { } } -func TestPreviewDialWritesAuditEvent(t *testing.T) { +func TestPreviewTouchWritesAuditEvent(t *testing.T) { eng := newFakeEngine() dir := t.TempDir() m, err := NewManager(t.Context(), &config.Config{DataDir: dir, AuditLog: true, Pools: []config.PoolSpec{}}, eng, testSecrets(t)) @@ -197,10 +197,8 @@ func TestPreviewDialWritesAuditEvent(t *testing.T) { t.Fatalf("setup manager: %v", err) } sb := mustClaim(t, m, testKey) - // The fake engine cannot complete the dial; the audit event records the - // access attempt against the live claim, before the engine is involved. - if _, dialErr := m.PreviewDial(t.Context(), sb.ID, 8080); dialErr == nil { - t.Fatal("fake engine dial unexpectedly succeeded") + if touchErr := m.PreviewTouch(t.Context(), sb.ID, 8080); touchErr != nil { + t.Fatalf("preview touch: %v", touchErr) } raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) @@ -216,7 +214,7 @@ func TestPreviewDialWritesAuditEvent(t *testing.T) { if err := json.Unmarshal([]byte(line), &ev); err != nil { t.Fatalf("bad audit line %q: %v", line, err) } - if ev.ID != sb.ID || ev.Op != "preview_dial" || ev.Port != 8080 { + if ev.ID != sb.ID || ev.Op != "preview" || ev.Port != 8080 { t.Errorf("audit event %+v", ev) } } diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 31967d06..1a01b4f5 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -30,6 +30,7 @@ type previewClaims struct { // PreviewManager is the slice of the pool manager the preview path needs. type PreviewManager interface { + PreviewTouch(ctx context.Context, id string, port uint16) error PreviewDial(ctx context.Context, id string, port uint16) (net.Conn, error) } @@ -48,10 +49,12 @@ func NewPreviewServer(secret, base, owner string, mgr PreviewManager) *PreviewSe return nil } p := &PreviewServer{secret: []byte(secret), base: base, owner: owner, mgr: mgr} - // Every request dials through PreviewDial so release revokes a URL even while - // VM removal is waiting for its eventual retry. + // One shared transport so a page's sub-resource fan-out reuses kept-alive + // guest conns; the Director keys each request's host to sandbox:port so + // the idle pool never mixes claims. Revocation rides PreviewTouch in + // serve — pooled conns skip this dial. p.transport = &http.Transport{ - DisableKeepAlives: true, + IdleConnTimeout: 90 * time.Second, DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { id, portStr, err := net.SplitHostPort(addr) if err != nil { @@ -98,10 +101,15 @@ func (p *PreviewServer) serve(w http.ResponseWriter, r *http.Request) { p.forward(w, r, claims.Owner) return } + if err := p.mgr.PreviewTouch(r.Context(), claims.ID, claims.Port); err != nil { + http.Error(w, "preview target unreachable", http.StatusBadGateway) + return + } p.proxyLocal(w, r, claims) } -// proxyLocal uses the live claim lookup as stateless revocation. +// proxyLocal reverse-proxies to the guest port over the pooled relay +// transport; serve's PreviewTouch has already authorized the request. func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claims previewClaims) { rp := &httputil.ReverseProxy{ Director: func(req *http.Request) { diff --git a/sandboxd/server/preview_test.go b/sandboxd/server/preview_test.go index a3c5988a..0b350da8 100644 --- a/sandboxd/server/preview_test.go +++ b/sandboxd/server/preview_test.go @@ -33,18 +33,14 @@ func TestPreviewTokenRoundTrip(t *testing.T) { func TestPreviewRejectsExpired(t *testing.T) { ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{}) - token := mintToken(ps, "sb_1", 8080, -time.Second) // already expired + token := mintToken(ps, "sb_1", 8080, -time.Second) if _, ok := ps.verify(token); ok { t.Error("expired token verified") } } func TestPreviewProxiesToGuest(t *testing.T) { - guest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, "guest saw "+r.URL.Path) - })) - t.Cleanup(guest.Close) - guestAddr := strings.TrimPrefix(guest.URL, "http://") + guestAddr := newGuestServer(t, func(r *http.Request) string { return "guest saw " + r.URL.Path }) dialed := false ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ @@ -91,21 +87,21 @@ func TestPreviewRevokedWhenDialFails(t *testing.T) { } func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { - guest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, "guest") - })) - t.Cleanup(guest.Close) - guestAddr := strings.TrimPrefix(guest.URL, "http://") + guestAddr := newGuestServer(t, func(*http.Request) string { return "guest" }) var live atomic.Bool - var dials atomic.Int32 + var touches, dials atomic.Int32 live.Store(true) ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { - dials.Add(1) + touch: func(string, uint16) error { + touches.Add(1) if !live.Load() { - return nil, net.ErrClosed + return net.ErrClosed } + return nil + }, + dial: func(string, uint16) (net.Conn, error) { + dials.Add(1) return net.Dial("tcp", guestAddr) }, }) @@ -132,17 +128,16 @@ func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { if resp.StatusCode != http.StatusBadGateway { t.Errorf("status after release = %d, want 502", resp.StatusCode) } - if got := dials.Load(); got != 2 { - t.Errorf("PreviewDial calls = %d, want one per request", got) + if got := touches.Load(); got != 2 { + t.Errorf("PreviewTouch calls = %d, want one per request", got) + } + if got := dials.Load(); got != 1 { + t.Errorf("PreviewDial calls = %d, want the pooled conn reused", got) } } func TestPreviewForwardsToOwner(t *testing.T) { - guest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, "guest saw "+r.URL.Path) - })) - t.Cleanup(guest.Close) - guestAddr := strings.TrimPrefix(guest.URL, "http://") + guestAddr := newGuestServer(t, func(r *http.Request) string { return "guest saw " + r.URL.Path }) owner := httptest.NewUnstartedServer(nil) ownerAddr := owner.Listener.Addr().String() @@ -171,13 +166,30 @@ func TestPreviewForwardsToOwner(t *testing.T) { } type fakePreviewMgr struct { - dial func(id string, port uint16) (net.Conn, error) + touch func(id string, port uint16) error + dial func(id string, port uint16) (net.Conn, error) +} + +func (f *fakePreviewMgr) PreviewTouch(_ context.Context, id string, port uint16) error { + if f.touch == nil { + return nil + } + return f.touch(id, port) } func (f *fakePreviewMgr) PreviewDial(_ context.Context, id string, port uint16) (net.Conn, error) { return f.dial(id, port) } +func newGuestServer(t *testing.T, body func(r *http.Request) string) string { + t.Helper() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, body(r)) + })) + t.Cleanup(ts.Close) + return strings.TrimPrefix(ts.URL, "http://") +} + func mintToken(ps *PreviewServer, id string, port uint16, ttl time.Duration) string { url := ps.Mint(id, port, ttl) return strings.TrimSuffix(url[strings.Index(url, "/p/")+3:], "/") From 718242a799c9f5c67912ace417e16f8001e45f82 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 02:12:54 +0800 Subject: [PATCH 04/26] review: tighten comments, converge relay close paths Reattach the preview_advertise doc to its field at one line, drop two edit-narration clauses and a restating test comment, compress the watch overflow WHYs, and make PortConn.Close match Pty.Close (the writer-side error value never reaches readers). --- sandboxd/config/config.go | 6 +++--- sandboxd/server/server.go | 4 ++-- sdk/go/port.go | 4 ++-- silkd/src/watch.rs | 8 +++----- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 7c2a6082..bf4d6ad3 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -221,9 +221,9 @@ type Config struct { ArchiveAfterSeconds int `json:"archive_after_seconds,omitempty"` ArchiveDeleteAfterSeconds int `json:"archive_delete_after_seconds,omitempty"` - // PreviewAdvertise is the browser-facing base URL and may be shared behind one proxy. - PreviewListen string `json:"preview_listen,omitempty"` - PreviewSecret string `json:"preview_secret,omitempty"` //nolint:gosec // config field, not a hardcoded credential + PreviewListen string `json:"preview_listen,omitempty"` + PreviewSecret string `json:"preview_secret,omitempty"` //nolint:gosec // config field, not a hardcoded credential + // PreviewAdvertise is the browser-facing base, shareable fleet-wide behind one proxy. PreviewAdvertise string `json:"preview_advertise,omitempty"` // CheckpointDir is where checkpoints live; defaults to diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 3da4416d..37454a00 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -376,7 +376,7 @@ func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req t // handleRelease releases a claimed sandbox. Two credentials authorize it: the // node's root api_token (the operator) may release any sandbox by id, so // aggregated/control-plane teardown works without holding the per-sandbox token; -// a per-sandbox token releases only its own claim, unchanged. A tenant token is +// a per-sandbox token releases only its own claim. A tenant token is // neither — it is not the root api_token, so it resolves as a (non-matching) // sandbox token and 404s. func (s *Server) handleRelease(w http.ResponseWriter, r *http.Request) { @@ -532,7 +532,7 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { // handleCheckpointProbe answers a peer's HEAD probe. With a probeKey // configured (an encrypted mesh), the caller must present a fresh MAC over // the id (peer.ProbeHeader) or the probe is rejected before the metadata -// read; without one, the id remains the only capability, same as before. +// read; without one, the id remains the only capability. func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") if len(s.probeKey) > 0 && !peer.VerifyProbeMAC(s.probeKey, id, r.Header.Get(peer.ProbeHeader)) { diff --git a/sdk/go/port.go b/sdk/go/port.go index b239f86b..c0b7198b 100644 --- a/sdk/go/port.go +++ b/sdk/go/port.go @@ -59,8 +59,8 @@ func (p *PortConn) Close() error { p.closeOnce.Do(func() { p.stop() // drain can be parked in a pipe write on an unread tail; only the - // reader side unblocks it, so teardown must close the pipe too. - _ = p.out.CloseWithError(net.ErrClosed) + // reader side unblocks it. + _ = p.out.Close() }) return nil } diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index 0e45b653..e2c300a1 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -48,8 +48,7 @@ where return Ok(()); } } - // While the watcher lives, only overflow drops the sender: the - // buffered prefix has all been delivered, then the terminal error. + // Only overflow drops the sender mid-watch; the buffered prefix is already out. None => return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await, }, // The client sends nothing during a watch, so any readable state — @@ -59,9 +58,8 @@ where } } -/// Forwards frames into the bounded channel; a full channel drops the sender, -/// so the closed channel itself is the overflow signal — a wakeup the async -/// loop cannot miss even while parked on an empty queue. +/// A full channel drops the sender: the closed channel is the overflow +/// signal, a wakeup the parked loop cannot miss. fn forward_frames(tx: &mut Option>, frames: Vec) { let Some(sender) = tx.as_ref() else { return }; for frame in frames { From 0f7cc33fd23e8aafba0bf3e988c1f9c2c911bec6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 02:50:21 +0800 Subject: [PATCH 05/26] fix(pool): scope promoted-template claims to the owning tenant resolveGolden decoded the template record and dropped it, so a tenant token could claim any other tenant's promoted template by name and boot its snapshot - proven with a test that saw the owner's content digest come back under a foreign tenant. Promote and delete were already tenant-scoped; only the read path was not. A foreign template now resolves as absent, which is byte-identical to a name that was never promoted, so existence stays unconfirmed. Root-promoted templates remain shared, as pools are. --- sandboxd/pool/claim.go | 2 +- sandboxd/pool/promote_test.go | 49 ++++++++++++++++++++++++++++++++++- sandboxd/pool/template.go | 13 ++++++---- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 5a4c1366..a3a2a26f 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -532,7 +532,7 @@ func (m *Manager) claimProvision(ctx context.Context, key types.PoolKey, ttl tim } reserved := applied defer func() { m.unreserveVolumes(reserved) }() - golden, err := m.resolveGolden(ctx, key) + golden, err := m.resolveGolden(ctx, key, tenant) if err != nil { return nil, fmt.Errorf("resolve template: %w", err) } diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index 916454e0..d285ab7f 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -200,7 +200,7 @@ func TestResolveGoldenSkipsPromotedEgressTemplate(t *testing.T) { if _, err = m.commitTemplate(t.Context(), staging, id, ""); err != nil { t.Fatalf("seed template: %v", err) } - golden, err := m.resolveGolden(t.Context(), egKey) + golden, err := m.resolveGolden(t.Context(), egKey, "") if err != nil { t.Fatalf("resolveGolden: %v", err) } @@ -312,6 +312,53 @@ func TestPromoteRefusesCrossTenantOverwrite(t *testing.T) { } } +func TestTemplateClaimIsTenantScoped(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + claim := func(tenant string) *types.Sandbox { + t.Helper() + sb, err := m.ClaimProvision(t.Context(), testKey, time.Hour, tenant, "", nil) + if err != nil { + t.Fatalf("claim %q: %v", tenant, err) + } + return sb + } + + a := claim("acme") + private, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") + if err != nil { + t.Fatalf("acme promote: %v", err) + } + r := claim("") + shared, _, err := m.Promote(t.Context(), r.ID, Cred{Token: r.Token}, "ops-shared", "") + if err != nil { + t.Fatalf("root promote: %v", err) + } + + if _, err := m.ClaimProvisionPromoted(t.Context(), private, time.Hour, "beta", "", nil); !errors.Is(err, ErrUnknownTemplate) { + t.Errorf("beta claiming acme's template: %v, want ErrUnknownTemplate", err) + } + for _, tc := range []struct { + name string + key types.PoolKey + tenant string + }{ + {"owner claims its own", private, "acme"}, + {"root claims a tenant's", private, ""}, + {"tenant claims a root template", shared, "beta"}, + } { + t.Run(tc.name, func(t *testing.T) { + sb, err := m.ClaimProvisionPromoted(t.Context(), tc.key, time.Hour, tc.tenant, "", nil) + if err != nil { + t.Fatalf("claim: %v", err) + } + if sb.TemplateDigest == "" { + t.Error("claim did not resolve from the promoted template") + } + }) + } +} + func TestTemplateHashesSortedForMeshCompare(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 4e559f61..042593e1 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -277,7 +277,7 @@ type goldenResolution struct { // golden (no release), else a promoted template fetched from the store; // empty dir cold-boots. Only a true absence cold-boots — a backend failure // propagates rather than silently booting a template name as an image ref. -func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey) (goldenResolution, error) { +func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant string) (goldenResolution, error) { m.mu.Lock() var dir string if p := m.pools[key]; p != nil { @@ -302,18 +302,21 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey) (goldenR } return goldenResolution{release: func() {}}, err } + cleanup := func() { release(); l.RUnlock(); m.recDone(id) } var rec templateRecord if err := json.Unmarshal(meta, &rec); err != nil { - release() - l.RUnlock() - m.recDone(id) + cleanup() return goldenResolution{release: func() {}}, fmt.Errorf("decode template metadata: %w", err) } + if rec.Tenant != "" && !tenantOwns(tenant, rec.Tenant) { + cleanup() + return goldenResolution{release: func() {}}, nil + } return goldenResolution{ dir: dir, templateDigest: digest, promoted: true, - release: func() { release(); l.RUnlock(); m.recDone(id) }, + release: cleanup, }, nil } From 90929bc15d935d3c1f188edf0a693ae70e47102c Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 02:50:21 +0800 Subject: [PATCH 06/26] fix(mesh): a departed node stays gone until it restarts forget dropped a dead peer from the view, but merge re-inserted any node it did not already know, so one push/pull from a peer that had not yet seen the death resurrected it - permanently, since SWIM never fires NotifyLeave twice, and the node was then re-gossiped cluster-wide. Tombstone the epoch it left at: a lagging peer replaying that epoch is ignored, while the node's own restart always seeds a higher epoch and clears the tombstone. --- sandboxd/mesh/mesh.go | 10 +++++++++- sandboxd/mesh/mesh_test.go | 26 +++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 8536aede..122ffefb 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -51,6 +51,8 @@ type Mesh struct { mu sync.Mutex self NodeState view map[string]NodeState // node_id → latest known state (includes self) + // departedEpoch tombstones a node at the epoch it left; a restart outranks it. + departedEpoch map[string]uint64 } // New starts a mesh member listening per cfg. selfAddr is the data-plane @@ -71,7 +73,8 @@ func New(ctx context.Context, cfg *memberlist.Config, nodeID, selfAddr string, s Epoch: epoch, Pools: map[string]int{}, }, - view: map[string]NodeState{}, + view: map[string]NodeState{}, + departedEpoch: map[string]uint64{}, } if err := m.persistEpoch(epoch); err != nil { return nil, fmt.Errorf("persist mesh epoch: %w", err) @@ -292,6 +295,7 @@ func (m *Mesh) forget(nodeID string) { m.mu.Lock() defer m.mu.Unlock() if nodeID != m.self.NodeID { + m.departedEpoch[nodeID] = m.view[nodeID].Epoch delete(m.view, nodeID) } } @@ -309,6 +313,10 @@ func (m *Mesh) merge(states []NodeState) { if ok && st.Epoch <= cur.Epoch { continue } + if st.Epoch <= m.departedEpoch[st.NodeID] { + continue + } + delete(m.departedEpoch, st.NodeID) // Warn on each distinct divergent digest (not once per lifetime): a // mismatched api_token/tenants/preview_secret/CA root 401s cross-node // redirects and fails interception. Warn-only — refusing would partition diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index 20e4968a..b3400ed0 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -95,6 +95,25 @@ func TestForgetPrunesDeadNode(t *testing.T) { } } +func TestForgottenNodeStaysGoneUntilItRestarts(t *testing.T) { + m := newTestMesh(t, "a") + dead := NodeState{NodeID: "b", Addr: "b:7777", Epoch: 4, Pools: map[string]int{"k": 3}} + m.merge([]NodeState{dead}) + m.forget("b") + + m.merge([]NodeState{dead}) + if got := m.Candidates("k"); got != nil { + t.Errorf("a lagging peer resurrected b: %v", got) + } + + restarted := dead + restarted.Epoch = 5 + m.merge([]NodeState{restarted}) + if len(m.Candidates("k")) != 1 { + t.Error("b restarted with a higher epoch and must be reinstated") + } +} + func TestCandidatesPowerOfTwo(t *testing.T) { m := newTestMesh(t, "a") m.merge([]NodeState{ @@ -210,9 +229,10 @@ func TestVolumeHoldersCountSelfAndPeers(t *testing.T) { func newTestMesh(t *testing.T, id string) *Mesh { t.Helper() return &Mesh{ - epochPath: filepath.Join(t.TempDir(), "mesh-epoch"), - self: NodeState{NodeID: id, Addr: id + ":7777", Pools: map[string]int{}}, - view: map[string]NodeState{id: {NodeID: id, Addr: id + ":7777"}}, + epochPath: filepath.Join(t.TempDir(), "mesh-epoch"), + self: NodeState{NodeID: id, Addr: id + ":7777", Pools: map[string]int{}}, + view: map[string]NodeState{id: {NodeID: id, Addr: id + ":7777"}}, + departedEpoch: map[string]uint64{}, } } From 8d9283d106dc76db739ddc7652b9ec74075aae15 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 02:50:21 +0800 Subject: [PATCH 07/26] docs: correct the preview audit op, bound verbs, and volume scope The audit op is preview, once per request, not preview_dial. pty_open, lsp_request and port_forward are connection-bound alongside fs_watch. Volume discovery returns the gossiped union only to root; a tenant sees this node's local entries its ACL permits. --- docs/deploy.md | 2 +- docs/sandboxd-api.md | 10 +++++----- docs/silkd.md | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 90bd8b15..fb67312a 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -103,7 +103,7 @@ sandboxd reads one JSON file (`-config`, default | `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (a replica a delete broadcast missed becomes eligible for expiry after it, and its next successful hourly sweep removes it — so it is the finite eligibility point, not an exact ceiling). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, normal warm-candidate placement applies, with volume claims limited to candidates holding every requested volume) | -| `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`); preview accesses record as op `preview_dial`. A request frame whose first line exceeds 4 KiB is skipped, never truncated | +| `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`); preview accesses record as op `preview`, one per request. A request frame whose first line exceeds 4 KiB is skipped, never truncated | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a claim with no data-plane connection for this long is hibernated; the next call wakes it transparently. Per-pool `idle_hibernate_seconds` (in a pool entry) does the same for that pool's claims — pooled keys ignore the node-wide value. Opt-in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call restores it transparently (a checkpoint restore's latency). Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived, reclaiming storage; the claim is then gone for good. Same node-wide/per-pool split | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index d268fbec..ba1e57a8 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -193,11 +193,11 @@ caller may use, without host paths or holder addresses: "size_bytes": 214748364800, "available": true, "nodes": 3}]} ``` -Root sees every entry; a tenant sees unrestricted entries plus those whose -access list names it. The response is the gossiped union: `nodes` counts members -advertising the name. `size_bytes` and `available` are a best-effort stat of the -answering node's image, so a peer-only entry remains discoverable with -`available: false`. Membership is eventually consistent by one gossip tick. +Root sees the gossiped union, including peer-only entries (`available: false`); +a tenant sees only entries this node declares locally and whose access list +permits it. `nodes` counts members advertising the name; `size_bytes` and +`available` are a best-effort stat of the answering node's image. Membership is +eventually consistent by one gossip tick. `writable` is the entry's catalog configuration, fleet-uniform like the access list; the field is emitted (as `true`) only for a writable entry and omitted otherwise, so a read-only entry's response is byte-identical to v1. diff --git a/docs/silkd.md b/docs/silkd.md index 3e36e9f0..befa4be9 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -19,8 +19,8 @@ ride base64 in `data` fields. Frames are capped at 8 MiB; requests carry `"v": 1` and unknown fields are ignored (the forward-compatibility story). Sessions and processes are server-side state addressed by id — a dropped -connection loses nothing (`attach` resumes). The only connection-bound verb -is `fs_watch`. +connection loses nothing (`attach` resumes). The connection-bound verbs are +`fs_watch`, `pty_open`, `lsp_request`, and `port_forward`. The authoritative wire contract is the shared fixture corpus in `protocol/wire/fixtures/v1`, round-tripped by both the Rust and Go test suites — From b0980911348a38af42abeb6917f2606447827e2f Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 02:55:07 +0800 Subject: [PATCH 08/26] test(peer): bound the in-flight wait and give the fan-out test teeth The Forget-mid-flight test spun on an atomic with no deadline, so a probe that never arrived hung until the binary timeout instead of failing; wait on a channel the handler closes. The swallows-failures test never checked the healthy peer was reached, so it passed even if the fan-out stopped at the first error - count that peer's hits. --- sandboxd/store/peer/broadcast_test.go | 5 +++++ sandboxd/store/peer/probe_test.go | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sandboxd/store/peer/broadcast_test.go b/sandboxd/store/peer/broadcast_test.go index 0797f6b3..d1aa8ed3 100644 --- a/sandboxd/store/peer/broadcast_test.go +++ b/sandboxd/store/peer/broadcast_test.go @@ -78,13 +78,18 @@ func TestBroadcastDeleteSwallowsFailures(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) })) defer fail.Close() + var reached atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached.Add(1) w.WriteHeader(http.StatusNoContent) })) defer ok.Close() b := &Broadcaster{Peers: func() []string { return []string{fail.URL, ok.URL, "127.0.0.1:1"} }} b.Delete(t.Context(), testID) // must return without panicking regardless of peer outcomes + if got := reached.Load(); got != 1 { + t.Errorf("healthy peer hit %d times, want 1 (fan-out continues past a failure)", got) + } } // TestBroadcastDeleteNoPeersNoOp: a single-node deployment must not dial diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index 425fd057..5d5d93b8 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -100,9 +100,12 @@ func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { // claims to a node that no longer holds it. func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { release := make(chan struct{}) + inFlight := make(chan struct{}) + probed := sync.OnceFunc(func() { close(inFlight) }) var hits atomic.Int32 owner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) + probed() <-release // hold the probe open so Forget can land mid-flight w.WriteHeader(http.StatusOK) })) @@ -113,9 +116,7 @@ func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { done := make(chan struct{}) go func() { owners = p.Owners(t.Context(), testID); close(done) }() - for hits.Load() == 0 { // wait until the probe is in flight - time.Sleep(time.Millisecond) - } + <-inFlight p.Forget(testID) // the delete lands while the flight is open close(release) <-done From cb6b293e96c693128c5eabf7b48cd2f88d8c69c7 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 02:59:49 +0800 Subject: [PATCH 09/26] fix(egress): a missing policy on either side denies, never inherits A tenant configured without an egress block inherited the pool's entire allow-list including its secret injections, so adding a tenant token handed that tenant the operator's credentials - the opposite of the documented intersection. Both single-sided branches now deny: a tenant reaches only what both layers allow, and a pool without a policy grants nothing to anyone. Root keeps the pool policy whole, having no tenant layer to intersect. --- docs/egress.md | 8 ++++++-- sandboxd/pool/egress.go | 19 +++++++++---------- sandboxd/pool/egress_test.go | 25 ++++++++++++++----------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/docs/egress.md b/docs/egress.md index dee6ab2f..46439e08 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -98,8 +98,12 @@ domain policy first; the allow-list widens the IP gate only. Policy is per pool and per tenant; the effective policy is their intersection (a request must pass both, and the pool rule's secret wins on a double allow). -Secrets are registered separately and referenced by name — the value comes from -the environment, never the config file. +A missing policy on either side is an empty allow-list, not a pass: a tenant +without its own `egress` block reaches nothing, so granting a tenant egress — +and the secret injection that rides it — is always an explicit act. Root +claims have no tenant layer and take the pool's policy whole. Secrets are +registered separately and referenced by name — the value comes from the +environment, never the config file. ```jsonc { diff --git a/sandboxd/pool/egress.go b/sandboxd/pool/egress.go index 5bd9544a..2544b2c2 100644 --- a/sandboxd/pool/egress.go +++ b/sandboxd/pool/egress.go @@ -183,23 +183,22 @@ func (m *Manager) poolIntercepts(key types.PoolKey) bool { return m.poolEgress[key].Intercepts() } -// effectivePolicy resolves a claim's egress evaluator: pool ∩ tenant (deny -// wins), or whichever single one is set; ok is false when neither applies. +// effectivePolicy resolves pool ∩ tenant; root has no tenant layer. func (m *Manager) effectivePolicy(sb *types.Sandbox) (egress.Evaluator, bool) { m.mu.Lock() defer m.mu.Unlock() poolPol := m.poolEgress[sb.Key] - tenantPol := m.tenantEgress[sb.Tenant] - switch { - case poolPol != nil && tenantPol != nil: - return egress.Compose(*poolPol, *tenantPol), true - case poolPol != nil: + if poolPol == nil { + return nil, false + } + if sb.Tenant == "" { return *poolPol, true - case tenantPol != nil: - return *tenantPol, true - default: + } + tenantPol := m.tenantEgress[sb.Tenant] + if tenantPol == nil { return nil, false } + return egress.Compose(*poolPol, *tenantPol), true } // newEgressDialer builds the proxy's upstream dialer: internal targets are diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index cd1506e9..52a9bef5 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -138,15 +138,18 @@ func TestEffectivePolicyComposition(t *testing.T) { tenantOnly := &egress.Policy{Allow: []egress.Rule{{Host: "b.test"}, {Host: "c.test"}}} cases := []struct { - name string - pool, tenant *egress.Policy - allow, deny string - wantArmed bool + name string + tenant string + pool, tnPol *egress.Policy + allow, deny string + wantArmed bool }{ - {"pool only", both, nil, "a.test", "z.test", true}, - {"tenant only", nil, tenantOnly, "c.test", "a.test", true}, - {"intersection", both, tenantOnly, "b.test", "a.test", true}, // a.test allowed by pool, denied by tenant - {"neither", nil, nil, "", "", false}, + {"root takes the pool policy whole", "", both, nil, "a.test", "z.test", true}, + {"root without a pool policy", "", nil, nil, "", "", false}, + {"tenant intersects", "acme", both, tenantOnly, "b.test", "a.test", true}, // a.test allowed by pool, denied by tenant + {"tenant declaring no policy", "acme", both, nil, "", "", false}, + {"tenant on a policyless pool", "acme", nil, tenantOnly, "", "", false}, + {"neither", "acme", nil, nil, "", "", false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -155,10 +158,10 @@ func TestEffectivePolicyComposition(t *testing.T) { m.poolEgress[testKey] = tc.pool } m.tenantEgress = map[string]*egress.Policy{} - if tc.tenant != nil { - m.tenantEgress["acme"] = tc.tenant + if tc.tnPol != nil { + m.tenantEgress["acme"] = tc.tnPol } - sb := &types.Sandbox{Key: testKey, Tenant: "acme"} + sb := &types.Sandbox{Key: testKey, Tenant: tc.tenant} eval, ok := m.effectivePolicy(sb) if ok != tc.wantArmed { t.Fatalf("armed=%v, want %v", ok, tc.wantArmed) From 7a85601535ec1b803b53b1e1dd5fcf0aa096d053 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:07:16 +0800 Subject: [PATCH 10/26] docs: correct the claim engine axis, metrics inventory, and config table Document engine as the fourth pool-key axis (claim body, pools table, and the three key examples that omitted it), list the archived/draining/digest-mismatch gauges and the archive counters /metrics actually emits, add the secrets and egress_internal_allow rows, name the 401 the keyed HEAD probe answers, and say the usage journal's key is the pool key's hash. Also: the audit record's egress decision/secret fields, sandboxd.dbg in the release assets, browser is amd64-only while base is multi-arch, browser in the os-image list, bench.sh and release.yml in the repo map, and the nft lock drops guest-initiated packets. Drops a third-party registry namespace from the android README. --- README.md | 8 +++++--- docs/cluster.md | 2 +- docs/deploy.md | 8 +++++--- docs/sandboxd-api.md | 30 ++++++++++++++++++------------ docs/security.md | 4 ++-- os-image/android/README.md | 6 +++--- os-image/browser/README.md | 3 ++- 7 files changed, 36 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 2ab717b8..0d80f5df 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ performance) — source in `/boot/vmlinuz-sandbox` + `/boot/initrd.img-sandbox` - `os-image/` — VM images consuming the boot artifact: `base` (layered, for builds), `rt` (base squashed to one layer — the default template in - examples), `python`, `python-rt`, and `android` -- `scripts/` — `boot-bench.sh` (boot phase timing) and `sandboxd-e2e.sh` - (bare-metal e2e, below) + examples), `python`, `python-rt`, `browser`, and `android` +- `scripts/` — `boot-bench.sh` (boot phase timing), `bench.sh` (the published + benchmark procedure), `sandboxd-e2e.sh` (bare-metal e2e, below), plus the + `archive`/`egress`/`intercept` e2e drivers ## Build & test @@ -118,6 +119,7 @@ TEMPLATE=rt:24.04 scripts/sandboxd-e2e.sh changed carriers (via `build-boot.yml` / `build-silkd.yml`, `workflow_call`) then os-images, in order, so the chain is deterministic - `build-os-images.yml` — bakes base + flavors FROM the sha-pinned carriers +- `release.yml` — on a version tag, builds the release binaries and archives - `publish-pypi.yml` — on an `sdk-*-v*` tag, builds and publishes the matching package via PyPI Trusted Publishing (OIDC, per-package environment) diff --git a/docs/cluster.md b/docs/cluster.md index 67f47955..7fa0e90d 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -123,7 +123,7 @@ curl -s -H "Authorization: Bearer $TOKEN" http://node-a:7777/v1/info | jq . ```json { "pools": [ - {"key": {"template": "base:24.04", "net": "none", "size": "small"}, + {"key": {"template": "base:24.04", "net": "none", "size": "small", "engine": "ch"}, "warm": 4, "refilling": 0, "target": 4, "golden": true} ], "claimed": 2, diff --git a/docs/deploy.md b/docs/deploy.md index fb67312a..efaa9c25 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -20,7 +20,7 @@ the cocoon CLI and needs a template image with silkd baked in. (pull via cocoon, or `cocoon image import` a tar) Prebuilt static linux/amd64 and linux/arm64 binaries (`sandboxd`, -`sandbox-mcp`, `silkd`, with `checksums.txt`) ship with every +`sandboxd.dbg`, `sandbox-mcp`, `silkd`, with `checksums.txt`) ship with every [GitHub release](https://github.com/cocoonstack/sandbox/releases); the boot artifact and the `base`/`rt`/`python` images are multi-arch manifests (`browser` and `android` remain amd64-only). Build from source with @@ -89,6 +89,8 @@ sandboxd reads one JSON file (`-config`, default | `advertise_addr` | = `listen` | the host:port clients reach this node at; returned as a claim's owner address and gossiped to peers. Must be routable when `listen` is a wildcard | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) needs `bridges` and rejects a CNI network at load | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, while every listed name must exist in the node's `tenants` config; root always has access. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | +| `secrets` | unset | node-side credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or tenant rule references the name; the value comes from the environment, never this file. See [egress](egress.md) | +| `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant). Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_ca` | unset | [HTTPS-interception](egress.md#https-interception) PKI: `root_cert` (the cluster root baked into intercepted guests; may bundle old+new roots during rotation) plus this node's `intermediate_cert`/`intermediate_key` from `sandboxd ca issue-intermediate`. Required when any pool rule sets `intercept` | | `api_token` | unset | the operator (root) credential: when set, guards the node-level endpoints (Bearer) with full access, including release-by-id cleanup. Per-sandbox tokens guard ordinary sandbox-scoped calls | | `tenants` | unset | multi-tenant tokens next to `api_token`: `[{"name": "acme", "token": "…", "max_claims": 50}]`. A tenant token reaches the resource-creating verbs (claim, fork, promote, checkpoint, preview), catalog discovery, and its own sandbox/checkpoint listings; everything it creates is stamped with the tenant name. Root-only surfaces (per-id sandbox reads, `GET /v1/info`, `PUT /v1/pools`, `POST/DELETE /v1/drain`, `/metrics`) answer it 403. `max_claims` (0 = unlimited) caps that tenant's live claims next to the node-wide cap. Requires `api_token` set. Names and tokens must be unique, tokens distinct from `api_token`. On a cluster all nodes must carry the same tenants set (the SDK replays whichever token authorized a redirect), and per-node caps mean a tenant's effective cluster limit is `max_claims` × nodes. Empty = exactly the single-token behavior | @@ -103,12 +105,12 @@ sandboxd reads one JSON file (`-config`, default | `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (a replica a delete broadcast missed becomes eligible for expiry after it, and its next successful hourly sweep removes it — so it is the finite eligibility point, not an exact ceiling). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, normal warm-candidate placement applies, with volume claims limited to candidates holding every requested volume) | -| `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`); preview accesses record as op `preview`, one per request. A request frame whose first line exceeds 4 KiB is skipped, never truncated | +| `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `decision` and `secret` (the ref name, never its value) on `egress` records; preview accesses record as op `preview`, one per request. A request frame whose first line exceeds 4 KiB is skipped, never truncated | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a claim with no data-plane connection for this long is hibernated; the next call wakes it transparently. Per-pool `idle_hibernate_seconds` (in a pool entry) does the same for that pool's claims — pooled keys ignore the node-wide value. Opt-in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call restores it transparently (a checkpoint restore's latency). Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived, reclaiming storage; the claim is then gone for good. Same node-wide/per-pool split | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | -| `pools[]` | — | warm pools. `warm` defaults to 4; `net` is `none` or `egress`; `size` is a tier, below. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | +| `pools[]` | — | warm pools, keyed by `(template, net, size, engine)`. `warm` defaults to 4; `net` is `none` or `egress`; `size` is a tier, below; `engine` is `ch` (default) or `fc` to cold-boot that key under Firecracker. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | Size tiers (free-form CPU/memory is deliberately not accepted — it would fragment the warm pools): diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index ba1e57a8..cf69fec9 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -32,7 +32,10 @@ Auth: `Authorization: Bearer ` (when configured). "require_promoted": false} ``` -- `net` defaults to `none`, `size` to `small` +- `net` defaults to `none`, `size` to `small`, `engine` to `ch`. The pool key + is `(template, net, size, engine)`; `engine: "fc"` cold-boots that key under + Firecracker, and clones inherit the hypervisor pinned in the golden's + snapshot - `ttl_seconds` 0 means the server default (5 minutes); capped at 24h. The owning node reaps the sandbox after the TTL even if the client vanishes - `claim_ref` is an optional opaque caller reference echoed by the scoped @@ -277,7 +280,7 @@ node (name-based calls route via gossip); a shared checkpoint store makes every node resolve it. Under exactly this key: ```json -{"key": {"template": "myproj:v1", "net": "none", "size": "small"}, +{"key": {"template": "myproj:v1", "net": "none", "size": "small", "engine": "ch"}, "content_digest": "sha256:…"} ``` @@ -403,7 +406,8 @@ part of the public API; an SDK caller has no reason to call it directly. 401 missing or unrecognized token, 403 a valid tenant token (authenticated but not the operator), 404 unknown checkpoint. - `HEAD` is the ownership probe: 200 when this node holds a branchable - (non-archive) copy, 404 otherwise. On a mesh with `cluster_key` set the + (non-archive) copy, 404 otherwise, and 401 on a keyed mesh when + `X-Cocoon-Probe` is absent or expired. On a mesh with `cluster_key` set the request must carry `X-Cocoon-Probe`, an HMAC over the id and a coarse time bucket keyed off a probe-specific derivation of the cluster key — verified before any disk is touched, replayable for roughly a minute at most. On a @@ -479,20 +483,22 @@ never mistaken for idle. 404 unknown id. ## GET /metrics Auth: root only (tenant tokens get 403). Prometheus text format, -hand-rendered: pool warm/target gauges, claimed/hibernated gauges, a -per-tenant live-claim gauge (`sandboxd_tenant_claims{tenant="…"}`, -configured tenants only), claims by tier (warm/clone/cold), -wake/hibernate/fork/checkpoint/promote/release/reap counters, and claim/wake -`*_seconds_total` for average latency. /metrics is a derived ops view; the -billing source of truth is the usage journal below. +hand-rendered: pool warm/target gauges, claimed/hibernated/archived/draining +gauges, a per-tenant live-claim gauge (`sandboxd_tenant_claims{tenant="…"}`, +configured tenants only), `sandboxd_config_digest_mismatch` on a mesh, claims +by tier (warm/clone/cold), +wake/hibernate/fork/checkpoint/promote/release/reap counters plus +archive/unarchive/archive-delete counters, and claim/wake `*_seconds_total` +for average latency. /metrics is a derived ops view; the billing source of +truth is the usage journal below. ## Usage journal (usage.jsonl) Always on: every lifecycle transition appends one JSONL event to `/usage.jsonl` — `{"t": , "ev": "claim|hibernate|wake|fork|checkpoint|promote|release|reap|archive|unarchive|archive_delete|egress", -"id": "sb_…", "vm": "sbx-…"}` plus `key` and `tenant` (the pool key and -owning tenant, claim events), `children` (fork) and `ref` (the promoted +"id": "sb_…", "vm": "sbx-…"}` plus `key` and `tenant` (the pool key's stable +hash and the owning tenant, claim events), `children` (fork) and `ref` (the promoted template / checkpoint id, or the egress host). A volume claim also carries `volumes`, the applied catalog names, and — omitted when empty — `volumes_rw`, the subset of those names claimed `rw`, so billing can discriminate write @@ -526,7 +532,7 @@ Auth: root only (tenant tokens get 403). Node pools, claim count, and mesh peers: ```json -{"pools": [{"key": {"template": "base:24.04", "net": "none", "size": "small"}, +{"pools": [{"key": {"template": "base:24.04", "net": "none", "size": "small", "engine": "ch"}, "warm": 4, "refilling": 0, "target": 4, "golden": true}], "claimed": 2, "hibernated": 1, diff --git a/docs/security.md b/docs/security.md index 124c6a3e..2b6576bb 100644 --- a/docs/security.md +++ b/docs/security.md @@ -25,8 +25,8 @@ exposing any part of a deployment beyond a single trusted host. toward the host, siblings, or the network beyond its lanes. - **What a compromised guest can reach.** On the none lane: nothing but vsock — the relay back to its own client and the guarded-egress proxy. - On the egress lane: the same vsock paths plus a NIC whose every packet - except IPv4 broadcast DHCP is dropped by an nftables lock in the host + On the egress lane: the same vsock paths plus a NIC whose every + guest-initiated packet except IPv4 broadcast DHCP is dropped by an nftables lock in the host root netns ([egress](egress.md)); the lock is fail-closed and applied before the claim is handed out. On both lanes the proxy refuses loopback, private, link-local (cloud metadata), CGN, and the diff --git a/os-image/android/README.md b/os-image/android/README.md index c0909150..48af3d14 100644 --- a/os-image/android/README.md +++ b/os-image/android/README.md @@ -27,9 +27,9 @@ The base image (15.0) pulls anonymously from ghcr at docker and `sys.boot_completed=1` with zygote64 + system_server alive (Cloud Hypervisor, no NIC, 4 CPU / 8G). -Do not revert to the previous `ghcr.io/jiaqing-simular/cocoon-android-vnc` -pin: that build ships a broken dexpreopt boot-image chain that -crash-loops zygote before the framework ever completes. +Do not revert to the previous third-party VNC android pin: that build ships a +broken dexpreopt boot-image chain that crash-loops zygote before the framework +ever completes. ## Constraints diff --git a/os-image/browser/README.md b/os-image/browser/README.md index fbf1b033..244d7551 100644 --- a/os-image/browser/README.md +++ b/os-image/browser/README.md @@ -14,7 +14,8 @@ pool shape is `size: large` (4 CPU / 4G) — a persistent Chromium idles at download Chrome for Testing pinned by version + SHA256 (the `install-agent.sh` idiom); bake `/usr/local/bin/chromium-cdp` and `chromium.service` (enabled, `multi-user.target`). -- `platforms` — `linux/amd64`; the base lineage is amd64-only. +- `platforms` — `linux/amd64`; this flavor is amd64-only (Chrome for Testing), + unlike the multi-arch `base`. Why Chrome for Testing: Ubuntu 24.04's `chromium` apt package is a transitional stub that pulls the snap, and snap cannot install inside a From a951bf81491f4d5b55f0cf0fdfae6ab8208a03b3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:21:00 +0800 Subject: [PATCH 11/26] fix: close preview and idle hibernate gaps Guest connection pooling let a reused preview request skip PreviewDial, and with it wakeResolved and the Transition lock that serializes a claim against its own hibernation: once the idle sweep passed its re-check and entered the seconds-long snapshot, a pooled request wrote into a freezing VM and never triggered the wake. Every request dials again, so authorization, the activity stamp, the audit record and the wake all ride the one path. Egress pools accepted idle_hibernate_seconds even though the lane refuses to hibernate at all, so the setting could only ever produce a failure. Config now rejects it, and the sweep skips egress claims of unpooled keys, which the config gate cannot reach - futile work whose only symptom was a logged error every tick, so nothing observable is left to assert. --- docs/deploy.md | 3 +-- sandboxd/config/config.go | 7 +++++-- sandboxd/config/config_test.go | 1 + sandboxd/pool/claim.go | 22 +++------------------- sandboxd/pool/hibernate.go | 2 +- sandboxd/pool/telemetry_test.go | 6 +++--- sandboxd/server/preview.go | 16 +--------------- sandboxd/server/preview_test.go | 29 +++++++---------------------- 8 files changed, 22 insertions(+), 64 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index efaa9c25..4db5d621 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -106,7 +106,7 @@ sandboxd reads one JSON file (`-config`, default | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, normal warm-candidate placement applies, with volume claims limited to candidates holding every requested volume) | | `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `decision` and `secret` (the ref name, never its value) on `egress` records; preview accesses record as op `preview`, one per request. A request frame whose first line exceeds 4 KiB is skipped, never truncated | -| `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a claim with no data-plane connection for this long is hibernated; the next call wakes it transparently. Per-pool `idle_hibernate_seconds` (in a pool entry) does the same for that pool's claims — pooled keys ignore the node-wide value. Opt-in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | +| `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim with no data-plane connection for this long is hibernated; the next call wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call restores it transparently (a checkpoint restore's latency). Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived, reclaiming storage; the claim is then gone for good. Same node-wide/per-pool split | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | @@ -297,7 +297,6 @@ here validates on load: "pools": [ {"template": "rt:24.04", "net": "none", "size": "small", "warm": 4, "warm_max": 12}, {"template": "rt:24.04", "net": "egress", "size": "medium", "warm": 2, - "idle_hibernate_seconds": 120, "archive_after_seconds": 900, "egress": {"allow": [ {"host": "api.github.com", "methods": ["GET", "POST"], "secret": "gh", "intercept": true}, {"host": "*.googleapis.com"} diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index bf4d6ad3..ec4973c7 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -49,7 +49,7 @@ type PoolSpec struct { // name a secret, the pool's is injected. Nil denies all egress. Egress *egress.Policy `json:"egress,omitempty"` - // IdleHibernateSeconds, when >0, hibernates this pool's idle claims + // IdleHibernateSeconds, when >0, hibernates this none-lane pool's idle claims // after that many seconds without a data-plane connection; the next // call wakes them transparently. Zero disables. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` @@ -76,6 +76,9 @@ func (s PoolSpec) ValidateLimits() error { if s.IdleHibernateSeconds < 0 { return fmt.Errorf("idle_hibernate_seconds must not be negative") } + if s.Net == types.NetEgress && s.IdleHibernateSeconds > 0 { + return fmt.Errorf("idle_hibernate_seconds is not supported for egress pools") + } return validateArchiveWindow(s.IdleHibernateSeconds, s.ArchiveAfterSeconds, s.ArchiveDeleteAfterSeconds) } @@ -210,7 +213,7 @@ type Config struct { // name; values come from the environment (value_env), never this file. Secrets []egress.SecretSpec `json:"secrets,omitempty"` - // IdleHibernateSeconds is the idle policy for claims of unpooled keys + // IdleHibernateSeconds is the idle policy for unpooled none-lane claims // (template and checkpoint claims); per-pool settings override it for // pooled keys. Zero disables. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 63242397..ac37e344 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -96,6 +96,7 @@ func TestLoadRejectsInvalid(t *testing.T) { {"bad restore mode", `{"restore_mode":"Mmap","pools":[]}`, "restore_mode"}, {"bad pool key", `{"pools":[{"template":"","net":"none","size":"small"}]}`, "pool"}, {"egress without attachment", `{"pools":[{"template":"rt:24.04","net":"egress","size":"small"}]}`, "egress lane needs"}, + {"egress idle hibernate", `{"bridges":["br0"],"pools":[{"template":"rt:24.04","net":"egress","size":"small","idle_hibernate_seconds":1}]}`, "not supported for egress"}, {"negative warm", `{"pools":[{"template":"rt:24.04","net":"none","size":"small","warm":-2}]}`, "negative"}, {"tenants without api_token", `{"pools":[],"tenants":[{"name":"acme","token":"t1"}]}`, "require api_token"}, {"empty tenant name", `{"api_token":"root","pools":[],"tenants":[{"name":"","token":"t1"}]}`, "tenant name"}, diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index a3a2a26f..f34f25e6 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -103,25 +103,7 @@ func (m *Manager) ClaimDeadline(id, token string) (time.Time, error) { return sb.Deadline, nil } -// PreviewTouch authorizes one preview request: the token is already verified, -// so the live-claim lookup is the whole check — a released sandbox is absent -// and its URL stops resolving. Stamps data-plane activity and writes the -// audit record (preview bypasses the relay's tap; this is the only trace). -func (m *Manager) PreviewTouch(ctx context.Context, id string, port uint16) error { - m.mu.Lock() - sb, ok := m.claimed[id] - m.mu.Unlock() - if !ok { - return ErrUnknownSandbox - } - sb.Touch() - m.recordAudit(ctx, id, auditFrame{Op: "preview", Port: port}) - return nil -} - -// PreviewDial opens a byte stream to a guest port for the preview proxy's -// connection pool; PreviewTouch authorizes each request separately. A -// hibernated sandbox wakes. +// PreviewDial authorizes one preview request and opens its guest connection. func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net.Conn, error) { m.mu.Lock() sb, ok := m.claimed[id] @@ -129,6 +111,8 @@ func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net. if !ok { return nil, ErrUnknownSandbox } + sb.Touch() + m.recordAudit(ctx, id, auditFrame{Op: "preview", Port: port}) sock, err := m.wakeResolved(ctx, sb) if err != nil { return nil, err diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index c97994aa..34e72ce6 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -205,7 +205,7 @@ func (m *Manager) idleOnce(ctx context.Context) { if p, pooled := m.activePool(sb.Key); pooled { idle = p.idle } - if idle <= 0 || hasAppliedVolumes(sb) || sb.HibernateSnap != "" || sb.ArchiveCk != "" || now.Sub(sb.LastSeen()) < idle { + if idle <= 0 || sb.Key.Net == types.NetEgress || hasAppliedVolumes(sb) || sb.HibernateSnap != "" || sb.ArchiveCk != "" || now.Sub(sb.LastSeen()) < idle { continue } victims = append(victims, victim{sb.ID, sb.Token}) diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index d8b5d876..f37bd7fa 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -189,7 +189,7 @@ func TestSandboxesIndexOmitsTokens(t *testing.T) { } } -func TestPreviewTouchWritesAuditEvent(t *testing.T) { +func TestPreviewDialWritesAuditEvent(t *testing.T) { eng := newFakeEngine() dir := t.TempDir() m, err := NewManager(t.Context(), &config.Config{DataDir: dir, AuditLog: true, Pools: []config.PoolSpec{}}, eng, testSecrets(t)) @@ -197,8 +197,8 @@ func TestPreviewTouchWritesAuditEvent(t *testing.T) { t.Fatalf("setup manager: %v", err) } sb := mustClaim(t, m, testKey) - if touchErr := m.PreviewTouch(t.Context(), sb.ID, 8080); touchErr != nil { - t.Fatalf("preview touch: %v", touchErr) + if _, dialErr := m.PreviewDial(t.Context(), sb.ID, 8080); dialErr == nil { + t.Fatal("fake engine dial unexpectedly succeeded") } raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 1a01b4f5..a5ba913c 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -20,7 +20,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// previewClaims is the signed guest target and owner route carried by a preview URL. type previewClaims struct { ID string `json:"id"` Port uint16 `json:"port"` @@ -30,7 +29,6 @@ type previewClaims struct { // PreviewManager is the slice of the pool manager the preview path needs. type PreviewManager interface { - PreviewTouch(ctx context.Context, id string, port uint16) error PreviewDial(ctx context.Context, id string, port uint16) (net.Conn, error) } @@ -49,12 +47,8 @@ func NewPreviewServer(secret, base, owner string, mgr PreviewManager) *PreviewSe return nil } p := &PreviewServer{secret: []byte(secret), base: base, owner: owner, mgr: mgr} - // One shared transport so a page's sub-resource fan-out reuses kept-alive - // guest conns; the Director keys each request's host to sandbox:port so - // the idle pool never mixes claims. Revocation rides PreviewTouch in - // serve — pooled conns skip this dial. p.transport = &http.Transport{ - IdleConnTimeout: 90 * time.Second, + DisableKeepAlives: true, DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { id, portStr, err := net.SplitHostPort(addr) if err != nil { @@ -101,15 +95,9 @@ func (p *PreviewServer) serve(w http.ResponseWriter, r *http.Request) { p.forward(w, r, claims.Owner) return } - if err := p.mgr.PreviewTouch(r.Context(), claims.ID, claims.Port); err != nil { - http.Error(w, "preview target unreachable", http.StatusBadGateway) - return - } p.proxyLocal(w, r, claims) } -// proxyLocal reverse-proxies to the guest port over the pooled relay -// transport; serve's PreviewTouch has already authorized the request. func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claims previewClaims) { rp := &httputil.ReverseProxy{ Director: func(req *http.Request) { @@ -130,7 +118,6 @@ func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claim rp.ServeHTTP(w, r) //nolint:gosec // target derived from an HMAC-signed token, not client input } -// forward relays the signed request to the owner node's main listener. func (p *PreviewServer) forward(w http.ResponseWriter, r *http.Request, owner string) { target := &url.URL{Scheme: "http", Host: owner} rp := httputil.NewSingleHostReverseProxy(target) @@ -163,7 +150,6 @@ func (p *PreviewServer) verify(token string) (previewClaims, bool) { return claims, true } -// handlePreview mints a preview URL bounded by the claim's remaining lease. func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) { if s.preview == nil { writeErr(w, http.StatusNotImplemented, "preview not configured") diff --git a/sandboxd/server/preview_test.go b/sandboxd/server/preview_test.go index 0b350da8..19755c5e 100644 --- a/sandboxd/server/preview_test.go +++ b/sandboxd/server/preview_test.go @@ -90,18 +90,14 @@ func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { guestAddr := newGuestServer(t, func(*http.Request) string { return "guest" }) var live atomic.Bool - var touches, dials atomic.Int32 + var dials atomic.Int32 live.Store(true) ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - touch: func(string, uint16) error { - touches.Add(1) - if !live.Load() { - return net.ErrClosed - } - return nil - }, dial: func(string, uint16) (net.Conn, error) { dials.Add(1) + if !live.Load() { + return nil, net.ErrClosed + } return net.Dial("tcp", guestAddr) }, }) @@ -128,11 +124,8 @@ func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { if resp.StatusCode != http.StatusBadGateway { t.Errorf("status after release = %d, want 502", resp.StatusCode) } - if got := touches.Load(); got != 2 { - t.Errorf("PreviewTouch calls = %d, want one per request", got) - } - if got := dials.Load(); got != 1 { - t.Errorf("PreviewDial calls = %d, want the pooled conn reused", got) + if got := dials.Load(); got != 2 { + t.Errorf("PreviewDial calls = %d, want one per request", got) } } @@ -166,15 +159,7 @@ func TestPreviewForwardsToOwner(t *testing.T) { } type fakePreviewMgr struct { - touch func(id string, port uint16) error - dial func(id string, port uint16) (net.Conn, error) -} - -func (f *fakePreviewMgr) PreviewTouch(_ context.Context, id string, port uint16) error { - if f.touch == nil { - return nil - } - return f.touch(id, port) + dial func(id string, port uint16) (net.Conn, error) } func (f *fakePreviewMgr) PreviewDial(_ context.Context, id string, port uint16) (net.Conn, error) { From 57e88c9eeed3c8e800a6efff1c72d64ff220e395 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:38:06 +0800 Subject: [PATCH 12/26] fix(mesh): let SWIM membership decide who is in the view The epoch tombstone the previous commit added carried two residuals of its own: it lived only in memory, so a restart forgot every death and a lagging peer could teach them back, and its entries were dropped only when the same node returned with a higher epoch, so a permanently retired node_id stayed forever. Membership already answers the question - merge now accepts gossip only about nodes SWIM currently reports, and NotifyJoin/NotifyLeave maintain that set. Both maps are bounded by live membership and rebuilt from SWIM on restart, and a node returns to the view by rejoining, not by out-numbering a tombstone. --- sandboxd/mesh/mesh.go | 26 +++++++++++------- sandboxd/mesh/mesh_test.go | 53 +++++++++++++++++++++++-------------- sandboxd/mesh/state_test.go | 4 +-- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 122ffefb..1b6a9b8e 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -51,8 +51,7 @@ type Mesh struct { mu sync.Mutex self NodeState view map[string]NodeState // node_id → latest known state (includes self) - // departedEpoch tombstones a node at the epoch it left; a restart outranks it. - departedEpoch map[string]uint64 + live map[string]struct{} // members SWIM reports; gossip about anyone else is ignored } // New starts a mesh member listening per cfg. selfAddr is the data-plane @@ -73,8 +72,8 @@ func New(ctx context.Context, cfg *memberlist.Config, nodeID, selfAddr string, s Epoch: epoch, Pools: map[string]int{}, }, - view: map[string]NodeState{}, - departedEpoch: map[string]uint64{}, + view: map[string]NodeState{}, + live: map[string]struct{}{}, } if err := m.persistEpoch(epoch); err != nil { return nil, fmt.Errorf("persist mesh epoch: %w", err) @@ -289,13 +288,21 @@ func (m *Mesh) owners(match func(NodeState) bool) []string { return owners } +// admit records a node SWIM has seen join; merge accepts gossip only about +// these, so a peer that has not noticed a death cannot reintroduce one. +func (m *Mesh) admit(nodeID string) { + m.mu.Lock() + defer m.mu.Unlock() + m.live[nodeID] = struct{}{} +} + // forget drops a departed node from the placement view so redirects stop // targeting a dead peer; SWIM detected the death, the view must follow. func (m *Mesh) forget(nodeID string) { m.mu.Lock() defer m.mu.Unlock() if nodeID != m.self.NodeID { - m.departedEpoch[nodeID] = m.view[nodeID].Epoch + delete(m.live, nodeID) delete(m.view, nodeID) } } @@ -309,14 +316,13 @@ func (m *Mesh) merge(states []NodeState) { if st.NodeID == m.self.NodeID { continue } - cur, ok := m.view[st.NodeID] - if ok && st.Epoch <= cur.Epoch { + if _, member := m.live[st.NodeID]; !member { continue } - if st.Epoch <= m.departedEpoch[st.NodeID] { + cur, ok := m.view[st.NodeID] + if ok && st.Epoch <= cur.Epoch { continue } - delete(m.departedEpoch, st.NodeID) // Warn on each distinct divergent digest (not once per lifetime): a // mismatched api_token/tenants/preview_secret/CA root 401s cross-node // redirects and fails interception. Warn-only — refusing would partition @@ -371,7 +377,7 @@ var _ memberlist.EventDelegate = (*eventDelegate)(nil) // dead peer stops attracting redirects. type eventDelegate Mesh -func (e *eventDelegate) NotifyJoin(*memberlist.Node) {} +func (e *eventDelegate) NotifyJoin(n *memberlist.Node) { (*Mesh)(e).admit(n.Name) } func (e *eventDelegate) NotifyUpdate(*memberlist.Node) {} func (e *eventDelegate) NotifyLeave(n *memberlist.Node) { (*Mesh)(e).forget(n.Name) diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index b3400ed0..b00aae50 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -15,9 +15,9 @@ import ( func TestMergeKeepsHigherEpoch(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}}) - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 3, Pools: map[string]int{"k": 5}}}) - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 9}}}) // stale + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}}) + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 3, Pools: map[string]int{"k": 5}}}) + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 9}}}) // stale got := 0 for _, st := range m.Members() { @@ -34,7 +34,7 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), map[string]int{"k": 3}, nil, nil) // A peer claiming to be "a" must not clobber our authoritative self entry. - m.merge([]NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) + mergeStates(t, m, []NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) for _, st := range m.Members() { if st.NodeID == "a" && (st.Addr != "a:7777" || st.Pools["k"] != 3) { @@ -46,7 +46,7 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil, nil) // self has warm, but is never a candidate - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, // no warm {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"other": 4}}, @@ -64,7 +64,7 @@ func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), nil, []string{"tpl"}, nil) // self holds it, but is never an owner candidate - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Templates: []string{"tpl", "other"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Templates: []string{"other"}}, }) @@ -79,7 +79,7 @@ func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { func TestForgetPrunesDeadNode(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 3}}}) + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 3}}}) if len(m.Candidates("k")) != 1 { t.Fatal("setup: b should be a candidate") } @@ -95,28 +95,33 @@ func TestForgetPrunesDeadNode(t *testing.T) { } } -func TestForgottenNodeStaysGoneUntilItRestarts(t *testing.T) { +func TestForgottenNodeStaysGoneUntilItRejoins(t *testing.T) { m := newTestMesh(t, "a") dead := NodeState{NodeID: "b", Addr: "b:7777", Epoch: 4, Pools: map[string]int{"k": 3}} - m.merge([]NodeState{dead}) + mergeStates(t, m, []NodeState{dead}) m.forget("b") m.merge([]NodeState{dead}) if got := m.Candidates("k"); got != nil { t.Errorf("a lagging peer resurrected b: %v", got) } - restarted := dead restarted.Epoch = 5 m.merge([]NodeState{restarted}) + if got := m.Candidates("k"); got != nil { + t.Errorf("a higher epoch alone resurrected b: %v", got) + } + + m.admit("b") + m.merge([]NodeState{restarted}) if len(m.Candidates("k")) != 1 { - t.Error("b restarted with a higher epoch and must be reinstated") + t.Error("b rejoined the cluster and must be reinstated") } } func TestCandidatesPowerOfTwo(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, @@ -163,7 +168,7 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { func TestVolumeOwnersRequireEveryNameAndExcludeSelf(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), nil, nil, []string{"dataset", "weights"}) - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Volumes: []string{"dataset", "weights"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Volumes: []string{"dataset"}}, {NodeID: "d", Addr: "d:7777", Epoch: 1, Volumes: []string{"weights"}}, @@ -182,7 +187,7 @@ func TestVolumeOwnersRequireEveryNameAndExcludeSelf(t *testing.T) { func TestVolumeCandidatesRequireWarmAndEveryVolume(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "both", Addr: "both:7777", Epoch: 1, Pools: map[string]int{"k": 2}, Volumes: []string{"dataset", "weights"}}, {NodeID: "partial", Addr: "partial:7777", Epoch: 1, Pools: map[string]int{"k": 3}, Volumes: []string{"dataset"}}, {NodeID: "cold", Addr: "cold:7777", Epoch: 1, Pools: map[string]int{"k": 0}, Volumes: []string{"dataset", "weights"}}, @@ -198,7 +203,7 @@ func TestVolumeCandidatesRequireWarmAndEveryVolume(t *testing.T) { func TestTemplateVolumeOwnersUseTrueIntersection(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "template", Addr: "template:7777", Epoch: 1, Templates: []string{"tpl"}}, {NodeID: "volume", Addr: "volume:7777", Epoch: 1, Volumes: []string{"dataset"}}, {NodeID: "both", Addr: "both:7777", Epoch: 1, Templates: []string{"tpl"}, Volumes: []string{"dataset"}}, @@ -215,7 +220,7 @@ func TestTemplateVolumeOwnersUseTrueIntersection(t *testing.T) { func TestVolumeHoldersCountSelfAndPeers(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), nil, nil, []string{"dataset"}) - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Volumes: []string{"dataset", "weights"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Volumes: []string{"weights"}}, }) @@ -226,13 +231,21 @@ func TestVolumeHoldersCountSelfAndPeers(t *testing.T) { } } +func mergeStates(t *testing.T, m *Mesh, states []NodeState) { + t.Helper() + for _, st := range states { + m.admit(st.NodeID) + } + m.merge(states) +} + func newTestMesh(t *testing.T, id string) *Mesh { t.Helper() return &Mesh{ - epochPath: filepath.Join(t.TempDir(), "mesh-epoch"), - self: NodeState{NodeID: id, Addr: id + ":7777", Pools: map[string]int{}}, - view: map[string]NodeState{id: {NodeID: id, Addr: id + ":7777"}}, - departedEpoch: map[string]uint64{}, + epochPath: filepath.Join(t.TempDir(), "mesh-epoch"), + self: NodeState{NodeID: id, Addr: id + ":7777", Pools: map[string]int{}}, + view: map[string]NodeState{id: {NodeID: id, Addr: id + ":7777"}}, + live: map[string]struct{}{}, } } diff --git a/sandboxd/mesh/state_test.go b/sandboxd/mesh/state_test.go index 3813b3e4..43c2b281 100644 --- a/sandboxd/mesh/state_test.go +++ b/sandboxd/mesh/state_test.go @@ -110,8 +110,8 @@ func TestUpdateSelfBumpsOnlyWhenVolumesChange(t *testing.T) { func TestConfigDigestMismatch(t *testing.T) { m := newTestMesh(t, "self") m.SetSelfDigest("self-digest") - m.merge([]NodeState{{NodeID: "peerA", Addr: "a:1", Epoch: 1, Digest: "self-digest"}}) - m.merge([]NodeState{{NodeID: "peerB", Addr: "b:1", Epoch: 1, Digest: "other-digest"}}) + mergeStates(t, m, []NodeState{{NodeID: "peerA", Addr: "a:1", Epoch: 1, Digest: "self-digest"}}) + mergeStates(t, m, []NodeState{{NodeID: "peerB", Addr: "b:1", Epoch: 1, Digest: "other-digest"}}) if n := m.ConfigMismatches(); n != 1 { t.Errorf("ConfigMismatches = %d, want 1 (only peerB diverges)", n) } From 05dda5463121574e56973345c9eb704ed3fbfaa9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:41:45 +0800 Subject: [PATCH 13/26] build: pin the toolchains that produce guest binaries, and gate shell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every base image was a floating tag, so the kernel build, sandbox-init and silkd were reproducible only by luck, and rust-toolchain named 'stable' — a new release could change codegen or fail -D warnings with nothing changed here. Bases are pinned by digest and both toolchains to the version the Linux gate actually runs. A pinned digest goes stale silently, so dependabot owns refreshing them (plus cargo, gomod and actions) rather than a human remembering. shellcheck had no gate at all: all seven scripts already pass, so this locks in what is already true. --- .github/dependabot.yml | 39 ++++++++++++++++++++++++++++++++++ .github/workflows/shell.yml | 28 ++++++++++++++++++++++++ Makefile | 5 ++++- boot/Dockerfile | 6 +++--- boot/init/rust-toolchain.toml | 2 +- os-image/base/24.04/Dockerfile | 2 +- silkd/Dockerfile | 2 +- silkd/rust-toolchain.toml | 2 +- 8 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/shell.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..9b0af65c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,39 @@ +# The guest binaries (sandbox-init, silkd) and every os-image are built from +# these bases, so their tags are pinned by digest for reproducibility and +# dependabot is what keeps the pins current — without it a pinned digest is a +# frozen, unpatched toolchain. +version: 2 +updates: + - package-ecosystem: docker + directories: + - /silkd + - /boot + - /os-image/base/24.04 + schedule: + interval: weekly + commit-message: + prefix: "build" + - package-ecosystem: cargo + directories: + - /silkd + - /boot/init + schedule: + interval: weekly + commit-message: + prefix: "build" + - package-ecosystem: gomod + directories: + - /sandboxd + - /sdk/go + - /e2e + - /mcp + schedule: + interval: weekly + commit-message: + prefix: "build" + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + commit-message: + prefix: "build" diff --git a/.github/workflows/shell.yml b/.github/workflows/shell.yml new file mode 100644 index 00000000..58005686 --- /dev/null +++ b/.github/workflows/shell.yml @@ -0,0 +1,28 @@ +# The e2e and bench drivers are shell, and they are what hardware rounds run; +# a broken quoting or trap change there costs a whole round. +name: shell + +on: + push: + branches: [main] + paths: + - "**.sh" + - ".github/workflows/shell.yml" + pull_request: + paths: + - "**.sh" + - ".github/workflows/shell.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: shellcheck + run: make sh-lint diff --git a/Makefile b/Makefile index 663b35a7..3d97b733 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ GOLANGCILINT_VERSION ?= v2.12.2 GOLANGCILINT_ROOT := $(LOCALBIN)/golangci-lint-$(GOLANGCILINT_VERSION) GOLANGCILINT := $(GOLANGCILINT_ROOT)/golangci-lint -.PHONY: help test lint boot boot-debug extract extract-debug silkd-image base python images \ +.PHONY: help test lint sh-lint boot boot-debug extract extract-debug silkd-image base python images \ sandboxd go-test go-lint bench cloc ## Tool download targets @@ -49,6 +49,9 @@ lint: ## Rust fmt --check + clippy -D warnings: boot/init + silkd cd boot/init && cargo fmt --check && cargo clippy --all-targets -- -D warnings cd silkd && cargo fmt --check && cargo clippy --all-targets -- -D warnings +sh-lint: ## shellcheck every tracked shell script + git ls-files '*.sh' | xargs shellcheck + sandboxd: ## build dist/sandboxd mkdir -p dist cd sandboxd && GOWORK=off go build -ldflags "-X main.version=$(SANDBOXD_VERSION)" -o ../dist/sandboxd . diff --git a/boot/Dockerfile b/boot/Dockerfile index 58511842..4e1b2595 100644 --- a/boot/Dockerfile +++ b/boot/Dockerfile @@ -4,7 +4,7 @@ # under /boot/ for os-image builds to COPY --from. Built per-platform on # native runners (a kernel build under QEMU emulation takes hours). -FROM debian:bookworm-slim AS kbuild +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS kbuild RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential bc bison flex libelf-dev libssl-dev \ xz-utils curl ca-certificates python3 \ @@ -75,14 +75,14 @@ RUN set -e; \ fi; \ gcc usr/gen_init_cpio.c -o /usr/local/bin/gen_init_cpio -FROM rust:1-alpine AS initbuild +FROM rust:1-alpine@sha256:3c38f3f82c2f3d73da3b38e18d279393a04cb43ddded0e35088a8c3324d40900 AS initbuild RUN apk add --no-cache musl-dev WORKDIR /build COPY init/Cargo.toml init/Cargo.lock ./ COPY init/src ./src RUN cargo build --release --locked && cp target/release/sandbox-init /sandbox-init -FROM busybox:musl AS bb +FROM busybox:musl@sha256:32b5cdad7cce41dfd53d0ae06baebcf8357a147ee7694dc706911c373bc30c37 AS bb FROM kbuild AS pack ARG INITRD_DEBUG=0 diff --git a/boot/init/rust-toolchain.toml b/boot/init/rust-toolchain.toml index 85f36062..5ec58696 100644 --- a/boot/init/rust-toolchain.toml +++ b/boot/init/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "stable" +channel = "1.97.1" components = ["rustfmt", "clippy"] profile = "minimal" diff --git a/os-image/base/24.04/Dockerfile b/os-image/base/24.04/Dockerfile index a4aad561..4d669a3f 100644 --- a/os-image/base/24.04/Dockerfile +++ b/os-image/base/24.04/Dockerfile @@ -10,7 +10,7 @@ ARG SILKD_IMAGE=ghcr.io/cocoonstack/sandbox/silkd:0.1.0 FROM ${BOOT_IMAGE} AS boot FROM ${SILKD_IMAGE} AS silkd -FROM ubuntu:24.04 +FROM ubuntu:24.04@sha256:561618e2c15bf2397621dd04f96926663a3b5616c189cf7e38db7e82f5c538ea ARG TARGETARCH ENV DEBIAN_FRONTEND=noninteractive # Content hash of the image dir's non-Dockerfile inputs (CI computes it). diff --git a/silkd/Dockerfile b/silkd/Dockerfile index 55491067..0af3c7c9 100644 --- a/silkd/Dockerfile +++ b/silkd/Dockerfile @@ -7,7 +7,7 @@ # /silkd-static is musl-static for guests without glibc, such as the android # flavor's non-systemd userspace. Built per-platform (amd64/arm64) on native # runners; TARGETARCH picks the musl target triple. -FROM rust:1-slim AS build +FROM rust:1-slim@sha256:8e8cf8f7fd54a2d23d5a743b3a03f56e26b6c774276c33fa0595111704ebb15c AS build ARG TARGETARCH RUN case "$TARGETARCH" in \ amd64) echo x86_64-unknown-linux-musl > /musl-target ;; \ diff --git a/silkd/rust-toolchain.toml b/silkd/rust-toolchain.toml index e3dcc8f6..261677a0 100644 --- a/silkd/rust-toolchain.toml +++ b/silkd/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "stable" +channel = "1.97.1" components = ["rustfmt", "clippy"] targets = ["x86_64-unknown-linux-musl"] profile = "minimal" From 0ca6c80eae188ba297e73ea1c359e0dac128c6c4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:50:08 +0800 Subject: [PATCH 14/26] feat(sdk): expose the reply status, and close the parity gaps httpError becomes APIError: the SDK's own redirect walk branches on the status, so a caller distinguishing gone from full from mid-heal needed the same field Python already exposes. WithHTTPClient lets a caller supply its own transport or deadline; the client sets no blanket one, because checkpoint and promote block for as long as the snapshot takes and ctx is where a Go caller says otherwise. claim_ref had no SDK at all despite being the aggregated apiserver's hook, and neither did the listing that reads it back, so both sides gain WithClaimRef / claim_ref and Sandboxes() / sandboxes(). Python gains attach, drain and uncordon: the OpenAI adapter was reaching around the missing attach by constructing a Sandbox by hand. MCP claimed with the node's 5-minute default and renewed nothing, so an agent session outliving it lost every sandbox mid-conversation; it now claims for an hour, says so in the tool description, accepts net and size, and releases what it claimed when the stdio session ends. --- mcp/server.go | 16 ++++++++++++ mcp/tools.go | 16 +++++++++--- sdk/go/client.go | 40 ++++++++++++++++++------------ sdk/go/info.go | 28 +++++++++++++++++++++ sdk/go/options.go | 6 +++++ sdk/python/cocoonsandbox/client.py | 29 +++++++++++++++++++--- 6 files changed, 112 insertions(+), 23 deletions(-) diff --git a/mcp/server.go b/mcp/server.go index be41b38f..23647e6e 100644 --- a/mcp/server.go +++ b/mcp/server.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "io" + "maps" + "slices" "sync" "time" @@ -15,6 +17,7 @@ import ( const ( protocolVersion = "2024-11-05" execTimeout = 5 * time.Minute + defaultToolTTL = time.Hour ) // server owns one sandboxd client and the handles minted over this stdio @@ -50,6 +53,7 @@ func newServer(addr, token, template string) (*server, error) { // stdio transport. Requests are handled strictly in order: sandbox tools are // stateful, and an agent's tool calls arrive sequentially anyway. func (s *server) serve(ctx context.Context, r *bufio.Reader, w io.Writer) error { + defer s.closeBoxes() for { line, err := r.ReadBytes('\n') if len(line) == 0 && err != nil { @@ -132,6 +136,18 @@ func (s *server) box(id string) (*sandbox.Sandbox, error) { return sb, nil } +// closeBoxes releases everything this session claimed; the lease would +// otherwise hold the VMs until it expires. +func (s *server) closeBoxes() { + s.mu.Lock() + boxes := slices.Collect(maps.Values(s.boxes)) + clear(s.boxes) + s.mu.Unlock() + for _, sb := range boxes { + _ = sb.Close() + } +} + func (s *server) trackBox(sb *sandbox.Sandbox) { s.mu.Lock() defer s.mu.Unlock() diff --git a/mcp/tools.go b/mcp/tools.go index ff151a13..18b630e5 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -14,7 +14,7 @@ import ( var tools = []tool{ { "create_sandbox", "Claim a fresh microVM sandbox; returns its id. Warm claims are milliseconds.", - schema(props{"template": str("template image ref; empty uses the server default"), "ttl_seconds": integer("sandbox lifetime; 0 = server default")}), toolCreateSandbox, + schema(props{"template": str("template image ref; empty uses the server default"), "net": str("network lane: none (default) or egress"), "size": str("resource tier: small (default), medium, large, xlarge"), "ttl_seconds": integer("sandbox lifetime in seconds; 0 means one hour, and nothing renews it")}), toolCreateSandbox, }, { "exec", "Run a command in a sandbox and return stdout/stderr/exit code. A hibernated sandbox wakes transparently.", @@ -94,14 +94,22 @@ func toolSpecs() []map[string]any { func toolCreateSandbox(ctx context.Context, s *server, raw json.RawMessage) (string, error) { var args struct { Template string `json:"template"` + Net string `json:"net"` + Size string `json:"size"` TTLSeconds int `json:"ttl_seconds"` } if err := parse(raw, &args); err != nil { return "", err } - var opts []sandbox.Option - if args.TTLSeconds > 0 { - opts = append(opts, sandbox.WithTimeout(time.Duration(args.TTLSeconds)*time.Second)) + // An agent session outlives the node's 5-minute default, and no tool + // renews a lease, so the sandbox would vanish mid-conversation. + ttl := time.Duration(cmp.Or(args.TTLSeconds, int(defaultToolTTL.Seconds()))) * time.Second + opts := []sandbox.Option{sandbox.WithTimeout(ttl)} + if args.Net != "" { + opts = append(opts, sandbox.WithNetwork(sandbox.NetShape(args.Net))) + } + if args.Size != "" { + opts = append(opts, sandbox.WithSize(sandbox.Size(args.Size))) } sb, err := s.client.New(ctx, cmp.Or(args.Template, s.template), opts...) if err != nil { diff --git a/sdk/go/client.go b/sdk/go/client.go index 5692e487..0023bb98 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -34,6 +34,12 @@ func WithAPIToken(token string) ClientOption { return func(c *Client) { c.apiToken = token } } +// WithHTTPClient replaces the control-plane HTTP client, for callers that need +// their own transport, proxy, or timeout. +func WithHTTPClient(hc *http.Client) ClientOption { + return func(c *Client) { c.hc = hc } +} + // Client talks to one sandboxd node. type Client struct { addr string @@ -194,7 +200,8 @@ func (c *Client) roundTrip(ctx context.Context, method, addr, path string, body // Connect returns a client for a sandboxd node. addr accepts a // comma-separated seed list for forward compatibility; v0 uses the first -// entry. +// entry. Calls are bounded by their ctx — checkpoint and promote run as long +// as the snapshot takes, so the client sets no blanket deadline. func Connect(addr string, opts ...ClientOption) (*Client, error) { first, _, _ := strings.Cut(addr, ",") first = strings.TrimSpace(first) @@ -269,8 +276,8 @@ func tryEach(candidates []string, call func(addr string) error, retry func(error // retryMiss retries a miss (the next candidate may own the record) or a // transport failure (dead peer); a served error is real and stops the walk. func retryMiss(err error) bool { - var he *httpError - return !errors.As(err, &he) || he.status == http.StatusNotFound + var he *APIError + return !errors.As(err, &he) || he.Status == http.StatusNotFound } // retryAny retries a redirect candidate's failure unconditionally: one @@ -285,11 +292,11 @@ func retryAny(error) bool { return true } // request, a forbidden token, or an egress conflict is definitive: the // origin would fail the same way. func retryTransient(err error) bool { - var he *httpError + var he *APIError if !errors.As(err, &he) { return true } - switch he.status { + switch he.Status { case http.StatusUnauthorized, http.StatusNotFound, http.StatusTooManyRequests, http.StatusServiceUnavailable, http.StatusInternalServerError, http.StatusBadGateway, http.StatusGatewayTimeout: @@ -401,26 +408,26 @@ func encodeBody(verb string, v any) ([]byte, error) { return body, nil } -// httpError is a non-2xx control-plane reply; redirect walks branch on the -// status (retryMiss). -type httpError struct { - verb string - status int - msg string +// APIError is a non-2xx control-plane reply. Status is exported because the +// SDK's own redirect walk branches on it, so callers need it too. +type APIError struct { + Verb string + Status int + Message string } -func (e *httpError) Error() string { - if e.msg != "" { - return fmt.Sprintf("%s: %s (http %d)", e.verb, e.msg, e.status) +func (e *APIError) Error() string { + if e.Message != "" { + return fmt.Sprintf("%s: %s (http %d)", e.Verb, e.Message, e.Status) } - return fmt.Sprintf("%s: http %d", e.verb, e.status) + return fmt.Sprintf("%s: http %d", e.Verb, e.Status) } // apiError surfaces the server's {"error": ...} body when present. func apiError(verb string, resp *http.Response) error { var er errorResponse _ = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&er) - return &httpError{verb: verb, status: resp.StatusCode, msg: er.Error} + return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} } // claimRequest mirrors sandboxd's wire type; duplicated so the SDK stays @@ -434,6 +441,7 @@ type claimRequest struct { TTLSeconds int `json:"ttl_seconds,omitempty"` NoRedirect bool `json:"no_redirect,omitempty"` RequirePromoted bool `json:"require_promoted,omitempty"` + ClaimRef string `json:"claim_ref,omitempty"` } // rejectPinnedAxes fails a snapshot claim (checkpoint, template) that passed diff --git a/sdk/go/info.go b/sdk/go/info.go index 9e672d7a..325767b9 100644 --- a/sdk/go/info.go +++ b/sdk/go/info.go @@ -36,11 +36,39 @@ type PoolStatus struct { Golden bool `json:"golden"` } +// SandboxSummary is one live claim as the scoped index reports it; never a +// token or a host path. +type SandboxSummary struct { + ID string `json:"id"` + Key PoolKey `json:"key"` + Deadline time.Time `json:"deadline"` + Hibernated bool `json:"hibernated"` + Archived bool `json:"archived,omitempty"` + FromCheckpoint string `json:"from_checkpoint,omitempty"` + Volumes []Volume `json:"volumes,omitempty"` + ClaimRef string `json:"claim_ref,omitempty"` +} + +// sandboxListResponse is the wire envelope of GET /v1/sandboxes. +type sandboxListResponse struct { + Sandboxes []SandboxSummary `json:"sandboxes"` +} + // Info reports the entry node's pools, claim counts, and mesh peers. func (c *Client) Info(ctx context.Context) (*NodeInfo, error) { return doJSONPtr[NodeInfo](ctx, c, http.MethodGet, c.addr, "/v1/info", nil, c.apiToken, "info") } +// Sandboxes lists the live claims this token may see — the read side of +// WithClaimRef, so a caller can map its own reference back to a sandbox. +func (c *Client) Sandboxes(ctx context.Context) ([]SandboxSummary, error) { + reply, err := doJSON[sandboxListResponse](ctx, c, http.MethodGet, c.addr, "/v1/sandboxes", nil, c.apiToken, "list sandboxes") + if err != nil { + return nil, err + } + return reply.Sandboxes, nil +} + // peers fetches the cluster's node addresses, best-effort (nil on failure). func (c *Client) peers(ctx context.Context) []string { addrs, _ := c.peersOrErr(ctx) diff --git a/sdk/go/options.go b/sdk/go/options.go index 3cb1cf9b..dde7eb63 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -86,6 +86,12 @@ func WithTimeout(d time.Duration) Option { } } +// WithClaimRef records an opaque caller reference on the claim (the aggregated +// apiserver passes its k8s "/"), echoed by Client.Sandboxes. +func WithClaimRef(ref string) Option { + return func(r *claimRequest) { r.ClaimRef = ref } +} + // ttlSeconds rounds a lease up to whole wire seconds. func ttlSeconds(d time.Duration) int { return int((d + time.Second - 1) / time.Second) diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 1cc017cb..4a187622 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -28,7 +28,7 @@ def __init__(self, addr: str, api_token: str = "", timeout: float = 120.0): self.api_token = api_token self.timeout = timeout - def new(self, template: str, net: str = "", size: str = "", ttl_seconds: int = 0, + def new(self, template: str, net: str = "", size: str = "", ttl_seconds: int = 0, claim_ref: str = "", volumes: list[str | Mapping[str, str]] | None = None, mount: bool = True) -> Sandbox: """Claims a sandbox; a warm hit is milliseconds. On a cluster a warm miss may redirect to a peer, followed transparently; if every @@ -37,7 +37,7 @@ def new(self, template: str, net: str = "", size: str = "", ttl_seconds: int = 0 volumes without mounting them, leaving that — and the flush — to the workload: releasing without a clean self-umount discards unsynced guest pages, since the sandbox performs no sync.""" - claim = _claim_body(template, net, size, ttl_seconds, volumes, mount) + claim = _claim_body(template, net, size, ttl_seconds, volumes, mount, claim_ref) return self._claim_from(self.addr, claim) def delete_template(self, template: str, net: str = "", size: str = "") -> None: @@ -70,6 +70,26 @@ def probe(addr: str) -> Sandbox: except APIError: raise APIError("lookup", 404, f"no owner found for {id}") from None + def attach(self, owner_addr: str, id: str, token: str) -> Sandbox: + """Binds a handle to an already-claimed sandbox whose owner address is + known (an apiserver annotation, say), with no lookup round-trip.""" + return Sandbox(client=self, id=id, token=token, owner=owner_addr) + + def sandboxes(self) -> list[dict]: + """Lists the claims this token may see: id, key, deadline, claim_ref — + never tokens or host paths.""" + reply = self._request(self.addr, "GET", "/v1/sandboxes", None, "list sandboxes") + return [dict(sb) for sb in reply.get("sandboxes") or []] + + def drain(self) -> dict: + """Cordons the node (root token): new claims are refused, live ones run + to their leases.""" + return self._request(self.addr, "POST", "/v1/drain", None, "drain") + + def uncordon(self) -> dict: + """Lifts a drain on the node (root token).""" + return self._request(self.addr, "DELETE", "/v1/drain", None, "uncordon") + def checkpoint(self, id: str) -> Checkpoint: """A handle for a known checkpoint id, bound to the entry node — no listing round-trip; an unknown id surfaces as 404 at claim time.""" @@ -166,7 +186,8 @@ def _request(self, addr: str, method: str, path: str, body, verb: str, bearer: s def _claim_body(template: str, net: str, size: str, ttl_seconds: int, - volumes: list[str | Mapping[str, str]] | None = None, mount: bool = True) -> dict: + volumes: list[str | Mapping[str, str]] | None = None, mount: bool = True, + claim_ref: str = "") -> dict: claim = {"template": template} if net: claim["net"] = net @@ -178,6 +199,8 @@ def _claim_body(template: str, net: str, size: str, ttl_seconds: int, claim["volumes"] = [_volume_body(volume, mount) for volume in volumes] if not mount: claim["volumes_attach_only"] = True + if claim_ref: + claim["claim_ref"] = claim_ref return claim From 5c7a85596caff01168f043752f0e0f43de1b03dc Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:50:08 +0800 Subject: [PATCH 15/26] docs: document the node-operation verbs, and stop restating delete semantics cluster.md told operators to call SetPools and Drain, which docs/sdk.md never mentioned despite claiming a per-method reference; both SDK pages now carry them alongside the new claim_ref and sandboxes surfaces. The checkpoint-delete lifecycle was written out in full in three places, so the API page keeps what a caller of that endpoint needs and links cluster.md for the rest. --- docs/sandboxd-api.md | 22 ++++++---------------- docs/sdk-python.md | 14 ++++++++++++++ docs/sdk.md | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index cf69fec9..e40fe87e 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -427,22 +427,12 @@ Auth: node API token. A tenant may delete only its own records — anything else is 404, never a hint the id exists; root deletes anything. 204 on success, 404 unknown. -**Delete removes the local record and then best-effort broadcasts to peers -so a healed replica does not outlive it — this is eventual best-effort -cleanup, not a fleet-wide revocation.** A peer that is offline or -partitioned during the broadcast keeps its copy until the checkpoint TTL -ages it out. A healed replica carries the source's original `CreatedAt`, so -it becomes eligible for expiry at the same instant on every node; the actual -removal is each node's own hourly sweep, which is independently phased and -retries on a later sweep if one fails. So a deleted checkpoint normally stops -being branchable within `checkpoint_ttl_hours` plus a sweep interval, but a -node whose sweeps keep failing holds its replica until one succeeds — the TTL -is the eligibility point, not a hard ceiling. The TTL must also match -fleet-wide, which the -[cluster-invariant config](cluster.md#cluster-invariant-config) digest -checks. A window always exists because `checkpoint_peer_heal` cannot be -enabled with `checkpoint_ttl_hours: 0` — a replica that can outlive a delete -must have a finite eligibility point. A shared +**Delete removes the local record, then best-effort broadcasts to peers so a +healed replica does not outlive it — eventual cleanup, not a fleet-wide +revocation.** A peer offline during the broadcast keeps its copy until the +checkpoint TTL ages it out, so an id-holder can still branch it for that +window; [placement lifecycle](cluster.md#delete-is-eventual-not-a-fleet-wide-revocation) +has the bound and why heal requires a nonzero, fleet-matching TTL. A shared checkpoint store skips the broadcast: every node already resolves every record directly, so there is no replica to chase. `?no_forward=1` marks a delete already arriving from another node's own broadcast, so it is not diff --git a/docs/sdk-python.md b/docs/sdk-python.md index e7508bf4..d5bde6ef 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -395,6 +395,20 @@ pty.resize(200, 50) pty.close() ``` +## Node operations + +```python +sb = client.new("rt:24.04", claim_ref="ns/workload") +client.sandboxes() # id, key, deadline, claim_ref — never tokens +client.drain() # cordon: refuse new claims, run leases out +client.uncordon() +client.attach(owner_addr, id, token) # bind a known handle, no lookup round-trip +``` + +`sandboxes()` is scoped to the calling token, so a tenant sees only its own +claims. `drain()` leaves live claims alone — poll `info()` until `claimed` is +zero. + ## Errors `SandboxError` is the base; catch the narrowest type you handle: diff --git a/docs/sdk.md b/docs/sdk.md index 262a45fb..27a7483b 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -480,6 +480,24 @@ err = pty.Resize(ctx, 200, 50) A PTY is a tracked guest process (`pty.PID`); closing the handle (or the ctx) tears the shell down. +## Node operations + +Root-token verbs for operating a node, plus the reference the aggregated +apiserver claims under: + +```go +sb, _ := client.New(ctx, "rt:24.04", sandbox.WithClaimRef("ns/workload")) +list, _ := client.Sandboxes(ctx) // id, key, deadline, claim_ref — never tokens +info, _ := client.Drain(ctx) // cordon: refuse new claims, run leases out +info, _ = client.Uncordon(ctx) +info, _ = client.SetPools(ctx, pools) // retune warm targets without a restart +info, _ = client.SetPoolsCluster(ctx, pools) +sb = client.Attach(ownerAddr, id, token) // bind a known handle, no lookup round-trip +``` + +`Sandboxes` is scoped to the calling token, so a tenant sees only its own +claims. `Drain` leaves live claims alone — poll `Info` until `Claimed` is zero. + ## Error handling - `*sandbox.ExitError` — non-zero exit from `Exec` (`Code`, `Stderr`) From a5c30828088927b0f890166889b78dd5a7aa8899 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:56:22 +0800 Subject: [PATCH 16/26] feat(sdk): finish the Go/Python surface parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exec() threw stdout away on a non-zero exit, so a failing build's log was lost where Go returns it alongside ExitError. Pty dropped the shell's exit code the exit frame carries, and Watcher ended iteration identically for a clean close and a dropped relay, so a caller could not tell them apart. Go gains Checkpoint(id), which is what Python already had and what MCP needed — it was listing every checkpoint on the node to resolve one id, and only ever saw the connected node's listing. The two adapters floored the base SDK at 0.1 while calling APIs that landed in 0.1.5. --- mcp/tools.go | 21 +++++++-------------- sdk/go/checkpoint.go | 6 ++++++ sdk/langchain/pyproject.toml | 2 +- sdk/openai/pyproject.toml | 2 +- sdk/python/cocoonsandbox/errors.py | 6 ++++-- sdk/python/cocoonsandbox/sandbox.py | 14 ++++++++++---- 6 files changed, 29 insertions(+), 22 deletions(-) diff --git a/mcp/tools.go b/mcp/tools.go index 18b630e5..4e280083 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -310,7 +310,7 @@ func toolCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string } func toolBranchCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string, error) { - ckpt, err := s.checkpointArg(ctx, raw) + ckpt, err := s.checkpointArg(raw) if err != nil { return "", err } @@ -335,7 +335,7 @@ func toolListCheckpoints(ctx context.Context, s *server, _ json.RawMessage) (str } func toolDeleteCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string, error) { - ckpt, err := s.checkpointArg(ctx, raw) + ckpt, err := s.checkpointArg(raw) if err != nil { return "", err } @@ -400,9 +400,8 @@ func (s *server) boxArg(raw json.RawMessage) (*sandbox.Sandbox, error) { } // checkpointArg resolves a checkpoint_id argument: a handle minted in this -// session when available, else a listing lookup (checkpoints outlive -// sessions). -func (s *server) checkpointArg(ctx context.Context, raw json.RawMessage) (*sandbox.Checkpoint, error) { +// session when available, else a fresh one — checkpoints outlive sessions. +func (s *server) checkpointArg(raw json.RawMessage) (*sandbox.Checkpoint, error) { var args struct { CheckpointID string `json:"checkpoint_id"` } @@ -412,16 +411,10 @@ func (s *server) checkpointArg(ctx context.Context, raw json.RawMessage) (*sandb if ckpt, ok := s.ckpt(args.CheckpointID); ok { return ckpt, nil } - ckpts, err := s.client.Checkpoints(ctx) - if err != nil { - return nil, err - } - for _, ck := range ckpts { - if ck.ID == args.CheckpointID { - return ck, nil - } + if args.CheckpointID == "" { + return nil, fmt.Errorf("checkpoint_id is required") } - return nil, fmt.Errorf("unknown checkpoint %q", args.CheckpointID) + return s.client.Checkpoint(args.CheckpointID), nil } func parse(raw json.RawMessage, v any) error { diff --git a/sdk/go/checkpoint.go b/sdk/go/checkpoint.go index 1751674f..ab6497a5 100644 --- a/sdk/go/checkpoint.go +++ b/sdk/go/checkpoint.go @@ -93,6 +93,12 @@ func (c *Client) Checkpoints(ctx context.Context) ([]*Checkpoint, error) { return ckpts, nil } +// Checkpoint returns a handle for a known checkpoint id, bound to the entry +// node — no listing round-trip; an unknown id surfaces as 404 at claim time. +func (c *Client) Checkpoint(id string) *Checkpoint { + return checkpointHandle(c, c.addr, checkpointRecord{ID: id}) +} + func checkpointHandle(c *Client, addr string, rec checkpointRecord) *Checkpoint { return &Checkpoint{ ID: rec.ID, Name: rec.Name, SandboxID: rec.SandboxID, CreatedAt: rec.CreatedAt, diff --git a/sdk/langchain/pyproject.toml b/sdk/langchain/pyproject.toml index cec2e448..852d47bb 100644 --- a/sdk/langchain/pyproject.toml +++ b/sdk/langchain/pyproject.toml @@ -9,7 +9,7 @@ description = "LangChain tools backed by cocoon microVM sandboxes" requires-python = ">=3.10" license = { text = "Apache-2.0" } readme = "README.md" -dependencies = ["langchain-core>=1.0", "pydantic>=2", "cocoonstack-sandbox>=0.1"] +dependencies = ["langchain-core>=1.0", "pydantic>=2", "cocoonstack-sandbox>=0.1.5"] [tool.setuptools.packages.find] include = ["cocoonsandbox_langchain*"] diff --git a/sdk/openai/pyproject.toml b/sdk/openai/pyproject.toml index b39dfb6b..569062f3 100644 --- a/sdk/openai/pyproject.toml +++ b/sdk/openai/pyproject.toml @@ -9,7 +9,7 @@ description = "OpenAI Agents SDK sandbox provider backed by cocoon microVMs" requires-python = ">=3.10" license = { text = "Apache-2.0" } readme = "README.md" -dependencies = ["openai-agents>=0.17", "cocoonstack-sandbox>=0.1"] +dependencies = ["openai-agents>=0.17", "cocoonstack-sandbox>=0.1.5"] [tool.setuptools.packages.find] include = ["cocoonsandbox_openai*"] diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index 3a3ac371..4b8f16eb 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -29,12 +29,14 @@ def __init__(self, kind: str, message: str): class ExitError(SandboxError): - """A command exited non-zero; carries the exit code and stderr.""" + """A command exited non-zero; carries the exit code, stderr, and whatever + stdout it had produced — a failing build's log is on stdout.""" - def __init__(self, code: int, stderr: str): + def __init__(self, code: int, stderr: str, stdout: str = ""): super().__init__(f"exit status {code}: {stderr.strip()}") self.code = code self.stderr = stderr + self.stdout = stdout class ProtocolError(SandboxError): diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 3f43a884..3eb019ff 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -54,7 +54,7 @@ def exec(self, *argv: str, cwd: str = "", env: dict | None = None, code = self.run(list(argv), cwd=cwd, env=env, user=user, session=session, stdin=stdin, on_stdout=out.extend, on_stderr=err.extend) if code != 0: - raise ExitError(code, err.decode(errors="replace")) + raise ExitError(code, err.decode(errors="replace"), out.decode(errors="replace")) return out.decode(errors="replace") def run(self, argv: list[str], cwd: str = "", env: dict | None = None, @@ -387,14 +387,17 @@ class Watcher(_Closeable): def __init__(self, conn: Conn): self._conn = conn + self.error: Exception | None = None def __iter__(self) -> Iterator[dict]: # Connection-bound: a close, drop, or undecodable frame ends iteration; - # a real server error frame (SilkdError) propagates. + # a real server error frame (SilkdError) propagates. error tells a + # clean close (None) from a relay that dropped mid-stream. while True: try: frame = self._conn.recv() - except (ProtocolError, OSError, ValueError): + except (ProtocolError, OSError, ValueError) as e: + self.error = e return if frame["type"] == "event": yield frame @@ -410,11 +413,14 @@ def __init__(self, sandbox: Sandbox, conn: Conn, pid: int): self._sandbox = sandbox self._conn = conn self.pid = pid + self.exit_code: int | None = None def read(self) -> bytes: - """The next output chunk; b'' once the shell exits.""" + """The next output chunk; b'' once the shell exits, after which + exit_code holds the shell's status.""" frame = self._conn.recv() if frame["type"] == "exit": + self.exit_code = frame.get("code") return b"" return frame.get("data") or b"" From 84cc8e73f1c69a809fd6dd4b684d42a95c99e930 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:56:50 +0800 Subject: [PATCH 17/26] docs: record the Python exit, pty and watch contracts ExitError now carries the stdout produced before the failure, Pty exposes the shell's exit code, and a Watcher tells a dropped relay from a clean close. --- docs/sdk-python.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/sdk-python.md b/docs/sdk-python.md index d5bde6ef..2f7df737 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -278,8 +278,8 @@ code = sb.run(["bash", "-c", "make test"], on_stderr=lambda b: sys.stderr.buffer.write(b)) ``` -`exec` returns stdout and raises `ExitError(code, stderr)` on a non-zero -exit. `run` streams raw bytes through the callbacks (chunk boundaries may +`exec` returns stdout and raises `ExitError` on a non-zero exit — carrying +`code`, `stderr`, and the `stdout` produced before it failed. `run` streams raw bytes through the callbacks (chunk boundaries may split multi-byte sequences) and returns the exit code. `user` de-escalates inside the guest; `session=` routes the command into a persistent session. @@ -363,7 +363,8 @@ w.close() `watch` returns once the guest acknowledges the watch is armed — events caused after it returns are guaranteed captured. A bad path fails synchronously; if the consumer falls too far behind, iteration raises the -terminal overflow instead of silently dropping events. +terminal overflow instead of silently dropping events. Iteration also ends +when the relay drops, which `w.error` tells apart from a clean close (`None`). ## Git @@ -391,6 +392,7 @@ pointing at `push`. pty = sb.open_pty(cols=120, rows=40) # context-manager; pty.pid is the guest process pty.write(b"make test\n") data = pty.read() # b"" when the shell exits +pty.exit_code # the shell's status, once read() returned b"" pty.resize(200, 50) pty.close() ``` @@ -416,7 +418,7 @@ zero. - `APIError(verb, status, message)` — control plane (HTTP status) - `SilkdError(kind, message)` — typed guest failure; `kind` is `bad_request` / `not_found` / `unimplemented` / `internal` -- `ExitError(code, stderr)` — non-zero exit from `exec` +- `ExitError(code, stderr, stdout)` — non-zero exit from `exec` - `ProtocolError` — broken stream ```python From bc7e7b35e6ffc528769e86d1062e4eea76e3e2e0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 03:58:19 +0800 Subject: [PATCH 18/26] review: cut the restating half of the admit godoc --- sandboxd/mesh/mesh.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 1b6a9b8e..16b4fd74 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -288,8 +288,8 @@ func (m *Mesh) owners(match func(NodeState) bool) []string { return owners } -// admit records a node SWIM has seen join; merge accepts gossip only about -// these, so a peer that has not noticed a death cannot reintroduce one. +// admit marks a node live: merge ignores gossip about anyone else, so a peer +// that has not yet noticed a death cannot reintroduce it. func (m *Mesh) admit(nodeID string) { m.mu.Lock() defer m.mu.Unlock() From 895aeef2ec42a2eabebc195d178702fab194ee56 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 04:03:17 +0800 Subject: [PATCH 19/26] fix: carry the engine axis through the SDK and metrics, and true up the docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR made engine a documented pool-key axis, which exposed two places that never had it. The SDK's PoolKey and PoolSpec omitted it, so an Info-then- SetPools round trip — a declarative full replace — silently drained every fc pool. The two pool gauges omitted it too, so two pools differing only by engine emitted the same label set and the scrape failed. The rest is truthfulness: resolveGolden's godoc still claimed only a true absence cold-boots, which the tenant gate changed; watch.rs still called itself the one connection-bound verb the docs page just stopped saying; the SetPools cluster example did not compile; ClaimRef claimed to be empty on warm-pool claims that in fact carry it; and the egress intersection change needed to say out loud that a tenant inheriting its pool's policy now reaches nothing. PortConn keeps CloseWithError(net.ErrClosed): it is a net.Conn, and callers test for that sentinel — only Pty, which is not, takes the plain Close. --- boot/init/src/boot.rs | 7 ++----- docs/egress.md | 3 ++- docs/sdk.md | 2 +- mcp/tools.go | 6 +++--- sandboxd/mesh/mesh.go | 5 +---- sandboxd/pool/hibernate.go | 10 +++++++++- sandboxd/pool/pool.go | 6 ++---- sandboxd/pool/template.go | 4 ++-- sandboxd/server/metrics.go | 4 ++-- sdk/go/client.go | 3 +-- sdk/go/info.go | 5 ++--- sdk/go/options.go | 7 +++++++ sdk/go/pools.go | 1 + sdk/go/port.go | 2 +- sdk/python/cocoonsandbox/sandbox.py | 7 +++---- silkd/src/watch.rs | 4 +--- 16 files changed, 40 insertions(+), 36 deletions(-) diff --git a/boot/init/src/boot.rs b/boot/init/src/boot.rs index 1034af29..bae1325a 100644 --- a/boot/init/src/boot.rs +++ b/boot/init/src/boot.rs @@ -242,11 +242,8 @@ fn scan_serials(ids: &[&str], found: &mut [Option]) { } fn record_serial(ids: &[&str], found: &mut [Option], serial: &str, device: &str) { - if !Path::new(device).exists() { - return; - } for (i, id) in ids.iter().enumerate() { - if found[i].is_none() && *id == serial { + if found[i].is_none() && *id == serial && Path::new(device).exists() { found[i] = Some(device.into()); } } @@ -264,7 +261,7 @@ mod tests { use super::*; #[test] - fn record_serial_waits_for_device_node() { + fn record_serial_skips_a_missing_device_node() { let ids = ["layer"]; let mut found = [None]; diff --git a/docs/egress.md b/docs/egress.md index 46439e08..4627e70c 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -99,7 +99,8 @@ domain policy first; the allow-list widens the IP gate only. Policy is per pool and per tenant; the effective policy is their intersection (a request must pass both, and the pool rule's secret wins on a double allow). A missing policy on either side is an empty allow-list, not a pass: a tenant -without its own `egress` block reaches nothing, so granting a tenant egress — +without its own `egress` block reaches nothing — upgrading, a tenant that +relied on inheriting its pool's policy must now declare one — so granting a tenant egress — and the secret injection that rides it — is always an explicit act. Root claims have no tenant layer and take the pool's policy whole. Secrets are registered separately and referenced by name — the value comes from the diff --git a/docs/sdk.md b/docs/sdk.md index 27a7483b..0e95b15c 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -491,7 +491,7 @@ list, _ := client.Sandboxes(ctx) // id, key, deadline, claim_ref — ne info, _ := client.Drain(ctx) // cordon: refuse new claims, run leases out info, _ = client.Uncordon(ctx) info, _ = client.SetPools(ctx, pools) // retune warm targets without a restart -info, _ = client.SetPoolsCluster(ctx, pools) +res, _ := client.SetPoolsCluster(ctx, pools) // per-node results; retry the failures sb = client.Attach(ownerAddr, id, token) // bind a known handle, no lookup round-trip ``` diff --git a/mcp/tools.go b/mcp/tools.go index 4e280083..899a721e 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -101,9 +101,9 @@ func toolCreateSandbox(ctx context.Context, s *server, raw json.RawMessage) (str if err := parse(raw, &args); err != nil { return "", err } - // An agent session outlives the node's 5-minute default, and no tool - // renews a lease, so the sandbox would vanish mid-conversation. - ttl := time.Duration(cmp.Or(args.TTLSeconds, int(defaultToolTTL.Seconds()))) * time.Second + // An agent session outlives the node's 5-minute default and nothing renews + // a lease, so the sandbox would vanish mid-conversation. + ttl := cmp.Or(time.Duration(args.TTLSeconds)*time.Second, defaultToolTTL) opts := []sandbox.Option{sandbox.WithTimeout(ttl)} if args.Net != "" { opts = append(opts, sandbox.WithNetwork(sandbox.NetShape(args.Net))) diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 16b4fd74..a8aab83f 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -288,8 +288,6 @@ func (m *Mesh) owners(match func(NodeState) bool) []string { return owners } -// admit marks a node live: merge ignores gossip about anyone else, so a peer -// that has not yet noticed a death cannot reintroduce it. func (m *Mesh) admit(nodeID string) { m.mu.Lock() defer m.mu.Unlock() @@ -373,8 +371,7 @@ func (d *delegate) MergeRemoteState(buf []byte, _ bool) { var _ memberlist.EventDelegate = (*eventDelegate)(nil) -// eventDelegate prunes the placement view when SWIM reports a node gone, so a -// dead peer stops attracting redirects. +// eventDelegate tracks SWIM membership: admit on join, prune the view on leave. type eventDelegate Mesh func (e *eventDelegate) NotifyJoin(n *memberlist.Node) { (*Mesh)(e).admit(n.Name) } diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 34e72ce6..02c470ee 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -205,7 +205,7 @@ func (m *Manager) idleOnce(ctx context.Context) { if p, pooled := m.activePool(sb.Key); pooled { idle = p.idle } - if idle <= 0 || sb.Key.Net == types.NetEgress || hasAppliedVolumes(sb) || sb.HibernateSnap != "" || sb.ArchiveCk != "" || now.Sub(sb.LastSeen()) < idle { + if skipIdle(sb, idle, now) { continue } victims = append(victims, victim{sb.ID, sb.Token}) @@ -343,3 +343,11 @@ func (m *Manager) recordHibernate(ctx context.Context, sb *types.Sandbox) { m.counters.hibernates.Add(1) m.recordUsage(ctx, usageEvent{Event: "hibernate", ID: sb.ID, VMName: sb.VMName}) } + +// skipIdle reports the claims an idle sweep must leave alone: the egress lane +// cannot resume, a mounted volume cannot be captured, and an already +// hibernated or archived claim has nothing left to do. +func skipIdle(sb *types.Sandbox, idle time.Duration, now time.Time) bool { + return idle <= 0 || sb.Key.Net == types.NetEgress || hasAppliedVolumes(sb) || + sb.HibernateSnap != "" || sb.ArchiveCk != "" || now.Sub(sb.LastSeen()) < idle +} diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index f78b0bac..bd517515 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -134,10 +134,8 @@ type SandboxSummary struct { Archived bool `json:"archived,omitempty"` FromCheckpoint string `json:"from_checkpoint,omitempty"` Volumes []types.Volume `json:"volumes,omitempty"` - // ClaimRef echoes the caller reference recorded at claim time (the - // aggregated apiserver's k8s "/"), so the operator index - // can map this sandbox back to the name it was claimed under. Empty for - // warm-pool, fork, and checkpoint-branch claims. + // ClaimRef echoes the caller reference recorded at claim time; empty for + // fork and checkpoint-branch claims. ClaimRef string `json:"claim_ref,omitempty"` } diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 042593e1..dea327a5 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -275,8 +275,8 @@ type goldenResolution struct { // resolveGolden resolves a key's clone source: the configured pool's local // golden (no release), else a promoted template fetched from the store; -// empty dir cold-boots. Only a true absence cold-boots — a backend failure -// propagates rather than silently booting a template name as an image ref. +// empty dir cold-boots. A cross-tenant or absent record cold-boots; a backend +// failure propagates rather than booting a template name as an image ref. func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant string) (goldenResolution, error) { m.mu.Lock() var dir string diff --git a/sandboxd/server/metrics.go b/sandboxd/server/metrics.go index 84597d92..f3d69e11 100644 --- a/sandboxd/server/metrics.go +++ b/sandboxd/server/metrics.go @@ -47,11 +47,11 @@ func (s *Server) handleMetrics(w http.ResponseWriter, _ *http.Request) { metric("pool_warm", "gauge", "claim-ready VMs per pool") for _, p := range pools { - _, _ = fmt.Fprintf(w, "sandboxd_pool_warm{template=%q,net=%q,size=%q} %d\n", p.Key.Template, p.Key.Net, p.Key.Size, p.Warm) + _, _ = fmt.Fprintf(w, "sandboxd_pool_warm{template=%q,net=%q,size=%q,engine=%q} %d\n", p.Key.Template, p.Key.Net, p.Key.Size, p.Key.Engine, p.Warm) } metric("pool_target", "gauge", "warm watermark per pool") for _, p := range pools { - _, _ = fmt.Fprintf(w, "sandboxd_pool_target{template=%q,net=%q,size=%q} %d\n", p.Key.Template, p.Key.Net, p.Key.Size, p.Target) + _, _ = fmt.Fprintf(w, "sandboxd_pool_target{template=%q,net=%q,size=%q,engine=%q} %d\n", p.Key.Template, p.Key.Net, p.Key.Size, p.Key.Engine, p.Target) } if s.placer != nil { diff --git a/sdk/go/client.go b/sdk/go/client.go index 0023bb98..8d76f760 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -408,8 +408,7 @@ func encodeBody(verb string, v any) ([]byte, error) { return body, nil } -// APIError is a non-2xx control-plane reply. Status is exported because the -// SDK's own redirect walk branches on it, so callers need it too. +// APIError is a non-2xx control-plane reply. type APIError struct { Verb string Status int diff --git a/sdk/go/info.go b/sdk/go/info.go index 325767b9..4b7107fa 100644 --- a/sdk/go/info.go +++ b/sdk/go/info.go @@ -25,6 +25,7 @@ type PoolKey struct { Template string `json:"template"` Net NetShape `json:"net"` Size Size `json:"size"` + Engine Engine `json:"engine,omitempty"` } // PoolStatus reports one warm pool on a node. @@ -49,7 +50,6 @@ type SandboxSummary struct { ClaimRef string `json:"claim_ref,omitempty"` } -// sandboxListResponse is the wire envelope of GET /v1/sandboxes. type sandboxListResponse struct { Sandboxes []SandboxSummary `json:"sandboxes"` } @@ -59,8 +59,7 @@ func (c *Client) Info(ctx context.Context) (*NodeInfo, error) { return doJSONPtr[NodeInfo](ctx, c, http.MethodGet, c.addr, "/v1/info", nil, c.apiToken, "info") } -// Sandboxes lists the live claims this token may see — the read side of -// WithClaimRef, so a caller can map its own reference back to a sandbox. +// Sandboxes lists the live claims this token may see. func (c *Client) Sandboxes(ctx context.Context) ([]SandboxSummary, error) { reply, err := doJSON[sandboxListResponse](ctx, c, http.MethodGet, c.addr, "/v1/sandboxes", nil, c.apiToken, "list sandboxes") if err != nil { diff --git a/sdk/go/options.go b/sdk/go/options.go index dde7eb63..a213ab0e 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -11,6 +11,10 @@ const ( // NetEgress attaches the node's bridge or CNI network. NetEgress NetShape = "egress" + // EngineCH is the default hypervisor; EngineFC cold-boots under Firecracker. + EngineCH Engine = "ch" + EngineFC Engine = "fc" + Small Size = "small" Medium Size = "medium" Large Size = "large" @@ -27,6 +31,9 @@ type NetShape string // node's warm pools. type Size string +// Engine is the pool key's hypervisor axis. +type Engine string + // Volume requests one catalog entry at an optional guest mount path and mode. type Volume struct { Name string `json:"name"` diff --git a/sdk/go/pools.go b/sdk/go/pools.go index 4f61767d..98b2f7cd 100644 --- a/sdk/go/pools.go +++ b/sdk/go/pools.go @@ -14,6 +14,7 @@ type PoolSpec struct { Template string `json:"template"` Net NetShape `json:"net,omitempty"` Size Size `json:"size,omitempty"` + Engine Engine `json:"engine,omitempty"` Warm int `json:"warm"` WarmMax int `json:"warm_max,omitempty"` IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` diff --git a/sdk/go/port.go b/sdk/go/port.go index c0b7198b..7c0a50ab 100644 --- a/sdk/go/port.go +++ b/sdk/go/port.go @@ -60,7 +60,7 @@ func (p *PortConn) Close() error { p.stop() // drain can be parked in a pipe write on an unread tail; only the // reader side unblocks it. - _ = p.out.Close() + _ = p.out.CloseWithError(net.ErrClosed) }) return nil } diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 3eb019ff..dcb2b23d 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -66,13 +66,12 @@ def run(self, argv: list[str], cwd: str = "", env: dict | None = None, with self._dial() as conn: conn.send("exec", argv=argv, cwd=cwd or None, env=env, user=user or None, session=session or None) - # The guest blocks writing output once its stdout buffer fills, and - # stops draining stdin while it does, so feeding stdin to completion - # before reading deadlocks on any payload past the socket buffers. + # The guest stops draining stdin while blocked writing stdout, so + # feeding it to completion before reading deadlocks. pump = threading.Thread(target=_feed_stdin, args=(conn, stdin), daemon=True) pump.start() code = _pump_stdio(conn, on_stdout, on_stderr) - pump.join() + pump.join() # the closed conn fails a stalled send, so this cannot hang if code is None: raise ProtocolError("exec stream ended without an exit frame") return code diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index e2c300a1..e5b120f2 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -1,5 +1,5 @@ //! `fs.watch`: stream filesystem events under a path until the client -//! disconnects. Watch is the one connection-bound verb — an event feed has no +//! disconnects. Like every connection-bound verb, an event feed has no //! meaningful detached state, so it lives only as long as its connection. use notify::{RecursiveMode, Watcher}; @@ -58,8 +58,6 @@ where } } -/// A full channel drops the sender: the closed channel is the overflow -/// signal, a wakeup the parked loop cannot miss. fn forward_frames(tx: &mut Option>, frames: Vec) { let Some(sender) = tx.as_ref() else { return }; for frame in frames { From 0c85e995a7c0a89f2639748121626a55988807b5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 04:07:28 +0800 Subject: [PATCH 20/26] fix(scripts): spell the daemon teardown as an if, not A && B || true The new shellcheck gate failed on its first CI run: SC2015 flags the cleanup idiom because C runs whether or not B did, which here is the intent but reads as if-then-else. Older shellcheck releases report it and the version this laptop has does not, so the code says what it means instead. --- scripts/archive-e2e.sh | 4 +++- scripts/bench.sh | 4 +++- scripts/egress-e2e.sh | 4 +++- scripts/intercept-e2e.sh | 4 +++- scripts/sandboxd-e2e.sh | 7 +++++-- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/archive-e2e.sh b/scripts/archive-e2e.sh index baad484e..76ecbe11 100644 --- a/scripts/archive-e2e.sh +++ b/scripts/archive-e2e.sh @@ -20,7 +20,9 @@ cleanup() { echo "== daemon log tail" tail -30 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | diff --git a/scripts/bench.sh b/scripts/bench.sh index c7e53c97..2809261f 100755 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -30,7 +30,9 @@ cleanup() { echo "== daemon log tail" tail -20 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | diff --git a/scripts/egress-e2e.sh b/scripts/egress-e2e.sh index 672e729a..efab59d8 100755 --- a/scripts/egress-e2e.sh +++ b/scripts/egress-e2e.sh @@ -34,7 +34,9 @@ cleanup() { echo "== daemon log tail" tail -30 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | diff --git a/scripts/intercept-e2e.sh b/scripts/intercept-e2e.sh index 85da6306..e0f64ef3 100755 --- a/scripts/intercept-e2e.sh +++ b/scripts/intercept-e2e.sh @@ -24,7 +24,9 @@ cleanup() { echo "== daemon log tail" tail -30 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | diff --git a/scripts/sandboxd-e2e.sh b/scripts/sandboxd-e2e.sh index b7a4a0f5..1e6f8cb6 100755 --- a/scripts/sandboxd-e2e.sh +++ b/scripts/sandboxd-e2e.sh @@ -56,7 +56,9 @@ cleanup() { echo "== daemon log tail" tail -20 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | @@ -223,7 +225,8 @@ claimed=$(api info | jq .claimed) echo "== restart: live claim re-adopts, warm VMs of the old life are replaced" "$DATA/demo" -addr "$ADDR" -token "$TOKEN" -template "$TEMPLATE" -n 1 -ttl 300 -leak -kill "$DAEMON_PID" && wait "$DAEMON_PID" 2>/dev/null || true +kill "$DAEMON_PID" 2>/dev/null || true +wait "$DAEMON_PID" 2>/dev/null || true start_daemon claimed=$(api info | jq .claimed) [[ $claimed == 1 ]] || { echo "claim not re-adopted: claimed=$claimed"; exit 1; } From 11988696ad8ec39a634ee35a78a9deed4c74fbdd Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 04:14:09 +0800 Subject: [PATCH 21/26] test(e2e): pin claim_ref and the engine axis on the wire The drift guard had no assertion for either, and engine is exactly the axis whose absence from the SDK key silently drained fc pools on an Info-SetPools round trip. Mutating the tag to json:"-" fails this test. --- e2e/e2e_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index be7bb876..3e174bd2 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -427,6 +427,30 @@ func TestVolumeModeWireShape(t *testing.T) { } } +func TestClaimRefRoundTrip(t *testing.T) { + stack := startStack(t, "node-token") + sb, err := stack.client.New(t.Context(), "rt:24.04", sandbox.WithClaimRef("ns/workload")) + if err != nil { + t.Fatalf("claim: %v", err) + } + defer sb.Close() + + list, err := stack.client.Sandboxes(t.Context()) + if err != nil { + t.Fatalf("list sandboxes: %v", err) + } + i := slices.IndexFunc(list, func(s sandbox.SandboxSummary) bool { return s.ID == sb.ID }) + if i < 0 { + t.Fatalf("claim %s missing from the index %+v", sb.ID, list) + } + if list[i].ClaimRef != "ns/workload" { + t.Errorf("claim_ref %q, want ns/workload", list[i].ClaimRef) + } + if list[i].Key.Engine != sandbox.EngineCH { + t.Errorf("engine %q, want the defaulted ch — the SDK key must carry the axis", list[i].Key.Engine) + } +} + // TestAttachOnlyVolumeEndToEnd drives one attach-only writable claim through // the whole stack: the device is attached writable and nothing else happens — // no mount, no marker, no unmount at release — while admission still excludes From a4f3ab978ab20509f29bb477b08091bb67a1fafb Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 04:23:28 +0800 Subject: [PATCH 22/26] fix(pool): routing must answer what a claim would about a template HasPromotedTemplate was tenant-blind while resolveGolden is not, so the two disagreed for a foreign tenant: routing saw a local golden, skipped the peer hop, and the claim cold-booted with routing believing a template had served it. Both now apply the same tenant test. The contract, stated precisely: a plain claim of a foreign name behaves exactly like a claim of a never-promoted name - the template axis is an image ref, so it cold-boots the public image or fails provisioning, and the owner's content is never touched. Answering 404 instead would be worse on two counts: it hands every tenant an existence oracle over other tenants' template names, and it lets one tenant deny everyone else a public image name by promoting a private template under it. Only require_promoted (the volume path, redirect retries) answers ErrUnknownTemplate. tplSet caches the owning tenant next to each id, so the ownership test stays in memory: the claim path must not grow an s3 read per warm miss. --- sandboxd/pool/pool.go | 13 +++++++------ sandboxd/pool/promote_test.go | 25 ++++++++++++++++++++++++- sandboxd/pool/template.go | 34 ++++++++++++++++++++++------------ sandboxd/pool/volume_test.go | 2 +- sandboxd/server/server.go | 12 ++++++------ sandboxd/server/server_test.go | 4 ++-- 6 files changed, 62 insertions(+), 28 deletions(-) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index bd517515..7279dac6 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -316,11 +316,12 @@ type Manager struct { ckptTTL time.Duration ckptSweeping atomic.Bool - // tplSet caches the template ids visible in the store so the 1s gossip - // tick never touches the backend (an s3 listing is network I/O); local - // promotes/deletes update it, startup loads it. + // tplSet caches each visible template id against its owning tenant so the + // 1s gossip tick and the claim-path ownership test never touch the backend + // (an s3 read is network I/O); local promotes/deletes update it, startup + // loads it. Empty value means the operator promoted it. tplMu sync.Mutex - tplSet map[string]struct{} + tplSet map[string]string // recLocks serializes same-id store record mutations and holds off a // re-publish swap while a clone reads the old generation (per id, RW). @@ -434,12 +435,12 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg return nil, err } m.ckptTTL = time.Duration(cfg.CheckpointTTLHours) * time.Hour - m.tplSet = map[string]struct{}{} + m.tplSet = map[string]string{} if metas, listErr := m.tpls.Metas(ctx); listErr == nil { for _, raw := range metas { var rec templateRecord if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { - m.tplSet[rec.ID] = struct{}{} + m.tplSet[rec.ID] = rec.Tenant } } } diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index d285ab7f..66798ee1 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -157,7 +157,7 @@ func TestDeleteTemplate(t *testing.T) { if err := m.DeleteTemplate(t.Context(), key, ""); err != nil { t.Fatalf("DeleteTemplate: %v", err) } - if m.HasGolden(t.Context(), key) { + if m.HasGolden(t.Context(), key, "") { t.Error("template still resolvable after delete") } // The next claim for the deleted template cold-boots instead of cloning. @@ -359,6 +359,29 @@ func TestTemplateClaimIsTenantScoped(t *testing.T) { } } +func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + a, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + if err != nil { + t.Fatalf("claim: %v", err) + } + key, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") + if err != nil { + t.Fatalf("promote: %v", err) + } + + // Routing must answer what a claim would: promising beta a golden here + // makes redirectClaim skip the peer hop and cold-boot the name as an image. + if m.HasPromotedTemplate(t.Context(), key, "beta") { + t.Error("beta sees acme's template as a local golden") + } + for _, tenant := range []string{"acme", ""} { + if !m.HasPromotedTemplate(t.Context(), key, tenant) { + t.Errorf("tenant %q lost its own template", tenant) + } + } +} + func TestTemplateHashesSortedForMeshCompare(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index dea327a5..53942729 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -144,8 +144,8 @@ func (m *Manager) TemplateHashes() []string { // boot — a configured pool golden or a promoted template in the store. The // tplSet answers without a store read; only a shared-store template promoted // elsewhere after startup falls through to the backend. -func (m *Manager) HasGolden(ctx context.Context, key types.PoolKey) bool { - return m.HasPoolGolden(key) || m.HasPromotedTemplate(ctx, key) +func (m *Manager) HasGolden(ctx context.Context, key types.PoolKey, tenant string) bool { + return m.HasPoolGolden(key) || m.HasPromotedTemplate(ctx, key, tenant) } // HasPoolGolden reports whether a configured pool can serve key from its own @@ -157,22 +157,32 @@ func (m *Manager) HasPoolGolden(key types.PoolKey) bool { return p != nil && p.goldenDir != "" } -// HasPromotedTemplate reports whether key resolves to a promoted template. -// A hash a configured pool owns is subtracted, as TemplateHashes does for the -// gossip: resolveGolden serves it from the pool golden, never the template. -func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey) bool { +// HasPromotedTemplate reports whether key resolves to a promoted template this +// tenant may claim. A hash a configured pool owns is subtracted, as +// TemplateHashes does for the gossip: resolveGolden serves it from the pool +// golden, never the template. The tenant test matches resolveGolden's, so +// routing cannot promise a golden the claim will then refuse. +func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool { if m.pooledHash(key.Hash()) { return false } id := store.TemplateID(key.Hash()) m.tplMu.Lock() - _, cached := m.tplSet[id] + owner, cached := m.tplSet[id] m.tplMu.Unlock() - if cached { - return true + if !cached { + // Only a shared-store template promoted elsewhere after startup. + raw, err := m.tpls.ReadMeta(ctx, id) + if err != nil { + return false + } + var rec templateRecord + if json.Unmarshal(raw, &rec) != nil { + return false + } + owner = rec.Tenant } - _, err := m.tpls.ReadMeta(ctx, id) - return err == nil + return owner == "" || tenantOwns(tenant, owner) } // pooledHash reports whether a configured pool occupies this hash — the @@ -353,7 +363,7 @@ func (m *Manager) commitTemplate(ctx context.Context, staging, id, tenant string return "", fmt.Errorf("publish template: %w", err) } m.tplMu.Lock() - m.tplSet[id] = struct{}{} + m.tplSet[id] = tenant m.tplMu.Unlock() return digest, nil } diff --git a/sandboxd/pool/volume_test.go b/sandboxd/pool/volume_test.go index 9fa56d16..6b6b89b1 100644 --- a/sandboxd/pool/volume_test.go +++ b/sandboxd/pool/volume_test.go @@ -348,7 +348,7 @@ func TestPooledKeyOutranksPromotedTemplate(t *testing.T) { } m.pools[key].goldenDir = "/goldens/pooled" - if m.HasPromotedTemplate(t.Context(), key) { + if m.HasPromotedTemplate(t.Context(), key, "") { t.Error("a pooled key reports as promoted, disagreeing with both resolveGolden and the gossip") } if hashes := m.TemplateHashes(); slices.Contains(hashes, key.Hash()) { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 37454a00..5d46153b 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -89,9 +89,9 @@ type Manager interface { FetchCheckpoint(ctx context.Context, ckptID string) (dir string, meta []byte, release func(), err error) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope pool.DeleteScope) error ClaimDeadline(id, token string) (time.Time, error) - HasGolden(ctx context.Context, key types.PoolKey) bool + HasGolden(ctx context.Context, key types.PoolKey, tenant string) bool HasPoolGolden(key types.PoolKey) bool - HasPromotedTemplate(ctx context.Context, key types.PoolKey) bool + HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool AgentSocket(id, token string) (string, error) WakeAgentSocket(ctx context.Context, id, token string) (string, error) SetPools(ctx context.Context, pools []config.PoolSpec) error @@ -252,7 +252,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { // this node provision (golden clone or cold boot). sb, err := s.mgr.ClaimWarm(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) if errors.Is(err, pool.ErrNoWarm) { - if s.redirectClaim(r.Context(), w, req, key, hash) { + if s.redirectClaim(r.Context(), w, req, key, hash, tenant) { return } sb, err = s.mgr.ClaimProvision(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) @@ -320,7 +320,7 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, return false, err } - localTemplate := s.mgr.HasPromotedTemplate(ctx, key) + localTemplate := s.mgr.HasPromotedTemplate(ctx, key, tenant) // Per-node content consistency: a peer's promoted template must not escalate // volume claims off the pool golden every other claim here resolves to. An // explicit RequirePromoted still passes through, for the manager to refuse. @@ -361,7 +361,7 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, // template owner when we lack a golden (so we don't cold-boot a nonexistent // image ref). A no_redirect request must resolve locally, never bounce again, // to avoid a two-node ping-pong. -func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req types.ClaimRequest, key types.PoolKey, hash string) bool { +func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req types.ClaimRequest, key types.PoolKey, hash, tenant string) bool { if s.placer == nil || req.NoRedirect { return false } @@ -370,7 +370,7 @@ func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req t } // TemplateOwners is in-memory; HasGolden can be a store round-trip. owners := s.placer.TemplateOwners(hash) - return len(owners) > 0 && !s.mgr.HasGolden(ctx, key) && writeRedirect(w, owners) + return len(owners) > 0 && !s.mgr.HasGolden(ctx, key, tenant) && writeRedirect(w, owners) } // handleRelease releases a claimed sandbox. Two credentials authorize it: the diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index f9cb721b..af243648 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2024,7 +2024,7 @@ func (f *fakeManager) DeleteTemplate(_ context.Context, key types.PoolKey, tenan return f.deleteGolden(key) } -func (f *fakeManager) HasGolden(context.Context, types.PoolKey) bool { +func (f *fakeManager) HasGolden(context.Context, types.PoolKey, string) bool { return f.hasGolden } @@ -2032,7 +2032,7 @@ func (f *fakeManager) HasPoolGolden(types.PoolKey) bool { return f.hasPoolGolden } -func (f *fakeManager) HasPromotedTemplate(context.Context, types.PoolKey) bool { +func (f *fakeManager) HasPromotedTemplate(context.Context, types.PoolKey, string) bool { return f.hasPromoted } From 5d8b083567028221bec158840e687005c1eab9a9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 10:54:35 +0800 Subject: [PATCH 23/26] fix(mcp): fork children and checkpoint branches get the session lease create_sandbox claims for an hour, but fork passed a zero TTL and a branch claim set no timeout, so both fell back to the node's 5-minute default while being session-tracked with nothing to renew them - the exact mid-conversation expiry the create fix closed. Both now use the same lease and their tool descriptions say so. Also drops deploy.md's stale CH-only tag: Firecracker is a live opt-in engine axis, and the sentence's real content is that old state is not converted. --- docs/deploy.md | 2 +- mcp/tools.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 4db5d621..7df8ebcc 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -29,7 +29,7 @@ reports what you are running. ## Upgrading -This CH-only release does not convert existing VM or snapshot state. Drain old +This release does not convert VM or snapshot state from older releases. Drain old claims and use fresh `data_dir` and `checkpoint_dir` locations when upgrading; older checkpoints and promoted templates must not be reused. diff --git a/mcp/tools.go b/mcp/tools.go index 899a721e..1ea838a0 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -49,7 +49,7 @@ var tools = []tool{ schema(props{"sandbox_id": str(""), "path": str("absolute path")}, "sandbox_id", "path"), toolListDir, }, { - "fork", "Clone a sandbox into N independent children carrying its exact memory and disk state.", + "fork", "Clone a sandbox into N independent children carrying its exact memory and disk state; children live one hour.", schema(props{"sandbox_id": str(""), "count": integer("children, 1-16")}, "sandbox_id", "count"), toolFork, }, { @@ -57,7 +57,7 @@ var tools = []tool{ schema(props{"sandbox_id": str(""), "name": str("optional label")}, "sandbox_id"), toolCheckpoint, }, { - "branch_checkpoint", "Claim a fresh sandbox branched from a checkpoint's exact captured moment.", + "branch_checkpoint", "Claim a fresh sandbox branched from a checkpoint's exact captured moment; it lives one hour.", schema(props{"checkpoint_id": str("")}, "checkpoint_id"), toolBranchCheckpoint, }, {"list_checkpoints", "List checkpoints on the node, newest first.", schema(props{}), toolListCheckpoints}, @@ -279,7 +279,7 @@ func toolFork(ctx context.Context, s *server, raw json.RawMessage) (string, erro if err != nil { return "", err } - children, err := sb.Fork(ctx, args.Count, 0) + children, err := sb.Fork(ctx, args.Count, defaultToolTTL) if err != nil { return "", err } @@ -314,7 +314,7 @@ func toolBranchCheckpoint(ctx context.Context, s *server, raw json.RawMessage) ( if err != nil { return "", err } - sb, err := ckpt.New(ctx) + sb, err := ckpt.New(ctx, sandbox.WithTimeout(defaultToolTTL)) if err != nil { return "", err } From 4931efc1ade6f79d4f8565aab4f1fce095098b1b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 11:12:22 +0800 Subject: [PATCH 24/26] review: compress the tenant-cache comments to their WHY --- sandboxd/pool/pool.go | 6 ++---- sandboxd/pool/template.go | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 7279dac6..7e9ec929 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -316,10 +316,8 @@ type Manager struct { ckptTTL time.Duration ckptSweeping atomic.Bool - // tplSet caches each visible template id against its owning tenant so the - // 1s gossip tick and the claim-path ownership test never touch the backend - // (an s3 read is network I/O); local promotes/deletes update it, startup - // loads it. Empty value means the operator promoted it. + // tplSet caches each template id against its owning tenant ("" = operator), + // so the gossip tick and the claim path never pay a store read. tplMu sync.Mutex tplSet map[string]string diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 53942729..255ff4a8 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -158,10 +158,8 @@ func (m *Manager) HasPoolGolden(key types.PoolKey) bool { } // HasPromotedTemplate reports whether key resolves to a promoted template this -// tenant may claim. A hash a configured pool owns is subtracted, as -// TemplateHashes does for the gossip: resolveGolden serves it from the pool -// golden, never the template. The tenant test matches resolveGolden's, so -// routing cannot promise a golden the claim will then refuse. +// tenant may claim — resolveGolden's test exactly, so routing never promises +// a golden the claim would then refuse. func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool { if m.pooledHash(key.Hash()) { return false From 855c67846d1c36cbb39c98276e0bec9a97304ec3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 11:27:32 +0800 Subject: [PATCH 25/26] sandboxd: drop the Firecracker engine axis (#86) The fc opt-in never carried its weight: it offered nothing CH does not (volumes, checkpoint, fork, hibernate and the restore modes are all CH; clone performance measured equal), it had zero e2e or hardware coverage, and the axis never made it through the template lifecycle - DELETE built its key without engine, so an fc-promoted template could never be deleted by name, and no SDK could send or read the axis at all. Rather than thread a one-value-in-practice axis through claim, promote, handles and delete, remove it: PoolKey is (template, net, size) again, the hash drops the engine component, the volumes-require-ch guard and the fc test rows go with it, and the pool gauges return to three labels. cocoon keeps --fc for anyone driving it directly. Old pool-key hashes change; upgrades already never convert state. --- docs/cluster.md | 2 +- docs/deploy.md | 5 ++--- docs/sandboxd-api.md | 15 ++++++--------- e2e/e2e_test.go | 5 +---- sandboxd/engine/engine.go | 5 ----- sandboxd/pool/claim.go | 4 ++-- sandboxd/pool/egress_test.go | 2 +- sandboxd/pool/intercept_test.go | 2 +- sandboxd/pool/pool_test.go | 2 +- sandboxd/pool/poolstore_test.go | 4 ++-- sandboxd/pool/promote_test.go | 8 ++++---- sandboxd/pool/template.go | 2 +- sandboxd/pool/volume.go | 5 +---- sandboxd/pool/volume_rw_test.go | 4 ++-- sandboxd/pool/volume_test.go | 3 +-- sandboxd/server/metrics.go | 4 ++-- sandboxd/server/server.go | 3 --- sandboxd/server/server_test.go | 5 ++--- sandboxd/types/api.go | 3 +-- sandboxd/types/types.go | 25 +------------------------ sdk/go/info.go | 1 - sdk/go/options.go | 7 ------- sdk/go/pools.go | 1 - 23 files changed, 32 insertions(+), 85 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 7fa0e90d..67f47955 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -123,7 +123,7 @@ curl -s -H "Authorization: Bearer $TOKEN" http://node-a:7777/v1/info | jq . ```json { "pools": [ - {"key": {"template": "base:24.04", "net": "none", "size": "small", "engine": "ch"}, + {"key": {"template": "base:24.04", "net": "none", "size": "small"}, "warm": 4, "refilling": 0, "target": 4, "golden": true} ], "claimed": 2, diff --git a/docs/deploy.md b/docs/deploy.md index 7df8ebcc..2a5d8731 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -110,7 +110,7 @@ sandboxd reads one JSON file (`-config`, default | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call restores it transparently (a checkpoint restore's latency). Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived, reclaiming storage; the claim is then gone for good. Same node-wide/per-pool split | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | -| `pools[]` | — | warm pools, keyed by `(template, net, size, engine)`. `warm` defaults to 4; `net` is `none` or `egress`; `size` is a tier, below; `engine` is `ch` (default) or `fc` to cold-boot that key under Firecracker. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | +| `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4; `net` is `none` or `egress`; `size` is a tier, below. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | Size tiers (free-form CPU/memory is deliberately not accepted — it would fragment the warm pools): @@ -157,8 +157,7 @@ mounts the device before finalizing the claim: `mode: "ro"` (the default) attaches and mounts read-only; `mode: "rw"` requires the catalog entry's `writable: true` and attaches and mounts read-write. Setup failure destroys the VM without quiescing — the claim was never handed out, so no workload -write happened — and a popped warm VM is refilled normally. Firecracker -volume claims are rejected. +write happened — and a popped warm VM is refilled normally. A multi-volume claim brings every volume up concurrently: each volume's own marker→attach→mount order is preserved, but cocoon serializes the hypervisor diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index e40fe87e..d2543423 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -32,10 +32,8 @@ Auth: `Authorization: Bearer ` (when configured). "require_promoted": false} ``` -- `net` defaults to `none`, `size` to `small`, `engine` to `ch`. The pool key - is `(template, net, size, engine)`; `engine: "fc"` cold-boots that key under - Firecracker, and clones inherit the hypervisor pinned in the golden's - snapshot +- `net` defaults to `none`, `size` to `small`; the pool key is + `(template, net, size)` - `ttl_seconds` 0 means the server default (5 minutes); capped at 24h. The owning node reaps the sandbox after the TTL even if the client vanishes - `claim_ref` is an optional opaque caller reference echoed by the scoped @@ -48,8 +46,7 @@ Auth: `Authorization: Bearer ` (when configured). defaults to `/volumes/`; a custom value must be absolute and clean, outside the guest OS tree, unique, and non-nesting within the request. `mode` is `"ro"` (default, omitted) or `"rw"`; `"rw"` requires the catalog - entry's `writable: true` (see [deploy](deploy.md#dataset-volumes)). Volumes - require Cloud Hypervisor + entry's `writable: true` (see [deploy](deploy.md#dataset-volumes)) - `volumes_attach_only` (default `false`) attaches every requested volume without mounting it, handing the whole mount contract to the workload. It requires at least one volume, and rejects any entry carrying a `mount` — @@ -175,7 +172,7 @@ Errors: 400 unknown template axis, invalid/duplicate volumes, `volumes_attach_only` with no volumes or with an entry carrying a `mount`, `mode: "rw"` against a non-writable entry, or a volume that is unknown or forbidden (the -latter two are deliberately indistinguishable), Firecracker with volumes, or +latter two are deliberately indistinguishable), or bad body; 401 bad api token; 409 egress requested on a node without an egress attachment, a writable name already claimed in a conflicting mode (volume busy — a live writer excludes every other claim for that name, live readers @@ -280,7 +277,7 @@ node (name-based calls route via gossip); a shared checkpoint store makes every node resolve it. Under exactly this key: ```json -{"key": {"template": "myproj:v1", "net": "none", "size": "small", "engine": "ch"}, +{"key": {"template": "myproj:v1", "net": "none", "size": "small"}, "content_digest": "sha256:…"} ``` @@ -522,7 +519,7 @@ Auth: root only (tenant tokens get 403). Node pools, claim count, and mesh peers: ```json -{"pools": [{"key": {"template": "base:24.04", "net": "none", "size": "small", "engine": "ch"}, +{"pools": [{"key": {"template": "base:24.04", "net": "none", "size": "small"}, "warm": 4, "refilling": 0, "target": 4, "golden": true}], "claimed": 2, "hibernated": 1, diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 3e174bd2..3a96f6ba 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -27,7 +27,7 @@ import ( sandbox "github.com/cocoonstack/sandbox/sdk/go" ) -var testKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} +var testKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} func TestEndToEnd(t *testing.T) { stack := startStack(t, "node-token", config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -446,9 +446,6 @@ func TestClaimRefRoundTrip(t *testing.T) { if list[i].ClaimRef != "ns/workload" { t.Errorf("claim_ref %q, want ns/workload", list[i].ClaimRef) } - if list[i].Key.Engine != sandbox.EngineCH { - t.Errorf("engine %q, want the defaulted ch — the SDK key must carry the axis", list[i].Key.Engine) - } } // TestAttachOnlyVolumeEndToEnd drives one attach-only writable claim through diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 43bfccca..06c3e9b8 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -358,11 +358,6 @@ func (e *Engine) restoreArgs() []string { func (e *Engine) runColdArgs(name string, key types.PoolKey) []string { spec, _ := key.Size.Spec() args := []string{"vm", "run", argName, name, argOutput, formatJSON, "--cpu", strconv.Itoa(spec.CPU), "--memory", spec.Memory, e.directIOArg()} - if key.Engine == types.EngineFC { - // Firecracker is a per-pool cold-boot choice; clones inherit the - // hypervisor from the golden's pinned snapshot, so only RunCold flags it. - args = append(args, "--fc") - } args = append(args, e.netArgs(name, key, true)...) return append(args, key.Template) } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index f34f25e6..cd42f0cc 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -30,7 +30,7 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Dur if err := m.validate(key); err != nil { return nil, err } - volumeSpecs, err := m.resolveVolumes(ctx, key, tenant, volumes) + volumeSpecs, err := m.resolveVolumes(ctx, tenant, volumes) if err != nil { return nil, err } @@ -506,7 +506,7 @@ func (m *Manager) claimProvision(ctx context.Context, key types.PoolKey, ttl tim if err := m.validate(key); err != nil { return nil, err } - volumeSpecs, err := m.resolveVolumes(ctx, key, tenant, volumes) + volumeSpecs, err := m.resolveVolumes(ctx, tenant, volumes) if err != nil { return nil, err } diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index 52a9bef5..8b547808 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -21,7 +21,7 @@ import ( ) var ( - egKey = types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall, Engine: types.EngineCH} + egKey = types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall} egPolicy = &egress.Policy{Allow: []egress.Rule{{Host: "example.com", Secret: "gh"}}} ) diff --git a/sandboxd/pool/intercept_test.go b/sandboxd/pool/intercept_test.go index 5bd01e8b..d210e1f2 100644 --- a/sandboxd/pool/intercept_test.go +++ b/sandboxd/pool/intercept_test.go @@ -11,7 +11,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -var interceptKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} +var interceptKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} func TestGoldenBuildInstallsCAForInterceptPool(t *testing.T) { eng := newFakeEngine() diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index e1ce3cd9..0ac304ea 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -21,7 +21,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -var testKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} +var testKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} func TestClaimWarmHitTransfersOwnership(t *testing.T) { eng := newFakeEngine() diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index c545b2fe..c0e6a101 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -12,8 +12,8 @@ import ( ) var ( - seedKey = types.PoolKey{Template: "seed:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} - apiKey = types.PoolKey{Template: "api:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} + seedKey = types.PoolKey{Template: "seed:24.04", Net: types.NetNone, Size: types.SizeSmall} + apiKey = types.PoolKey{Template: "api:24.04", Net: types.NetNone, Size: types.SizeSmall} ) func TestPersistedPoolsSurviveRestart(t *testing.T) { diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index 66798ee1..bab58ef8 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -26,7 +26,7 @@ func TestPromoteThenClaimClonesFromTemplate(t *testing.T) { if len(eng.snapSaves) != 1 || !slices.Contains(eng.snapRemoves, eng.snapSaves[0]) { t.Errorf("snapSaves=%v snapRemoves=%v, want one transient snapshot dropped", eng.snapSaves, eng.snapRemoves) } - key := types.PoolKey{Template: "tpl:x", Net: parent.Key.Net, Size: parent.Key.Size, Engine: parent.Key.Engine} + key := types.PoolKey{Template: "tpl:x", Net: parent.Key.Net, Size: parent.Key.Size} if gotKey != key { t.Errorf("returned key %+v, want %+v (the parent's axes)", gotKey, key) } @@ -146,12 +146,12 @@ func TestDeleteTemplate(t *testing.T) { if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:del", ""); err != nil { t.Fatalf("Promote: %v", err) } - key := types.PoolKey{Template: "tpl:del", Net: testKey.Net, Size: testKey.Size, Engine: testKey.Engine} + key := types.PoolKey{Template: "tpl:del", Net: testKey.Net, Size: testKey.Size} if err := m.DeleteTemplate(t.Context(), testKey, ""); !errors.Is(err, ErrPooledTemplate) { t.Errorf("pooled delete: %v, want ErrPooledTemplate", err) } - if err := m.DeleteTemplate(t.Context(), types.PoolKey{Template: "nope", Net: testKey.Net, Size: testKey.Size, Engine: testKey.Engine}, ""); !errors.Is(err, ErrUnknownTemplate) { + if err := m.DeleteTemplate(t.Context(), types.PoolKey{Template: "nope", Net: testKey.Net, Size: testKey.Size}, ""); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("unknown delete: %v, want ErrUnknownTemplate", err) } if err := m.DeleteTemplate(t.Context(), key, ""); err != nil { @@ -276,7 +276,7 @@ func TestPromoteFailsClosedOnMetaError(t *testing.T) { if _, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "acme"); err != nil { t.Fatalf("promote: %v", err) } - key := types.PoolKey{Template: "shared:v1", Net: testKey.Net, Size: testKey.Size, Engine: testKey.Engine} + key := types.PoolKey{Template: "shared:v1", Net: testKey.Net, Size: testKey.Size} meta := filepath.Join(m.dataDir, "checkpoints", store.TemplateID(key.Hash()), store.MetaFile) if err := os.Chmod(meta, 0o000); err != nil { t.Fatalf("chmod: %v", err) diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 255ff4a8..57622300 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -42,7 +42,7 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t if !sb.Key.Capturable() { return types.PoolKey{}, "", ErrNoEgressFork } - key := types.PoolKey{Template: template, Net: sb.Key.Net, Size: sb.Key.Size, Engine: sb.Key.Engine} + key := types.PoolKey{Template: template, Net: sb.Key.Net, Size: sb.Key.Size} if m.pooledHash(key.Hash()) { // A configured pool owns this key — promoting over it would // silently change what refills produce. diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 42f1616d..3401326d 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -135,7 +135,7 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri return local, nil } -func (m *Manager) resolveVolumes(ctx context.Context, key types.PoolKey, tenant string, requested []types.Volume) ([]resolvedVolume, error) { +func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested []types.Volume) ([]resolvedVolume, error) { if len(requested) == 0 { return nil, nil } @@ -145,9 +145,6 @@ func (m *Manager) resolveVolumes(ctx context.Context, key types.PoolKey, tenant if err != nil { return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) } - if key.Engine != types.EngineCH { - return nil, fmt.Errorf("%w: volumes require engine ch", ErrBadVolume) - } resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { entry, ok := m.volumes[volume.Name] diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index 9bd8ea13..2e20d820 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -147,7 +147,7 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { // The reader resolves a clean image; a writable claim then fails between // that resolve and admission, leaving its marker but no hold behind. - resolved, err := m.resolveVolumes(t.Context(), testKey, "", readOnly) + resolved, err := m.resolveVolumes(t.Context(), "", readOnly) if err != nil { t.Fatalf("resolveVolumes: %v", err) } @@ -166,7 +166,7 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { } m.unreserveVolumes(appliedVolumes(resolved)) - writable, err := m.resolveVolumes(t.Context(), testKey, "", []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW}}) + writable, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW}}) if err != nil { t.Fatalf("resolveVolumes writable: %v", err) } diff --git a/sandboxd/pool/volume_test.go b/sandboxd/pool/volume_test.go index 6b6b89b1..617cc1f0 100644 --- a/sandboxd/pool/volume_test.go +++ b/sandboxd/pool/volume_test.go @@ -268,7 +268,6 @@ func TestClaimProvisionRejectsInvalidVolumesBeforeProvision(t *testing.T) { {"too many", testKey, nil, tooMany}, {"unknown", testKey, []config.VolumeSpec{{Name: "data", Path: path}}, []types.Volume{{Name: "other"}}}, {"invalid mount", testKey, []config.VolumeSpec{{Name: "data", Path: path}}, []types.Volume{{Name: "data", Mount: "relative"}}}, - {"firecracker", types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineFC}, []config.VolumeSpec{{Name: "data", Path: path}}, []types.Volume{{Name: "data"}}}, } { t.Run(tt.name, func(t *testing.T) { eng := newFakeEngine() @@ -472,7 +471,7 @@ func TestVolumePlacementChecksAccessAndLocalAvailability(t *testing.T) { if local, err := m.VolumePlacement(testKey, "", []string{"peer-only"}); err != nil || local { t.Errorf("root peer-only placement=(%v, %v), want false, nil", local, err) } - badKey := types.PoolKey{Template: "rt:24.04", Net: "lan", Size: types.SizeSmall, Engine: types.EngineCH} + badKey := types.PoolKey{Template: "rt:24.04", Net: "lan", Size: types.SizeSmall} if _, err := m.VolumePlacement(badKey, "", []string{"local"}); !errors.Is(err, ErrBadKey) { t.Errorf("invalid key error=%v, want ErrBadKey", err) } diff --git a/sandboxd/server/metrics.go b/sandboxd/server/metrics.go index f3d69e11..84597d92 100644 --- a/sandboxd/server/metrics.go +++ b/sandboxd/server/metrics.go @@ -47,11 +47,11 @@ func (s *Server) handleMetrics(w http.ResponseWriter, _ *http.Request) { metric("pool_warm", "gauge", "claim-ready VMs per pool") for _, p := range pools { - _, _ = fmt.Fprintf(w, "sandboxd_pool_warm{template=%q,net=%q,size=%q,engine=%q} %d\n", p.Key.Template, p.Key.Net, p.Key.Size, p.Key.Engine, p.Warm) + _, _ = fmt.Fprintf(w, "sandboxd_pool_warm{template=%q,net=%q,size=%q} %d\n", p.Key.Template, p.Key.Net, p.Key.Size, p.Warm) } metric("pool_target", "gauge", "warm watermark per pool") for _, p := range pools { - _, _ = fmt.Fprintf(w, "sandboxd_pool_target{template=%q,net=%q,size=%q,engine=%q} %d\n", p.Key.Template, p.Key.Net, p.Key.Size, p.Key.Engine, p.Target) + _, _ = fmt.Fprintf(w, "sandboxd_pool_target{template=%q,net=%q,size=%q} %d\n", p.Key.Template, p.Key.Net, p.Key.Size, p.Target) } if s.placer != nil { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 5d46153b..b24f54e2 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -270,9 +270,6 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { func (s *Server) handleVolumeClaim(w http.ResponseWriter, r *http.Request, req types.ClaimRequest, key types.PoolKey, hash, tenant string) { volumes, err := types.ValidateVolumes(req.Volumes, req.VolumesAttachOnly) - if err == nil && key.Engine != types.EngineCH { - err = errors.New("volumes require engine ch") - } if err != nil { writeErr(w, http.StatusBadRequest, fmt.Errorf("%w: %v", pool.ErrBadVolume, err).Error()) return diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index af243648..093aa951 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -56,7 +56,7 @@ func TestClaimHappyPath(t *testing.T) { if cr.TemplateDigest != "sha256:claim-digest" { t.Errorf("template digest %q, want sha256:claim-digest", cr.TemplateDigest) } - want := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} + want := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} if gotKey != want { t.Errorf("key %+v, want defaults %+v", gotKey, want) } @@ -738,7 +738,7 @@ func TestPromoteAndDeleteTemplateFlow(t *testing.T) { if got := del("Bearer sekret", "template=tpl:x&net=none&size=small"); got != http.StatusNoContent { t.Errorf("delete status %d, want 204", got) } - want := types.PoolKey{Template: "tpl:x", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} + want := types.PoolKey{Template: "tpl:x", Net: types.NetNone, Size: types.SizeSmall} if gotKey != want { t.Errorf("delete key %+v, want %+v (claim defaults applied)", gotKey, want) } @@ -1394,7 +1394,6 @@ func TestVolumeClaimRejectsShapeBeforePlacement(t *testing.T) { `{"template":"rt:24.04","volumes":["data"]}`, `{"template":"rt:24.04","volumes":[{"name":"data"},{"name":"data"}]}`, `{"template":"rt:24.04","volumes":[{"name":"cocoon-data"}]}`, - `{"template":"rt:24.04","engine":"fc","volumes":[{"name":"data"}]}`, `{"template":"rt:24.04","volumes":[{"name":"data","mount":"relative"}]}`, `{"template":"rt:24.04","volumes":[{"name":"data","mount":"/datasets"},{"name":"other","mount":"/datasets/nested"}]}`, `{"template":"rt:24.04","volumes":[{"name":"a"},{"name":"b"},{"name":"c"},{"name":"d"},{"name":"e"},{"name":"f"},{"name":"g"},{"name":"h"},{"name":"i"}]}`, diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 49b578f0..880ee138 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -22,7 +22,6 @@ type ClaimRequest struct { Template string `json:"template"` Net NetShape `json:"net,omitempty"` Size Size `json:"size,omitempty"` - Engine Engine `json:"engine,omitempty"` Volumes []Volume `json:"volumes,omitempty"` // VolumesAttachOnly attaches every requested volume without mounting it: // the workload finds the device by its serial and owns the mount contract. @@ -40,7 +39,7 @@ type ClaimRequest struct { // Key resolves the requested pool key with the wire defaults filled. func (r ClaimRequest) Key() PoolKey { - return PoolKey{Template: r.Template, Net: r.Net, Size: r.Size, Engine: r.Engine}.Defaulted() + return PoolKey{Template: r.Template, Net: r.Net, Size: r.Size}.Defaulted() } // ClaimResponse is the wire reply of POST /v1/claim. A successful claim diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 8dad0341..dde79926 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -29,9 +29,6 @@ const ( RestoreOnDemand RestoreMode = "ondemand" RestoreMmap RestoreMode = "mmap" - EngineCH Engine = "ch" - EngineFC Engine = "fc" - MaxClaimVolumes = 8 // Also the guest `mount -o` option literals (engine.MountVolume): renaming @@ -82,21 +79,6 @@ func (m RestoreMode) Validate() error { } } -// Engine selects the hypervisor backend cocoon boots a pool's VMs on: Cloud -// Hypervisor (default) or Firecracker. It is a pool axis so a CH pool and an -// FC pool with the same template/net/size stay distinct goldens. -type Engine string - -// Validate accepts the empty default (resolved to EngineCH) plus known engines. -func (e Engine) Validate() error { - switch e { - case "", EngineCH, EngineFC: - return nil - default: - return fmt.Errorf("unknown engine %q", e) - } -} - // Size is a T-shirt resource tier. type Size string @@ -120,7 +102,6 @@ type PoolKey struct { Template string `json:"template"` Net NetShape `json:"net"` Size Size `json:"size"` - Engine Engine `json:"engine,omitempty"` } // Capturable reports whether state capture (fork, checkpoint, promote) is @@ -135,7 +116,6 @@ func (k PoolKey) Capturable() bool { func (k PoolKey) Defaulted() PoolKey { k.Net = cmp.Or(k.Net, NetNone) k.Size = cmp.Or(k.Size, SizeSmall) - k.Engine = cmp.Or(k.Engine, EngineCH) return k } @@ -144,7 +124,7 @@ func (k PoolKey) Defaulted() PoolKey { // names, so a targeted collision with a configured pool's hash must stay a // second-preimage problem, never a brute-forceable one. func (k PoolKey) Hash() string { - sum := sha256.Sum256([]byte(k.Template + "|" + string(k.Net) + "|" + string(k.Size) + "|" + string(k.Engine))) + sum := sha256.Sum256([]byte(k.Template + "|" + string(k.Net) + "|" + string(k.Size))) return hex.EncodeToString(sum[:16]) } @@ -161,9 +141,6 @@ func (k PoolKey) Validate() error { if _, ok := k.Size.Spec(); !ok { return fmt.Errorf("unknown size %q", k.Size) } - if err := k.Engine.Validate(); err != nil { - return err - } return nil } diff --git a/sdk/go/info.go b/sdk/go/info.go index 4b7107fa..42c3c233 100644 --- a/sdk/go/info.go +++ b/sdk/go/info.go @@ -25,7 +25,6 @@ type PoolKey struct { Template string `json:"template"` Net NetShape `json:"net"` Size Size `json:"size"` - Engine Engine `json:"engine,omitempty"` } // PoolStatus reports one warm pool on a node. diff --git a/sdk/go/options.go b/sdk/go/options.go index a213ab0e..dde7eb63 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -11,10 +11,6 @@ const ( // NetEgress attaches the node's bridge or CNI network. NetEgress NetShape = "egress" - // EngineCH is the default hypervisor; EngineFC cold-boots under Firecracker. - EngineCH Engine = "ch" - EngineFC Engine = "fc" - Small Size = "small" Medium Size = "medium" Large Size = "large" @@ -31,9 +27,6 @@ type NetShape string // node's warm pools. type Size string -// Engine is the pool key's hypervisor axis. -type Engine string - // Volume requests one catalog entry at an optional guest mount path and mode. type Volume struct { Name string `json:"name"` diff --git a/sdk/go/pools.go b/sdk/go/pools.go index 98b2f7cd..4f61767d 100644 --- a/sdk/go/pools.go +++ b/sdk/go/pools.go @@ -14,7 +14,6 @@ type PoolSpec struct { Template string `json:"template"` Net NetShape `json:"net,omitempty"` Size Size `json:"size,omitempty"` - Engine Engine `json:"engine,omitempty"` Warm int `json:"warm"` WarmMax int `json:"warm_max,omitempty"` IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` From 89304e5dab23af2f9ad1c9aba3baec4789b9d248 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 14 Aug 2026 12:00:01 +0800 Subject: [PATCH 26/26] fix(mesh): tenant-scope the template gossip hashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TemplateOwners answered from raw key hashes, so a foreign tenant's promoted template escalated another tenant's claim: a volume claim that would have cold-booted locally was forced onto the promoted path and refused, and a plain claim earned a useless redirect that also signalled the name exists. The local check is tenant-aware; the remote one had nothing to be aware with — gossip deliberately carries no tenant. Scope the hash instead: gossip advertises hash(keyHash|owner), and an owner query probes the requester's own scope plus the operator's (root probes every configured tenant). A foreign template simply never matches, which is exactly how the claim path treats it, and the wire still carries nothing but opaque hashes. Salting costs two sha256 on the warm-miss redirect path only; mixed-version meshes disagree on hashes, and upgrades are lockstep. --- sandboxd/pool/promote_test.go | 25 +++++++++++++ sandboxd/pool/template.go | 4 +- sandboxd/server/server.go | 31 ++++++++++++++-- sandboxd/server/server_test.go | 67 +++++++++++++++++++++++++++++++++- sandboxd/types/types.go | 7 ++++ 5 files changed, 126 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index bab58ef8..c54f5711 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -382,6 +382,31 @@ func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { } } +func TestTemplateHashesAreTenantScoped(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + a, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + if err != nil { + t.Fatalf("claim: %v", err) + } + key, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") + if err != nil { + t.Fatalf("promote: %v", err) + } + + hashes := m.TemplateHashes() + if want := types.TemplateGossipHash(key.Hash(), "acme"); !slices.Contains(hashes, want) { + t.Errorf("gossip %v lacks the owner-scoped hash %s", hashes, want) + } + for name, bad := range map[string]string{ + "raw": key.Hash(), + "foreign": types.TemplateGossipHash(key.Hash(), "beta"), + } { + if slices.Contains(hashes, bad) { + t.Errorf("gossip %v carries the %s hash — a foreign tenant could match it", hashes, name) + } + } +} + func TestTemplateHashesSortedForMeshCompare(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 57622300..90c36512 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -129,10 +129,10 @@ func (m *Manager) TemplateHashes() []string { m.mu.Unlock() m.tplMu.Lock() hashes := make([]string, 0, len(m.tplSet)) - for id := range m.tplSet { + for id, tenant := range m.tplSet { hash := store.TemplateHash(id) if _, ok := pooled[hash]; !ok { - hashes = append(hashes, hash) + hashes = append(hashes, types.TemplateGossipHash(hash, tenant)) } } m.tplMu.Unlock() diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index b24f54e2..5da79caa 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -15,6 +15,7 @@ import ( "io" "net" "net/http" + "slices" "sync" "time" @@ -324,7 +325,7 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, pooled := s.mgr.HasPoolGolden(key) var templateOwners []string if s.placer != nil && !pooled { - templateOwners = s.placer.TemplateOwners(hash) + templateOwners = s.templateOwners(s.placer.TemplateOwners, hash, tenant) } promoted := req.RequirePromoted || localTemplate || len(templateOwners) > 0 req.RequirePromoted = promoted @@ -340,7 +341,9 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, // A shared template store lets a volume holder resolve the template // even before that node has advertised the newly published hash. The // no_redirect target re-checks both resources before provisioning. - owners = s.placer.TemplateVolumeOwners(hash, names) + owners = s.templateOwners(func(probe string) []string { + return s.placer.TemplateVolumeOwners(probe, names) + }, hash, tenant) } else { owners = s.placer.VolumeCandidates(hash, names) } @@ -354,6 +357,26 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, return true, nil } +func (s *Server) templateOwners(query func(string) []string, hash, tenant string) []string { + probes := []string{types.TemplateGossipHash(hash, tenant)} + if tenant == "" { + for _, tn := range s.tenants { + probes = append(probes, types.TemplateGossipHash(hash, tn.Name)) + } + } else { + probes = append(probes, types.TemplateGossipHash(hash, "")) + } + var owners []string + for _, probe := range probes { + for _, owner := range query(probe) { + if !slices.Contains(owners, owner) { + owners = append(owners, owner) + } + } + } + return owners +} + // redirectClaim redirects a warm-miss to a better peer — a warm holder, or the // template owner when we lack a golden (so we don't cold-boot a nonexistent // image ref). A no_redirect request must resolve locally, never bounce again, @@ -366,7 +389,7 @@ func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req t return true } // TemplateOwners is in-memory; HasGolden can be a store round-trip. - owners := s.placer.TemplateOwners(hash) + owners := s.templateOwners(s.placer.TemplateOwners, hash, tenant) return len(owners) > 0 && !s.mgr.HasGolden(ctx, key, tenant) && writeRedirect(w, owners) } @@ -593,7 +616,7 @@ func (s *Server) handleDeleteTemplate(w http.ResponseWriter, r *http.Request) { // owner. no_redirect mirrors the claim protocol — a redirected retry // carries it, so the owner answers for itself and never bounces again. if errors.Is(err, pool.ErrUnknownTemplate) && s.placer != nil && q.Get("no_redirect") == "" && - writeRedirect(w, s.placer.TemplateOwners(key.Hash())) { + writeRedirect(w, s.templateOwners(s.placer.TemplateOwners, key.Hash(), tenantFrom(r.Context()))) { return } writeResult(w, r, "delete template", req.Template, "delete template failed", err, func() { diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 093aa951..fab3b1c0 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -1337,6 +1337,62 @@ func TestVolumeClaimKeepsPoolContentAgainstPeerTemplates(t *testing.T) { } } +func TestForeignTemplateGossipNeverEscalates(t *testing.T) { + hash := types.ClaimRequest{Template: "tpl"}.Key().Hash() + tenants := []config.TenantSpec{{Name: "acme", Token: "acme-tok"}, {Name: "beta", Token: "beta-tok"}} + for _, tt := range []struct { + name, token string + volumes []types.Volume + wantRedirect bool + wantProvision int + }{ + {"foreign volume claim cold-boots locally", "beta-tok", []types.Volume{{Name: "imagenet"}}, false, 1}, + {"owner volume claim escalates to its template", "acme-tok", []types.Volume{{Name: "imagenet"}}, true, 0}, + {"foreign plain claim stays local, no existence signal", "beta-tok", nil, false, 1}, + {"owner plain claim follows its template", "acme-tok", nil, true, 0}, + } { + t.Run(tt.name, func(t *testing.T) { + mgr := &fakeManager{ + volumePlacement: func(types.PoolKey, string, []string) (bool, error) { return true, nil }, + } + placer := &fakePlacer{ownersByProbe: map[string][]string{ + types.TemplateGossipHash(hash, "acme"): {"peer:7777"}, + }} + srv := New("root-tok", tenants, "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + + body, _ := json.Marshal(types.ClaimRequest{Template: "tpl", Volumes: tt.volumes}) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/claim", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+tt.token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("claim: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status=%d, want 200", resp.StatusCode) + } + var cr types.ClaimResponse + if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil { + t.Fatalf("decode: %v", err) + } + if got := len(cr.Redirect) > 0; got != tt.wantRedirect { + t.Errorf("redirect=%v (%v), want %v", got, cr.Redirect, tt.wantRedirect) + } + if wantPromoted := tt.wantRedirect && tt.volumes != nil; cr.RequirePromoted != wantPromoted { + t.Errorf("require_promoted=%v, want %v (only the volume path pins it)", cr.RequirePromoted, wantPromoted) + } + if mgr.provisionCalls != tt.wantProvision { + t.Errorf("provisions=%d, want %d", mgr.provisionCalls, tt.wantProvision) + } + if mgr.gotRequirePromoted { + t.Error("a local resolution must not be forced onto the promoted path") + } + }) + } +} + func TestVolumeClaimValidatesKeyBeforePlacement(t *testing.T) { mgr := &fakeManager{ volumePlacement: func(types.PoolKey, string, []string) (bool, error) { @@ -2185,6 +2241,7 @@ func (f *fakeDialer) DialSilkd(ctx context.Context, sock string) (net.Conn, erro type fakePlacer struct { addrs []string owners []string + ownersByProbe map[string][]string volumeCandidates []string volumeOwners []string templateVolumeOwners []string @@ -2208,8 +2265,11 @@ func (f *fakePlacer) VolumeCandidates(_ string, names []string) []string { return f.volumeCandidates } -func (f *fakePlacer) TemplateOwners(string) []string { +func (f *fakePlacer) TemplateOwners(probe string) []string { f.templateOwnerCalls++ + if f.ownersByProbe != nil { + return f.ownersByProbe[probe] + } return f.owners } @@ -2218,8 +2278,11 @@ func (f *fakePlacer) VolumeOwners([]string) []string { return f.volumeOwners } -func (f *fakePlacer) TemplateVolumeOwners(string, []string) []string { +func (f *fakePlacer) TemplateVolumeOwners(probe string, _ []string) []string { f.templateVolumeCalls++ + if f.ownersByProbe != nil { + return f.ownersByProbe[probe] + } return f.templateVolumeOwners } func (f *fakePlacer) VolumeHolders() map[string]int { return f.volumeHolders } diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index dde79926..55fccda5 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -144,6 +144,13 @@ func (k PoolKey) Validate() error { return nil } +// TemplateGossipHash scopes a key hash to its owner for the gossip wire, so +// foreign templates never match an owner query; the tenant itself never travels. +func TemplateGossipHash(keyHash, tenant string) string { + sum := sha256.Sum256([]byte(keyHash + "|" + tenant)) + return hex.EncodeToString(sum[:16]) +} + // Sandbox is the node-local record of one pooled or claimed VM. type Sandbox struct { ID string `json:"id"`