diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..9b0af65c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,39 @@ +# The guest binaries (sandbox-init, silkd) and every os-image are built from +# these bases, so their tags are pinned by digest for reproducibility and +# dependabot is what keeps the pins current — without it a pinned digest is a +# frozen, unpatched toolchain. +version: 2 +updates: + - package-ecosystem: docker + directories: + - /silkd + - /boot + - /os-image/base/24.04 + schedule: + interval: weekly + commit-message: + prefix: "build" + - package-ecosystem: cargo + directories: + - /silkd + - /boot/init + schedule: + interval: weekly + commit-message: + prefix: "build" + - package-ecosystem: gomod + directories: + - /sandboxd + - /sdk/go + - /e2e + - /mcp + schedule: + interval: weekly + commit-message: + prefix: "build" + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + commit-message: + prefix: "build" diff --git a/.github/workflows/shell.yml b/.github/workflows/shell.yml new file mode 100644 index 00000000..58005686 --- /dev/null +++ b/.github/workflows/shell.yml @@ -0,0 +1,28 @@ +# The e2e and bench drivers are shell, and they are what hardware rounds run; +# a broken quoting or trap change there costs a whole round. +name: shell + +on: + push: + branches: [main] + paths: + - "**.sh" + - ".github/workflows/shell.yml" + pull_request: + paths: + - "**.sh" + - ".github/workflows/shell.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: shellcheck + run: make sh-lint diff --git a/Makefile b/Makefile index 663b35a7..3d97b733 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ GOLANGCILINT_VERSION ?= v2.12.2 GOLANGCILINT_ROOT := $(LOCALBIN)/golangci-lint-$(GOLANGCILINT_VERSION) GOLANGCILINT := $(GOLANGCILINT_ROOT)/golangci-lint -.PHONY: help test lint boot boot-debug extract extract-debug silkd-image base python images \ +.PHONY: help test lint sh-lint boot boot-debug extract extract-debug silkd-image base python images \ sandboxd go-test go-lint bench cloc ## Tool download targets @@ -49,6 +49,9 @@ lint: ## Rust fmt --check + clippy -D warnings: boot/init + silkd cd boot/init && cargo fmt --check && cargo clippy --all-targets -- -D warnings cd silkd && cargo fmt --check && cargo clippy --all-targets -- -D warnings +sh-lint: ## shellcheck every tracked shell script + git ls-files '*.sh' | xargs shellcheck + sandboxd: ## build dist/sandboxd mkdir -p dist cd sandboxd && GOWORK=off go build -ldflags "-X main.version=$(SANDBOXD_VERSION)" -o ../dist/sandboxd . diff --git a/README.md b/README.md index 2ab717b8..0d80f5df 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ performance) — source in `/boot/vmlinuz-sandbox` + `/boot/initrd.img-sandbox` - `os-image/` — VM images consuming the boot artifact: `base` (layered, for builds), `rt` (base squashed to one layer — the default template in - examples), `python`, `python-rt`, and `android` -- `scripts/` — `boot-bench.sh` (boot phase timing) and `sandboxd-e2e.sh` - (bare-metal e2e, below) + examples), `python`, `python-rt`, `browser`, and `android` +- `scripts/` — `boot-bench.sh` (boot phase timing), `bench.sh` (the published + benchmark procedure), `sandboxd-e2e.sh` (bare-metal e2e, below), plus the + `archive`/`egress`/`intercept` e2e drivers ## Build & test @@ -118,6 +119,7 @@ TEMPLATE=rt:24.04 scripts/sandboxd-e2e.sh changed carriers (via `build-boot.yml` / `build-silkd.yml`, `workflow_call`) then os-images, in order, so the chain is deterministic - `build-os-images.yml` — bakes base + flavors FROM the sha-pinned carriers +- `release.yml` — on a version tag, builds the release binaries and archives - `publish-pypi.yml` — on an `sdk-*-v*` tag, builds and publishes the matching package via PyPI Trusted Publishing (OIDC, per-package environment) diff --git a/boot/Dockerfile b/boot/Dockerfile index 58511842..4e1b2595 100644 --- a/boot/Dockerfile +++ b/boot/Dockerfile @@ -4,7 +4,7 @@ # under /boot/ for os-image builds to COPY --from. Built per-platform on # native runners (a kernel build under QEMU emulation takes hours). -FROM debian:bookworm-slim AS kbuild +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS kbuild RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential bc bison flex libelf-dev libssl-dev \ xz-utils curl ca-certificates python3 \ @@ -75,14 +75,14 @@ RUN set -e; \ fi; \ gcc usr/gen_init_cpio.c -o /usr/local/bin/gen_init_cpio -FROM rust:1-alpine AS initbuild +FROM rust:1-alpine@sha256:3c38f3f82c2f3d73da3b38e18d279393a04cb43ddded0e35088a8c3324d40900 AS initbuild RUN apk add --no-cache musl-dev WORKDIR /build COPY init/Cargo.toml init/Cargo.lock ./ COPY init/src ./src RUN cargo build --release --locked && cp target/release/sandbox-init /sandbox-init -FROM busybox:musl AS bb +FROM busybox:musl@sha256:32b5cdad7cce41dfd53d0ae06baebcf8357a147ee7694dc706911c373bc30c37 AS bb FROM kbuild AS pack ARG INITRD_DEBUG=0 diff --git a/boot/init/rust-toolchain.toml b/boot/init/rust-toolchain.toml index 85f36062..5ec58696 100644 --- a/boot/init/rust-toolchain.toml +++ b/boot/init/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "stable" +channel = "1.97.1" components = ["rustfmt", "clippy"] profile = "minimal" diff --git a/boot/init/src/boot.rs b/boot/init/src/boot.rs index ec520295..bae1325a 100644 --- a/boot/init/src/boot.rs +++ b/boot/init/src/boot.rs @@ -2,6 +2,7 @@ //! switch_root → exec. use std::fs; +use std::path::Path; use std::time::{Duration, Instant}; use crate::cfg::{self, BootCfg}; @@ -235,12 +236,15 @@ fn scan_serials(ids: &[&str], found: &mut [Option]) { let Ok(serial) = fs::read_to_string(&path) else { continue; }; - let serial = serial.trim_end(); - for (i, id) in ids.iter().enumerate() { - if found[i].is_none() && *id == serial { - found[i] = Some(format!("/dev/{name}")); - } - } + record_serial(ids, found, serial.trim_end(), &format!("/dev/{name}")); + } + } +} + +fn record_serial(ids: &[&str], found: &mut [Option], serial: &str, device: &str) { + for (i, id) in ids.iter().enumerate() { + if found[i].is_none() && *id == serial && Path::new(device).exists() { + found[i] = Some(device.into()); } } } @@ -251,3 +255,25 @@ fn uptime() -> String { .and_then(|s| s.split_ascii_whitespace().next().map(String::from)) .unwrap_or_else(|| "?".into()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn record_serial_skips_a_missing_device_node() { + let ids = ["layer"]; + let mut found = [None]; + + record_serial( + &ids, + &mut found, + "layer", + "/dev/sandbox-init-missing-device", + ); + assert!(found[0].is_none()); + + record_serial(&ids, &mut found, "layer", "/dev/null"); + assert_eq!(found[0].as_deref(), Some("/dev/null")); + } +} diff --git a/docs/deploy.md b/docs/deploy.md index 24048d6a..2a5d8731 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -20,7 +20,7 @@ the cocoon CLI and needs a template image with silkd baked in. (pull via cocoon, or `cocoon image import` a tar) Prebuilt static linux/amd64 and linux/arm64 binaries (`sandboxd`, -`sandbox-mcp`, `silkd`, with `checksums.txt`) ship with every +`sandboxd.dbg`, `sandbox-mcp`, `silkd`, with `checksums.txt`) ship with every [GitHub release](https://github.com/cocoonstack/sandbox/releases); the boot artifact and the `base`/`rt`/`python` images are multi-arch manifests (`browser` and `android` remain amd64-only). Build from source with @@ -29,7 +29,7 @@ reports what you are running. ## Upgrading -This CH-only release does not convert existing VM or snapshot state. Drain old +This release does not convert VM or snapshot state from older releases. Drain old claims and use fresh `data_dir` and `checkpoint_dir` locations when upgrading; older checkpoints and promoted templates must not be reused. @@ -89,6 +89,8 @@ sandboxd reads one JSON file (`-config`, default | `advertise_addr` | = `listen` | the host:port clients reach this node at; returned as a claim's owner address and gossiped to peers. Must be routable when `listen` is a wildcard | | `bridges` / `networks` | unset | egress-lane attachment: a list of host bridge devices, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. A Linux bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so an N-entry list raises the node's egress ceiling to N×1024 — VMs spread over the list by a stable hash of the VM name, so size it with headroom (the spread is statistical, not exact). `bridges` keeps the raw TAP-on-bridge attachment (taps in the root netns, no per-VM network namespace or CNI plugin execution); `networks` runs the CNI chain per VM. [Guarded egress](egress.md) needs `bridges` and rejects a CNI network at load | | `volumes` | unset | node-local catalog of operator-managed dataset images: `[ {"name":"imagenet","path":"/srv/datasets/imagenet.img","directio":"off","tenants":["acme"]}, {"name":"scratch-db","path":"/srv/datasets/scratch.img","writable":true} ]`. Names match `^[a-z][a-z0-9_-]{0,19}$` and cannot start with `cocoon-`; paths are absolute; `directio` is `on`, `off`, or `auto` and defaults to `off` for both read-only and writable entries. `tenants` is an optional access list: empty means every authenticated scope, while every listed name must exist in the node's `tenants` config; root always has access. `writable` (default `false`) lets a claim request `mode: "rw"` on that entry — see [Dataset volumes](#dataset-volumes). The catalog is intentionally not part of the cluster digest | +| `secrets` | unset | node-side credentials the egress proxy injects by name: `[{"name": "gh", "header": "Authorization", "value_env": "GH_TOKEN"}]`. A pool or tenant rule references the name; the value comes from the environment, never this file. See [egress](egress.md) | +| `egress_internal_allow` | unset | CIDR prefixes re-admitted through the egress proxy's SSRF guard, node-wide (every pool and tenant). Prefixes, not a permit-private switch — the guest bridges are themselves ULA/RFC1918. See [egress](egress.md) | | `egress_ca` | unset | [HTTPS-interception](egress.md#https-interception) PKI: `root_cert` (the cluster root baked into intercepted guests; may bundle old+new roots during rotation) plus this node's `intermediate_cert`/`intermediate_key` from `sandboxd ca issue-intermediate`. Required when any pool rule sets `intercept` | | `api_token` | unset | the operator (root) credential: when set, guards the node-level endpoints (Bearer) with full access, including release-by-id cleanup. Per-sandbox tokens guard ordinary sandbox-scoped calls | | `tenants` | unset | multi-tenant tokens next to `api_token`: `[{"name": "acme", "token": "…", "max_claims": 50}]`. A tenant token reaches the resource-creating verbs (claim, fork, promote, checkpoint, preview), catalog discovery, and its own sandbox/checkpoint listings; everything it creates is stamped with the tenant name. Root-only surfaces (per-id sandbox reads, `GET /v1/info`, `PUT /v1/pools`, `POST/DELETE /v1/drain`, `/metrics`) answer it 403. `max_claims` (0 = unlimited) caps that tenant's live claims next to the node-wide cap. Requires `api_token` set. Names and tokens must be unique, tokens distinct from `api_token`. On a cluster all nodes must carry the same tenants set (the SDK replays whichever token authorized a redirect), and per-node caps mean a tenant's effective cluster limit is `max_claims` × nodes. Empty = exactly the single-token behavior | @@ -96,19 +98,19 @@ sandboxd reads one JSON file (`-config`, default | `refill_concurrency` | 0 (auto) | concurrent VM provisioning budget, shared by warm-pool refills, fork clones, and the reap/hibernate/reconcile engine batches. 0 sizes it from the node: `NumCPU*2/3` clamped to [4, 256] — a 384-core node gets 256; small nodes keep a floor of 4 | | `preview_listen` | (off) | address for a preview HTTP server that serves guest ports under signed URLs; needs `preview_secret` | | `preview_secret` | — | cluster-shared HMAC secret signing preview tokens (all nodes share one) | -| `preview_advertise` | = `preview_listen` | the base URL a browser/proxy reaches this node's preview server at | +| `preview_advertise` | = `preview_listen` | the browser-facing preview base URL; nodes behind one TLS proxy may share it, while signed tokens route internally through each owner's `advertise_addr` | | `checkpoint_dir` | `/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — records are generation-addressed with `meta.json` as the atomic commit pointer, so no cross-node locking is required of the filesystem. A re-publish retains its superseded export generation for at least ~1h and until a following hourly sweep so an in-flight clone that resolved the old metadata can finish; budget the current generation plus every generation retained across that grace-and-sweep window. An explicit delete can make a concurrent clone fail visibly. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide) | | `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. Re-publish retains prior export generations until Delete so an in-flight fetch that selected old metadata can finish; budget storage for those generations. An explicit S3 Delete can still make a concurrent fetch that has not finished materializing fail visibly. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` | | `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the expiry eligibility point for a healed replica a delete broadcast missed, after which its next successful hourly sweep removes it; persistent sweep failure extends retention until one succeeds, so it is not a hard ceiling | | `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (a replica a delete broadcast missed becomes eligible for expiry after it, and its next successful hourly sweep removes it — so it is the finite eligibility point, not an exact ceiling). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, normal warm-candidate placement applies, with volume claims limited to candidates holding every requested volume) | -| `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`); preview accesses record as op `preview_dial`. A request frame whose first line exceeds 4 KiB is skipped, never truncated | -| `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a claim with no data-plane connection for this long is hibernated; the next call wakes it transparently. Per-pool `idle_hibernate_seconds` (in a pool entry) does the same for that pool's claims — pooled keys ignore the node-wide value. Opt-in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | +| `audit_log` | false | append every relayed request frame's op + addressing fields (never payloads) to `/audit.jsonl`, size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `decision` and `secret` (the ref name, never its value) on `egress` records; preview accesses record as op `preview`, one per request. A request frame whose first line exceeds 4 KiB is skipped, never truncated | +| `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim with no data-plane connection for this long is hibernated; the next call wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call restores it transparently (a checkpoint restore's latency). Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived, reclaiming storage; the claim is then gone for good. Same node-wide/per-pool split | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | -| `pools[]` | — | warm pools. `warm` defaults to 4; `net` is `none` or `egress`; `size` is a tier, below. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | +| `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4; `net` is `none` or `egress`; `size` is a tier, below. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | Size tiers (free-form CPU/memory is deliberately not accepted — it would fragment the warm pools): @@ -155,8 +157,7 @@ mounts the device before finalizing the claim: `mode: "ro"` (the default) attaches and mounts read-only; `mode: "rw"` requires the catalog entry's `writable: true` and attaches and mounts read-write. Setup failure destroys the VM without quiescing — the claim was never handed out, so no workload -write happened — and a popped warm VM is refilled normally. Firecracker -volume claims are rejected. +write happened — and a popped warm VM is refilled normally. A multi-volume claim brings every volume up concurrently: each volume's own marker→attach→mount order is preserved, but cocoon serializes the hypervisor @@ -295,7 +296,6 @@ here validates on load: "pools": [ {"template": "rt:24.04", "net": "none", "size": "small", "warm": 4, "warm_max": 12}, {"template": "rt:24.04", "net": "egress", "size": "medium", "warm": 2, - "idle_hibernate_seconds": 120, "archive_after_seconds": 900, "egress": {"allow": [ {"host": "api.github.com", "methods": ["GET", "POST"], "secret": "gh", "intercept": true}, {"host": "*.googleapis.com"} @@ -452,12 +452,18 @@ sandboxd: life is clamped to the claim's lease. - **Serving**: any node's preview listener verifies the token (no shared state), then reverse-proxies to the guest port over the relay if it owns - the sandbox, or forwards to the owner node otherwise. A released sandbox - is gone from the claim map, so its URL stops resolving — revocation is - the liveness lookup, not a list. + the sandbox, or forwards over HTTP to the owner node's `advertise_addr` + otherwise. A released sandbox is gone from the claim map, so its URL stops + resolving — revocation is the liveness lookup, not a list. - **The public entry point is a commodity dumb proxy.** Because any node can accept and forward, front the nodes with whatever terminates TLS and round-robins: a cloud HTTPS load balancer with a managed wildcard cert (GCP/AWS) in production, or a plain nginx/Caddy for self-hosting. It understands nothing about tokens — not sandboxd's code. Dev and e2e hit `preview_listen` directly over HTTP. +- **Sharing scope**: a preview URL is a bearer link, and its token payload + (sandbox id, port, owner `advertise_addr`) is readable by whoever holds + it — keep `advertise_addr` off browser-routable networks. All URLs under + one `preview_advertise` share one browser origin, so different claims' + apps are same-origin to the browser; workloads needing browser-side + isolation need per-sandbox subdomains on the fronting proxy. diff --git a/docs/egress.md b/docs/egress.md index dee6ab2f..4627e70c 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -98,8 +98,13 @@ domain policy first; the allow-list widens the IP gate only. Policy is per pool and per tenant; the effective policy is their intersection (a request must pass both, and the pool rule's secret wins on a double allow). -Secrets are registered separately and referenced by name — the value comes from -the environment, never the config file. +A missing policy on either side is an empty allow-list, not a pass: a tenant +without its own `egress` block reaches nothing — upgrading, a tenant that +relied on inheriting its pool's policy must now declare one — so granting a tenant egress — +and the secret injection that rides it — is always an explicit act. Root +claims have no tenant layer and take the pool's policy whole. Secrets are +registered separately and referenced by name — the value comes from the +environment, never the config file. ```jsonc { diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 0095ac8b..d2543423 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -32,7 +32,8 @@ Auth: `Authorization: Bearer ` (when configured). "require_promoted": false} ``` -- `net` defaults to `none`, `size` to `small` +- `net` defaults to `none`, `size` to `small`; the pool key is + `(template, net, size)` - `ttl_seconds` 0 means the server default (5 minutes); capped at 24h. The owning node reaps the sandbox after the TTL even if the client vanishes - `claim_ref` is an optional opaque caller reference echoed by the scoped @@ -45,8 +46,7 @@ Auth: `Authorization: Bearer ` (when configured). defaults to `/volumes/`; a custom value must be absolute and clean, outside the guest OS tree, unique, and non-nesting within the request. `mode` is `"ro"` (default, omitted) or `"rw"`; `"rw"` requires the catalog - entry's `writable: true` (see [deploy](deploy.md#dataset-volumes)). Volumes - require Cloud Hypervisor + entry's `writable: true` (see [deploy](deploy.md#dataset-volumes)) - `volumes_attach_only` (default `false`) attaches every requested volume without mounting it, handing the whole mount contract to the workload. It requires at least one volume, and rejects any entry carrying a `mount` — @@ -99,9 +99,21 @@ can lag the serial match the same way it does for an eager mount's device settle. Confirm both before mounting: ```sh -for dev in /sys/block/*/serial; do - [ "$(cat "$dev")" = scratch-db ] && echo "/dev/$(basename "$(dirname "$dev")")" +device= +tries=0 +while [ "$tries" -lt 200 ]; do + for serial in /sys/block/*/serial /sys/block/*/device/serial; do + [ -r "$serial" ] || continue + [ "$(cat "$serial")" = scratch-db ] || continue + block=${serial#/sys/block/} + candidate="/dev/${block%%/*}" + [ -b "$candidate" ] && { device="$candidate"; break 2; } + done + tries=$((tries + 1)) + sleep 0.01 done +[ -n "$device" ] || { echo "scratch-db device not ready" >&2; exit 1; } +printf '%s\n' "$device" ``` Then mount it however the workload needs. A `ro` entry is attached @@ -160,7 +172,7 @@ Errors: 400 unknown template axis, invalid/duplicate volumes, `volumes_attach_only` with no volumes or with an entry carrying a `mount`, `mode: "rw"` against a non-writable entry, or a volume that is unknown or forbidden (the -latter two are deliberately indistinguishable), Firecracker with volumes, or +latter two are deliberately indistinguishable), or bad body; 401 bad api token; 409 egress requested on a node without an egress attachment, a writable name already claimed in a conflicting mode (volume busy — a live writer excludes every other claim for that name, live readers @@ -181,11 +193,11 @@ caller may use, without host paths or holder addresses: "size_bytes": 214748364800, "available": true, "nodes": 3}]} ``` -Root sees every entry; a tenant sees unrestricted entries plus those whose -access list names it. The response is the gossiped union: `nodes` counts members -advertising the name. `size_bytes` and `available` are a best-effort stat of the -answering node's image, so a peer-only entry remains discoverable with -`available: false`. Membership is eventually consistent by one gossip tick. +Root sees the gossiped union, including peer-only entries (`available: false`); +a tenant sees only entries this node declares locally and whose access list +permits it. `nodes` counts members advertising the name; `size_bytes` and +`available` are a best-effort stat of the answering node's image. Membership is +eventually consistent by one gossip tick. `writable` is the entry's catalog configuration, fleet-uniform like the access list; the field is emitted (as `true`) only for a writable entry and omitted otherwise, so a read-only entry's response is byte-identical to v1. @@ -333,9 +345,9 @@ guest HTTP port from a browser: body `{"token": "...", "port": 8080, "ttl_seconds": 0}` → `{"url": "http:///p//"}`. The URL's life is clamped to the claim's remaining lease. 501 when the node has no `preview_listen`. The signed token embeds the sandbox id, port, and -owner node, so any node's preview listener can serve it (forwarding to the -owner) and a released sandbox's URL simply stops resolving — no revocation -list. See [deploy](deploy.md#preview-urls). +owner `advertise_addr`, so any node's preview listener can serve it (forwarding +to the owner's main listener) and a released sandbox's URL simply stops +resolving — no revocation list. See [deploy](deploy.md#preview-urls). ## POST /v1/sandboxes/{id}/checkpoint @@ -391,7 +403,8 @@ part of the public API; an SDK caller has no reason to call it directly. 401 missing or unrecognized token, 403 a valid tenant token (authenticated but not the operator), 404 unknown checkpoint. - `HEAD` is the ownership probe: 200 when this node holds a branchable - (non-archive) copy, 404 otherwise. On a mesh with `cluster_key` set the + (non-archive) copy, 404 otherwise, and 401 on a keyed mesh when + `X-Cocoon-Probe` is absent or expired. On a mesh with `cluster_key` set the request must carry `X-Cocoon-Probe`, an HMAC over the id and a coarse time bucket keyed off a probe-specific derivation of the cluster key — verified before any disk is touched, replayable for roughly a minute at most. On a @@ -411,22 +424,12 @@ Auth: node API token. A tenant may delete only its own records — anything else is 404, never a hint the id exists; root deletes anything. 204 on success, 404 unknown. -**Delete removes the local record and then best-effort broadcasts to peers -so a healed replica does not outlive it — this is eventual best-effort -cleanup, not a fleet-wide revocation.** A peer that is offline or -partitioned during the broadcast keeps its copy until the checkpoint TTL -ages it out. A healed replica carries the source's original `CreatedAt`, so -it becomes eligible for expiry at the same instant on every node; the actual -removal is each node's own hourly sweep, which is independently phased and -retries on a later sweep if one fails. So a deleted checkpoint normally stops -being branchable within `checkpoint_ttl_hours` plus a sweep interval, but a -node whose sweeps keep failing holds its replica until one succeeds — the TTL -is the eligibility point, not a hard ceiling. The TTL must also match -fleet-wide, which the -[cluster-invariant config](cluster.md#cluster-invariant-config) digest -checks. A window always exists because `checkpoint_peer_heal` cannot be -enabled with `checkpoint_ttl_hours: 0` — a replica that can outlive a delete -must have a finite eligibility point. A shared +**Delete removes the local record, then best-effort broadcasts to peers so a +healed replica does not outlive it — eventual cleanup, not a fleet-wide +revocation.** A peer offline during the broadcast keeps its copy until the +checkpoint TTL ages it out, so an id-holder can still branch it for that +window; [placement lifecycle](cluster.md#delete-is-eventual-not-a-fleet-wide-revocation) +has the bound and why heal requires a nonzero, fleet-matching TTL. A shared checkpoint store skips the broadcast: every node already resolves every record directly, so there is no replica to chase. `?no_forward=1` marks a delete already arriving from another node's own broadcast, so it is not @@ -467,20 +470,22 @@ never mistaken for idle. 404 unknown id. ## GET /metrics Auth: root only (tenant tokens get 403). Prometheus text format, -hand-rendered: pool warm/target gauges, claimed/hibernated gauges, a -per-tenant live-claim gauge (`sandboxd_tenant_claims{tenant="…"}`, -configured tenants only), claims by tier (warm/clone/cold), -wake/hibernate/fork/checkpoint/promote/release/reap counters, and claim/wake -`*_seconds_total` for average latency. /metrics is a derived ops view; the -billing source of truth is the usage journal below. +hand-rendered: pool warm/target gauges, claimed/hibernated/archived/draining +gauges, a per-tenant live-claim gauge (`sandboxd_tenant_claims{tenant="…"}`, +configured tenants only), `sandboxd_config_digest_mismatch` on a mesh, claims +by tier (warm/clone/cold), +wake/hibernate/fork/checkpoint/promote/release/reap counters plus +archive/unarchive/archive-delete counters, and claim/wake `*_seconds_total` +for average latency. /metrics is a derived ops view; the billing source of +truth is the usage journal below. ## Usage journal (usage.jsonl) Always on: every lifecycle transition appends one JSONL event to `/usage.jsonl` — `{"t": , "ev": "claim|hibernate|wake|fork|checkpoint|promote|release|reap|archive|unarchive|archive_delete|egress", -"id": "sb_…", "vm": "sbx-…"}` plus `key` and `tenant` (the pool key and -owning tenant, claim events), `children` (fork) and `ref` (the promoted +"id": "sb_…", "vm": "sbx-…"}` plus `key` and `tenant` (the pool key's stable +hash and the owning tenant, claim events), `children` (fork) and `ref` (the promoted template / checkpoint id, or the egress host). A volume claim also carries `volumes`, the applied catalog names, and — omitted when empty — `volumes_rw`, the subset of those names claimed `rw`, so billing can discriminate write diff --git a/docs/sdk-python.md b/docs/sdk-python.md index f6864e86..2f7df737 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -278,8 +278,8 @@ code = sb.run(["bash", "-c", "make test"], on_stderr=lambda b: sys.stderr.buffer.write(b)) ``` -`exec` returns stdout and raises `ExitError(code, stderr)` on a non-zero -exit. `run` streams raw bytes through the callbacks (chunk boundaries may +`exec` returns stdout and raises `ExitError` on a non-zero exit — carrying +`code`, `stderr`, and the `stdout` produced before it failed. `run` streams raw bytes through the callbacks (chunk boundaries may split multi-byte sequences) and returns the exit code. `user` de-escalates inside the guest; `session=` routes the command into a persistent session. @@ -362,7 +362,9 @@ w.close() `watch` returns once the guest acknowledges the watch is armed — events caused after it returns are guaranteed captured. A bad path fails -synchronously. +synchronously; if the consumer falls too far behind, iteration raises the +terminal overflow instead of silently dropping events. Iteration also ends +when the relay drops, which `w.error` tells apart from a clean close (`None`). ## Git @@ -390,10 +392,25 @@ pointing at `push`. pty = sb.open_pty(cols=120, rows=40) # context-manager; pty.pid is the guest process pty.write(b"make test\n") data = pty.read() # b"" when the shell exits +pty.exit_code # the shell's status, once read() returned b"" pty.resize(200, 50) pty.close() ``` +## Node operations + +```python +sb = client.new("rt:24.04", claim_ref="ns/workload") +client.sandboxes() # id, key, deadline, claim_ref — never tokens +client.drain() # cordon: refuse new claims, run leases out +client.uncordon() +client.attach(owner_addr, id, token) # bind a known handle, no lookup round-trip +``` + +`sandboxes()` is scoped to the calling token, so a tenant sees only its own +claims. `drain()` leaves live claims alone — poll `info()` until `claimed` is +zero. + ## Errors `SandboxError` is the base; catch the narrowest type you handle: @@ -401,7 +418,7 @@ pty.close() - `APIError(verb, status, message)` — control plane (HTTP status) - `SilkdError(kind, message)` — typed guest failure; `kind` is `bad_request` / `not_found` / `unimplemented` / `internal` -- `ExitError(code, stderr)` — non-zero exit from `exec` +- `ExitError(code, stderr, stdout)` — non-zero exit from `exec` - `ProtocolError` — broken stream ```python diff --git a/docs/sdk.md b/docs/sdk.md index ba49dad4..0e95b15c 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -443,7 +443,8 @@ err = w.Err() // why the stream ended; nil after Close `Watch` returns once the guest acknowledges the watch is armed — events caused after it returns are guaranteed captured. A bad path fails -synchronously. +synchronously; if the consumer falls too far behind, `Err` reports a terminal +overflow instead of the stream silently dropping events. ## Git @@ -479,6 +480,24 @@ err = pty.Resize(ctx, 200, 50) A PTY is a tracked guest process (`pty.PID`); closing the handle (or the ctx) tears the shell down. +## Node operations + +Root-token verbs for operating a node, plus the reference the aggregated +apiserver claims under: + +```go +sb, _ := client.New(ctx, "rt:24.04", sandbox.WithClaimRef("ns/workload")) +list, _ := client.Sandboxes(ctx) // id, key, deadline, claim_ref — never tokens +info, _ := client.Drain(ctx) // cordon: refuse new claims, run leases out +info, _ = client.Uncordon(ctx) +info, _ = client.SetPools(ctx, pools) // retune warm targets without a restart +res, _ := client.SetPoolsCluster(ctx, pools) // per-node results; retry the failures +sb = client.Attach(ownerAddr, id, token) // bind a known handle, no lookup round-trip +``` + +`Sandboxes` is scoped to the calling token, so a tenant sees only its own +claims. `Drain` leaves live claims alone — poll `Info` until `Claimed` is zero. + ## Error handling - `*sandbox.ExitError` — non-zero exit from `Exec` (`Code`, `Stderr`) diff --git a/docs/security.md b/docs/security.md index 124c6a3e..2b6576bb 100644 --- a/docs/security.md +++ b/docs/security.md @@ -25,8 +25,8 @@ exposing any part of a deployment beyond a single trusted host. toward the host, siblings, or the network beyond its lanes. - **What a compromised guest can reach.** On the none lane: nothing but vsock — the relay back to its own client and the guarded-egress proxy. - On the egress lane: the same vsock paths plus a NIC whose every packet - except IPv4 broadcast DHCP is dropped by an nftables lock in the host + On the egress lane: the same vsock paths plus a NIC whose every + guest-initiated packet except IPv4 broadcast DHCP is dropped by an nftables lock in the host root netns ([egress](egress.md)); the lock is fail-closed and applied before the claim is handed out. On both lanes the proxy refuses loopback, private, link-local (cloud metadata), CGN, and the diff --git a/docs/silkd.md b/docs/silkd.md index 51b17057..befa4be9 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -19,8 +19,8 @@ ride base64 in `data` fields. Frames are capped at 8 MiB; requests carry `"v": 1` and unknown fields are ignored (the forward-compatibility story). Sessions and processes are server-side state addressed by id — a dropped -connection loses nothing (`attach` resumes). The only connection-bound verb -is `fs_watch`. +connection loses nothing (`attach` resumes). The connection-bound verbs are +`fs_watch`, `pty_open`, `lsp_request`, and `port_forward`. The authoritative wire contract is the shared fixture corpus in `protocol/wire/fixtures/v1`, round-tripped by both the Rust and Go test suites — @@ -36,7 +36,7 @@ a frame only one side can parse fails CI. | fs | `fs_write {path, mode?}` (+`data`/`data_end` frames) / `fs_read` / `fs_list` / `fs_stat` / `fs_mkdir {parents?}` / `fs_rm {recursive?}` / `fs_rename {from, to}` | streaming both directions; write commits atomically via temp+rename and inherits an overwritten file's mode; `fs_list` streams 4096-entry batches | | tree | `fs_push {dest}` (+tar as `data` frames) / `fs_pull {path}` | whole trees as tar streams through the guest tar | | search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files | -| watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher errors arrive as a terminal `error` | +| watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | | pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | a shell under a pseudo-terminal; output as `stdout` frames, input as `stdin` frames, exit terminal. PTYs register in the proc table like any exec | | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | structured results (porcelain-v2 status, commit hash, branch list). `auth` is injected as an in-memory header, never written to guest disk | | port | `port_forward {port}` | relays guest TCP 127.0.0.1:port over this connection: `ready` once connected, then `data` both ways (`data_end` half-closes the guest socket); the server closing ends the stream with `done`. Works on both lanes — the no-network lane's only way in | diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index be7bb876..3a96f6ba 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -27,7 +27,7 @@ import ( sandbox "github.com/cocoonstack/sandbox/sdk/go" ) -var testKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} +var testKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} func TestEndToEnd(t *testing.T) { stack := startStack(t, "node-token", config.PoolSpec{PoolKey: testKey, Warm: 1}) @@ -427,6 +427,27 @@ func TestVolumeModeWireShape(t *testing.T) { } } +func TestClaimRefRoundTrip(t *testing.T) { + stack := startStack(t, "node-token") + sb, err := stack.client.New(t.Context(), "rt:24.04", sandbox.WithClaimRef("ns/workload")) + if err != nil { + t.Fatalf("claim: %v", err) + } + defer sb.Close() + + list, err := stack.client.Sandboxes(t.Context()) + if err != nil { + t.Fatalf("list sandboxes: %v", err) + } + i := slices.IndexFunc(list, func(s sandbox.SandboxSummary) bool { return s.ID == sb.ID }) + if i < 0 { + t.Fatalf("claim %s missing from the index %+v", sb.ID, list) + } + if list[i].ClaimRef != "ns/workload" { + t.Errorf("claim_ref %q, want ns/workload", list[i].ClaimRef) + } +} + // TestAttachOnlyVolumeEndToEnd drives one attach-only writable claim through // the whole stack: the device is attached writable and nothing else happens — // no mount, no marker, no unmount at release — while admission still excludes diff --git a/mcp/server.go b/mcp/server.go index be41b38f..23647e6e 100644 --- a/mcp/server.go +++ b/mcp/server.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "io" + "maps" + "slices" "sync" "time" @@ -15,6 +17,7 @@ import ( const ( protocolVersion = "2024-11-05" execTimeout = 5 * time.Minute + defaultToolTTL = time.Hour ) // server owns one sandboxd client and the handles minted over this stdio @@ -50,6 +53,7 @@ func newServer(addr, token, template string) (*server, error) { // stdio transport. Requests are handled strictly in order: sandbox tools are // stateful, and an agent's tool calls arrive sequentially anyway. func (s *server) serve(ctx context.Context, r *bufio.Reader, w io.Writer) error { + defer s.closeBoxes() for { line, err := r.ReadBytes('\n') if len(line) == 0 && err != nil { @@ -132,6 +136,18 @@ func (s *server) box(id string) (*sandbox.Sandbox, error) { return sb, nil } +// closeBoxes releases everything this session claimed; the lease would +// otherwise hold the VMs until it expires. +func (s *server) closeBoxes() { + s.mu.Lock() + boxes := slices.Collect(maps.Values(s.boxes)) + clear(s.boxes) + s.mu.Unlock() + for _, sb := range boxes { + _ = sb.Close() + } +} + func (s *server) trackBox(sb *sandbox.Sandbox) { s.mu.Lock() defer s.mu.Unlock() diff --git a/mcp/tools.go b/mcp/tools.go index ff151a13..1ea838a0 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -14,7 +14,7 @@ import ( var tools = []tool{ { "create_sandbox", "Claim a fresh microVM sandbox; returns its id. Warm claims are milliseconds.", - schema(props{"template": str("template image ref; empty uses the server default"), "ttl_seconds": integer("sandbox lifetime; 0 = server default")}), toolCreateSandbox, + schema(props{"template": str("template image ref; empty uses the server default"), "net": str("network lane: none (default) or egress"), "size": str("resource tier: small (default), medium, large, xlarge"), "ttl_seconds": integer("sandbox lifetime in seconds; 0 means one hour, and nothing renews it")}), toolCreateSandbox, }, { "exec", "Run a command in a sandbox and return stdout/stderr/exit code. A hibernated sandbox wakes transparently.", @@ -49,7 +49,7 @@ var tools = []tool{ schema(props{"sandbox_id": str(""), "path": str("absolute path")}, "sandbox_id", "path"), toolListDir, }, { - "fork", "Clone a sandbox into N independent children carrying its exact memory and disk state.", + "fork", "Clone a sandbox into N independent children carrying its exact memory and disk state; children live one hour.", schema(props{"sandbox_id": str(""), "count": integer("children, 1-16")}, "sandbox_id", "count"), toolFork, }, { @@ -57,7 +57,7 @@ var tools = []tool{ schema(props{"sandbox_id": str(""), "name": str("optional label")}, "sandbox_id"), toolCheckpoint, }, { - "branch_checkpoint", "Claim a fresh sandbox branched from a checkpoint's exact captured moment.", + "branch_checkpoint", "Claim a fresh sandbox branched from a checkpoint's exact captured moment; it lives one hour.", schema(props{"checkpoint_id": str("")}, "checkpoint_id"), toolBranchCheckpoint, }, {"list_checkpoints", "List checkpoints on the node, newest first.", schema(props{}), toolListCheckpoints}, @@ -94,14 +94,22 @@ func toolSpecs() []map[string]any { func toolCreateSandbox(ctx context.Context, s *server, raw json.RawMessage) (string, error) { var args struct { Template string `json:"template"` + Net string `json:"net"` + Size string `json:"size"` TTLSeconds int `json:"ttl_seconds"` } if err := parse(raw, &args); err != nil { return "", err } - var opts []sandbox.Option - if args.TTLSeconds > 0 { - opts = append(opts, sandbox.WithTimeout(time.Duration(args.TTLSeconds)*time.Second)) + // An agent session outlives the node's 5-minute default and nothing renews + // a lease, so the sandbox would vanish mid-conversation. + ttl := cmp.Or(time.Duration(args.TTLSeconds)*time.Second, defaultToolTTL) + opts := []sandbox.Option{sandbox.WithTimeout(ttl)} + if args.Net != "" { + opts = append(opts, sandbox.WithNetwork(sandbox.NetShape(args.Net))) + } + if args.Size != "" { + opts = append(opts, sandbox.WithSize(sandbox.Size(args.Size))) } sb, err := s.client.New(ctx, cmp.Or(args.Template, s.template), opts...) if err != nil { @@ -271,7 +279,7 @@ func toolFork(ctx context.Context, s *server, raw json.RawMessage) (string, erro if err != nil { return "", err } - children, err := sb.Fork(ctx, args.Count, 0) + children, err := sb.Fork(ctx, args.Count, defaultToolTTL) if err != nil { return "", err } @@ -302,11 +310,11 @@ func toolCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string } func toolBranchCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string, error) { - ckpt, err := s.checkpointArg(ctx, raw) + ckpt, err := s.checkpointArg(raw) if err != nil { return "", err } - sb, err := ckpt.New(ctx) + sb, err := ckpt.New(ctx, sandbox.WithTimeout(defaultToolTTL)) if err != nil { return "", err } @@ -327,7 +335,7 @@ func toolListCheckpoints(ctx context.Context, s *server, _ json.RawMessage) (str } func toolDeleteCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string, error) { - ckpt, err := s.checkpointArg(ctx, raw) + ckpt, err := s.checkpointArg(raw) if err != nil { return "", err } @@ -392,9 +400,8 @@ func (s *server) boxArg(raw json.RawMessage) (*sandbox.Sandbox, error) { } // checkpointArg resolves a checkpoint_id argument: a handle minted in this -// session when available, else a listing lookup (checkpoints outlive -// sessions). -func (s *server) checkpointArg(ctx context.Context, raw json.RawMessage) (*sandbox.Checkpoint, error) { +// session when available, else a fresh one — checkpoints outlive sessions. +func (s *server) checkpointArg(raw json.RawMessage) (*sandbox.Checkpoint, error) { var args struct { CheckpointID string `json:"checkpoint_id"` } @@ -404,16 +411,10 @@ func (s *server) checkpointArg(ctx context.Context, raw json.RawMessage) (*sandb if ckpt, ok := s.ckpt(args.CheckpointID); ok { return ckpt, nil } - ckpts, err := s.client.Checkpoints(ctx) - if err != nil { - return nil, err - } - for _, ck := range ckpts { - if ck.ID == args.CheckpointID { - return ck, nil - } + if args.CheckpointID == "" { + return nil, fmt.Errorf("checkpoint_id is required") } - return nil, fmt.Errorf("unknown checkpoint %q", args.CheckpointID) + return s.client.Checkpoint(args.CheckpointID), nil } func parse(raw json.RawMessage, v any) error { diff --git a/os-image/android/README.md b/os-image/android/README.md index c0909150..48af3d14 100644 --- a/os-image/android/README.md +++ b/os-image/android/README.md @@ -27,9 +27,9 @@ The base image (15.0) pulls anonymously from ghcr at docker and `sys.boot_completed=1` with zygote64 + system_server alive (Cloud Hypervisor, no NIC, 4 CPU / 8G). -Do not revert to the previous `ghcr.io/jiaqing-simular/cocoon-android-vnc` -pin: that build ships a broken dexpreopt boot-image chain that -crash-loops zygote before the framework ever completes. +Do not revert to the previous third-party VNC android pin: that build ships a +broken dexpreopt boot-image chain that crash-loops zygote before the framework +ever completes. ## Constraints diff --git a/os-image/base/24.04/Dockerfile b/os-image/base/24.04/Dockerfile index a4aad561..4d669a3f 100644 --- a/os-image/base/24.04/Dockerfile +++ b/os-image/base/24.04/Dockerfile @@ -10,7 +10,7 @@ ARG SILKD_IMAGE=ghcr.io/cocoonstack/sandbox/silkd:0.1.0 FROM ${BOOT_IMAGE} AS boot FROM ${SILKD_IMAGE} AS silkd -FROM ubuntu:24.04 +FROM ubuntu:24.04@sha256:561618e2c15bf2397621dd04f96926663a3b5616c189cf7e38db7e82f5c538ea ARG TARGETARCH ENV DEBIAN_FRONTEND=noninteractive # Content hash of the image dir's non-Dockerfile inputs (CI computes it). diff --git a/os-image/browser/README.md b/os-image/browser/README.md index fbf1b033..244d7551 100644 --- a/os-image/browser/README.md +++ b/os-image/browser/README.md @@ -14,7 +14,8 @@ pool shape is `size: large` (4 CPU / 4G) — a persistent Chromium idles at download Chrome for Testing pinned by version + SHA256 (the `install-agent.sh` idiom); bake `/usr/local/bin/chromium-cdp` and `chromium.service` (enabled, `multi-user.target`). -- `platforms` — `linux/amd64`; the base lineage is amd64-only. +- `platforms` — `linux/amd64`; this flavor is amd64-only (Chrome for Testing), + unlike the multi-arch `base`. Why Chrome for Testing: Ubuntu 24.04's `chromium` apt package is a transitional stub that pulls the snap, and snap cannot install inside a diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 16c4367e..ec4973c7 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -49,7 +49,7 @@ type PoolSpec struct { // name a secret, the pool's is injected. Nil denies all egress. Egress *egress.Policy `json:"egress,omitempty"` - // IdleHibernateSeconds, when >0, hibernates this pool's idle claims + // IdleHibernateSeconds, when >0, hibernates this none-lane pool's idle claims // after that many seconds without a data-plane connection; the next // call wakes them transparently. Zero disables. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` @@ -76,6 +76,9 @@ func (s PoolSpec) ValidateLimits() error { if s.IdleHibernateSeconds < 0 { return fmt.Errorf("idle_hibernate_seconds must not be negative") } + if s.Net == types.NetEgress && s.IdleHibernateSeconds > 0 { + return fmt.Errorf("idle_hibernate_seconds is not supported for egress pools") + } return validateArchiveWindow(s.IdleHibernateSeconds, s.ArchiveAfterSeconds, s.ArchiveDeleteAfterSeconds) } @@ -210,7 +213,7 @@ type Config struct { // name; values come from the environment (value_env), never this file. Secrets []egress.SecretSpec `json:"secrets,omitempty"` - // IdleHibernateSeconds is the idle policy for claims of unpooled keys + // IdleHibernateSeconds is the idle policy for unpooled none-lane claims // (template and checkpoint claims); per-pool settings override it for // pooled keys. Zero disables. IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitempty"` @@ -221,12 +224,9 @@ type Config struct { ArchiveAfterSeconds int `json:"archive_after_seconds,omitempty"` ArchiveDeleteAfterSeconds int `json:"archive_delete_after_seconds,omitempty"` - // PreviewListen, when set, starts a preview HTTP server on that address - // serving guest ports under signed URLs. PreviewSecret (cluster-shared) - // signs the tokens; PreviewAdvertise is the base a browser/proxy reaches - // this node's preview server at, defaulting to PreviewListen. - PreviewListen string `json:"preview_listen,omitempty"` - PreviewSecret string `json:"preview_secret,omitempty"` //nolint:gosec // config field, not a hardcoded credential + PreviewListen string `json:"preview_listen,omitempty"` + PreviewSecret string `json:"preview_secret,omitempty"` //nolint:gosec // config field, not a hardcoded credential + // PreviewAdvertise is the browser-facing base, shareable fleet-wide behind one proxy. PreviewAdvertise string `json:"preview_advertise,omitempty"` // CheckpointDir is where checkpoints live; defaults to diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 63242397..ac37e344 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -96,6 +96,7 @@ func TestLoadRejectsInvalid(t *testing.T) { {"bad restore mode", `{"restore_mode":"Mmap","pools":[]}`, "restore_mode"}, {"bad pool key", `{"pools":[{"template":"","net":"none","size":"small"}]}`, "pool"}, {"egress without attachment", `{"pools":[{"template":"rt:24.04","net":"egress","size":"small"}]}`, "egress lane needs"}, + {"egress idle hibernate", `{"bridges":["br0"],"pools":[{"template":"rt:24.04","net":"egress","size":"small","idle_hibernate_seconds":1}]}`, "not supported for egress"}, {"negative warm", `{"pools":[{"template":"rt:24.04","net":"none","size":"small","warm":-2}]}`, "negative"}, {"tenants without api_token", `{"pools":[],"tenants":[{"name":"acme","token":"t1"}]}`, "require api_token"}, {"empty tenant name", `{"api_token":"root","pools":[],"tenants":[{"name":"","token":"t1"}]}`, "tenant name"}, diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 43bfccca..06c3e9b8 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -358,11 +358,6 @@ func (e *Engine) restoreArgs() []string { func (e *Engine) runColdArgs(name string, key types.PoolKey) []string { spec, _ := key.Size.Spec() args := []string{"vm", "run", argName, name, argOutput, formatJSON, "--cpu", strconv.Itoa(spec.CPU), "--memory", spec.Memory, e.directIOArg()} - if key.Engine == types.EngineFC { - // Firecracker is a per-pool cold-boot choice; clones inherit the - // hypervisor from the golden's pinned snapshot, so only RunCold flags it. - args = append(args, "--fc") - } args = append(args, e.netArgs(name, key, true)...) return append(args, key.Template) } diff --git a/sandboxd/main.go b/sandboxd/main.go index 904b3fbb..14a58c92 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -132,7 +132,7 @@ func main() { var preview *server.PreviewServer if cfg.PreviewListen != "" { - preview = server.NewPreviewServer(cfg.PreviewSecret, cfg.PreviewAdvertise, mgr) + preview = server.NewPreviewServer(cfg.PreviewSecret, cfg.PreviewAdvertise, cfg.AdvertiseAddr, mgr) } srv := server.New(cfg.APIToken, cfg.Tenants, cfg.AdvertiseAddr, mgr, eng, placer, prober, probeKey, preview) httpSrv := &http.Server{ diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 8536aede..a8aab83f 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -51,6 +51,7 @@ type Mesh struct { mu sync.Mutex self NodeState view map[string]NodeState // node_id → latest known state (includes self) + live map[string]struct{} // members SWIM reports; gossip about anyone else is ignored } // New starts a mesh member listening per cfg. selfAddr is the data-plane @@ -72,6 +73,7 @@ func New(ctx context.Context, cfg *memberlist.Config, nodeID, selfAddr string, s Pools: map[string]int{}, }, view: map[string]NodeState{}, + live: map[string]struct{}{}, } if err := m.persistEpoch(epoch); err != nil { return nil, fmt.Errorf("persist mesh epoch: %w", err) @@ -286,12 +288,19 @@ func (m *Mesh) owners(match func(NodeState) bool) []string { return owners } +func (m *Mesh) admit(nodeID string) { + m.mu.Lock() + defer m.mu.Unlock() + m.live[nodeID] = struct{}{} +} + // forget drops a departed node from the placement view so redirects stop // targeting a dead peer; SWIM detected the death, the view must follow. func (m *Mesh) forget(nodeID string) { m.mu.Lock() defer m.mu.Unlock() if nodeID != m.self.NodeID { + delete(m.live, nodeID) delete(m.view, nodeID) } } @@ -305,6 +314,9 @@ func (m *Mesh) merge(states []NodeState) { if st.NodeID == m.self.NodeID { continue } + if _, member := m.live[st.NodeID]; !member { + continue + } cur, ok := m.view[st.NodeID] if ok && st.Epoch <= cur.Epoch { continue @@ -359,11 +371,10 @@ func (d *delegate) MergeRemoteState(buf []byte, _ bool) { var _ memberlist.EventDelegate = (*eventDelegate)(nil) -// eventDelegate prunes the placement view when SWIM reports a node gone, so a -// dead peer stops attracting redirects. +// eventDelegate tracks SWIM membership: admit on join, prune the view on leave. type eventDelegate Mesh -func (e *eventDelegate) NotifyJoin(*memberlist.Node) {} +func (e *eventDelegate) NotifyJoin(n *memberlist.Node) { (*Mesh)(e).admit(n.Name) } func (e *eventDelegate) NotifyUpdate(*memberlist.Node) {} func (e *eventDelegate) NotifyLeave(n *memberlist.Node) { (*Mesh)(e).forget(n.Name) diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index 20e4968a..b00aae50 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -15,9 +15,9 @@ import ( func TestMergeKeepsHigherEpoch(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}}) - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 3, Pools: map[string]int{"k": 5}}}) - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 9}}}) // stale + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}}) + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 3, Pools: map[string]int{"k": 5}}}) + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 2, Pools: map[string]int{"k": 9}}}) // stale got := 0 for _, st := range m.Members() { @@ -34,7 +34,7 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), map[string]int{"k": 3}, nil, nil) // A peer claiming to be "a" must not clobber our authoritative self entry. - m.merge([]NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) + mergeStates(t, m, []NodeState{{NodeID: "a", Addr: "evil:9999", Epoch: 999, Pools: map[string]int{"k": 0}}}) for _, st := range m.Members() { if st.NodeID == "a" && (st.Addr != "a:7777" || st.Pools["k"] != 3) { @@ -46,7 +46,7 @@ func TestMergeNeverOverwritesSelf(t *testing.T) { func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), map[string]int{"k": 5}, nil, nil) // self has warm, but is never a candidate - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 2}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 0}}, // no warm {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"other": 4}}, @@ -64,7 +64,7 @@ func TestCandidatesExcludeSelfAndEmpty(t *testing.T) { func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), nil, []string{"tpl"}, nil) // self holds it, but is never an owner candidate - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Templates: []string{"tpl", "other"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Templates: []string{"other"}}, }) @@ -79,7 +79,7 @@ func TestTemplateOwnersExcludeSelfAndUnknown(t *testing.T) { func TestForgetPrunesDeadNode(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 3}}}) + mergeStates(t, m, []NodeState{{NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 3}}}) if len(m.Candidates("k")) != 1 { t.Fatal("setup: b should be a candidate") } @@ -95,9 +95,33 @@ func TestForgetPrunesDeadNode(t *testing.T) { } } +func TestForgottenNodeStaysGoneUntilItRejoins(t *testing.T) { + m := newTestMesh(t, "a") + dead := NodeState{NodeID: "b", Addr: "b:7777", Epoch: 4, Pools: map[string]int{"k": 3}} + mergeStates(t, m, []NodeState{dead}) + m.forget("b") + + m.merge([]NodeState{dead}) + if got := m.Candidates("k"); got != nil { + t.Errorf("a lagging peer resurrected b: %v", got) + } + restarted := dead + restarted.Epoch = 5 + m.merge([]NodeState{restarted}) + if got := m.Candidates("k"); got != nil { + t.Errorf("a higher epoch alone resurrected b: %v", got) + } + + m.admit("b") + m.merge([]NodeState{restarted}) + if len(m.Candidates("k")) != 1 { + t.Error("b rejoined the cluster and must be reinstated") + } +} + func TestCandidatesPowerOfTwo(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, {NodeID: "d", Addr: "d:7777", Epoch: 1, Pools: map[string]int{"k": 1}}, @@ -144,7 +168,7 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { func TestVolumeOwnersRequireEveryNameAndExcludeSelf(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), nil, nil, []string{"dataset", "weights"}) - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Volumes: []string{"dataset", "weights"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Volumes: []string{"dataset"}}, {NodeID: "d", Addr: "d:7777", Epoch: 1, Volumes: []string{"weights"}}, @@ -163,7 +187,7 @@ func TestVolumeOwnersRequireEveryNameAndExcludeSelf(t *testing.T) { func TestVolumeCandidatesRequireWarmAndEveryVolume(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "both", Addr: "both:7777", Epoch: 1, Pools: map[string]int{"k": 2}, Volumes: []string{"dataset", "weights"}}, {NodeID: "partial", Addr: "partial:7777", Epoch: 1, Pools: map[string]int{"k": 3}, Volumes: []string{"dataset"}}, {NodeID: "cold", Addr: "cold:7777", Epoch: 1, Pools: map[string]int{"k": 0}, Volumes: []string{"dataset", "weights"}}, @@ -179,7 +203,7 @@ func TestVolumeCandidatesRequireWarmAndEveryVolume(t *testing.T) { func TestTemplateVolumeOwnersUseTrueIntersection(t *testing.T) { m := newTestMesh(t, "a") - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "template", Addr: "template:7777", Epoch: 1, Templates: []string{"tpl"}}, {NodeID: "volume", Addr: "volume:7777", Epoch: 1, Volumes: []string{"dataset"}}, {NodeID: "both", Addr: "both:7777", Epoch: 1, Templates: []string{"tpl"}, Volumes: []string{"dataset"}}, @@ -196,7 +220,7 @@ func TestTemplateVolumeOwnersUseTrueIntersection(t *testing.T) { func TestVolumeHoldersCountSelfAndPeers(t *testing.T) { m := newTestMesh(t, "a") m.UpdateSelf(t.Context(), nil, nil, []string{"dataset"}) - m.merge([]NodeState{ + mergeStates(t, m, []NodeState{ {NodeID: "b", Addr: "b:7777", Epoch: 1, Volumes: []string{"dataset", "weights"}}, {NodeID: "c", Addr: "c:7777", Epoch: 1, Volumes: []string{"weights"}}, }) @@ -207,12 +231,21 @@ func TestVolumeHoldersCountSelfAndPeers(t *testing.T) { } } +func mergeStates(t *testing.T, m *Mesh, states []NodeState) { + t.Helper() + for _, st := range states { + m.admit(st.NodeID) + } + m.merge(states) +} + func newTestMesh(t *testing.T, id string) *Mesh { t.Helper() return &Mesh{ epochPath: filepath.Join(t.TempDir(), "mesh-epoch"), self: NodeState{NodeID: id, Addr: id + ":7777", Pools: map[string]int{}}, view: map[string]NodeState{id: {NodeID: id, Addr: id + ":7777"}}, + live: map[string]struct{}{}, } } diff --git a/sandboxd/mesh/state_test.go b/sandboxd/mesh/state_test.go index 3813b3e4..43c2b281 100644 --- a/sandboxd/mesh/state_test.go +++ b/sandboxd/mesh/state_test.go @@ -110,8 +110,8 @@ func TestUpdateSelfBumpsOnlyWhenVolumesChange(t *testing.T) { func TestConfigDigestMismatch(t *testing.T) { m := newTestMesh(t, "self") m.SetSelfDigest("self-digest") - m.merge([]NodeState{{NodeID: "peerA", Addr: "a:1", Epoch: 1, Digest: "self-digest"}}) - m.merge([]NodeState{{NodeID: "peerB", Addr: "b:1", Epoch: 1, Digest: "other-digest"}}) + mergeStates(t, m, []NodeState{{NodeID: "peerA", Addr: "a:1", Epoch: 1, Digest: "self-digest"}}) + mergeStates(t, m, []NodeState{{NodeID: "peerB", Addr: "b:1", Epoch: 1, Digest: "other-digest"}}) if n := m.ConfigMismatches(); n != 1 { t.Errorf("ConfigMismatches = %d, want 1 (only peerB diverges)", n) } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index bb9f9195..cd42f0cc 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -30,7 +30,7 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Dur if err := m.validate(key); err != nil { return nil, err } - volumeSpecs, err := m.resolveVolumes(ctx, key, tenant, volumes) + volumeSpecs, err := m.resolveVolumes(ctx, tenant, volumes) if err != nil { return nil, err } @@ -103,10 +103,7 @@ func (m *Manager) ClaimDeadline(id, token string) (time.Time, error) { return sb.Deadline, nil } -// PreviewDial opens a byte stream to a guest port for the preview server. The -// caller has already verified the signed preview token, so no sandbox token -// is needed; the live-claim lookup is the revocation check — a released or -// reaped sandbox is absent and this fails. A hibernated sandbox wakes. +// PreviewDial authorizes one preview request and opens its guest connection. func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net.Conn, error) { m.mu.Lock() sb, ok := m.claimed[id] @@ -114,11 +111,8 @@ func (m *Manager) PreviewDial(ctx context.Context, id string, port uint16) (net. if !ok { return nil, ErrUnknownSandbox } - sb.Touch() // a live preview stream is data-plane activity - // Preview bypasses the relay's audit tap (it dials the engine directly), - // so record the access here — the only data-plane entry that would - // otherwise leave no audit trace. - m.recordAudit(ctx, id, auditFrame{Op: "preview_dial", Port: port}) + sb.Touch() + m.recordAudit(ctx, id, auditFrame{Op: "preview", Port: port}) sock, err := m.wakeResolved(ctx, sb) if err != nil { return nil, err @@ -512,7 +506,7 @@ func (m *Manager) claimProvision(ctx context.Context, key types.PoolKey, ttl tim if err := m.validate(key); err != nil { return nil, err } - volumeSpecs, err := m.resolveVolumes(ctx, key, tenant, volumes) + volumeSpecs, err := m.resolveVolumes(ctx, tenant, volumes) if err != nil { return nil, err } @@ -522,7 +516,7 @@ func (m *Manager) claimProvision(ctx context.Context, key types.PoolKey, ttl tim } reserved := applied defer func() { m.unreserveVolumes(reserved) }() - golden, err := m.resolveGolden(ctx, key) + golden, err := m.resolveGolden(ctx, key, tenant) if err != nil { return nil, fmt.Errorf("resolve template: %w", err) } diff --git a/sandboxd/pool/egress.go b/sandboxd/pool/egress.go index 5bd9544a..2544b2c2 100644 --- a/sandboxd/pool/egress.go +++ b/sandboxd/pool/egress.go @@ -183,23 +183,22 @@ func (m *Manager) poolIntercepts(key types.PoolKey) bool { return m.poolEgress[key].Intercepts() } -// effectivePolicy resolves a claim's egress evaluator: pool ∩ tenant (deny -// wins), or whichever single one is set; ok is false when neither applies. +// effectivePolicy resolves pool ∩ tenant; root has no tenant layer. func (m *Manager) effectivePolicy(sb *types.Sandbox) (egress.Evaluator, bool) { m.mu.Lock() defer m.mu.Unlock() poolPol := m.poolEgress[sb.Key] - tenantPol := m.tenantEgress[sb.Tenant] - switch { - case poolPol != nil && tenantPol != nil: - return egress.Compose(*poolPol, *tenantPol), true - case poolPol != nil: + if poolPol == nil { + return nil, false + } + if sb.Tenant == "" { return *poolPol, true - case tenantPol != nil: - return *tenantPol, true - default: + } + tenantPol := m.tenantEgress[sb.Tenant] + if tenantPol == nil { return nil, false } + return egress.Compose(*poolPol, *tenantPol), true } // newEgressDialer builds the proxy's upstream dialer: internal targets are diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index cd1506e9..8b547808 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -21,7 +21,7 @@ import ( ) var ( - egKey = types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall, Engine: types.EngineCH} + egKey = types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeSmall} egPolicy = &egress.Policy{Allow: []egress.Rule{{Host: "example.com", Secret: "gh"}}} ) @@ -138,15 +138,18 @@ func TestEffectivePolicyComposition(t *testing.T) { tenantOnly := &egress.Policy{Allow: []egress.Rule{{Host: "b.test"}, {Host: "c.test"}}} cases := []struct { - name string - pool, tenant *egress.Policy - allow, deny string - wantArmed bool + name string + tenant string + pool, tnPol *egress.Policy + allow, deny string + wantArmed bool }{ - {"pool only", both, nil, "a.test", "z.test", true}, - {"tenant only", nil, tenantOnly, "c.test", "a.test", true}, - {"intersection", both, tenantOnly, "b.test", "a.test", true}, // a.test allowed by pool, denied by tenant - {"neither", nil, nil, "", "", false}, + {"root takes the pool policy whole", "", both, nil, "a.test", "z.test", true}, + {"root without a pool policy", "", nil, nil, "", "", false}, + {"tenant intersects", "acme", both, tenantOnly, "b.test", "a.test", true}, // a.test allowed by pool, denied by tenant + {"tenant declaring no policy", "acme", both, nil, "", "", false}, + {"tenant on a policyless pool", "acme", nil, tenantOnly, "", "", false}, + {"neither", "acme", nil, nil, "", "", false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -155,10 +158,10 @@ func TestEffectivePolicyComposition(t *testing.T) { m.poolEgress[testKey] = tc.pool } m.tenantEgress = map[string]*egress.Policy{} - if tc.tenant != nil { - m.tenantEgress["acme"] = tc.tenant + if tc.tnPol != nil { + m.tenantEgress["acme"] = tc.tnPol } - sb := &types.Sandbox{Key: testKey, Tenant: "acme"} + sb := &types.Sandbox{Key: testKey, Tenant: tc.tenant} eval, ok := m.effectivePolicy(sb) if ok != tc.wantArmed { t.Fatalf("armed=%v, want %v", ok, tc.wantArmed) diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index c97994aa..02c470ee 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -205,7 +205,7 @@ func (m *Manager) idleOnce(ctx context.Context) { if p, pooled := m.activePool(sb.Key); pooled { idle = p.idle } - if idle <= 0 || hasAppliedVolumes(sb) || sb.HibernateSnap != "" || sb.ArchiveCk != "" || now.Sub(sb.LastSeen()) < idle { + if skipIdle(sb, idle, now) { continue } victims = append(victims, victim{sb.ID, sb.Token}) @@ -343,3 +343,11 @@ func (m *Manager) recordHibernate(ctx context.Context, sb *types.Sandbox) { m.counters.hibernates.Add(1) m.recordUsage(ctx, usageEvent{Event: "hibernate", ID: sb.ID, VMName: sb.VMName}) } + +// skipIdle reports the claims an idle sweep must leave alone: the egress lane +// cannot resume, a mounted volume cannot be captured, and an already +// hibernated or archived claim has nothing left to do. +func skipIdle(sb *types.Sandbox, idle time.Duration, now time.Time) bool { + return idle <= 0 || sb.Key.Net == types.NetEgress || hasAppliedVolumes(sb) || + sb.HibernateSnap != "" || sb.ArchiveCk != "" || now.Sub(sb.LastSeen()) < idle +} diff --git a/sandboxd/pool/intercept_test.go b/sandboxd/pool/intercept_test.go index 5bd01e8b..d210e1f2 100644 --- a/sandboxd/pool/intercept_test.go +++ b/sandboxd/pool/intercept_test.go @@ -11,7 +11,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -var interceptKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} +var interceptKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} func TestGoldenBuildInstallsCAForInterceptPool(t *testing.T) { eng := newFakeEngine() diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index f78b0bac..7e9ec929 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -134,10 +134,8 @@ type SandboxSummary struct { Archived bool `json:"archived,omitempty"` FromCheckpoint string `json:"from_checkpoint,omitempty"` Volumes []types.Volume `json:"volumes,omitempty"` - // ClaimRef echoes the caller reference recorded at claim time (the - // aggregated apiserver's k8s "/"), so the operator index - // can map this sandbox back to the name it was claimed under. Empty for - // warm-pool, fork, and checkpoint-branch claims. + // ClaimRef echoes the caller reference recorded at claim time; empty for + // fork and checkpoint-branch claims. ClaimRef string `json:"claim_ref,omitempty"` } @@ -318,11 +316,10 @@ type Manager struct { ckptTTL time.Duration ckptSweeping atomic.Bool - // tplSet caches the template ids visible in the store so the 1s gossip - // tick never touches the backend (an s3 listing is network I/O); local - // promotes/deletes update it, startup loads it. + // tplSet caches each template id against its owning tenant ("" = operator), + // so the gossip tick and the claim path never pay a store read. tplMu sync.Mutex - tplSet map[string]struct{} + tplSet map[string]string // recLocks serializes same-id store record mutations and holds off a // re-publish swap while a clone reads the old generation (per id, RW). @@ -436,12 +433,12 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg return nil, err } m.ckptTTL = time.Duration(cfg.CheckpointTTLHours) * time.Hour - m.tplSet = map[string]struct{}{} + m.tplSet = map[string]string{} if metas, listErr := m.tpls.Metas(ctx); listErr == nil { for _, raw := range metas { var rec templateRecord if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { - m.tplSet[rec.ID] = struct{}{} + m.tplSet[rec.ID] = rec.Tenant } } } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index e1ce3cd9..0ac304ea 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -21,7 +21,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -var testKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} +var testKey = types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} func TestClaimWarmHitTransfersOwnership(t *testing.T) { eng := newFakeEngine() diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index c545b2fe..c0e6a101 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -12,8 +12,8 @@ import ( ) var ( - seedKey = types.PoolKey{Template: "seed:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} - apiKey = types.PoolKey{Template: "api:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} + seedKey = types.PoolKey{Template: "seed:24.04", Net: types.NetNone, Size: types.SizeSmall} + apiKey = types.PoolKey{Template: "api:24.04", Net: types.NetNone, Size: types.SizeSmall} ) func TestPersistedPoolsSurviveRestart(t *testing.T) { diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index 916454e0..c54f5711 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -26,7 +26,7 @@ func TestPromoteThenClaimClonesFromTemplate(t *testing.T) { if len(eng.snapSaves) != 1 || !slices.Contains(eng.snapRemoves, eng.snapSaves[0]) { t.Errorf("snapSaves=%v snapRemoves=%v, want one transient snapshot dropped", eng.snapSaves, eng.snapRemoves) } - key := types.PoolKey{Template: "tpl:x", Net: parent.Key.Net, Size: parent.Key.Size, Engine: parent.Key.Engine} + key := types.PoolKey{Template: "tpl:x", Net: parent.Key.Net, Size: parent.Key.Size} if gotKey != key { t.Errorf("returned key %+v, want %+v (the parent's axes)", gotKey, key) } @@ -146,18 +146,18 @@ func TestDeleteTemplate(t *testing.T) { if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:del", ""); err != nil { t.Fatalf("Promote: %v", err) } - key := types.PoolKey{Template: "tpl:del", Net: testKey.Net, Size: testKey.Size, Engine: testKey.Engine} + key := types.PoolKey{Template: "tpl:del", Net: testKey.Net, Size: testKey.Size} if err := m.DeleteTemplate(t.Context(), testKey, ""); !errors.Is(err, ErrPooledTemplate) { t.Errorf("pooled delete: %v, want ErrPooledTemplate", err) } - if err := m.DeleteTemplate(t.Context(), types.PoolKey{Template: "nope", Net: testKey.Net, Size: testKey.Size, Engine: testKey.Engine}, ""); !errors.Is(err, ErrUnknownTemplate) { + if err := m.DeleteTemplate(t.Context(), types.PoolKey{Template: "nope", Net: testKey.Net, Size: testKey.Size}, ""); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("unknown delete: %v, want ErrUnknownTemplate", err) } if err := m.DeleteTemplate(t.Context(), key, ""); err != nil { t.Fatalf("DeleteTemplate: %v", err) } - if m.HasGolden(t.Context(), key) { + if m.HasGolden(t.Context(), key, "") { t.Error("template still resolvable after delete") } // The next claim for the deleted template cold-boots instead of cloning. @@ -200,7 +200,7 @@ func TestResolveGoldenSkipsPromotedEgressTemplate(t *testing.T) { if _, err = m.commitTemplate(t.Context(), staging, id, ""); err != nil { t.Fatalf("seed template: %v", err) } - golden, err := m.resolveGolden(t.Context(), egKey) + golden, err := m.resolveGolden(t.Context(), egKey, "") if err != nil { t.Fatalf("resolveGolden: %v", err) } @@ -276,7 +276,7 @@ func TestPromoteFailsClosedOnMetaError(t *testing.T) { if _, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "acme"); err != nil { t.Fatalf("promote: %v", err) } - key := types.PoolKey{Template: "shared:v1", Net: testKey.Net, Size: testKey.Size, Engine: testKey.Engine} + key := types.PoolKey{Template: "shared:v1", Net: testKey.Net, Size: testKey.Size} meta := filepath.Join(m.dataDir, "checkpoints", store.TemplateID(key.Hash()), store.MetaFile) if err := os.Chmod(meta, 0o000); err != nil { t.Fatalf("chmod: %v", err) @@ -312,6 +312,101 @@ func TestPromoteRefusesCrossTenantOverwrite(t *testing.T) { } } +func TestTemplateClaimIsTenantScoped(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + claim := func(tenant string) *types.Sandbox { + t.Helper() + sb, err := m.ClaimProvision(t.Context(), testKey, time.Hour, tenant, "", nil) + if err != nil { + t.Fatalf("claim %q: %v", tenant, err) + } + return sb + } + + a := claim("acme") + private, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") + if err != nil { + t.Fatalf("acme promote: %v", err) + } + r := claim("") + shared, _, err := m.Promote(t.Context(), r.ID, Cred{Token: r.Token}, "ops-shared", "") + if err != nil { + t.Fatalf("root promote: %v", err) + } + + if _, err := m.ClaimProvisionPromoted(t.Context(), private, time.Hour, "beta", "", nil); !errors.Is(err, ErrUnknownTemplate) { + t.Errorf("beta claiming acme's template: %v, want ErrUnknownTemplate", err) + } + for _, tc := range []struct { + name string + key types.PoolKey + tenant string + }{ + {"owner claims its own", private, "acme"}, + {"root claims a tenant's", private, ""}, + {"tenant claims a root template", shared, "beta"}, + } { + t.Run(tc.name, func(t *testing.T) { + sb, err := m.ClaimProvisionPromoted(t.Context(), tc.key, time.Hour, tc.tenant, "", nil) + if err != nil { + t.Fatalf("claim: %v", err) + } + if sb.TemplateDigest == "" { + t.Error("claim did not resolve from the promoted template") + } + }) + } +} + +func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + a, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + if err != nil { + t.Fatalf("claim: %v", err) + } + key, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") + if err != nil { + t.Fatalf("promote: %v", err) + } + + // Routing must answer what a claim would: promising beta a golden here + // makes redirectClaim skip the peer hop and cold-boot the name as an image. + if m.HasPromotedTemplate(t.Context(), key, "beta") { + t.Error("beta sees acme's template as a local golden") + } + for _, tenant := range []string{"acme", ""} { + if !m.HasPromotedTemplate(t.Context(), key, tenant) { + t.Errorf("tenant %q lost its own template", tenant) + } + } +} + +func TestTemplateHashesAreTenantScoped(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + a, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + if err != nil { + t.Fatalf("claim: %v", err) + } + key, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") + if err != nil { + t.Fatalf("promote: %v", err) + } + + hashes := m.TemplateHashes() + if want := types.TemplateGossipHash(key.Hash(), "acme"); !slices.Contains(hashes, want) { + t.Errorf("gossip %v lacks the owner-scoped hash %s", hashes, want) + } + for name, bad := range map[string]string{ + "raw": key.Hash(), + "foreign": types.TemplateGossipHash(key.Hash(), "beta"), + } { + if slices.Contains(hashes, bad) { + t.Errorf("gossip %v carries the %s hash — a foreign tenant could match it", hashes, name) + } + } +} + func TestTemplateHashesSortedForMeshCompare(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index d573cf32..f37bd7fa 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -197,8 +197,6 @@ func TestPreviewDialWritesAuditEvent(t *testing.T) { t.Fatalf("setup manager: %v", err) } sb := mustClaim(t, m, testKey) - // The fake engine cannot complete the dial; the audit event records the - // access attempt against the live claim, before the engine is involved. if _, dialErr := m.PreviewDial(t.Context(), sb.ID, 8080); dialErr == nil { t.Fatal("fake engine dial unexpectedly succeeded") } @@ -216,7 +214,7 @@ func TestPreviewDialWritesAuditEvent(t *testing.T) { if err := json.Unmarshal([]byte(line), &ev); err != nil { t.Fatalf("bad audit line %q: %v", line, err) } - if ev.ID != sb.ID || ev.Op != "preview_dial" || ev.Port != 8080 { + if ev.ID != sb.ID || ev.Op != "preview" || ev.Port != 8080 { t.Errorf("audit event %+v", ev) } } diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 4e559f61..90c36512 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -42,7 +42,7 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t if !sb.Key.Capturable() { return types.PoolKey{}, "", ErrNoEgressFork } - key := types.PoolKey{Template: template, Net: sb.Key.Net, Size: sb.Key.Size, Engine: sb.Key.Engine} + key := types.PoolKey{Template: template, Net: sb.Key.Net, Size: sb.Key.Size} if m.pooledHash(key.Hash()) { // A configured pool owns this key — promoting over it would // silently change what refills produce. @@ -129,10 +129,10 @@ func (m *Manager) TemplateHashes() []string { m.mu.Unlock() m.tplMu.Lock() hashes := make([]string, 0, len(m.tplSet)) - for id := range m.tplSet { + for id, tenant := range m.tplSet { hash := store.TemplateHash(id) if _, ok := pooled[hash]; !ok { - hashes = append(hashes, hash) + hashes = append(hashes, types.TemplateGossipHash(hash, tenant)) } } m.tplMu.Unlock() @@ -144,8 +144,8 @@ func (m *Manager) TemplateHashes() []string { // boot — a configured pool golden or a promoted template in the store. The // tplSet answers without a store read; only a shared-store template promoted // elsewhere after startup falls through to the backend. -func (m *Manager) HasGolden(ctx context.Context, key types.PoolKey) bool { - return m.HasPoolGolden(key) || m.HasPromotedTemplate(ctx, key) +func (m *Manager) HasGolden(ctx context.Context, key types.PoolKey, tenant string) bool { + return m.HasPoolGolden(key) || m.HasPromotedTemplate(ctx, key, tenant) } // HasPoolGolden reports whether a configured pool can serve key from its own @@ -157,22 +157,30 @@ func (m *Manager) HasPoolGolden(key types.PoolKey) bool { return p != nil && p.goldenDir != "" } -// HasPromotedTemplate reports whether key resolves to a promoted template. -// A hash a configured pool owns is subtracted, as TemplateHashes does for the -// gossip: resolveGolden serves it from the pool golden, never the template. -func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey) bool { +// HasPromotedTemplate reports whether key resolves to a promoted template this +// tenant may claim — resolveGolden's test exactly, so routing never promises +// a golden the claim would then refuse. +func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool { if m.pooledHash(key.Hash()) { return false } id := store.TemplateID(key.Hash()) m.tplMu.Lock() - _, cached := m.tplSet[id] + owner, cached := m.tplSet[id] m.tplMu.Unlock() - if cached { - return true + if !cached { + // Only a shared-store template promoted elsewhere after startup. + raw, err := m.tpls.ReadMeta(ctx, id) + if err != nil { + return false + } + var rec templateRecord + if json.Unmarshal(raw, &rec) != nil { + return false + } + owner = rec.Tenant } - _, err := m.tpls.ReadMeta(ctx, id) - return err == nil + return owner == "" || tenantOwns(tenant, owner) } // pooledHash reports whether a configured pool occupies this hash — the @@ -275,9 +283,9 @@ type goldenResolution struct { // resolveGolden resolves a key's clone source: the configured pool's local // golden (no release), else a promoted template fetched from the store; -// empty dir cold-boots. Only a true absence cold-boots — a backend failure -// propagates rather than silently booting a template name as an image ref. -func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey) (goldenResolution, error) { +// empty dir cold-boots. A cross-tenant or absent record cold-boots; a backend +// failure propagates rather than booting a template name as an image ref. +func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant string) (goldenResolution, error) { m.mu.Lock() var dir string if p := m.pools[key]; p != nil { @@ -302,18 +310,21 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey) (goldenR } return goldenResolution{release: func() {}}, err } + cleanup := func() { release(); l.RUnlock(); m.recDone(id) } var rec templateRecord if err := json.Unmarshal(meta, &rec); err != nil { - release() - l.RUnlock() - m.recDone(id) + cleanup() return goldenResolution{release: func() {}}, fmt.Errorf("decode template metadata: %w", err) } + if rec.Tenant != "" && !tenantOwns(tenant, rec.Tenant) { + cleanup() + return goldenResolution{release: func() {}}, nil + } return goldenResolution{ dir: dir, templateDigest: digest, promoted: true, - release: func() { release(); l.RUnlock(); m.recDone(id) }, + release: cleanup, }, nil } @@ -350,7 +361,7 @@ func (m *Manager) commitTemplate(ctx context.Context, staging, id, tenant string return "", fmt.Errorf("publish template: %w", err) } m.tplMu.Lock() - m.tplSet[id] = struct{}{} + m.tplSet[id] = tenant m.tplMu.Unlock() return digest, nil } diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 42f1616d..3401326d 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -135,7 +135,7 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri return local, nil } -func (m *Manager) resolveVolumes(ctx context.Context, key types.PoolKey, tenant string, requested []types.Volume) ([]resolvedVolume, error) { +func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested []types.Volume) ([]resolvedVolume, error) { if len(requested) == 0 { return nil, nil } @@ -145,9 +145,6 @@ func (m *Manager) resolveVolumes(ctx context.Context, key types.PoolKey, tenant if err != nil { return nil, fmt.Errorf("%w: %v", ErrBadVolume, err) } - if key.Engine != types.EngineCH { - return nil, fmt.Errorf("%w: volumes require engine ch", ErrBadVolume) - } resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { entry, ok := m.volumes[volume.Name] diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index 9bd8ea13..2e20d820 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -147,7 +147,7 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { // The reader resolves a clean image; a writable claim then fails between // that resolve and admission, leaving its marker but no hold behind. - resolved, err := m.resolveVolumes(t.Context(), testKey, "", readOnly) + resolved, err := m.resolveVolumes(t.Context(), "", readOnly) if err != nil { t.Fatalf("resolveVolumes: %v", err) } @@ -166,7 +166,7 @@ func TestConfirmVolumesCleanCatchesMarkerAfterAdmission(t *testing.T) { } m.unreserveVolumes(appliedVolumes(resolved)) - writable, err := m.resolveVolumes(t.Context(), testKey, "", []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW}}) + writable, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW}}) if err != nil { t.Fatalf("resolveVolumes writable: %v", err) } diff --git a/sandboxd/pool/volume_test.go b/sandboxd/pool/volume_test.go index 9fa56d16..617cc1f0 100644 --- a/sandboxd/pool/volume_test.go +++ b/sandboxd/pool/volume_test.go @@ -268,7 +268,6 @@ func TestClaimProvisionRejectsInvalidVolumesBeforeProvision(t *testing.T) { {"too many", testKey, nil, tooMany}, {"unknown", testKey, []config.VolumeSpec{{Name: "data", Path: path}}, []types.Volume{{Name: "other"}}}, {"invalid mount", testKey, []config.VolumeSpec{{Name: "data", Path: path}}, []types.Volume{{Name: "data", Mount: "relative"}}}, - {"firecracker", types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineFC}, []config.VolumeSpec{{Name: "data", Path: path}}, []types.Volume{{Name: "data"}}}, } { t.Run(tt.name, func(t *testing.T) { eng := newFakeEngine() @@ -348,7 +347,7 @@ func TestPooledKeyOutranksPromotedTemplate(t *testing.T) { } m.pools[key].goldenDir = "/goldens/pooled" - if m.HasPromotedTemplate(t.Context(), key) { + if m.HasPromotedTemplate(t.Context(), key, "") { t.Error("a pooled key reports as promoted, disagreeing with both resolveGolden and the gossip") } if hashes := m.TemplateHashes(); slices.Contains(hashes, key.Hash()) { @@ -472,7 +471,7 @@ func TestVolumePlacementChecksAccessAndLocalAvailability(t *testing.T) { if local, err := m.VolumePlacement(testKey, "", []string{"peer-only"}); err != nil || local { t.Errorf("root peer-only placement=(%v, %v), want false, nil", local, err) } - badKey := types.PoolKey{Template: "rt:24.04", Net: "lan", Size: types.SizeSmall, Engine: types.EngineCH} + badKey := types.PoolKey{Template: "rt:24.04", Net: "lan", Size: types.SizeSmall} if _, err := m.VolumePlacement(badKey, "", []string{"local"}); !errors.Is(err, ErrBadKey) { t.Errorf("invalid key error=%v, want ErrBadKey", err) } diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 1c34d7d7..a5ba913c 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -20,9 +20,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// previewClaims is the signed payload of a preview URL: it authorizes serving -// guest `Port` of sandbox `ID`, owned by the node reachable at preview base -// `Owner`, until `Exp`. Signed so any node can verify without shared state. type previewClaims struct { ID string `json:"id"` Port uint16 `json:"port"` @@ -35,31 +32,23 @@ type PreviewManager interface { PreviewDial(ctx context.Context, id string, port uint16) (net.Conn, error) } -// PreviewServer serves guest HTTP apps under signed, expiring URLs, built on -// the port relay. The whole mechanism lives here: a signed token needs no -// revocation list (a released sandbox simply isn't in the claim map), and -// because the token carries its owner's preview address, any node can accept -// a request and proxy it to the owner — so the public entry point is a dumb -// TLS proxy, not a stateful gateway. +// PreviewServer serves signed guest HTTP URLs and forwards requests to their owner node. type PreviewServer struct { secret []byte - advertise string + base string + owner string mgr PreviewManager transport *http.Transport } -// NewPreviewServer returns nil when preview is not configured (empty secret). -// advertise is this node's preview base (host:port a browser/proxy reaches). -func NewPreviewServer(secret, advertise string, mgr PreviewManager) *PreviewServer { +// NewPreviewServer returns nil when preview is not configured. +func NewPreviewServer(secret, base, owner string, mgr PreviewManager) *PreviewServer { if secret == "" { return nil } - p := &PreviewServer{secret: []byte(secret), advertise: advertise, mgr: mgr} - // One shared transport so a page's sub-resource fan-out reuses kept-alive - // guest conns instead of re-dialing per request; the Director keys each - // request's host to the sandbox+port so the idle pool never mixes claims. + p := &PreviewServer{secret: []byte(secret), base: base, owner: owner, mgr: mgr} p.transport = &http.Transport{ - IdleConnTimeout: 90 * time.Second, + DisableKeepAlives: true, DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { id, portStr, err := net.SplitHostPort(addr) if err != nil { @@ -75,23 +64,20 @@ func NewPreviewServer(secret, advertise string, mgr PreviewManager) *PreviewServ return p } -// Mint returns a preview URL for a guest port, valid for ttl (bounded by the -// caller to the claim's deadline). Called on the owner node with the sandbox -// already authorized, so the token names this node as owner. +// Mint returns a preview URL for a guest port, valid for ttl. func (p *PreviewServer) Mint(id string, port uint16, ttl time.Duration) string { - claims := previewClaims{ID: id, Port: port, Owner: p.advertise, Exp: time.Now().Add(ttl).Unix()} + claims := previewClaims{ID: id, Port: port, Owner: p.owner, Exp: time.Now().Add(ttl).Unix()} payload, _ := json.Marshal(claims) enc := base64.RawURLEncoding.EncodeToString(payload) token := enc + "." + p.sign(enc) - base := p.advertise + base := p.base if !strings.Contains(base, "://") { base = "http://" + base } return fmt.Sprintf("%s/p/%s/", strings.TrimRight(base, "/"), token) } -// Handler serves the preview_listen address: verify the token, then either -// reverse-proxy to the guest port locally or forward to the owner node. +// Handler serves preview requests and health checks on preview_listen. func (p *PreviewServer) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/p/{token}/", p.serve) @@ -105,27 +91,21 @@ func (p *PreviewServer) serve(w http.ResponseWriter, r *http.Request) { http.Error(w, "invalid or expired preview token", http.StatusForbidden) return } - if claims.Owner != p.advertise { + if claims.Owner != p.owner { p.forward(w, r, claims.Owner) return } p.proxyLocal(w, r, claims) } -// proxyLocal reverse-proxies to the guest port over a relay connection whose -// liveness lookup is the revocation check: a released sandbox is gone from -// the claim map, so PreviewDial fails and the URL 404s. func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claims previewClaims) { rp := &httputil.ReverseProxy{ Director: func(req *http.Request) { req.URL.Scheme = "http" - // Host = sandbox:port so the shared transport's idle pool keys per - // claim; DialContext parses it back to PreviewDial. + // The synthetic host carries PreviewDial's target. req.URL.Host = fmt.Sprintf("%s:%d", claims.ID, claims.Port) req.URL.Path = "/" + strings.TrimPrefix(req.URL.Path, "/p/"+r.PathValue("token")+"/") - // The preview URL is bearer-authorized by its token; never hand - // the browser's ambient credentials for the preview domain to - // untrusted guest code. + // Browser credentials for the preview domain must not reach guest code. req.Header.Del("Cookie") req.Header.Del("Authorization") }, @@ -138,8 +118,6 @@ func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claim rp.ServeHTTP(w, r) //nolint:gosec // target derived from an HMAC-signed token, not client input } -// forward relays the request to the owner node's preview_listen — the token -// self-authorizes, so this node is a dumb hop. func (p *PreviewServer) forward(w http.ResponseWriter, r *http.Request, owner string) { target := &url.URL{Scheme: "http", Host: owner} rp := httputil.NewSingleHostReverseProxy(target) @@ -172,8 +150,6 @@ func (p *PreviewServer) verify(token string) (previewClaims, bool) { return claims, true } -// handlePreview mints a preview URL for a claimed sandbox's port. The sandbox -// token authorizes it; the TTL is clamped to the claim's remaining lease. func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) { if s.preview == nil { writeErr(w, http.StatusNotImplemented, "preview not configured") diff --git a/sandboxd/server/preview_test.go b/sandboxd/server/preview_test.go index 6a7691fd..19755c5e 100644 --- a/sandboxd/server/preview_test.go +++ b/sandboxd/server/preview_test.go @@ -7,45 +7,43 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" ) func TestPreviewTokenRoundTrip(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", &fakePreviewMgr{}) + ps := NewPreviewServer("secret", "https://preview.example.com", "node:7777", &fakePreviewMgr{}) token := mintToken(ps, "sb_1", 8080, time.Hour) claims, ok := ps.verify(token) - if !ok || claims.ID != "sb_1" || claims.Port != 8080 || claims.Owner != "node:9000" { + if !ok || claims.ID != "sb_1" || claims.Port != 8080 || claims.Owner != "node:7777" { t.Fatalf("verify %+v ok=%v", claims, ok) } + if url := ps.Mint("sb_1", 8080, time.Hour); !strings.HasPrefix(url, "https://preview.example.com/p/") { + t.Errorf("url %q, want public preview base", url) + } if _, ok := ps.verify(token + "x"); ok { t.Error("tampered token verified") } - if _, ok := NewPreviewServer("other-secret", "node:9000", &fakePreviewMgr{}).verify(token); ok { + if _, ok := NewPreviewServer("other-secret", "https://preview.example.com", "node:7777", &fakePreviewMgr{}).verify(token); ok { t.Error("token verified under the wrong secret") } } func TestPreviewRejectsExpired(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", &fakePreviewMgr{}) - token := mintToken(ps, "sb_1", 8080, -time.Second) // already expired + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{}) + token := mintToken(ps, "sb_1", 8080, -time.Second) if _, ok := ps.verify(token); ok { t.Error("expired token verified") } } func TestPreviewProxiesToGuest(t *testing.T) { - // A real HTTP server stands in for the guest app; PreviewDial hands the - // proxy a raw conn to it. - guest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, "guest saw "+r.URL.Path) - })) - t.Cleanup(guest.Close) - guestAddr := strings.TrimPrefix(guest.URL, "http://") + guestAddr := newGuestServer(t, func(r *http.Request) string { return "guest saw " + r.URL.Path }) dialed := false - ps := NewPreviewServer("secret", "node:9000", &fakePreviewMgr{ + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ dial: func(id string, port uint16) (net.Conn, error) { dialed = true if id != "sb_1" || port != 8080 { @@ -71,8 +69,7 @@ func TestPreviewProxiesToGuest(t *testing.T) { } func TestPreviewRevokedWhenDialFails(t *testing.T) { - // A released sandbox: PreviewDial errors, so the URL 502s — statelessly. - ps := NewPreviewServer("secret", "node:9000", &fakePreviewMgr{ + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ dial: func(string, uint16) (net.Conn, error) { return nil, net.ErrClosed }, }) ts := httptest.NewServer(ps.Handler()) @@ -89,31 +86,75 @@ func TestPreviewRevokedWhenDialFails(t *testing.T) { } } +func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { + guestAddr := newGuestServer(t, func(*http.Request) string { return "guest" }) + + var live atomic.Bool + var dials atomic.Int32 + live.Store(true) + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ + dial: func(string, uint16) (net.Conn, error) { + dials.Add(1) + if !live.Load() { + return nil, net.ErrClosed + } + return net.Dial("tcp", guestAddr) + }, + }) + ts := httptest.NewServer(ps.Handler()) + t.Cleanup(ts.Close) + url := ts.URL + "/p/" + mintToken(ps, "sb_1", 8080, time.Hour) + "/" + + resp, err := http.Get(url) + if err != nil { + t.Fatalf("first get: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("first status = %d, want 200", resp.StatusCode) + } + + live.Store(false) + resp, err = http.Get(url) + if err != nil { + t.Fatalf("get after release: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadGateway { + t.Errorf("status after release = %d, want 502", resp.StatusCode) + } + if got := dials.Load(); got != 2 { + t.Errorf("PreviewDial calls = %d, want one per request", got) + } +} + func TestPreviewForwardsToOwner(t *testing.T) { - // A token owned by a different node is proxied to that node verbatim. - var forwarded bool - owner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - forwarded = true - _, _ = io.WriteString(w, "owner served") - })) - t.Cleanup(owner.Close) - ownerAddr := strings.TrimPrefix(owner.URL, "http://") + guestAddr := newGuestServer(t, func(r *http.Request) string { return "guest saw " + r.URL.Path }) - // Mint on the owner, serve on a different node. - ownerPS := NewPreviewServer("secret", ownerAddr, &fakePreviewMgr{}) - token := mintToken(ownerPS, "sb_1", 8080, time.Hour) + owner := httptest.NewUnstartedServer(nil) + ownerAddr := owner.Listener.Addr().String() + ownerPS := NewPreviewServer("secret", "https://preview.example.com", ownerAddr, &fakePreviewMgr{ + dial: func(string, uint16) (net.Conn, error) { return net.Dial("tcp", guestAddr) }, + }) + ownerSrv := New("", nil, ownerAddr, &fakeManager{}, &fakeDialer{}, nil, nil, nil, ownerPS) + owner.Config.Handler = ownerSrv.Handler() + owner.Start() + t.Cleanup(func() { owner.Close(); ownerSrv.CloseRelays() }) - entry := NewPreviewServer("secret", "entry:9000", &fakePreviewMgr{}) + token := mintToken(ownerPS, "sb_1", 8080, time.Hour) + entry := NewPreviewServer("secret", "https://preview.example.com", "entry:7777", &fakePreviewMgr{}) ts := httptest.NewServer(entry.Handler()) t.Cleanup(ts.Close) - resp, err := http.Get(ts.URL + "/p/" + token + "/") + resp, err := http.Get(ts.URL + "/p/" + token + "/via-owner") if err != nil { t.Fatalf("get: %v", err) } defer resp.Body.Close() - if !forwarded { - t.Error("request not forwarded to the owner node") + body, _ := io.ReadAll(resp.Body) + if string(body) != "guest saw /via-owner" { + t.Errorf("body %q, want request forwarded through owner to guest", body) } } @@ -125,6 +166,15 @@ func (f *fakePreviewMgr) PreviewDial(_ context.Context, id string, port uint16) return f.dial(id, port) } +func newGuestServer(t *testing.T, body func(r *http.Request) string) string { + t.Helper() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, body(r)) + })) + t.Cleanup(ts.Close) + return strings.TrimPrefix(ts.URL, "http://") +} + func mintToken(ps *PreviewServer, id string, port uint16, ttl time.Duration) string { url := ps.Mint(id, port, ttl) return strings.TrimSuffix(url[strings.Index(url, "/p/")+3:], "/") diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index ad7d0d5a..5da79caa 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -15,6 +15,7 @@ import ( "io" "net" "net/http" + "slices" "sync" "time" @@ -89,9 +90,9 @@ type Manager interface { FetchCheckpoint(ctx context.Context, ckptID string) (dir string, meta []byte, release func(), err error) DeleteCheckpoint(ctx context.Context, ckptID, tenant string, scope pool.DeleteScope) error ClaimDeadline(id, token string) (time.Time, error) - HasGolden(ctx context.Context, key types.PoolKey) bool + HasGolden(ctx context.Context, key types.PoolKey, tenant string) bool HasPoolGolden(key types.PoolKey) bool - HasPromotedTemplate(ctx context.Context, key types.PoolKey) bool + HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool AgentSocket(id, token string) (string, error) WakeAgentSocket(ctx context.Context, id, token string) (string, error) SetPools(ctx context.Context, pools []config.PoolSpec) error @@ -227,6 +228,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /v1/sandboxes", s.requireToken(s.handleSandboxes)) mux.HandleFunc("GET /metrics", s.requireRoot(s.handleMetrics)) mux.HandleFunc("GET /healthz", s.handleHealthz) + if s.preview != nil { + mux.HandleFunc("/p/{token}/", s.preview.serve) + } return mux } @@ -249,7 +253,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { // this node provision (golden clone or cold boot). sb, err := s.mgr.ClaimWarm(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) if errors.Is(err, pool.ErrNoWarm) { - if s.redirectClaim(r.Context(), w, req, key, hash) { + if s.redirectClaim(r.Context(), w, req, key, hash, tenant) { return } sb, err = s.mgr.ClaimProvision(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) @@ -267,9 +271,6 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { func (s *Server) handleVolumeClaim(w http.ResponseWriter, r *http.Request, req types.ClaimRequest, key types.PoolKey, hash, tenant string) { volumes, err := types.ValidateVolumes(req.Volumes, req.VolumesAttachOnly) - if err == nil && key.Engine != types.EngineCH { - err = errors.New("volumes require engine ch") - } if err != nil { writeErr(w, http.StatusBadRequest, fmt.Errorf("%w: %v", pool.ErrBadVolume, err).Error()) return @@ -317,14 +318,14 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, return false, err } - localTemplate := s.mgr.HasPromotedTemplate(ctx, key) + localTemplate := s.mgr.HasPromotedTemplate(ctx, key, tenant) // Per-node content consistency: a peer's promoted template must not escalate // volume claims off the pool golden every other claim here resolves to. An // explicit RequirePromoted still passes through, for the manager to refuse. pooled := s.mgr.HasPoolGolden(key) var templateOwners []string if s.placer != nil && !pooled { - templateOwners = s.placer.TemplateOwners(hash) + templateOwners = s.templateOwners(s.placer.TemplateOwners, hash, tenant) } promoted := req.RequirePromoted || localTemplate || len(templateOwners) > 0 req.RequirePromoted = promoted @@ -340,7 +341,9 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, // A shared template store lets a volume holder resolve the template // even before that node has advertised the newly published hash. The // no_redirect target re-checks both resources before provisioning. - owners = s.placer.TemplateVolumeOwners(hash, names) + owners = s.templateOwners(func(probe string) []string { + return s.placer.TemplateVolumeOwners(probe, names) + }, hash, tenant) } else { owners = s.placer.VolumeCandidates(hash, names) } @@ -354,11 +357,31 @@ func (s *Server) redirectVolumeClaim(ctx context.Context, w http.ResponseWriter, return true, nil } +func (s *Server) templateOwners(query func(string) []string, hash, tenant string) []string { + probes := []string{types.TemplateGossipHash(hash, tenant)} + if tenant == "" { + for _, tn := range s.tenants { + probes = append(probes, types.TemplateGossipHash(hash, tn.Name)) + } + } else { + probes = append(probes, types.TemplateGossipHash(hash, "")) + } + var owners []string + for _, probe := range probes { + for _, owner := range query(probe) { + if !slices.Contains(owners, owner) { + owners = append(owners, owner) + } + } + } + return owners +} + // redirectClaim redirects a warm-miss to a better peer — a warm holder, or the // template owner when we lack a golden (so we don't cold-boot a nonexistent // image ref). A no_redirect request must resolve locally, never bounce again, // to avoid a two-node ping-pong. -func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req types.ClaimRequest, key types.PoolKey, hash string) bool { +func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req types.ClaimRequest, key types.PoolKey, hash, tenant string) bool { if s.placer == nil || req.NoRedirect { return false } @@ -366,14 +389,14 @@ func (s *Server) redirectClaim(ctx context.Context, w http.ResponseWriter, req t return true } // TemplateOwners is in-memory; HasGolden can be a store round-trip. - owners := s.placer.TemplateOwners(hash) - return len(owners) > 0 && !s.mgr.HasGolden(ctx, key) && writeRedirect(w, owners) + owners := s.templateOwners(s.placer.TemplateOwners, hash, tenant) + return len(owners) > 0 && !s.mgr.HasGolden(ctx, key, tenant) && writeRedirect(w, owners) } // handleRelease releases a claimed sandbox. Two credentials authorize it: the // node's root api_token (the operator) may release any sandbox by id, so // aggregated/control-plane teardown works without holding the per-sandbox token; -// a per-sandbox token releases only its own claim, unchanged. A tenant token is +// a per-sandbox token releases only its own claim. A tenant token is // neither — it is not the root api_token, so it resolves as a (non-matching) // sandbox token and 404s. func (s *Server) handleRelease(w http.ResponseWriter, r *http.Request) { @@ -529,7 +552,7 @@ func (s *Server) handleCheckpointBlob(w http.ResponseWriter, r *http.Request) { // handleCheckpointProbe answers a peer's HEAD probe. With a probeKey // configured (an encrypted mesh), the caller must present a fresh MAC over // the id (peer.ProbeHeader) or the probe is rejected before the metadata -// read; without one, the id remains the only capability, same as before. +// read; without one, the id remains the only capability. func (s *Server) handleCheckpointProbe(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") if len(s.probeKey) > 0 && !peer.VerifyProbeMAC(s.probeKey, id, r.Header.Get(peer.ProbeHeader)) { @@ -593,7 +616,7 @@ func (s *Server) handleDeleteTemplate(w http.ResponseWriter, r *http.Request) { // owner. no_redirect mirrors the claim protocol — a redirected retry // carries it, so the owner answers for itself and never bounces again. if errors.Is(err, pool.ErrUnknownTemplate) && s.placer != nil && q.Get("no_redirect") == "" && - writeRedirect(w, s.placer.TemplateOwners(key.Hash())) { + writeRedirect(w, s.templateOwners(s.placer.TemplateOwners, key.Hash(), tenantFrom(r.Context()))) { return } writeResult(w, r, "delete template", req.Template, "delete template failed", err, func() { diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index e39c7637..fab3b1c0 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -56,7 +56,7 @@ func TestClaimHappyPath(t *testing.T) { if cr.TemplateDigest != "sha256:claim-digest" { t.Errorf("template digest %q, want sha256:claim-digest", cr.TemplateDigest) } - want := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} + want := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeSmall} if gotKey != want { t.Errorf("key %+v, want defaults %+v", gotKey, want) } @@ -738,7 +738,7 @@ func TestPromoteAndDeleteTemplateFlow(t *testing.T) { if got := del("Bearer sekret", "template=tpl:x&net=none&size=small"); got != http.StatusNoContent { t.Errorf("delete status %d, want 204", got) } - want := types.PoolKey{Template: "tpl:x", Net: types.NetNone, Size: types.SizeSmall, Engine: types.EngineCH} + want := types.PoolKey{Template: "tpl:x", Net: types.NetNone, Size: types.SizeSmall} if gotKey != want { t.Errorf("delete key %+v, want %+v (claim defaults applied)", gotKey, want) } @@ -1337,6 +1337,62 @@ func TestVolumeClaimKeepsPoolContentAgainstPeerTemplates(t *testing.T) { } } +func TestForeignTemplateGossipNeverEscalates(t *testing.T) { + hash := types.ClaimRequest{Template: "tpl"}.Key().Hash() + tenants := []config.TenantSpec{{Name: "acme", Token: "acme-tok"}, {Name: "beta", Token: "beta-tok"}} + for _, tt := range []struct { + name, token string + volumes []types.Volume + wantRedirect bool + wantProvision int + }{ + {"foreign volume claim cold-boots locally", "beta-tok", []types.Volume{{Name: "imagenet"}}, false, 1}, + {"owner volume claim escalates to its template", "acme-tok", []types.Volume{{Name: "imagenet"}}, true, 0}, + {"foreign plain claim stays local, no existence signal", "beta-tok", nil, false, 1}, + {"owner plain claim follows its template", "acme-tok", nil, true, 0}, + } { + t.Run(tt.name, func(t *testing.T) { + mgr := &fakeManager{ + volumePlacement: func(types.PoolKey, string, []string) (bool, error) { return true, nil }, + } + placer := &fakePlacer{ownersByProbe: map[string][]string{ + types.TemplateGossipHash(hash, "acme"): {"peer:7777"}, + }} + srv := New("root-tok", tenants, "node-a:7777", mgr, &fakeDialer{}, placer, nil, nil, nil) + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) + + body, _ := json.Marshal(types.ClaimRequest{Template: "tpl", Volumes: tt.volumes}) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/claim", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+tt.token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("claim: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status=%d, want 200", resp.StatusCode) + } + var cr types.ClaimResponse + if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil { + t.Fatalf("decode: %v", err) + } + if got := len(cr.Redirect) > 0; got != tt.wantRedirect { + t.Errorf("redirect=%v (%v), want %v", got, cr.Redirect, tt.wantRedirect) + } + if wantPromoted := tt.wantRedirect && tt.volumes != nil; cr.RequirePromoted != wantPromoted { + t.Errorf("require_promoted=%v, want %v (only the volume path pins it)", cr.RequirePromoted, wantPromoted) + } + if mgr.provisionCalls != tt.wantProvision { + t.Errorf("provisions=%d, want %d", mgr.provisionCalls, tt.wantProvision) + } + if mgr.gotRequirePromoted { + t.Error("a local resolution must not be forced onto the promoted path") + } + }) + } +} + func TestVolumeClaimValidatesKeyBeforePlacement(t *testing.T) { mgr := &fakeManager{ volumePlacement: func(types.PoolKey, string, []string) (bool, error) { @@ -1394,7 +1450,6 @@ func TestVolumeClaimRejectsShapeBeforePlacement(t *testing.T) { `{"template":"rt:24.04","volumes":["data"]}`, `{"template":"rt:24.04","volumes":[{"name":"data"},{"name":"data"}]}`, `{"template":"rt:24.04","volumes":[{"name":"cocoon-data"}]}`, - `{"template":"rt:24.04","engine":"fc","volumes":[{"name":"data"}]}`, `{"template":"rt:24.04","volumes":[{"name":"data","mount":"relative"}]}`, `{"template":"rt:24.04","volumes":[{"name":"data","mount":"/datasets"},{"name":"other","mount":"/datasets/nested"}]}`, `{"template":"rt:24.04","volumes":[{"name":"a"},{"name":"b"},{"name":"c"},{"name":"d"},{"name":"e"},{"name":"f"},{"name":"g"},{"name":"h"},{"name":"i"}]}`, @@ -1472,7 +1527,7 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { mgr := &fakeManager{claimDeadline: func(string, string) (time.Time, error) { return time.Time{}, nil }} - ps := NewPreviewServer("secret", "node:7777", &fakePreviewMgr{}) + ps := NewPreviewServer("secret", "node:7777", "node:7777", &fakePreviewMgr{}) srv := New("", nil, "node:7777", mgr, &fakeDialer{}, nil, nil, nil, ps) ts := httptest.NewServer(srv.Handler()) t.Cleanup(func() { ts.Close(); srv.CloseRelays() }) @@ -2024,7 +2079,7 @@ func (f *fakeManager) DeleteTemplate(_ context.Context, key types.PoolKey, tenan return f.deleteGolden(key) } -func (f *fakeManager) HasGolden(context.Context, types.PoolKey) bool { +func (f *fakeManager) HasGolden(context.Context, types.PoolKey, string) bool { return f.hasGolden } @@ -2032,7 +2087,7 @@ func (f *fakeManager) HasPoolGolden(types.PoolKey) bool { return f.hasPoolGolden } -func (f *fakeManager) HasPromotedTemplate(context.Context, types.PoolKey) bool { +func (f *fakeManager) HasPromotedTemplate(context.Context, types.PoolKey, string) bool { return f.hasPromoted } @@ -2186,6 +2241,7 @@ func (f *fakeDialer) DialSilkd(ctx context.Context, sock string) (net.Conn, erro type fakePlacer struct { addrs []string owners []string + ownersByProbe map[string][]string volumeCandidates []string volumeOwners []string templateVolumeOwners []string @@ -2209,8 +2265,11 @@ func (f *fakePlacer) VolumeCandidates(_ string, names []string) []string { return f.volumeCandidates } -func (f *fakePlacer) TemplateOwners(string) []string { +func (f *fakePlacer) TemplateOwners(probe string) []string { f.templateOwnerCalls++ + if f.ownersByProbe != nil { + return f.ownersByProbe[probe] + } return f.owners } @@ -2219,8 +2278,11 @@ func (f *fakePlacer) VolumeOwners([]string) []string { return f.volumeOwners } -func (f *fakePlacer) TemplateVolumeOwners(string, []string) []string { +func (f *fakePlacer) TemplateVolumeOwners(probe string, _ []string) []string { f.templateVolumeCalls++ + if f.ownersByProbe != nil { + return f.ownersByProbe[probe] + } return f.templateVolumeOwners } func (f *fakePlacer) VolumeHolders() map[string]int { return f.volumeHolders } diff --git a/sandboxd/store/peer/broadcast_test.go b/sandboxd/store/peer/broadcast_test.go index 0797f6b3..d1aa8ed3 100644 --- a/sandboxd/store/peer/broadcast_test.go +++ b/sandboxd/store/peer/broadcast_test.go @@ -78,13 +78,18 @@ func TestBroadcastDeleteSwallowsFailures(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) })) defer fail.Close() + var reached atomic.Int32 ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached.Add(1) w.WriteHeader(http.StatusNoContent) })) defer ok.Close() b := &Broadcaster{Peers: func() []string { return []string{fail.URL, ok.URL, "127.0.0.1:1"} }} b.Delete(t.Context(), testID) // must return without panicking regardless of peer outcomes + if got := reached.Load(); got != 1 { + t.Errorf("healthy peer hit %d times, want 1 (fan-out continues past a failure)", got) + } } // TestBroadcastDeleteNoPeersNoOp: a single-node deployment must not dial diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index 425fd057..5d5d93b8 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -100,9 +100,12 @@ func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { // claims to a node that no longer holds it. func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { release := make(chan struct{}) + inFlight := make(chan struct{}) + probed := sync.OnceFunc(func() { close(inFlight) }) var hits atomic.Int32 owner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) + probed() <-release // hold the probe open so Forget can land mid-flight w.WriteHeader(http.StatusOK) })) @@ -113,9 +116,7 @@ func TestForgetDuringFlightPreventsStaleCache(t *testing.T) { done := make(chan struct{}) go func() { owners = p.Owners(t.Context(), testID); close(done) }() - for hits.Load() == 0 { // wait until the probe is in flight - time.Sleep(time.Millisecond) - } + <-inFlight p.Forget(testID) // the delete lands while the flight is open close(release) <-done diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 49b578f0..880ee138 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -22,7 +22,6 @@ type ClaimRequest struct { Template string `json:"template"` Net NetShape `json:"net,omitempty"` Size Size `json:"size,omitempty"` - Engine Engine `json:"engine,omitempty"` Volumes []Volume `json:"volumes,omitempty"` // VolumesAttachOnly attaches every requested volume without mounting it: // the workload finds the device by its serial and owns the mount contract. @@ -40,7 +39,7 @@ type ClaimRequest struct { // Key resolves the requested pool key with the wire defaults filled. func (r ClaimRequest) Key() PoolKey { - return PoolKey{Template: r.Template, Net: r.Net, Size: r.Size, Engine: r.Engine}.Defaulted() + return PoolKey{Template: r.Template, Net: r.Net, Size: r.Size}.Defaulted() } // ClaimResponse is the wire reply of POST /v1/claim. A successful claim diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 8dad0341..55fccda5 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -29,9 +29,6 @@ const ( RestoreOnDemand RestoreMode = "ondemand" RestoreMmap RestoreMode = "mmap" - EngineCH Engine = "ch" - EngineFC Engine = "fc" - MaxClaimVolumes = 8 // Also the guest `mount -o` option literals (engine.MountVolume): renaming @@ -82,21 +79,6 @@ func (m RestoreMode) Validate() error { } } -// Engine selects the hypervisor backend cocoon boots a pool's VMs on: Cloud -// Hypervisor (default) or Firecracker. It is a pool axis so a CH pool and an -// FC pool with the same template/net/size stay distinct goldens. -type Engine string - -// Validate accepts the empty default (resolved to EngineCH) plus known engines. -func (e Engine) Validate() error { - switch e { - case "", EngineCH, EngineFC: - return nil - default: - return fmt.Errorf("unknown engine %q", e) - } -} - // Size is a T-shirt resource tier. type Size string @@ -120,7 +102,6 @@ type PoolKey struct { Template string `json:"template"` Net NetShape `json:"net"` Size Size `json:"size"` - Engine Engine `json:"engine,omitempty"` } // Capturable reports whether state capture (fork, checkpoint, promote) is @@ -135,7 +116,6 @@ func (k PoolKey) Capturable() bool { func (k PoolKey) Defaulted() PoolKey { k.Net = cmp.Or(k.Net, NetNone) k.Size = cmp.Or(k.Size, SizeSmall) - k.Engine = cmp.Or(k.Engine, EngineCH) return k } @@ -144,7 +124,7 @@ func (k PoolKey) Defaulted() PoolKey { // names, so a targeted collision with a configured pool's hash must stay a // second-preimage problem, never a brute-forceable one. func (k PoolKey) Hash() string { - sum := sha256.Sum256([]byte(k.Template + "|" + string(k.Net) + "|" + string(k.Size) + "|" + string(k.Engine))) + sum := sha256.Sum256([]byte(k.Template + "|" + string(k.Net) + "|" + string(k.Size))) return hex.EncodeToString(sum[:16]) } @@ -161,12 +141,16 @@ func (k PoolKey) Validate() error { if _, ok := k.Size.Spec(); !ok { return fmt.Errorf("unknown size %q", k.Size) } - if err := k.Engine.Validate(); err != nil { - return err - } return nil } +// TemplateGossipHash scopes a key hash to its owner for the gossip wire, so +// foreign templates never match an owner query; the tenant itself never travels. +func TemplateGossipHash(keyHash, tenant string) string { + sum := sha256.Sum256([]byte(keyHash + "|" + tenant)) + return hex.EncodeToString(sum[:16]) +} + // Sandbox is the node-local record of one pooled or claimed VM. type Sandbox struct { ID string `json:"id"` diff --git a/scripts/archive-e2e.sh b/scripts/archive-e2e.sh index baad484e..76ecbe11 100644 --- a/scripts/archive-e2e.sh +++ b/scripts/archive-e2e.sh @@ -20,7 +20,9 @@ cleanup() { echo "== daemon log tail" tail -30 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | diff --git a/scripts/bench.sh b/scripts/bench.sh index c7e53c97..2809261f 100755 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -30,7 +30,9 @@ cleanup() { echo "== daemon log tail" tail -20 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | diff --git a/scripts/egress-e2e.sh b/scripts/egress-e2e.sh index 672e729a..efab59d8 100755 --- a/scripts/egress-e2e.sh +++ b/scripts/egress-e2e.sh @@ -34,7 +34,9 @@ cleanup() { echo "== daemon log tail" tail -30 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | diff --git a/scripts/intercept-e2e.sh b/scripts/intercept-e2e.sh index 85da6306..e0f64ef3 100755 --- a/scripts/intercept-e2e.sh +++ b/scripts/intercept-e2e.sh @@ -24,7 +24,9 @@ cleanup() { echo "== daemon log tail" tail -30 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | diff --git a/scripts/sandboxd-e2e.sh b/scripts/sandboxd-e2e.sh index b7a4a0f5..1e6f8cb6 100755 --- a/scripts/sandboxd-e2e.sh +++ b/scripts/sandboxd-e2e.sh @@ -56,7 +56,9 @@ cleanup() { echo "== daemon log tail" tail -20 "$DATA/daemon.log" fi - [[ -n $DAEMON_PID ]] && kill "$DAEMON_PID" 2>/dev/null || true + if [[ -n $DAEMON_PID ]]; then + kill "$DAEMON_PID" 2>/dev/null || true + fi wait 2>/dev/null || true cocoon vm list --format json 2>/dev/null | jq -r '.[] | select(.config.name | startswith("sbx-")) | .config.name' | @@ -223,7 +225,8 @@ claimed=$(api info | jq .claimed) echo "== restart: live claim re-adopts, warm VMs of the old life are replaced" "$DATA/demo" -addr "$ADDR" -token "$TOKEN" -template "$TEMPLATE" -n 1 -ttl 300 -leak -kill "$DAEMON_PID" && wait "$DAEMON_PID" 2>/dev/null || true +kill "$DAEMON_PID" 2>/dev/null || true +wait "$DAEMON_PID" 2>/dev/null || true start_daemon claimed=$(api info | jq .claimed) [[ $claimed == 1 ]] || { echo "claim not re-adopted: claimed=$claimed"; exit 1; } diff --git a/sdk/go/checkpoint.go b/sdk/go/checkpoint.go index 1751674f..ab6497a5 100644 --- a/sdk/go/checkpoint.go +++ b/sdk/go/checkpoint.go @@ -93,6 +93,12 @@ func (c *Client) Checkpoints(ctx context.Context) ([]*Checkpoint, error) { return ckpts, nil } +// Checkpoint returns a handle for a known checkpoint id, bound to the entry +// node — no listing round-trip; an unknown id surfaces as 404 at claim time. +func (c *Client) Checkpoint(id string) *Checkpoint { + return checkpointHandle(c, c.addr, checkpointRecord{ID: id}) +} + func checkpointHandle(c *Client, addr string, rec checkpointRecord) *Checkpoint { return &Checkpoint{ ID: rec.ID, Name: rec.Name, SandboxID: rec.SandboxID, CreatedAt: rec.CreatedAt, diff --git a/sdk/go/client.go b/sdk/go/client.go index 5692e487..8d76f760 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -34,6 +34,12 @@ func WithAPIToken(token string) ClientOption { return func(c *Client) { c.apiToken = token } } +// WithHTTPClient replaces the control-plane HTTP client, for callers that need +// their own transport, proxy, or timeout. +func WithHTTPClient(hc *http.Client) ClientOption { + return func(c *Client) { c.hc = hc } +} + // Client talks to one sandboxd node. type Client struct { addr string @@ -194,7 +200,8 @@ func (c *Client) roundTrip(ctx context.Context, method, addr, path string, body // Connect returns a client for a sandboxd node. addr accepts a // comma-separated seed list for forward compatibility; v0 uses the first -// entry. +// entry. Calls are bounded by their ctx — checkpoint and promote run as long +// as the snapshot takes, so the client sets no blanket deadline. func Connect(addr string, opts ...ClientOption) (*Client, error) { first, _, _ := strings.Cut(addr, ",") first = strings.TrimSpace(first) @@ -269,8 +276,8 @@ func tryEach(candidates []string, call func(addr string) error, retry func(error // retryMiss retries a miss (the next candidate may own the record) or a // transport failure (dead peer); a served error is real and stops the walk. func retryMiss(err error) bool { - var he *httpError - return !errors.As(err, &he) || he.status == http.StatusNotFound + var he *APIError + return !errors.As(err, &he) || he.Status == http.StatusNotFound } // retryAny retries a redirect candidate's failure unconditionally: one @@ -285,11 +292,11 @@ func retryAny(error) bool { return true } // request, a forbidden token, or an egress conflict is definitive: the // origin would fail the same way. func retryTransient(err error) bool { - var he *httpError + var he *APIError if !errors.As(err, &he) { return true } - switch he.status { + switch he.Status { case http.StatusUnauthorized, http.StatusNotFound, http.StatusTooManyRequests, http.StatusServiceUnavailable, http.StatusInternalServerError, http.StatusBadGateway, http.StatusGatewayTimeout: @@ -401,26 +408,25 @@ func encodeBody(verb string, v any) ([]byte, error) { return body, nil } -// httpError is a non-2xx control-plane reply; redirect walks branch on the -// status (retryMiss). -type httpError struct { - verb string - status int - msg string +// APIError is a non-2xx control-plane reply. +type APIError struct { + Verb string + Status int + Message string } -func (e *httpError) Error() string { - if e.msg != "" { - return fmt.Sprintf("%s: %s (http %d)", e.verb, e.msg, e.status) +func (e *APIError) Error() string { + if e.Message != "" { + return fmt.Sprintf("%s: %s (http %d)", e.Verb, e.Message, e.Status) } - return fmt.Sprintf("%s: http %d", e.verb, e.status) + return fmt.Sprintf("%s: http %d", e.Verb, e.Status) } // apiError surfaces the server's {"error": ...} body when present. func apiError(verb string, resp *http.Response) error { var er errorResponse _ = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&er) - return &httpError{verb: verb, status: resp.StatusCode, msg: er.Error} + return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} } // claimRequest mirrors sandboxd's wire type; duplicated so the SDK stays @@ -434,6 +440,7 @@ type claimRequest struct { TTLSeconds int `json:"ttl_seconds,omitempty"` NoRedirect bool `json:"no_redirect,omitempty"` RequirePromoted bool `json:"require_promoted,omitempty"` + ClaimRef string `json:"claim_ref,omitempty"` } // rejectPinnedAxes fails a snapshot claim (checkpoint, template) that passed diff --git a/sdk/go/info.go b/sdk/go/info.go index 9e672d7a..42c3c233 100644 --- a/sdk/go/info.go +++ b/sdk/go/info.go @@ -36,11 +36,37 @@ type PoolStatus struct { Golden bool `json:"golden"` } +// SandboxSummary is one live claim as the scoped index reports it; never a +// token or a host path. +type SandboxSummary struct { + ID string `json:"id"` + Key PoolKey `json:"key"` + Deadline time.Time `json:"deadline"` + Hibernated bool `json:"hibernated"` + Archived bool `json:"archived,omitempty"` + FromCheckpoint string `json:"from_checkpoint,omitempty"` + Volumes []Volume `json:"volumes,omitempty"` + ClaimRef string `json:"claim_ref,omitempty"` +} + +type sandboxListResponse struct { + Sandboxes []SandboxSummary `json:"sandboxes"` +} + // Info reports the entry node's pools, claim counts, and mesh peers. func (c *Client) Info(ctx context.Context) (*NodeInfo, error) { return doJSONPtr[NodeInfo](ctx, c, http.MethodGet, c.addr, "/v1/info", nil, c.apiToken, "info") } +// Sandboxes lists the live claims this token may see. +func (c *Client) Sandboxes(ctx context.Context) ([]SandboxSummary, error) { + reply, err := doJSON[sandboxListResponse](ctx, c, http.MethodGet, c.addr, "/v1/sandboxes", nil, c.apiToken, "list sandboxes") + if err != nil { + return nil, err + } + return reply.Sandboxes, nil +} + // peers fetches the cluster's node addresses, best-effort (nil on failure). func (c *Client) peers(ctx context.Context) []string { addrs, _ := c.peersOrErr(ctx) diff --git a/sdk/go/options.go b/sdk/go/options.go index 3cb1cf9b..dde7eb63 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -86,6 +86,12 @@ func WithTimeout(d time.Duration) Option { } } +// WithClaimRef records an opaque caller reference on the claim (the aggregated +// apiserver passes its k8s "/"), echoed by Client.Sandboxes. +func WithClaimRef(ref string) Option { + return func(r *claimRequest) { r.ClaimRef = ref } +} + // ttlSeconds rounds a lease up to whole wire seconds. func ttlSeconds(d time.Duration) int { return int((d + time.Second - 1) / time.Second) diff --git a/sdk/go/port.go b/sdk/go/port.go index b239f86b..7c0a50ab 100644 --- a/sdk/go/port.go +++ b/sdk/go/port.go @@ -59,7 +59,7 @@ func (p *PortConn) Close() error { p.closeOnce.Do(func() { p.stop() // drain can be parked in a pipe write on an unread tail; only the - // reader side unblocks it, so teardown must close the pipe too. + // reader side unblocks it. _ = p.out.CloseWithError(net.ErrClosed) }) return nil diff --git a/sdk/go/pty.go b/sdk/go/pty.go index c340ce62..a3a31624 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -27,13 +27,13 @@ type Pty struct { stop func() out *io.PipeReader - mu sync.Mutex - exitCode int - exited bool + mu sync.Mutex + exitCode int + exited bool + closeOnce sync.Once } -// Read returns terminal output; io.EOF signals the shell has exited (check -// ExitCode after). +// Read returns terminal output; io.EOF means ExitCode is ready. func (p *Pty) Read(b []byte) (int, error) { return p.out.Read(b) } @@ -61,21 +61,21 @@ func (p *Pty) Resize(ctx context.Context, cols, rows uint16) error { // Close ends the pty session (silkd sees the disconnect and kills the shell). func (p *Pty) Close() error { - p.stop() + p.closeOnce.Do(func() { + p.stop() + _ = p.out.Close() + }) return nil } -// ExitCode reports the shell's exit code once Read has returned io.EOF; ok is -// false while the shell is still running. +// ExitCode reports the shell's exit code after Read returns io.EOF. func (p *Pty) ExitCode() (code int, ok bool) { p.mu.Lock() defer p.mu.Unlock() return p.exitCode, p.exited } -// drain relays response frames: output into the pipe, the exit code into -// state, a terminal error to unblock Read; an OpenPty-ctx cancel surfaces as -// ctx.Err(), matching Watcher and PortConn. +// drain relays response frames into the output pipe and terminal state. func (p *Pty) drain(ctx context.Context, pw *io.PipeWriter) { for { resp, err := recv(ctx, p.conn) @@ -105,8 +105,7 @@ func (p *Pty) drain(ctx context.Context, pw *io.PipeWriter) { } } -// OpenPty starts a shell under a pty. The ctx governs the pty's lifetime: -// canceling it (or calling Close) tears the session down. +// OpenPty starts a shell whose lifetime is governed by ctx or Close. func (s *Sandbox) OpenPty(ctx context.Context, opts PtyOpts) (*Pty, error) { req := &wire.PtyOpen{Cols: opts.Cols, Rows: opts.Rows, Cwd: opts.Cwd, Env: opts.Env, User: opts.User} conn, done, err := s.call(ctx, req) diff --git a/sdk/go/pty_test.go b/sdk/go/pty_test.go index 389b6fe6..b65e19b2 100644 --- a/sdk/go/pty_test.go +++ b/sdk/go/pty_test.go @@ -19,7 +19,6 @@ func TestPtyEchoAndExit(t *testing.T) { } defer pty.Close() - // The fake echoes input; write a line and read it back. if _, err = pty.Write([]byte("ping\n")); err != nil { t.Fatalf("write: %v", err) } @@ -35,7 +34,6 @@ func TestPtyEchoAndExit(t *testing.T) { t.Fatalf("resize: %v", err) } - // "exit" ends the shell → Read returns EOF and ExitCode is set. if _, err := pty.Write([]byte("exit\n")); err != nil { t.Fatalf("write exit: %v", err) } @@ -48,8 +46,6 @@ func TestPtyEchoAndExit(t *testing.T) { } } -// TestPtyCtxCancelIsTyped: canceling the OpenPty ctx must surface as a typed -// context error through Read, matching Watcher and PortConn. func TestPtyCtxCancelIsTyped(t *testing.T) { sb := fakeSandbox(t) ctx, cancel := context.WithCancel(t.Context()) @@ -65,3 +61,30 @@ func TestPtyCtxCancelIsTyped(t *testing.T) { t.Errorf("Read after ctx cancel = %v, want context.Canceled", err) } } + +func TestPtyCloseUnblocksUnreadOutput(t *testing.T) { + sb := fakeSandbox(t) + pty, err := sb.OpenPty(t.Context(), PtyOpts{Cols: 80, Rows: 24}) + if err != nil { + t.Fatalf("OpenPty: %v", err) + } + if _, err = pty.Write([]byte("unread\n")); err != nil { + t.Fatalf("Write: %v", err) + } + stops := 0 + stop := pty.stop + pty.stop = func() { stops++; stop() } + + if err = pty.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if err = pty.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } + if _, err = pty.Read(make([]byte, 1)); !errors.Is(err, io.ErrClosedPipe) { + t.Errorf("Read after Close = %v, want io.ErrClosedPipe", err) + } + if stops != 1 { + t.Errorf("stop called %d times, want 1", stops) + } +} diff --git a/sdk/langchain/pyproject.toml b/sdk/langchain/pyproject.toml index cec2e448..852d47bb 100644 --- a/sdk/langchain/pyproject.toml +++ b/sdk/langchain/pyproject.toml @@ -9,7 +9,7 @@ description = "LangChain tools backed by cocoon microVM sandboxes" requires-python = ">=3.10" license = { text = "Apache-2.0" } readme = "README.md" -dependencies = ["langchain-core>=1.0", "pydantic>=2", "cocoonstack-sandbox>=0.1"] +dependencies = ["langchain-core>=1.0", "pydantic>=2", "cocoonstack-sandbox>=0.1.5"] [tool.setuptools.packages.find] include = ["cocoonsandbox_langchain*"] diff --git a/sdk/openai/pyproject.toml b/sdk/openai/pyproject.toml index b39dfb6b..569062f3 100644 --- a/sdk/openai/pyproject.toml +++ b/sdk/openai/pyproject.toml @@ -9,7 +9,7 @@ description = "OpenAI Agents SDK sandbox provider backed by cocoon microVMs" requires-python = ">=3.10" license = { text = "Apache-2.0" } readme = "README.md" -dependencies = ["openai-agents>=0.17", "cocoonstack-sandbox>=0.1"] +dependencies = ["openai-agents>=0.17", "cocoonstack-sandbox>=0.1.5"] [tool.setuptools.packages.find] include = ["cocoonsandbox_openai*"] diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 1cc017cb..4a187622 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -28,7 +28,7 @@ def __init__(self, addr: str, api_token: str = "", timeout: float = 120.0): self.api_token = api_token self.timeout = timeout - def new(self, template: str, net: str = "", size: str = "", ttl_seconds: int = 0, + def new(self, template: str, net: str = "", size: str = "", ttl_seconds: int = 0, claim_ref: str = "", volumes: list[str | Mapping[str, str]] | None = None, mount: bool = True) -> Sandbox: """Claims a sandbox; a warm hit is milliseconds. On a cluster a warm miss may redirect to a peer, followed transparently; if every @@ -37,7 +37,7 @@ def new(self, template: str, net: str = "", size: str = "", ttl_seconds: int = 0 volumes without mounting them, leaving that — and the flush — to the workload: releasing without a clean self-umount discards unsynced guest pages, since the sandbox performs no sync.""" - claim = _claim_body(template, net, size, ttl_seconds, volumes, mount) + claim = _claim_body(template, net, size, ttl_seconds, volumes, mount, claim_ref) return self._claim_from(self.addr, claim) def delete_template(self, template: str, net: str = "", size: str = "") -> None: @@ -70,6 +70,26 @@ def probe(addr: str) -> Sandbox: except APIError: raise APIError("lookup", 404, f"no owner found for {id}") from None + def attach(self, owner_addr: str, id: str, token: str) -> Sandbox: + """Binds a handle to an already-claimed sandbox whose owner address is + known (an apiserver annotation, say), with no lookup round-trip.""" + return Sandbox(client=self, id=id, token=token, owner=owner_addr) + + def sandboxes(self) -> list[dict]: + """Lists the claims this token may see: id, key, deadline, claim_ref — + never tokens or host paths.""" + reply = self._request(self.addr, "GET", "/v1/sandboxes", None, "list sandboxes") + return [dict(sb) for sb in reply.get("sandboxes") or []] + + def drain(self) -> dict: + """Cordons the node (root token): new claims are refused, live ones run + to their leases.""" + return self._request(self.addr, "POST", "/v1/drain", None, "drain") + + def uncordon(self) -> dict: + """Lifts a drain on the node (root token).""" + return self._request(self.addr, "DELETE", "/v1/drain", None, "uncordon") + def checkpoint(self, id: str) -> Checkpoint: """A handle for a known checkpoint id, bound to the entry node — no listing round-trip; an unknown id surfaces as 404 at claim time.""" @@ -166,7 +186,8 @@ def _request(self, addr: str, method: str, path: str, body, verb: str, bearer: s def _claim_body(template: str, net: str, size: str, ttl_seconds: int, - volumes: list[str | Mapping[str, str]] | None = None, mount: bool = True) -> dict: + volumes: list[str | Mapping[str, str]] | None = None, mount: bool = True, + claim_ref: str = "") -> dict: claim = {"template": template} if net: claim["net"] = net @@ -178,6 +199,8 @@ def _claim_body(template: str, net: str, size: str, ttl_seconds: int, claim["volumes"] = [_volume_body(volume, mount) for volume in volumes] if not mount: claim["volumes_attach_only"] = True + if claim_ref: + claim["claim_ref"] = claim_ref return claim diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index 3a3ac371..4b8f16eb 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -29,12 +29,14 @@ def __init__(self, kind: str, message: str): class ExitError(SandboxError): - """A command exited non-zero; carries the exit code and stderr.""" + """A command exited non-zero; carries the exit code, stderr, and whatever + stdout it had produced — a failing build's log is on stdout.""" - def __init__(self, code: int, stderr: str): + def __init__(self, code: int, stderr: str, stdout: str = ""): super().__init__(f"exit status {code}: {stderr.strip()}") self.code = code self.stderr = stderr + self.stdout = stdout class ProtocolError(SandboxError): diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 3f43a884..dcb2b23d 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -54,7 +54,7 @@ def exec(self, *argv: str, cwd: str = "", env: dict | None = None, code = self.run(list(argv), cwd=cwd, env=env, user=user, session=session, stdin=stdin, on_stdout=out.extend, on_stderr=err.extend) if code != 0: - raise ExitError(code, err.decode(errors="replace")) + raise ExitError(code, err.decode(errors="replace"), out.decode(errors="replace")) return out.decode(errors="replace") def run(self, argv: list[str], cwd: str = "", env: dict | None = None, @@ -66,13 +66,12 @@ def run(self, argv: list[str], cwd: str = "", env: dict | None = None, with self._dial() as conn: conn.send("exec", argv=argv, cwd=cwd or None, env=env, user=user or None, session=session or None) - # The guest blocks writing output once its stdout buffer fills, and - # stops draining stdin while it does, so feeding stdin to completion - # before reading deadlocks on any payload past the socket buffers. + # The guest stops draining stdin while blocked writing stdout, so + # feeding it to completion before reading deadlocks. pump = threading.Thread(target=_feed_stdin, args=(conn, stdin), daemon=True) pump.start() code = _pump_stdio(conn, on_stdout, on_stderr) - pump.join() + pump.join() # the closed conn fails a stalled send, so this cannot hang if code is None: raise ProtocolError("exec stream ended without an exit frame") return code @@ -387,14 +386,17 @@ class Watcher(_Closeable): def __init__(self, conn: Conn): self._conn = conn + self.error: Exception | None = None def __iter__(self) -> Iterator[dict]: # Connection-bound: a close, drop, or undecodable frame ends iteration; - # a real server error frame (SilkdError) propagates. + # a real server error frame (SilkdError) propagates. error tells a + # clean close (None) from a relay that dropped mid-stream. while True: try: frame = self._conn.recv() - except (ProtocolError, OSError, ValueError): + except (ProtocolError, OSError, ValueError) as e: + self.error = e return if frame["type"] == "event": yield frame @@ -410,11 +412,14 @@ def __init__(self, sandbox: Sandbox, conn: Conn, pid: int): self._sandbox = sandbox self._conn = conn self.pid = pid + self.exit_code: int | None = None def read(self) -> bytes: - """The next output chunk; b'' once the shell exits.""" + """The next output chunk; b'' once the shell exits, after which + exit_code holds the shell's status.""" frame = self._conn.recv() if frame["type"] == "exit": + self.exit_code = frame.get("code") return b"" return frame.get("data") or b"" diff --git a/silkd/Dockerfile b/silkd/Dockerfile index 55491067..0af3c7c9 100644 --- a/silkd/Dockerfile +++ b/silkd/Dockerfile @@ -7,7 +7,7 @@ # /silkd-static is musl-static for guests without glibc, such as the android # flavor's non-systemd userspace. Built per-platform (amd64/arm64) on native # runners; TARGETARCH picks the musl target triple. -FROM rust:1-slim AS build +FROM rust:1-slim@sha256:8e8cf8f7fd54a2d23d5a743b3a03f56e26b6c774276c33fa0595111704ebb15c AS build ARG TARGETARCH RUN case "$TARGETARCH" in \ amd64) echo x86_64-unknown-linux-musl > /musl-target ;; \ diff --git a/silkd/rust-toolchain.toml b/silkd/rust-toolchain.toml index e3dcc8f6..261677a0 100644 --- a/silkd/rust-toolchain.toml +++ b/silkd/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "stable" +channel = "1.97.1" components = ["rustfmt", "clippy"] targets = ["x86_64-unknown-linux-musl"] profile = "minimal" diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index 353126d3..e5b120f2 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -1,5 +1,5 @@ //! `fs.watch`: stream filesystem events under a path until the client -//! disconnects. Watch is the one connection-bound verb — an event feed has no +//! disconnects. Like every connection-bound verb, an event feed has no //! meaningful detached state, so it lives only as long as its connection. use notify::{RecursiveMode, Watcher}; @@ -8,15 +8,9 @@ use tokio::sync::mpsc; use crate::proto::{self, ErrorKind, EventKind, Response}; -/// Watches `path` (recursively when set): a `ready` frame once the watch is -/// armed (events after it are guaranteed captured), then `event` frames until -/// the client disconnects or the watcher dies. The blocking notify watcher -/// runs on its own thread and forwards frames into an async channel; the -/// client half is polled concurrently so a disconnect ends the watch even -/// when no event is pending (otherwise an abandoned quiet watch would leak -/// the task and the notify thread). A watcher error (inotify overflow, -/// watcher death) arrives as an `error` frame, which is terminal. Dropping -/// the watcher on return stops it. +const OVERFLOW_MESSAGE: &str = "watch event queue overflow"; + +/// Watches `path`, writing `ready`, ordered events, or a terminal error until disconnect. pub async fn watch( reader: &mut R, w: &mut W, @@ -28,12 +22,9 @@ where W: AsyncWrite + Unpin, { let (tx, mut rx) = mpsc::channel::(256); + let mut tx = Some(tx); let mut watcher = match notify::recommended_watcher(move |res| { - for frame in to_frames(res) { - // Best-effort: a full channel means the client is slower than the - // filesystem; drop rather than block the notify thread. - let _ = tx.try_send(frame); - } + forward_frames(&mut tx, to_frames(res)); }) { Ok(watcher) => watcher, Err(e) => return proto::error_frame(w, ErrorKind::Internal, e.to_string()).await, @@ -57,7 +48,8 @@ where return Ok(()); } } - None => return Ok(()), + // Only overflow drops the sender mid-watch; the buffered prefix is already out. + None => return proto::error_frame(w, ErrorKind::Internal, OVERFLOW_MESSAGE).await, }, // The client sends nothing during a watch, so any readable state — // EOF (disconnect), a stray frame, or an error — ends the watch. @@ -66,8 +58,16 @@ where } } -/// Maps one notify result to the frames to stream: `event` frames for a good -/// event, a single terminal `error` frame for a watcher error. +fn forward_frames(tx: &mut Option>, frames: Vec) { + let Some(sender) = tx.as_ref() else { return }; + for frame in frames { + if sender.try_send(frame).is_err() { + *tx = None; + return; + } + } +} + fn to_frames(res: notify::Result) -> Vec { use notify::EventKind as N; let event = match res { @@ -126,4 +126,19 @@ mod tests { other => panic!("expected event frame, got {other:?}"), } } + + #[test] + fn full_channel_drops_sender_after_the_delivered_prefix() { + let (tx, mut rx) = mpsc::channel(1); + let mut tx = Some(tx); + + forward_frames(&mut tx, vec![Response::Ready, Response::Ready]); + + assert!(tx.is_none()); + assert!(matches!(rx.try_recv(), Ok(Response::Ready))); + assert!(matches!( + rx.try_recv(), + Err(mpsc::error::TryRecvError::Disconnected) + )); + } }