diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bc3d6c7..d69e23a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -12,13 +12,10 @@ "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ], - "forwardPorts": [8080, 8765], + "forwardPorts": [8765], "portsAttributes": { - "8080": { - "label": "Pilot client" - }, "8765": { - "label": "Ito server" + "label": "Ito application and Pilot client" } } } diff --git a/CONTEXT.md b/CONTEXT.md index 195efee..9029aab 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -1,233 +1,50 @@ -# Ito +# Ito domain language -Ito is an immersive teleoperation system for people who directly pilot robots. +**Pilot** +The person who perceives through and controls the configured robot. -## Language +**Pilot client** +The WebXR application served by Ito. It connects to the same Ito endpoint in +both deployment modes. -**Pilot**: -The person who perceives through and controls a robot in real time through Ito. -_Avoid_: User, operator +**Ito application / Ito app** +The single Python application that hosts the client, terminates the +pilot-facing protocol, runs reconstruction, and integrates one robot. -**Robot**: -The physical or simulated embodied machine controlled by a pilot. -_Avoid_: Device +**Robot adapter** +The narrow internal boundary through which Ito starts/stops control and moves +pilot input and sensor frames. There is one adapter per Ito application. -**Robot Type**: -A coarse Ito-known embodiment category used to orient the pilot to the robot they are about to inhabit, including likely control style. Current types are Mecha, Android Robot, Droid, Drone, Car, and Plane. -_Avoid_: Hardware model, driver type, capture modality, capability contract +**Local robot adapter** +The default in-process adapter. It receives pilot input and publishes sensor +frames using ordinary calls and memory. -**Control Convention**: -The expected control style for a Robot Type. Robots of the same type should feel similar to pilot, but each robot driver remains responsible for its concrete control mapping and any robot-specific controls. -_Avoid_: Control profile, capability contract, universal mapping +**Remote robot adapter** +The optional Ito-side proxy for one lightweight remote robot driver. -**Mecha**: -A large humanoid robot. -_Avoid_: Android Robot, Droid +**Remote robot driver** +The robot-local fallback component used when Ito runs externally. It forwards +sensors, accepts control targets, and retains robot-local safety guarantees. -**Android Robot**: -A human-sized humanoid robot. Use the full term when ambiguity with the Android operating system is possible. -_Avoid_: Android device, Droid +**Control active / control paused / control stopped** +The direct lifecycle of pilot authority. These are connection and safety states, +not allocated resources. -**Droid**: -A small humanoid robot. -_Avoid_: Android Robot, device +**Pilot Input Snapshot** +A replaceable full-state pose and controller sample. Newest input wins. -**Drone**: -A quadcopter or similar flying robot that can hover. -_Avoid_: Plane +**Robot-local safety** +Timeouts, rate limits, neutralization, emergency stop, and safe resumption that +continue to work without the pilot or external Ito connection. -**Car**: -A robot on three or more wheels. -_Avoid_: Rover +**Reconstruction** +The live 3D representation produced from robot sensors while control is active. +Raw local frames stay onboard and enter reconstruction directly. -**Plane**: -A winged aircraft that relies on forward momentum to stay at altitude. -_Avoid_: Drone +**Splat Batch / Splat Scene** +A binary incremental reconstruction update and the bounded client-owned scene +assembled from those updates. -**Ito Droid**: -Ito's physical reference robot family for validating the complete piloting experience. -_Avoid_: Ito platform, required robot architecture - -**Robot Catalog**: -The server-memory set of robots known to Ito from robot-driver reports and offered to a pilot for selection. -_Avoid_: Persistent registry, static robot inventory - -**Robot Identity**: -The stable machine-readable identity a robot driver reports for catalog and session bookkeeping. In v1 it is not proof of authenticity. -_Avoid_: Robot name, credential, authentication - -**Session Identity**: -The server-generated identity for a piloting session. -_Avoid_: Client session token, WebRTC connection ID - -**Driver Status Report**: -A repeated robot-driver-to-server report that refreshes the server's in-memory Robot Catalog entry for that robot. -_Avoid_: Robot authentication, proof of identity - -**Driver Status Watchdog**: -The server-owned duration after which a robot becomes Unavailable if its driver has not sent fresh status over its control connection. -_Avoid_: Pilot input timeout, session cleanup timeout, control-loss timeout - -**Robot Status**: -Current availability/session state used to present a catalog entry. V1 states are Available, Occupied, and Unavailable. Available and Unavailable come from driver availability plus server connectivity; Occupied is assigned by the Ito Server when it reserves or allocates the robot for a pilot. -_Avoid_: Catalog metadata - -**Occupied Robot**: -A robot that the Ito Server has reserved for an in-progress acquisition or allocated to a piloting session. -_Avoid_: Connected robot, busy robot - -**Robot Driver**: -The robot-side Ito component that connects a robot to Ito, reports robot availability, translates pilot input into robot commands, and forwards sensor feeds needed by Ito. -_Avoid_: Robot app, server driver - -**Ito Protocol**: -The shared message and media contract between the Pilot Client, Ito Server, and Robot Driver. -_Avoid_: ROS API, WebRTC glue, implementation class model - -**Ito Protocol Version**: -The exact Ito Protocol version identifier advertised by Ito programs. V1 requires an exact version match. -_Avoid_: API version, feature negotiation - -**Piloting Session**: -A bounded, exclusive period in which the Ito Server allocates one pilot control authority over one robot. It can survive temporary loss or replacement of its network connection. -_Avoid_: WebRTC session, peer session - -**Session Authority**: -The responsibility for deciding whether a pilot may acquire a robot and for serializing competing acquisition attempts. The Ito Server is the session authority and may reserve a robot during acquisition to prevent races; the robot driver remains the control safety authority. -_Avoid_: Control safety authority, robot driver authority - -**Driver-Terminated Session**: -A piloting session ended by the robot driver because the driver or robot can no longer satisfy the session's required behavior. It is reserved for non-recoverable driver or robot conditions, not ordinary transient control loss. -_Avoid_: Disconnect, control loss - -**Session Termination Reason**: -A driver- or server-provided display value explaining why a session ended. It may be a localization resource key or free text. -_Avoid_: Log line, robot status - -**Display Reason**: -A displayable reason value supplied across Ito IPC, represented as either a localization resource key or free text fallback. -_Avoid_: Exception, log message, internal error - -**Request Timeout**: -The configurable duration after which an Ito WebSocket request expecting a response is treated as failed if no correlated response arrives. -_Avoid_: Pilot input timeout, visual freshness timeout, session cleanup timeout - -**Pilot Input**: -Robot-independent, time-varying data produced by the pilot, including tracked body poses and controller input. -_Avoid_: Robot command, control message - -**Pilot Input Snapshot**: -A complete current-state sample of pilot input, including relevant pose, controller axes, and button state at that moment. It replaces earlier snapshots rather than depending on delivery of every prior message. -_Avoid_: Input delta, command event - -**Pilot Input Rate**: -The frequency at which the Pilot Client sends Pilot Input Snapshots during an active session. V1 defaults to 60 Hz. -_Avoid_: Render frame rate, pilot-input timeout - -**Driver Control Tick**: -One iteration of a robot driver's control loop. The driver uses the newest available Pilot Input Snapshot at each tick and discards older queued snapshots. Ito Droid v1 defaults to 60 Hz. -_Avoid_: Pilot input rate, render frame - -**Pilot Input Timeout**: -The robot-driver-owned duration after which missing fresh pilot input causes control loss. For v1 Ito Droid, this defaults to 2 seconds. -_Avoid_: Visual freshness timeout, network timeout - -**Menu Pause**: -A Pilot Client state in which an in-VR menu is open and robot-directed pilot input is withheld while UI interaction remains active. -_Avoid_: Visual freshness timeout, session end - -**Pilot Frame**: -The piloting-session-relative coordinate frame used to interpret pilot input. For v1 camera-pan control, headset yaw at session start defines zero yaw. -_Avoid_: World frame, headset absolute pose - -**Pilot Client**: -The WebXR application used by the pilot to perceive reconstruction and produce pilot input. -_Avoid_: Viewer, dashboard - -**Robot Command**: -A robot-specific target or action derived from pilot input. -_Avoid_: Pilot input - -**Control Mapping**: -The robot-driver-owned translation from pilot input in the Pilot Frame into robot commands, including robot-specific limits, smoothing, and rate limits. -_Avoid_: Control convention, universal control scheme, raw pose forwarding - -**Sensor Feed**: -A live robot-provided observation stream, such as a camera feed, that Ito consumes for reconstruction or direct view. Producing and configuring the feed is outside Ito's responsibility. -_Avoid_: Ito camera, reconstruction output - -**Mock Robot**: -A test double that behaves like a robot driver in every relevant Ito interaction, including reporting to the Robot Catalog, providing a Sensor Feed from a video file, and accepting pilot input. -_Avoid_: Hidden test mode, replay mode - -**Reconstruction**: -A session-scoped, evolving three-dimensional representation derived from robot sensor input. It gives the pilot a locally responsive viewpoint despite robot-motion and network latency. -_Avoid_: Persistent map, processed view, processor output - -**Reconstruction Algorithm**: -A server-internal method for turning a compatible Capture Modality into Splat Batches for a piloting session. -_Avoid_: Deployed processor, visual processor, client renderer - -**Reconstruction Failure**: -A failure of the session-scoped reconstruction pipeline that prevents Ito from producing fresh Splat Batches for that session. It should stop the affected piloting session without crashing the Ito Server. -_Avoid_: Server crash, robot failure - -**Capture Modality**: -The form of visual observations available for reconstruction, such as monocular RGB, calibrated stereo RGB, or RGB-D. -_Avoid_: Camera model, reconstruction algorithm - -**Splat Scene**: -The client-owned reconstruction presented to the pilot as a rolling collection of Gaussian splats. -_Avoid_: Splat file, point cloud, processor output - -**Splat Batch**: -A unit of newly reconstructed splat data that the client receives, adds to its Splat Scene, and later evicts as a unit. -_Avoid_: Splat file, scene snapshot - -**Splat Budget**: -The maximum number of splats a client retains in its Splat Scene for its performance constraints. -_Avoid_: Frame-rate limit, level of detail - -**Splat Lifetime**: -The maximum age for which a client retains a Splat Batch in its Splat Scene. -_Avoid_: Session lifetime, visual-freshness timeout - -**Direct View**: -An optional pilot view that presents robot camera footage without reconstruction. -_Avoid_: Raw mode, visual processor - -**Visual Freshness**: -The recency of reconstruction output successfully applied by the Pilot Client. In v1 this is measured as time since the last Splat Batch completed the client's normal receive-and-apply path, not as a claim about exactly what geometry is currently in the pilot's view. -_Avoid_: Render frame rate, message latency, scene quality - -**Visual Freshness Timeout**: -The client-side duration after which missing fresh reconstruction output causes the Pilot Client to pause controls and withhold pilot input. V1 defaults this timeout to 2 seconds. -_Avoid_: Splat lifetime, network timeout - -**Control Loss**: -The condition in which sufficiently fresh, valid pilot input is no longer available during active control. It may be recoverable during a recovery period and is distinct from a driver-terminated session. -_Avoid_: Disconnect, timeout - -**Control-Loss Response**: -The robot-specific behavior used to keep a robot safe and stable after control loss, such as braking, hovering, balancing, or descending. -_Avoid_: Safe stop, universal failsafe - -**Control Resumption**: -The transition from control loss back to active control after fresh pilot input resumes. The robot driver owns this transition and must prevent sudden dangerous motion while reconciling differences between pilot pose and robot pose. -_Avoid_: Reconnect, snap back - -**Control Safety Authority**: -The responsibility for deciding how robot control behaves when inputs or conditions are unsafe. The robot driver is primary; the pilot client is secondary by deciding whether to continue sending pilot input; the Ito Server does not make robot-control decisions. -_Avoid_: Server safety policy, central control authority - -**Recovery Period**: -The driver-defined interval after connection loss during which a piloting session remains occupied and can be resumed by the same pilot. -_Avoid_: Connection timeout, control-loss response - -**Session Cleanup Timeout**: -The server-owned duration after which Ito removes stale session state when the responsible endpoint is gone. It is bookkeeping for resource cleanup, not a robot-safety mechanism. -_Avoid_: Control-loss timeout, pilot-input timeout - -**Control Authority**: -The exclusive permission conferred by a piloting session for one pilot's input to affect its robot. It has the same lifecycle as the piloting session and is allocated by the Ito Server. -_Avoid_: Connection, enable flag +Terms intentionally absent from the current product include robot catalog, +robot identity, acquisition, reservation, occupied robot, session authority, +and fleet state. diff --git a/README.md b/README.md index 08ae94f..6edb074 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,64 @@ # Ito -Ito is immersive teleoperation software built entirely for piloting robots. +Ito is immersive teleoperation software for one pilot operating one robot. -Most teleoperation software treats the pilot experience as secondary. It is -usually a basic tool for collecting demonstrations and training robot policies. -Ito takes a different approach: it is not designed to train AI. Its sole purpose -is to make remotely operating a robot comfortable for the human pilot. +The normal deployment is one Python application running on the robot. It hosts +the WebXR pilot client, accepts pilot input, integrates with robot control and +sensors in-process, runs reconstruction, and streams binary scene updates back +to the pilot. -We envision a future where people pilot every type of robot from their home or -office. This could enable disabled people to act through robots in places their -bodies cannot easily take them, and allow people to explore or work in -environments that are hostile to humans. +```text +Pilot WebXR client <── network ──> Ito application on robot + ├── web hosting and WebRTC + ├── local robot adapter + ├── camera and sensor ingress + └── reconstruction +``` -Ito is intended to support humanoids, droids, vehicles, mechas, and robot forms -that do not fit an existing category. It translates the pilot's tracked pose and -controller input into control instructions appropriate to the piloted robot. -In the other direction, it translates the robot's sensor input into a -comfortable immersive 3D reconstruction of its surroundings. +Robots that cannot run Ito or reconstruction onboard use the same application +on an external machine. A lightweight remote driver stays on the robot for +sensor forwarding, actuation, and robot-local safety. The pilot still connects +to one Ito URL and sees the same controls. -## Codebase +## Run + +The default Compose deployment starts one container and serves the client at +`http://localhost:8765/`: + +```bash +docker compose up --build ito +``` + +The default local adapter is an in-process integration seam. Robot-specific +code supplies `ReconstructionFrame` values through +`LocalRobotAdapter.publish_sensor_frame()` and receives pilot snapshots through +its control sink. The adapter owns pilot-input timeout, newest-input rate +limiting, neutral stop, and an emergency-stop latch; robot-specific callbacks +provide the actual actuation and neutralization. No loopback transport is +involved. + +The stock container has no robot-specific hardware binding and therefore +reports the local robot as not ready. A robot image wires the adapter callbacks +and a `ReconstructionProcessor` factory into the same `ItoApplication`; this is +still one process and introduces no local network boundary. -Ito is being reset around the v1 design in `docs/v1.md`. Protocol seams are -documented in `docs/protocol.md`, and architectural decisions are recorded in -`docs/adr/`. Local Docker Compose operation is documented in -`docs/local-v1.md`, and the current v1 acceptance record is in -`docs/acceptance-v1.md`. +For the external-driver fallback: + +```bash +ITO_ROBOT_BACKEND=remote docker compose --profile remote up --build +``` + +Set `ITO_MOCK_ROBOT_CAMERA_VIDEO_HOST` to an H.264 sample file when using the +mock remote driver. + +## Codebase -The main source directories are: +- `server/ito/`: the Ito application and its local/remote robot adapters. +- `server/processors/`: in-application reconstruction algorithms. +- `client/`: the WebXR pilot client served by Ito. +- `drivers/`: lightweight remote-driver implementations for external mode. +- `docs/`: product, protocol, deployment, and architectural decisions. -- `server/`: Ito Server code. -- `server/processors`: 3D reconstruction algorithms, applied. -- `client/`: WebXR Pilot Client code. -- `drivers/`: robot-side drivers and robot reference material. -- `docs/`: design notes, protocol notes, and architectural decisions. +See `docs/v1.md`, `docs/protocol.md`, and `docs/local-v1.md` for the current +design. Multi-user, multi-robot, discovery, allocation, and fleet coordination +are intentionally out of scope. diff --git a/client/README.md b/client/README.md index 3291894..7c7d567 100644 --- a/client/README.md +++ b/client/README.md @@ -1,42 +1,16 @@ -# Pilot Client +# Pilot client -The Pilot Client is the WebXR application used by the pilot to perceive through -and control a robot. +The WebXR client is served by the Ito Python application. Open Ito's root URL, +enter VR, connect to the same-origin `/ws` endpoint, and explicitly start +control. There is no robot browser or allocation step. -The v1 client is a static, plain-JavaScript A-Frame/WebXR application. The -non-VR page only exposes the browser-required Enter VR launch action; catalog, -acquisition, settings, session state, and session end controls are rendered in -VR. +The client sends replaceable Pilot Input Snapshots and receives binary Splat +Batches. It pauses outgoing robot input while its menu is open or reconstruction +is stale. Robot and reconstruction placement is neither visible to the client +nor part of its workflow. -## Run locally +Run tests inside the Dev Container: -From this directory: - -```sh -python -m http.server 8080 -``` - -Then open `http://localhost:8080/`. The client defaults to the Ito Server -control WebSocket at `ws://:8765` and stores runtime settings in -browser Local Storage under `ito.pilotClient.settings.v1`. - -## Tests - -The client uses Node's built-in test runner and has no npm dependencies. - -```sh -npm test +```bash +cd client && npm test ``` - -## Implementation notes - -- WebSocket control-plane messages are MessagePack-encoded Ito envelopes. -- Pilot-facing text is loaded from `resources/en/default.json` and resolved by - resource key before falling back to driver/server free text. -- The Splat Scene is client-owned. `src/splat-scene.js` includes the Spark.JS - adapter seam and v1 Splat Batch binary header parser. Exact Spark insertion - performance still needs Pico 4 validation. -- Pilot Input Snapshots are generated at the configured rate with headset yaw - relative to session start plus current controller state. `src/webrtc.js` - creates non-trickle WebRTC offers for pilot-input and Splat Batch data - channels over the WebSocket control plane. diff --git a/client/resources/en/default.json b/client/resources/en/default.json index eacd9be..99755c6 100644 --- a/client/resources/en/default.json +++ b/client/resources/en/default.json @@ -5,33 +5,30 @@ "webxrUnavailable": "Immersive VR is not available in this browser.", "loading": "Loading..." }, - "catalog": { - "title": "Robot Catalog", - "refresh": "Refresh", - "empty": "No robots are reporting yet.", - "acquire": "Pilot", - "unavailable": "Unavailable" - }, "connection": { "connecting": "Connecting...", "connected": "Connected", - "failed": "Connection failed" + "failed": "Connection failed", + "retry": "Retry" }, - "session": { - "connecting": "Connecting to {{name}}...", - "active": "Piloting {{name}}", - "ending": "Ending session...", - "ended": "Session ended", - "returnToCatalog": "Return to Catalog", + "control": { + "ready": "Ready", + "notReady": "Robot not ready", + "start": "Start control", + "starting": "Starting control...", + "active": "Control active", + "stopping": "Stopping control...", + "stopped": "Control stopped", + "returnToReady": "Return to ready", "visualPaused": "Visual feed lost. Robot controls paused.", "menu": "Menu", "resume": "Resume", - "end": "End Session", + "stop": "Stop control", "settings": "Settings" }, "settings": { "title": "Settings", - "serverUrl": "Server", + "serverUrl": "Ito endpoint", "visualFreshnessTimeoutMs": "Visual timeout", "pilotInputRateHz": "Input rate", "splatBudget": "Splat budget", @@ -39,49 +36,32 @@ "save": "Save", "reset": "Reset" }, - "enum": { - "robotType": { - "Mecha": "Mecha", - "Android Robot": "Android Robot", - "Droid": "Droid", - "Drone": "Drone", - "Car": "Car", - "Plane": "Plane" - }, - "robotStatus": { - "Available": "Available", - "Occupied": "Occupied", - "Unavailable": "Unavailable" - } - }, "reason": { "unknown": "Unknown reason.", "request": { "timeout": "The request timed out." }, "robot": { - "unavailable": "The robot is unavailable.", - "driver_disconnected": "The robot driver disconnected.", - "driver_status_timeout": "The robot stopped reporting status." + "not_ready": "The configured robot is not ready." }, - "session": { - "acquire": { - "robot_unavailable": "That robot is not available." - }, - "ended": { - "requested": "The session was ended.", - "pilot_requested": "The pilot ended the session.", - "endpoint_disappeared": "A required endpoint disconnected.", - "reconstruction_failed": "Reconstruction failed." - }, - "resume_unavailable": "The previous session can no longer be resumed." + "control": { + "stopped": { + "requested": "Control was stopped.", + "pilot_requested": "You stopped control.", + "pilot_disconnected": "The pilot connection was lost.", + "robot_driver_disconnected": "The robot driver disconnected.", + "reconstruction_failed": "Reconstruction failed.", + "transport_failed": "A live transport failed.", + "start_failed": "Control could not start." + } }, "connection": { - "hello_required": "The server rejected the connection handshake.", - "invalid_role": "The server rejected the client role." + "hello_required": "Ito rejected the connection handshake.", + "invalid_role": "Ito rejected the client role.", + "pilot_already_connected": "Another pilot is connected." }, "protocol": { - "invalid_message": "The server rejected an invalid protocol message.", + "invalid_message": "Ito rejected an invalid protocol message.", "version_mismatch": "The Ito protocol versions do not match." } } diff --git a/client/src/app.js b/client/src/app.js index 31eff16..c25a91a 100644 --- a/client/src/app.js +++ b/client/src/app.js @@ -1,10 +1,11 @@ -import { ClientSettingsStore, mergeSessionConfig } from "./config.js"; +import { ClientSettingsStore, mergeControlConfig } from "./config.js"; import { ItoControlClient, DisplayableError } from "./control-client.js"; import { TextResources } from "./i18n.js"; import { DataChannelPilotInputTransport, PilotInputLoop } from "./pilot-input.js"; -import { displayReason, ROBOT_STATUS_AVAILABLE } from "./protocol.js"; +import { displayReason } from "./protocol.js"; import { SparkJsSplatAdapter, SplatSceneOwner } from "./splat-scene.js"; import { VisualFreshnessMonitor } from "./visual-freshness.js"; +import { PilotInputPeer, SplatBatchPeer } from "./webrtc.js"; import { VrUi } from "./vr-ui.js"; export class ItoPilotApp { @@ -19,18 +20,19 @@ export class ItoPilotApp { this.text = new TextResources(); this.ui = null; this.control = null; - this.catalogRobots = []; - this.selectedRobot = null; - this.session = null; + this.endpoint = null; + this.controlActive = false; + this.controlConfig = null; this.menuOpen = false; this.visualPaused = false; this.splatScene = null; this.freshness = null; + this.pilotPeer = null; + this.splatPeer = null; this.pilotInput = new PilotInputLoop({ transport: new DataChannelPilotInputTransport(), rateHz: this.settings.pilotInputRateHz, }); - this.xrReferenceSpace = null; this.controllerMenuHandlersInstalled = false; } @@ -41,7 +43,7 @@ export class ItoPilotApp { this.statusElement.textContent = ""; this.launchButton.addEventListener("click", () => this.enterVr()); this.scene.addEventListener("click", (event) => this.handleClick(event)); - this.scene.addEventListener("enter-vr", () => this.connectAndShowCatalog()); + this.scene.addEventListener("enter-vr", () => this.connectAndShowReady()); if (this.scene.hasLoaded) this.installControllerMenuHandlers(); else this.scene.addEventListener("loaded", () => this.installControllerMenuHandlers(), { once: true }); this.scene.addEventListener("xrframe", (event) => this.onXrFrame(event.detail)); @@ -59,149 +61,117 @@ export class ItoPilotApp { this.scene.enterVR(); } - async connectAndShowCatalog() { + async connectAndShowReady() { try { this.renderStatus(this.text.t("connection.connecting")); + this.control?.close(); this.control = new ItoControlClient({ serverUrl: this.settings.serverUrl, requestTimeoutMs: this.settings.requestTimeoutMs, }); - this.control.addEventListener("sessionended", (event) => this.handleSessionEnded(event.detail)); - await this.control.connect(); - await this.showCatalog(); + this.control.addEventListener("controlstopped", (event) => this.handleControlStopped(event.detail)); + this.control.addEventListener("robotready", (event) => this.handleRobotReady(event.detail)); + this.endpoint = await this.control.connect(); + this.showReady(); } catch (error) { this.renderStatus(this.reasonText(error)); } } - async showCatalog(reason = null) { - this.session = null; - this.selectedRobot = null; - this.menuOpen = false; - this.visualPaused = false; - this.pilotInput.stop(); - this.splatScene?.clear(); - this.splatScene = null; - + showReady(reason = null) { + this.resetControl(); const panel = this.ui.panel({ - title: this.text.t("catalog.title"), + title: this.text.t("control.ready"), subtitle: reason ? this.text.displayReason(reason) : "", }); this.ui.button(panel, { - label: this.text.t("catalog.refresh"), - position: "1.05 0.82 0.02", - action: "catalog.refresh", - width: 0.72, + label: this.endpoint?.robotReady ? this.text.t("control.start") : this.text.t("control.notReady"), + position: "-0.45 -0.22 0.02", + enabled: Boolean(this.endpoint?.robotReady), + action: "control.start", + width: 1.15, }); this.ui.button(panel, { - label: this.text.t("session.settings"), - position: "0.2 0.82 0.02", + label: this.text.t("control.settings"), + position: "0.72 -0.22 0.02", action: "settings.open", - width: 0.72, + width: 0.82, }); - - try { - this.catalogRobots = await this.control.getCatalog(); - this.renderCatalogRows(panel); - } catch (error) { - this.ui.label(panel, this.reasonText(error), "-1.42 0.38 0.02", { color: "#ffd2c9" }); - } } - renderCatalogRows(panel) { - if (this.catalogRobots.length === 0) { - this.ui.label(panel, this.text.t("catalog.empty"), "-1.42 0.32 0.02"); - return; - } - this.catalogRobots.slice(0, 6).forEach((robot, index) => { - const y = 0.46 - index * 0.28; - const type = this.text.enumLabel("robotType", robot.type); - const status = this.text.enumLabel("robotStatus", robot.status); - const detail = robot.availabilityDetail ? ` - ${this.text.displayReason(robot.availabilityDetail)}` : ""; - this.ui.label(panel, `${robot.name} ${type} ${status}${detail}`, "-1.42 " + y + " 0.02", { - width: 2.2, - color: robot.status === ROBOT_STATUS_AVAILABLE ? "#f7fbff" : "#98a7b7", - }); - this.ui.button(panel, { - label: robot.status === ROBOT_STATUS_AVAILABLE ? this.text.t("catalog.acquire") : this.text.t("catalog.unavailable"), - position: `1.02 ${y + 0.02} 0.02`, - enabled: robot.status === ROBOT_STATUS_AVAILABLE, - action: "catalog.acquire", - detail: robot, - width: 0.72, - }); - }); + handleRobotReady(envelope) { + if (!this.endpoint || typeof envelope.payload?.ready !== "boolean") return; + this.endpoint.robotReady = envelope.payload.ready; + if (!this.controlActive) this.showReady(); } - async acquireRobot(robot) { - this.selectedRobot = robot; - const panel = this.ui.panel({ - title: this.text.t("connection.connecting"), - subtitle: this.text.t("session.connecting", { name: robot.name }), - }); - this.ui.label(panel, this.text.t("connection.connecting"), "-1.42 0.35 0.02"); + async startControl() { + this.ui.panel({ title: this.text.t("control.starting") }); try { - const acquisition = await this.control.acquire(robot.robotId); - this.startSession(robot, acquisition); + const result = await this.control.startControl(); + this.controlConfig = mergeControlConfig(this.settings, result.controlConfig || this.endpoint.controlConfig || {}); + this.splatScene = new SplatSceneOwner({ + adapter: new SparkJsSplatAdapter(this.splatRoot), + budget: this.controlConfig.splatBudget, + lifetimeMs: this.controlConfig.splatLifetimeMs, + }); + this.freshness = new VisualFreshnessMonitor({ timeoutMs: this.controlConfig.visualFreshnessTimeoutMs }); + this.freshness.addEventListener("stale", () => this.setVisualPaused(true)); + this.freshness.addEventListener("fresh", () => this.setVisualPaused(false)); + this.pilotPeer = new PilotInputPeer({ + controlClient: this.control, + dataChannelProfile: this.controlConfig.pilotInputDataChannel, + }); + this.splatPeer = new SplatBatchPeer({ + controlClient: this.control, + dataChannelProfile: this.controlConfig.splatBatchDataChannel, + }); + this.splatPeer.addEventListener("splatbatch", (event) => this.receiveSplatBatch(event.detail)); + const [pilotChannel] = await Promise.all([this.pilotPeer.negotiate(), this.splatPeer.negotiate()]); + this.pilotInput.transport.attach(pilotChannel); + this.pilotInput.rateHz = this.controlConfig.pilotInputRateHz; + this.controlActive = true; + this.freshness.markFresh(); + this.pilotInput.start(); + this.renderControl(); } catch (error) { - await this.showCatalog(error.reason); + try { + await this.control.stopControl(displayReason("control.stopped.start_failed")); + } catch {} + this.showReady(error.reason || displayReason("control.stopped.start_failed", error.message)); } } - startSession(robot, acquisition) { - const sessionConfig = mergeSessionConfig(this.settings, acquisition.sessionConfig || {}); - this.session = { - sessionId: acquisition.sessionId, - robotId: acquisition.robotId, - robot, - sessionConfig, - ended: false, - }; - this.splatScene = new SplatSceneOwner({ - adapter: new SparkJsSplatAdapter(this.splatRoot), - budget: sessionConfig.splatBudget, - lifetimeMs: sessionConfig.splatLifetimeMs, - }); - this.freshness = new VisualFreshnessMonitor({ timeoutMs: sessionConfig.visualFreshnessTimeoutMs }); - this.freshness.addEventListener("stale", () => this.setVisualPaused(true)); - this.freshness.addEventListener("fresh", () => this.setVisualPaused(false)); - this.freshness.markFresh(); - this.pilotInput.rateHz = sessionConfig.pilotInputRateHz; - this.pilotInput.start(); - this.renderSession(); - } - receiveSplatBatch(payload, metadata = {}) { - if (!this.session || this.session.ended || !this.splatScene) return null; - if (this.visualPaused) this.freshness?.markFresh(); + if (!this.controlActive || !this.splatScene) return null; const batch = this.splatScene.applySplatBatch(payload, metadata); if (batch) this.freshness?.markFresh(); return batch; } - renderSession() { + renderControl() { const panel = this.ui.panel({ - title: this.text.t("session.active", { name: this.session.robot.name }), - subtitle: this.visualPaused ? this.text.t("session.visualPaused") : this.text.t("connection.connected"), + title: this.text.t("control.active"), + subtitle: this.visualPaused ? this.text.t("control.visualPaused") : this.text.t("connection.connected"), width: 2.4, height: this.menuOpen ? 1.52 : 0.72, position: "0 1.85 -2.8", }); this.ui.button(panel, { - label: this.menuOpen ? this.text.t("session.resume") : this.text.t("session.menu"), + label: this.menuOpen ? this.text.t("control.resume") : this.text.t("control.menu"), position: "-0.48 0.0 0.02", - action: "session.menu.toggle", + action: "control.menu.toggle", width: 0.82, }); if (this.menuOpen) { this.ui.button(panel, { - label: this.text.t("session.end"), + label: this.text.t("control.stop"), position: "0.48 0.0 0.02", - action: "session.end", + action: "control.stop", width: 0.82, }); this.ui.button(panel, { - label: this.text.t("session.settings"), + label: this.text.t("control.settings"), position: "0 -0.34 0.02", action: "settings.open", width: 0.92, @@ -213,86 +183,72 @@ export class ItoPilotApp { this.visualPaused = paused; this.splatScene?.setFrozen(paused); if (paused) this.pilotInput.stop(); - if (!paused && !this.menuOpen && !this.session?.ended) this.pilotInput.start(); - if (this.session && !this.session.ended) this.renderSession(); + if (!paused && !this.menuOpen && this.controlActive) this.pilotInput.start(); + if (this.controlActive) this.renderControl(); } toggleMenu() { - if (!this.session?.sessionId || this.session.ended) return; + if (!this.controlActive) return; this.menuOpen = !this.menuOpen; if (this.menuOpen) this.pilotInput.stop(); if (!this.menuOpen && !this.visualPaused) this.pilotInput.start(); - this.renderSession(); + this.renderControl(); } - async endSession() { - if (!this.session?.sessionId) return; + async stopControl() { + if (!this.controlActive) return; this.pilotInput.stop(); - this.ui.panel({ title: this.text.t("session.ending"), subtitle: this.session.robot.name }); + this.ui.panel({ title: this.text.t("control.stopping") }); try { - await this.control.endSession(this.session.sessionId); + await this.control.stopControl(); } catch (error) { - this.handleSessionEnded({ - payload: { reason: error.reason || displayReason("session.ended.requested"), endedBy: "pilotClient", clean: false }, - sessionId: this.session.sessionId, - }); + this.handleControlStopped({ payload: { reason: error.reason || displayReason("control.stopped.pilot_requested") } }); } } - handleSessionEnded(envelope) { - if (!this.session) return; - this.session.ended = true; - this.pilotInput.stop(); - this.splatScene?.setFrozen(true); - const reason = envelope.payload?.reason || displayReason("session.ended.requested"); + handleControlStopped(envelope) { + const reason = envelope.payload?.reason || displayReason("control.stopped.requested"); + this.resetControl(); const panel = this.ui.panel({ - title: this.text.t("session.ended"), + title: this.text.t("control.stopped"), subtitle: this.text.displayReason(reason), width: 2.8, height: 1.35, position: "0 1.65 -2.3", }); this.ui.button(panel, { - label: this.text.t("session.returnToCatalog"), + label: this.text.t("control.returnToReady"), position: "0 -0.28 0.02", - action: "session.returnCatalog", + action: "control.ready", width: 1.35, }); } + resetControl() { + this.controlActive = false; + this.menuOpen = false; + this.visualPaused = false; + this.pilotInput?.stop(); + this.pilotPeer?.close(); + this.splatPeer?.close(); + this.pilotPeer = null; + this.splatPeer = null; + this.splatScene?.clear(); + this.splatScene = null; + this.freshness = null; + } + showSettings() { - const panel = this.ui.panel({ - title: this.text.t("settings.title"), - subtitle: this.settings.serverUrl, - width: 3.2, - height: 2.25, - }); - const rows = [ - ["visualFreshnessTimeoutMs", 250], - ["pilotInputRateHz", 5], - ["splatBudget", 20], - ["splatLifetimeMs", 5000], - ]; + const panel = this.ui.panel({ title: this.text.t("settings.title"), subtitle: this.settings.serverUrl, width: 3.2, height: 2.25 }); + const rows = [["visualFreshnessTimeoutMs", 250], ["pilotInputRateHz", 5], ["splatBudget", 20], ["splatLifetimeMs", 5000]]; rows.forEach(([key, step], index) => { const y = 0.48 - index * 0.3; - this.ui.label(panel, `${this.text.t(`settings.${key}`)}: ${this.settings[key]}`, "-1.42 " + y + " 0.02", { - width: 1.7, - }); + this.ui.label(panel, `${this.text.t(`settings.${key}`)}: ${this.settings[key]}`, `-1.42 ${y} 0.02`, { width: 1.7 }); this.ui.button(panel, { label: "-", position: `0.52 ${y + 0.02} 0.02`, action: "settings.adjust", detail: { key, delta: -step }, width: 0.22 }); this.ui.button(panel, { label: "+", position: `0.84 ${y + 0.02} 0.02`, action: "settings.adjust", detail: { key, delta: step }, width: 0.22 }); }); - this.ui.button(panel, { - label: this.text.t("settings.save"), - position: "-0.42 -0.82 0.02", - action: "settings.save", - width: 0.72, - }); - this.ui.button(panel, { - label: this.session ? this.text.t("session.resume") : this.text.t("session.returnToCatalog"), - position: "0.48 -0.82 0.02", - action: "settings.close", - width: 1.05, - }); + this.ui.button(panel, { label: this.text.t("settings.save"), position: "-0.42 -0.82 0.02", action: "settings.save", width: 0.72 }); + this.ui.button(panel, { label: this.text.t("control.resume"), position: "0.48 -0.82 0.02", action: "settings.close", width: 1.05 }); } handleClick(event) { @@ -300,11 +256,11 @@ export class ItoPilotApp { const action = target?.getAttribute("data-action"); if (!action) return; const detail = target.itoActionDetail; - if (action === "catalog.refresh") this.showCatalog(); - if (action === "catalog.acquire") this.acquireRobot(detail); - if (action === "session.menu.toggle") this.toggleMenu(); - if (action === "session.end") this.endSession(); - if (action === "session.returnCatalog") this.showCatalog(); + if (action === "connection.retry") this.connectAndShowReady(); + if (action === "control.start") this.startControl(); + if (action === "control.menu.toggle") this.toggleMenu(); + if (action === "control.stop") this.stopControl(); + if (action === "control.ready") this.showReady(); if (action === "settings.open") this.showSettings(); if (action === "settings.adjust") { this.settings[detail.key] += detail.delta; @@ -313,24 +269,23 @@ export class ItoPilotApp { } if (action === "settings.save") { this.settings = this.settingsStore.save(this.settings); - if (this.session?.sessionConfig && this.splatScene) { - this.session.sessionConfig = mergeSessionConfig(this.settings, this.session.sessionConfig); - this.splatScene.setLimits(this.session.sessionConfig); + if (this.controlConfig && this.splatScene) { + this.controlConfig = mergeControlConfig(this.settings, this.controlConfig); + this.splatScene.setLimits(this.controlConfig); } this.showSettings(); } if (action === "settings.close") { - if (this.session && !this.session.ended) this.renderSession(); - else this.showCatalog(); + if (this.controlActive) this.renderControl(); + else this.showReady(); } } onXrFrame({ frame, referenceSpace }) { - if (!frame || !referenceSpace || !this.session || this.session.ended) return; - this.xrReferenceSpace = referenceSpace; + if (!frame || !referenceSpace || !this.controlActive) return; this.freshness?.tick(); this.splatScene?.evict(); - this.pilotInput.maybeSend(frame, referenceSpace, this.session.sessionId); + this.pilotInput.maybeSend(frame, referenceSpace); } installControllerMenuHandlers() { @@ -358,12 +313,7 @@ export class ItoPilotApp { renderStatus(message) { const panel = this.ui.panel({ title: message || this.text.t("app.title"), height: 1.0 }); - this.ui.button(panel, { - label: this.text.t("catalog.refresh"), - position: "0 -0.2 0.02", - action: "catalog.refresh", - width: 0.85, - }); + this.ui.button(panel, { label: this.text.t("connection.retry"), position: "0 -0.2 0.02", action: "connection.retry", width: 0.85 }); } reasonText(error) { diff --git a/client/src/config.js b/client/src/config.js index 0eb4ab9..aeab6d1 100644 --- a/client/src/config.js +++ b/client/src/config.js @@ -56,17 +56,17 @@ export function normalizeSettings(settings) { return normalized; } -export function mergeSessionConfig(settings, sessionConfig = {}) { +export function mergeControlConfig(settings, controlConfig = {}) { return { - ...sessionConfig, + ...controlConfig, pilotInputRateHz: clampInteger( - sessionConfig.pilotInputRateHz ?? settings.pilotInputRateHz, + controlConfig.pilotInputRateHz ?? settings.pilotInputRateHz, settings.pilotInputRateHz, SETTING_LIMITS.pilotInputRateHz[0], SETTING_LIMITS.pilotInputRateHz[1], ), visualFreshnessTimeoutMs: clampInteger( - sessionConfig.visualFreshnessTimeoutMs ?? settings.visualFreshnessTimeoutMs, + controlConfig.visualFreshnessTimeoutMs ?? settings.visualFreshnessTimeoutMs, settings.visualFreshnessTimeoutMs, SETTING_LIMITS.visualFreshnessTimeoutMs[0], SETTING_LIMITS.visualFreshnessTimeoutMs[1], @@ -84,7 +84,7 @@ function clampInteger(value, fallback, minimum, maximum) { function defaultServerUrl() { const location = globalThis.location; - if (!location?.host) return "ws://localhost:8765"; + if (!location?.host) return "ws://localhost:8765/ws"; const scheme = location.protocol === "https:" ? "wss:" : "ws:"; - return `${scheme}//${location.hostname}:8765`; + return `${scheme}//${location.host}/ws`; } diff --git a/client/src/control-client.js b/client/src/control-client.js index 6fd5faf..6e61665 100644 --- a/client/src/control-client.js +++ b/client/src/control-client.js @@ -9,11 +9,10 @@ import { } from "./protocol.js"; export class ItoControlClient extends EventTarget { - constructor({ serverUrl, requestTimeoutMs = 5000, sessionId = null, WebSocketImpl = globalThis.WebSocket }) { + constructor({ serverUrl, requestTimeoutMs = 5000, WebSocketImpl = globalThis.WebSocket }) { super(); this.serverUrl = serverUrl; this.requestTimeoutMs = requestTimeoutMs; - this.sessionId = sessionId; this.WebSocketImpl = WebSocketImpl; this.websocket = null; this.pending = new Map(); @@ -32,9 +31,11 @@ export class ItoControlClient extends EventTarget { this.websocket.addEventListener("error", reject, { once: true }); }); - const payload = { role: ROLE_PILOT_CLIENT }; - if (this.sessionId) payload.sessionId = this.sessionId; - const hello = await this.request(MESSAGE_TYPES.CONNECTION_HELLO, payload, MESSAGE_TYPES.CONNECTION_HELLO_RESULT); + const hello = await this.request( + MESSAGE_TYPES.CONNECTION_HELLO, + { role: ROLE_PILOT_CLIENT }, + MESSAGE_TYPES.CONNECTION_HELLO_RESULT, + ); if (!hello.ok) throw new DisplayableError(resultReason(hello)); return hello.value; } @@ -49,44 +50,28 @@ export class ItoControlClient extends EventTarget { this.pending.clear(); } - async getCatalog() { - const result = await this.request( - MESSAGE_TYPES.CATALOG_GET, - { includeUnavailable: true }, - MESSAGE_TYPES.CATALOG_GET_RESULT, - ); - if (!result.ok) throw new DisplayableError(resultReason(result)); - return result.value.robots || []; - } - - async acquire(robotId) { + async startControl() { const result = await this.request( - MESSAGE_TYPES.SESSION_ACQUIRE, - { robotId }, - MESSAGE_TYPES.SESSION_ACQUIRE_RESULT, - { robotId }, + MESSAGE_TYPES.CONTROL_START, + {}, + MESSAGE_TYPES.CONTROL_START_RESULT, ); if (!result.ok) throw new DisplayableError(resultReason(result)); - this.sessionId = result.value.sessionId; return result.value; } - async endSession(sessionId, reason = displayReason("session.ended.pilot_requested"), clean = true) { + async stopControl(reason = displayReason("control.stopped.pilot_requested")) { const result = await this.request( - MESSAGE_TYPES.SESSION_END, - { reason, clean }, - MESSAGE_TYPES.SESSION_END_RESULT, - { sessionId }, + MESSAGE_TYPES.CONTROL_STOP, + { reason }, + MESSAGE_TYPES.CONTROL_STOP_RESULT, ); if (!result.ok) throw new DisplayableError(resultReason(result)); return result.value; } - request(type, payload, expectedType, options = {}) { - const envelope = makeEnvelope(type, payload, { - robotId: options.robotId, - sessionId: options.sessionId, - }); + request(type, payload, expectedType) { + const envelope = makeEnvelope(type, payload); return new Promise((resolve, reject) => { const timeoutId = setTimeout(() => { this.pending.delete(envelope.messageId); @@ -125,9 +110,13 @@ export class ItoControlClient extends EventTarget { return; } - if (envelope.type === MESSAGE_TYPES.SESSION_ENDED) { - this.sessionId = null; - this.dispatchEvent(new CustomEvent("sessionended", { detail: envelope })); + if (envelope.type === MESSAGE_TYPES.CONTROL_STOPPED) { + this.dispatchEvent(new CustomEvent("controlstopped", { detail: envelope })); + return; + } + + if (envelope.type === MESSAGE_TYPES.ROBOT_READY) { + this.dispatchEvent(new CustomEvent("robotready", { detail: envelope })); return; } diff --git a/client/src/pilot-input.js b/client/src/pilot-input.js index c7dd114..cf72cdd 100644 --- a/client/src/pilot-input.js +++ b/client/src/pilot-input.js @@ -22,26 +22,25 @@ export class PilotInputLoop { this.enabled = false; } - maybeSend(frame, referenceSpace, sessionId) { + maybeSend(frame, referenceSpace) { if (!this.enabled || !this.transport?.canSend()) return null; const now = this.now(); const intervalMs = 1000 / this.rateHz; if (now - this.lastSentAt < intervalMs) return null; const pose = frame.getViewerPose(referenceSpace); if (!pose) return null; - const snapshot = this.createSnapshot(pose, frame.session.inputSources, sessionId, now); + const snapshot = this.createSnapshot(pose, frame.session.inputSources, now); this.transport.sendSnapshot(snapshot); this.lastSentAt = now; return snapshot; } - createSnapshot(viewerPose, inputSources, sessionId, timestampMs = this.now()) { + createSnapshot(viewerPose, inputSources, timestampMs = this.now()) { const absoluteYaw = yawFromViewerPose(viewerPose); if (this.zeroYawRad === null) this.zeroYawRad = absoluteYaw; const headsetYawRad = normalizeRadians(absoluteYaw - this.zeroYawRad); return { protocolVersion: "ito.v1", - sessionId, sequence: ++this.sequence, timestampMs: Math.round(timestampMs), headsetYawRad, diff --git a/client/src/protocol.js b/client/src/protocol.js index a7dc193..f5529f1 100644 --- a/client/src/protocol.js +++ b/client/src/protocol.js @@ -3,28 +3,22 @@ import { decodeMessagePack, encodeMessagePack } from "./msgpack.js"; export const PROTOCOL_VERSION = "ito.v1"; export const MESSAGE_TYPES = Object.freeze({ - CATALOG_GET: "catalog.get", - CATALOG_GET_RESULT: "catalog.get.result", CONNECTION_HELLO: "connection.hello", CONNECTION_HELLO_RESULT: "connection.hello.result", - SESSION_ACQUIRE: "session.acquire", - SESSION_ACQUIRE_RESULT: "session.acquire.result", - SESSION_END: "session.end", - SESSION_END_RESULT: "session.end.result", - SESSION_ENDED: "session.ended", + CONTROL_START: "control.start", + CONTROL_START_RESULT: "control.start.result", + CONTROL_STOP: "control.stop", + CONTROL_STOP_RESULT: "control.stop.result", + CONTROL_STOPPED: "control.stopped", + ROBOT_READY: "robot.ready", WEBRTC_OFFER: "webrtc.offer", WEBRTC_ANSWER: "webrtc.answer", }); export const ROLE_PILOT_CLIENT = "pilotClient"; -export const ROBOT_STATUS_AVAILABLE = "Available"; -export const ROBOT_STATUS_OCCUPIED = "Occupied"; -export const ROBOT_STATUS_UNAVAILABLE = "Unavailable"; export function makeMessageId() { - if (globalThis.crypto?.randomUUID) { - return globalThis.crypto.randomUUID(); - } + if (globalThis.crypto?.randomUUID) return globalThis.crypto.randomUUID(); return `client-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; } @@ -36,8 +30,6 @@ export function makeEnvelope(type, payload = {}, options = {}) { payload, }; if (options.replyToMessageId) envelope.replyToMessageId = options.replyToMessageId; - if (options.robotId) envelope.robotId = options.robotId; - if (options.sessionId) envelope.sessionId = options.sessionId; validateEnvelope(envelope); return envelope; } @@ -72,11 +64,8 @@ export function validateEnvelope(envelope) { if (envelope.replyToMessageId !== undefined && typeof envelope.replyToMessageId !== "string") { throw new Error("replyToMessageId must be a string"); } - if (envelope.robotId !== undefined && typeof envelope.robotId !== "string") { - throw new Error("robotId must be a string"); - } - if (envelope.sessionId !== undefined && typeof envelope.sessionId !== "string") { - throw new Error("sessionId must be a string"); + if ("robotId" in envelope || "sessionId" in envelope) { + throw new Error("robot and session identifiers aren't part of this protocol"); } } diff --git a/client/src/webrtc.js b/client/src/webrtc.js index d62b0d2..0a3732b 100644 --- a/client/src/webrtc.js +++ b/client/src/webrtc.js @@ -31,10 +31,8 @@ export async function applyNonTrickleAnswer(peerConnection, sdp) { } export class PilotInputPeer { - constructor({ controlClient, sessionId, robotId, dataChannelProfile = {}, RTCPeerConnectionImpl = globalThis.RTCPeerConnection }) { + constructor({ controlClient, dataChannelProfile = {}, RTCPeerConnectionImpl = globalThis.RTCPeerConnection }) { this.controlClient = controlClient; - this.sessionId = sessionId; - this.robotId = robotId; this.peerConnection = new RTCPeerConnectionImpl({ iceServers: [] }); this.dataChannel = this.peerConnection.createDataChannel("ito.pilotInput", dataChannelProfile); } @@ -45,7 +43,6 @@ export class PilotInputPeer { MESSAGE_TYPES.WEBRTC_OFFER, { path: LIVE_PATHS.PILOT_INPUT, sdp }, MESSAGE_TYPES.WEBRTC_ANSWER, - { robotId: this.robotId, sessionId: this.sessionId }, ); await applyNonTrickleAnswer(this.peerConnection, result.sdp); return this.dataChannel; @@ -58,10 +55,9 @@ export class PilotInputPeer { } export class SplatBatchPeer extends EventTarget { - constructor({ controlClient, sessionId, dataChannelProfile = {}, RTCPeerConnectionImpl = globalThis.RTCPeerConnection }) { + constructor({ controlClient, dataChannelProfile = {}, RTCPeerConnectionImpl = globalThis.RTCPeerConnection }) { super(); this.controlClient = controlClient; - this.sessionId = sessionId; this.peerConnection = new RTCPeerConnectionImpl({ iceServers: [] }); this.dataChannelProfile = dataChannelProfile; } @@ -73,7 +69,6 @@ export class SplatBatchPeer extends EventTarget { MESSAGE_TYPES.WEBRTC_OFFER, { path: LIVE_PATHS.SPLAT_BATCHES, sdp }, MESSAGE_TYPES.WEBRTC_ANSWER, - { sessionId: this.sessionId }, ); await applyNonTrickleAnswer(this.peerConnection, result.sdp); } diff --git a/client/tests/app.test.js b/client/tests/app.test.js index 0be9a45..d0b34f0 100644 --- a/client/tests/app.test.js +++ b/client/tests/app.test.js @@ -3,6 +3,39 @@ import test from "node:test"; import { ItoPilotApp } from "../src/app.js"; +test("ready screen offers direct control without a robot catalog", () => { + const buttons = []; + const app = { + endpoint: { robotReady: true }, + ui: { + panel(options) { + assert.equal(options.title, "Ready"); + return {}; + }, + button(_panel, options) { + buttons.push(options); + }, + }, + text: { + t(key) { + return { + "control.ready": "Ready", + "control.start": "Start control", + "control.settings": "Settings", + }[key] || key; + }, + displayReason() { + return ""; + }, + }, + resetControl() {}, + }; + + ItoPilotApp.prototype.showReady.call(app); + + assert.deepEqual(buttons.map((button) => button.action), ["control.start", "settings.open"]); +}); + test("enterVr enters immediately when A-Frame has already loaded", async () => { let entered = 0; let listenerAdded = false; diff --git a/client/tests/config.test.js b/client/tests/config.test.js index e75fd94..2e71205 100644 --- a/client/tests/config.test.js +++ b/client/tests/config.test.js @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { ClientSettingsStore, DEFAULT_SETTINGS, mergeSessionConfig, normalizeSettings } from "../src/config.js"; +import { ClientSettingsStore, DEFAULT_SETTINGS, mergeControlConfig, normalizeSettings } from "../src/config.js"; class MemoryStorage { constructor() { @@ -34,9 +34,9 @@ test("settings persist through local storage and clamp unsafe values", () => { assert.deepEqual(store.load(), saved); }); -test("session config merges server data channel profiles with local client settings", () => { +test("control config merges Ito data channel profiles with local client settings", () => { const settings = normalizeSettings({ ...DEFAULT_SETTINGS, pilotInputRateHz: 30, splatBudget: 25 }); - const merged = mergeSessionConfig(settings, { pilotInputDataChannel: { ordered: false } }); + const merged = mergeControlConfig(settings, { pilotInputDataChannel: { ordered: false } }); assert.equal(merged.pilotInputRateHz, 30); assert.equal(merged.splatBudget, 25); diff --git a/client/tests/control-client.test.js b/client/tests/control-client.test.js new file mode 100644 index 0000000..8e177bb --- /dev/null +++ b/client/tests/control-client.test.js @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { ItoControlClient } from "../src/control-client.js"; +import { makeEnvelope, packEnvelope } from "../src/protocol.js"; + +test("control client starts and stops direct control", async () => { + const requests = []; + const client = new ItoControlClient({ serverUrl: "ws://ito/ws", WebSocketImpl: class {} }); + client.request = async (...args) => { + requests.push(args); + return { ok: true, value: { controlConfig: {} } }; + }; + + await client.startControl(); + await client.stopControl(); + + assert.deepEqual(requests.map(([type]) => type), ["control.start", "control.stop"]); + assert.equal(requests.flat().some((value) => value === "catalog.get" || value === "session.acquire"), false); +}); + +test("control client exposes robot readiness events", () => { + const client = new ItoControlClient({ serverUrl: "ws://ito/ws", WebSocketImpl: class {} }); + let ready = null; + client.addEventListener("robotready", (event) => { + ready = event.detail.payload.ready; + }); + + client.handleMessage( + packEnvelope(makeEnvelope("robot.ready", { ready: true })), + ); + + assert.equal(ready, true); +}); diff --git a/client/tests/i18n.test.js b/client/tests/i18n.test.js index 6d23c69..58aed82 100644 --- a/client/tests/i18n.test.js +++ b/client/tests/i18n.test.js @@ -4,9 +4,9 @@ import test from "node:test"; import { TextResources } from "../src/i18n.js"; test("text resources resolve nested keys and template values", () => { - const text = new TextResources({ session: { active: "Piloting {{name}}" } }); + const text = new TextResources({ control: { active: "Controlling {{name}}" } }); - assert.equal(text.t("session.active", { name: "Dory" }), "Piloting Dory"); + assert.equal(text.t("control.active", { name: "Dory" }), "Controlling Dory"); }); test("display reasons prefer localized resource keys and fall back to free text", () => { diff --git a/client/tests/pilot-input.test.js b/client/tests/pilot-input.test.js index b18e0a2..df688b4 100644 --- a/client/tests/pilot-input.test.js +++ b/client/tests/pilot-input.test.js @@ -3,13 +3,13 @@ import test from "node:test"; import { PilotInputLoop } from "../src/pilot-input.js"; -test("pilot input snapshots use headset yaw relative to session start", () => { +test("pilot input snapshots use headset yaw relative to control start", () => { let yaw = Math.PI / 4; const loop = new PilotInputLoop({ transport: null, now: () => 0 }); - const first = loop.createSnapshot(poseWithYaw(yaw), [], "session-1", 0); + const first = loop.createSnapshot(poseWithYaw(yaw), [], 0); yaw = Math.PI / 2; - const second = loop.createSnapshot(poseWithYaw(yaw), [], "session-1", 16); + const second = loop.createSnapshot(poseWithYaw(yaw), [], 16); assert.equal(first.headsetYawRad, 0); assert.ok(Math.abs(second.headsetYawRad - Math.PI / 4) < 0.000001); @@ -30,7 +30,6 @@ test("pilot input snapshots include controller full state", () => { }, }, ], - "session-1", 0, ); @@ -40,6 +39,7 @@ test("pilot input snapshots include controller full state", () => { buttons: [{ pressed: true, touched: true, value: 1 }], axes: [0.1, -0.2], }); + assert.equal("sessionId" in snapshot, false); }); function poseWithYaw(yaw) { diff --git a/client/tests/protocol.test.js b/client/tests/protocol.test.js index 7f84d41..fc7afd2 100644 --- a/client/tests/protocol.test.js +++ b/client/tests/protocol.test.js @@ -1,27 +1,19 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { decodeMessagePack, encodeMessagePack } from "../src/msgpack.js"; import { MESSAGE_TYPES, makeEnvelope, packEnvelope, unpackEnvelope } from "../src/protocol.js"; -test("MessagePack codec round-trips Ito envelope values", () => { - const value = { - protocolVersion: "ito.v1", - messageId: "message-1", - type: "catalog.get.result", - payload: { - ok: true, - value: { - robots: [{ robotId: "droid-1", status: "Available", unavailable: false, score: 1.5 }], - }, - }, - }; - - assert.deepEqual(decodeMessagePack(encodeMessagePack(value)), value); +test("pilot protocol exposes direct control without catalog or acquisition", () => { + assert.equal(MESSAGE_TYPES.CONTROL_START, "control.start"); + assert.equal(MESSAGE_TYPES.CONTROL_STOP, "control.stop"); + assert.equal(Object.values(MESSAGE_TYPES).some((type) => type.includes("catalog")), false); + assert.equal(Object.values(MESSAGE_TYPES).some((type) => type.includes("acquire")), false); }); -test("Ito envelopes pack and unpack as MessagePack", () => { - const envelope = makeEnvelope(MESSAGE_TYPES.CATALOG_GET, { includeUnavailable: true }, { messageId: "cat-1" }); +test("Ito envelopes have no robot or session identity", () => { + const envelope = makeEnvelope(MESSAGE_TYPES.CONNECTION_HELLO, { role: "pilotClient" }, { messageId: "hello-1" }); assert.deepEqual(unpackEnvelope(packEnvelope(envelope)), envelope); + assert.equal("robotId" in envelope, false); + assert.equal("sessionId" in envelope, false); }); diff --git a/client/tests/webrtc.test.js b/client/tests/webrtc.test.js index 4845065..2f0a322 100644 --- a/client/tests/webrtc.test.js +++ b/client/tests/webrtc.test.js @@ -38,7 +38,6 @@ test("SplatBatchPeer negotiates non-trickle offer over control client", async () }; const peer = new SplatBatchPeer({ controlClient, - sessionId: "session-1", RTCPeerConnectionImpl: FakePeerConnection, }); @@ -48,5 +47,6 @@ test("SplatBatchPeer negotiates non-trickle offer over control client", async () assert.equal(requests[0].payload.path, "splatBatches"); assert.equal(requests[0].payload.sdp, "local offer"); assert.equal(requests[0].expectedType, "webrtc.answer"); + assert.deepEqual(requests[0].options, undefined); assert.deepEqual(peer.peerConnection.remoteDescription, { type: "answer", sdp: "server answer" }); }); diff --git a/compose.yaml b/compose.yaml index 84d2551..08e5b08 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,40 +1,29 @@ services: - ito-server: + ito: build: context: . dockerfile: server/Dockerfile environment: - ITO_SERVER_HOST: 0.0.0.0 - ITO_SERVER_PORT: "8765" + ITO_HOST: 0.0.0.0 + ITO_PORT: "8765" + ITO_ROBOT_BACKEND: ${ITO_ROBOT_BACKEND:-local} ITO_REQUEST_TIMEOUT_MS: "5000" - ITO_DRIVER_STATUS_WATCHDOG_MS: "2000" - ITO_SESSION_CLEANUP_TIMEOUT_MS: "30000" ITO_PILOT_INPUT_ORDERED: "false" ITO_PILOT_INPUT_MAX_RETRANSMITS: "0" ITO_SPLAT_BATCH_ORDERED: "true" ports: - "8765:8765" - pilot-client: - image: nginx:1.27-alpine - depends_on: - - ito-server - volumes: - - ./client:/usr/share/nginx/html:ro,Z - ports: - - "8080:80" - - mock-robot: + remote-driver: + profiles: + - remote build: context: . dockerfile: drivers/mock-robot/Dockerfile depends_on: - - ito-server + - ito environment: - ITO_SERVER_URL: ws://host.containers.internal:8765 - ITO_MOCK_ROBOT_ID: mock-robot-1 - ITO_MOCK_ROBOT_NAME: Mock Robot - ITO_MOCK_ROBOT_STATUS_INTERVAL_MS: "1000" + ITO_URL: ws://ito:8765/ws ITO_MOCK_ROBOT_CAMERA_VIDEO: /data/mock-camera.mp4 ITO_MOCK_ROBOT_CAMERA_LOOP: "true" volumes: @@ -48,9 +37,7 @@ services: dockerfile: drivers/ito-droid/Dockerfile network_mode: host environment: - ITO_SERVER_URL: ${ITO_SERVER_URL:-ws://127.0.0.1:8765} - ITO_DROID_ROBOT_ID: ${ITO_DROID_ROBOT_ID:-ito-droid-1} - ITO_DROID_NAME: ${ITO_DROID_NAME:-Ito Droid} + ITO_URL: ${ITO_URL:-ws://127.0.0.1:8765/ws} ITO_DROID_ROS_CAMERA_TOPIC: ${ITO_DROID_ROS_CAMERA_TOPIC:-/image_raw} ITO_DROID_ROS_SERVO_COMMAND_TOPIC: ${ITO_DROID_ROS_SERVO_COMMAND_TOPIC:-/ito_droid/camera_pan/command} diff --git a/docs/acceptance-v1.md b/docs/acceptance-v1.md index a4bad32..4c251b0 100644 --- a/docs/acceptance-v1.md +++ b/docs/acceptance-v1.md @@ -1,76 +1,21 @@ -# v1 Acceptance Pass - -This file records the current v1 acceptance pass against the core outcome in -`docs/v1.md`. Hardware-only acceptance remains unchecked until it is performed -with a physical Ito Droid and Pico 4 browser. - -## Latest Local Run - -Run on July 9, 2026: - -- [x] `pytest -q`: 40 passed, 1 skipped. The skipped test is the Mock Robot - `aiortc` WebRTC e2e test because this local Python environment does not have - `aiortc` installed. -- [x] `npm test` from `client/`: 15 passed. -- [ ] `docker compose config`: not run because Docker is not installed in this - environment. -- [ ] `python -m pip install -r server/requirements.txt -r drivers/mock-robot/requirements.txt`: - attempted, but PyAV built from source and failed because FFmpeg development - libraries were unavailable through `pkg-config`. - -## Local Acceptance - -- [x] Ito Protocol control-plane envelope tests pass for MessagePack binary - WebSocket messages, exact `ito.v1` protocol-version validation, Display - Reason helpers, and standard result payloads. -- [x] Ito Server unit tests cover pilot hello, catalog responses, driver status, - duplicate `robotId` handling, serialized acquisition, session allocation, - session end fan-out, disappeared-endpoint cleanup, and pilot reconnect resume - or rejection. -- [x] Mock Robot unit tests cover status reporting, video-file-backed camera - opening/closing, session start/end handling, session-start failure without a - configured camera file, and Pilot Input Snapshot logging. -- [x] Mock Robot end-to-end test covers real MessagePack WebSocket control - connections through the Ito Server, catalog acquisition, relayed - `pilotInput` WebRTC offer/answer signaling, and delivery of a Pilot Input - Snapshot over a real `aiortc` data channel when `aiortc` is installed. -- [x] Ito Droid unit tests cover environment configuration, status reporting, - ROS camera frame ingress seams, yaw-to-servo mapping, control tick behavior, - pilot-input timeout, safe resumption ramping, session-start neutralization, - and clean session-end neutralization. -- [x] Pilot Client Node tests cover config persistence, i18n fallback, - protocol helpers, Pilot Input Snapshot generation, Splat Batch parsing, - Splat Scene ownership/eviction, visual-freshness timeout, and WebRTC - non-trickle offer handling seams. - -## Hardware-Only Acceptance Still Required - -- [ ] Verify the Pilot Client on Pico 4's built-in browser, including Enter VR, - controller-ray catalog interaction, acquisition, in-VR settings/menu pause, - session end, and session-ended popup behavior. -- [ ] Run the Ito Droid driver on physical robot hardware with the configured - ROS camera topic available and the camera-pan servo command topic connected. -- [ ] Confirm the physical session-start procedure moves the camera-pan servo to - neutral before accepting pilot input. -- [ ] Confirm pilot headset yaw controls the physical camera-pan servo within - configured limits and smoothing/rate limits. -- [ ] Confirm recoverable control loss by withholding pilot input for at least - the configured timeout; the servo should hold the last commanded position. -- [ ] Confirm safe control resumption ramps correction velocity rather than - snapping the servo after control loss. -- [ ] Confirm clean session end attempts to return the physical camera-pan servo - to neutral. -- [ ] Confirm end-to-end robot camera media over WebRTC H.264 once TODO 23 is - complete, then confirm reconstruction frames and Splat Batches reach the Pico - 4 client with the visual-freshness behavior described in `docs/v1.md`. - -## Current Gaps - -- TODO 23 is still open: driver-to-server WebRTC H.264 media transport is not - complete, so the full live reconstruction loop cannot be accepted locally or - on hardware yet. -- TODO 27-30 and TODO 32 are still open: the representative reconstruction - sequences, algorithm spike/selection, and selected processor integration are - not complete. -- TODO 35 and TODO 51 are still open: Spark.JS insertion performance and Pico 4 - browser acceptance require the headset. +# Ito v1 acceptance + +Automated acceptance currently proves: + +- one Ito application serves the client and accepts the pilot connection; +- the client presents direct start/stop control with no browsing or allocation; +- a second simultaneous pilot is rejected; +- pilot disconnect invokes adapter stop; +- local sensor frames and pilot input cross the adapter boundary in-process; +- external mode connects one lightweight driver through the same pilot endpoint; +- control messages and live signaling contain no robot or allocation identity; +- Ito Droid retains local input timeout and neutralization behavior; +- Python and client unit/integration suites pass in the Dev Container. + +Hardware acceptance still required: + +- Pico 4 enters VR and negotiates both live channels; +- the selected reconstruction algorithm produces a usable scene; +- control loss, emergency stop, and safe resumption behave correctly on the + physical robot; +- onboard latency/copy measurements meet the selected robot's limits. diff --git a/docs/adr/0001-robot-driver-owns-control-translation.md b/docs/adr/0001-robot-driver-owns-control-translation.md deleted file mode 100644 index 4162dd3..0000000 --- a/docs/adr/0001-robot-driver-owns-control-translation.md +++ /dev/null @@ -1,11 +0,0 @@ -# Robot drivers own control translation - -Ito clients send robot-independent pilot input. Each robot driver translates that input into robot-specific commands, applies robot-specific constraints, and owns the response to control loss and control resumption. The pilot client may decide to withhold pilot input when local checks fail or UI state requires it, and may resume sending input when those checks pass again or the UI state closes, but the Ito Server does not make robot-control decisions. Control-loss and resumption behavior is robot-specific: stopping is not safe or feasible for every robot, and snapping directly back to the pilot's pose can be dangerous. This keeps embodiment and safety-policy knowledge out of the server and avoids inserting a required control-processing service into the latency- and safety-critical path. - -Pilot input is sent as repeated full-state snapshots rather than deltas. This lets the driver use the newest available state and tolerate dropped stale samples. V1 defaults to sending snapshots at 60 Hz, configurable by the Pilot Client. The robot driver runs its own control loop and uses only the newest available snapshot on each control tick, discarding older queued snapshots. Ito Droid v1 defaults this control tick rate to 60 Hz, configurable by driver environment variable. - -Each driver also owns its pilot-input timeout. Ito Droid v1 defaults this timeout to 2 seconds, but other robot types may need different thresholds and responses. Any client-side reason for withholding robot-directed pilot input, including visual freshness timeout or Menu Pause, reaches the driver as missing pilot input and uses the driver's normal control-loss behavior. - -For Ito Droid v1, safe control resumption means ramping the allowed camera-pan servo correction velocity from a low value back to normal over a configurable duration, rather than snapping immediately to the current pilot yaw. - -Ito Droid v1's recoverable control-loss response holds the camera-pan servo at its last commanded position. Returning the servo toward neutral happens on session end, not during recoverable control loss. diff --git a/docs/adr/0002-piloting-sessions-survive-transport-loss.md b/docs/adr/0002-piloting-sessions-survive-transport-loss.md deleted file mode 100644 index e6a32ca..0000000 --- a/docs/adr/0002-piloting-sessions-survive-transport-loss.md +++ /dev/null @@ -1,5 +0,0 @@ -# Piloting sessions survive transport loss - -A piloting session is a logical resource allocated by the Ito Server, not a WebRTC peer connection. When its connection fails, the robot driver begins its control-loss response while the server retains the occupied session for a recovery period; a client reconnecting on the trusted private network can attempt to resume by sending `connection.hello` with the existing `sessionId`. Explicit release or recovery expiry ends the session. The Ito Server may also expire stale session records when responsible endpoints disappear, defaulting to 30 seconds in v1, but that is bookkeeping for resource cleanup, not robot safety. This prevents transient or changing cellular and Wi-Fi paths from transferring an active robot to another pilot while keeping physical safety decisions in the robot driver. - -Robot Drivers reconnect forever with backoff. Pilot Clients reconnect while the app is open and let the server decide whether the session can resume. If the server no longer has a resumable session, it returns a failed hello result with a Display Reason and the client returns to the Robot Catalog. On successful resume, v1 recreates the relevant WebRTC peer connections with fresh non-trickle offer/answer signaling rather than preserving old peer connections. The session survives transport loss; the transports themselves are disposable. Because v1 has no application-layer authentication, this reconnect behavior is trusted-network behavior rather than proof of pilot identity. diff --git a/docs/adr/0003-use-a-monolithic-ito-server.md b/docs/adr/0003-use-a-monolithic-ito-server.md deleted file mode 100644 index 22cf5ac..0000000 --- a/docs/adr/0003-use-a-monolithic-ito-server.md +++ /dev/null @@ -1,11 +0,0 @@ -# Use a monolithic Ito server - -Ito has one central server program, process, and IPC API. It maintains the Robot Catalog and current Robot Status from robot-driver status reports, acts as session authority for piloting-session allocation and recovery, and performs visual reconstruction. Robot drivers run on their robots rather than as server-side services. The Pilot Client is a static website that may be hosted separately, such as by nginx; static web hosting is not part of Ito IPC. Remote robot drivers should not require inbound reachability, and Ito's server-side behavior belongs to one cohesive application. - -Decoding received robot camera media into frames for reconstruction is inside the monolithic Ito Server boundary. The server may use internal libraries or subprocess-style tooling inside its container, but v1 does not introduce a separate media ingestion or decoding service. Reconstruction may run in-process or as a subprocess inside the same server container; the selected path must avoid large unnecessary copies between decode, reconstruction, and Splat Batch output, and the reconstruction spike should measure that overhead. - -Reconstruction algorithm implementations live under `server/processors/` as server-internal modules. They share a server-internal interface, but individual implementations may accept different Capture Modalities such as monocular RGB, calibrated stereo RGB, or RGB-D. - -Reconstruction failure is session-scoped. The desired behavior is to stop the affected piloting session while keeping the Ito Server alive, preserving the path toward future multi-robot and multi-pilot operation. This makes failure isolation a requirement even though reconstruction remains inside the monolithic server boundary. - -The server is deployed as a Docker container. Runtime settings are supplied through environment variables in Docker Compose rather than through separate configuration services or mutable in-app configuration. diff --git a/docs/adr/0004-route-live-data-to-its-responsible-endpoint.md b/docs/adr/0004-route-live-data-to-its-responsible-endpoint.md deleted file mode 100644 index 424e8d7..0000000 --- a/docs/adr/0004-route-live-data-to-its-responsible-endpoint.md +++ /dev/null @@ -1,7 +0,0 @@ -# Route live data to its responsible endpoint - -Pilot input flows directly from the web client to the robot driver because the driver owns low-latency control translation. Robot camera footage normally flows to the Ito Server, which reconstructs an evolving Gaussian-splat scene and sends that scene to the client; the server is the intended processing endpoint, not a transparent relay. An optional Direct View may instead send camera footage directly from driver to client. TURN remains an allowed connectivity fallback for otherwise direct paths. - -This directional flow applies to live session data, not to session lifecycle/control-plane messages. The Ito Server communicates session lifecycle directly with both the Pilot Client and robot driver. - -V1 carries these live paths over three WebRTC peer connections: Pilot Client ↔ robot driver for pilot input, robot driver ↔ Ito Server for robot camera/media transport, and Ito Server ↔ Pilot Client for Splat Batches. V1 robot camera transport uses a WebRTC media track encoded as H.264, while bitrate, congestion behavior, and other media tuning are left to WebRTC defaults. The Ito Server owns decoding received camera media into frames for reconstruction inside the monolithic server boundary, using internal libraries or tools as needed. Reconstruction may be in-process or a subprocess inside the same server container, but not a separately deployed service. WebSocket handles Ito control-plane IPC and non-trickle WebRTC offer/answer signaling. diff --git a/docs/adr/0005-scope-reconstruction-to-a-piloting-session.md b/docs/adr/0005-scope-reconstruction-to-a-piloting-session.md deleted file mode 100644 index 6da2684..0000000 --- a/docs/adr/0005-scope-reconstruction-to-a-piloting-session.md +++ /dev/null @@ -1,13 +0,0 @@ -# Scope reconstruction to a piloting session - -The Ito Server creates reconstruction state for a piloting session and discards it when the session ends. Reconstructions are never shared across sessions, persisted as maps, or maintained for unoccupied robots. They exist to provide a responsive pilot view despite network and robot-motion latency, not to make Ito a mapping system. - -During temporary connection recovery, the Ito Server should retain reconstruction state only to the extent the selected reconstruction algorithm can safely continue after transport or camera discontinuity. Some algorithms may not tolerate the camera effectively "jumping" after media reconnection. In that case, the server may reset reconstruction state within the same piloting session rather than ending the session, as long as fresh reconstruction can resume. - -The first reconstruction spike should observe each candidate algorithm's behavior after short media interruptions and resumed feed from a slightly changed pose. Built-in recovery is a selection benefit, but Ito does not make it a hard algorithm responsibility for v1 because server-side reconstruction reset remains an acceptable recovery mechanism within the same session. - -If reconstruction fails, the failure applies to that piloting session. The Ito Server should stop the affected session rather than crash the whole server. - -Reconstruction should be testable against the same feed boundary used by real sessions. Unit tests may inject decoded frames or an abstract feed directly. Integration tests may use a Mock Robot that supplies a video-file-backed Sensor Feed over the real WebRTC H.264 media path, exercising transport and decoding without adding a production replay mode to the reconstruction module. - -The Mock Robot should behave like a robot driver in every relevant Ito interaction. If it runs and reports to the Ito Server, it appears in the Robot Catalog like any other robot. It accepts pilot input and logs received Pilot Input Snapshots to stdout, but does not maintain an internal fake robot pose unless a later test needs that state for a concrete assertion. diff --git a/docs/adr/0006-stale-reconstruction-causes-control-loss.md b/docs/adr/0006-stale-reconstruction-causes-control-loss.md deleted file mode 100644 index 97b7d17..0000000 --- a/docs/adr/0006-stale-reconstruction-causes-control-loss.md +++ /dev/null @@ -1,3 +0,0 @@ -# Stale reconstruction causes control loss - -The client uses a fixed timeout based on time since the last Splat Batch completed its normal receive-and-apply path and withdraws pilot input when that timeout expires. V1 defaults this client-side timeout to 2 seconds, with the value configurable in browser Local Storage and backed by a sane code default. It also freezes the last known Splat Scene, greys or fades the reconstructed view, and shows that robot controls are paused because the visual feed is lost. In-VR UI remains usable while robot controls are paused, including settings and ending the session. This gives the pilot something stable to look at while making the stale state explicit. This causes the robot driver to enter its control-loss response while retaining or terminating the piloting session according to the driver's policy. If Splat Batches resume while the session still exists, the client unfreezes the Splat Scene and automatically resumes sending pilot input; the driver owns safe control resumption and must avoid sudden dangerous correction. The Ito Server reports and transports reconstruction state, but it does not decide whether robot control remains safe. Direct View is not a fallback for this failure because it depends on the same live camera path. V1 does not require a separate protocol-validity metric or a metric for what geometry is visible to the pilot, because that varies by sensor type and reconstruction algorithm. Robot-specific thresholds and end-to-end frame-age tracking are deferred. diff --git a/docs/adr/0007-trust-a-private-vpn-for-v1.md b/docs/adr/0007-trust-a-private-vpn-for-v1.md deleted file mode 100644 index 8a776e8..0000000 --- a/docs/adr/0007-trust-a-private-vpn-for-v1.md +++ /dev/null @@ -1,3 +0,0 @@ -# Trust a private network for v1 - -Ito v1 runs only inside a private network whose members are trusted, such as local Wi-Fi during early development or a private VPN when the headset, server, and robot are not on the same LAN. The server, web client, and robot drivers do not implement application-layer authentication or authorization, and the server must not be exposed for public control. This keeps the first system small while making public deployment explicitly dependent on the future driver-authentication, pilot-authorization, and transport-security design. diff --git a/docs/adr/0008-build-ito-droid-on-ros.md b/docs/adr/0008-build-ito-droid-on-ros.md deleted file mode 100644 index 190be2c..0000000 --- a/docs/adr/0008-build-ito-droid-on-ros.md +++ /dev/null @@ -1,5 +0,0 @@ -# Build Ito Droid on ROS - -Ito Droid uses ROS as its robot software platform because it provides a widely adopted ecosystem for cameras, actuators, and future robot capabilities. Its robot driver adapts between Ito and ROS; ROS concepts do not become requirements of Ito's client, server, or protocol, so non-ROS robots remain first-class integration targets. - -The Ito Droid driver is deployed as a container built on a ROS base image. It participates in the robot's ROS graph and consumes existing ROS topics for sensor feeds, such as camera frames, instead of owning camera setup itself. The driver is configured through Docker Compose environment variables, including the Ito Server URL, ROS topic names, pilot-input timeout, servo neutral angle, servo limits, smoothing, and control-resumption ramp rates. This keeps Ito's responsibility at the teleoperation integration boundary while letting robot-specific ROS packages handle device drivers and hardware bring-up. diff --git a/docs/adr/0009-require-an-nvidia-gpu-for-v1.md b/docs/adr/0009-require-an-nvidia-gpu-for-v1.md deleted file mode 100644 index 1c7f343..0000000 --- a/docs/adr/0009-require-an-nvidia-gpu-for-v1.md +++ /dev/null @@ -1,3 +0,0 @@ -# Require an NVIDIA GPU for v1 - -Ito v1 targets an x86_64 server with an NVIDIA GPU and runs in one container with NVIDIA Container Runtime access. Live neural reconstruction is the primary technical risk, so v1 will use the CUDA ecosystem directly rather than dilute that work with a CPU fallback or hardware-portability layer. Broader accelerator support can follow evidence of need. diff --git a/docs/adr/0010-use-gaussian-splats-as-the-client-scene.md b/docs/adr/0010-use-gaussian-splats-as-the-client-scene.md deleted file mode 100644 index bb3be7b..0000000 --- a/docs/adr/0010-use-gaussian-splats-as-the-client-scene.md +++ /dev/null @@ -1,5 +0,0 @@ -# Use Gaussian splats as the client scene - -Every reconstruction algorithm normalizes its output into incremental Splat Batches, which are the only reconstructed-scene representation sent to the WebXR client. The client uses A-Frame and SparkJS to render a rolling Splat Scene; algorithm-specific depth, point maps, and internal state remain on the Ito Server. Direct View may later add a camera track, but it does not replace this canonical reconstruction contract. - -Splat Batches are binary data channel payloads, not JSON. The v1 default Splat Batch data channel is reliable and ordered because batches mutate the client-owned Splat Scene. The reliability/ordering profile is configurable through server container environment variables and sent to the Pilot Client as session configuration. The v1 binary layout should be selected based on the SparkJS insertion path: minimize client-side parsing, decoding, and conversion before adding the batch to the Splat Scene. The exact layout should not be frozen until the client spike proves which representation SparkJS can ingest with the least work on Pico 4. diff --git a/docs/adr/0011-let-the-client-own-the-splat-scene.md b/docs/adr/0011-let-the-client-own-the-splat-scene.md deleted file mode 100644 index fd84761..0000000 --- a/docs/adr/0011-let-the-client-own-the-splat-scene.md +++ /dev/null @@ -1,11 +0,0 @@ -# Let the client own the Splat Scene - -The Ito Server sends newly reconstructed Splat Batches but does not maintain an authoritative copy of the pilot's rendered scene. The client adds each batch, then evicts batches that exceed its configurable Splat Lifetime and removes the oldest remaining batches when its configurable Splat Budget is exceeded. Client runtime settings are stored in browser Local Storage with sane code defaults. Splat Batch payloads are binary and should be shaped around efficient SparkJS ingestion. The v1 default Splat Batch data channel is reliable and ordered, with the profile configurable through server container environment variables. This avoids server/client scene synchronization and lets weaker headsets bound their own resource use, at the accepted cost that incorrect geometry can remain visible until its batch is evicted. - -When visual freshness times out, the client freezes the last known Splat Scene while showing a stale/paused state. If fresh Splat Batches resume while the session still exists, the client unfreezes the scene and resumes normal batch application and eviction. - -If the server resets reconstruction state within the same piloting session, v1 does not require an explicit client-side Splat Scene clear. The client continues applying normal Splat Batch insertion, Splat Lifetime, and Splat Budget eviction; new splats gradually replace older splats as they do during normal operation. The selected reconstruction algorithm should therefore produce post-reset Splat Batches that remain usable in the session's visual frame, or this assumption must be revisited. - -The server does not send a `reconstruction.reset` control-plane event in v1. If fresh splats continue arriving, the client does not need to know that the server reset internal reconstruction state. If fresh splats stop arriving, the existing Visual Freshness Timeout handles the pilot-visible state. If reconstruction cannot restart, the session ends with a Session Termination Reason. - -When a session ends, the client initially freezes the last known Splat Scene and overlays the Session Termination Reason plus a return-to-catalog action. The catalog replaces the scene only after the pilot chooses to return. diff --git a/docs/adr/0012-keep-the-pilot-ui-in-vr.md b/docs/adr/0012-keep-the-pilot-ui-in-vr.md deleted file mode 100644 index cccd582..0000000 --- a/docs/adr/0012-keep-the-pilot-ui-in-vr.md +++ /dev/null @@ -1,5 +0,0 @@ -# Keep the pilot UI in VR - -Ito minimizes non-VR UI. The web page outside XR exists only to satisfy browser requirements for entering an immersive WebXR session, such as the required user gesture for an Enter VR button. Robot catalog browsing, robot acquisition, piloting-session state, and client settings are presented inside VR. This increases v1 VR UI work but keeps Ito's product surface aligned with immersive teleoperation instead of splitting the pilot workflow across flat browser pages and VR. - -V1 uses controller-ray interaction for VR UI. During an active session, a controller button opens the in-VR menu. While the menu is open, the client withholds robot-directed pilot input but keeps UI interaction active. If Menu Pause lasts long enough to exceed the driver's pilot-input timeout, the driver enters its normal control-loss response. When the menu closes, the client resumes sending pilot input automatically; the driver owns safe control resumption and must not instantly snap the robot to a substantially different pilot pose. diff --git a/docs/adr/0013-make-the-server-session-authority.md b/docs/adr/0013-make-the-server-session-authority.md deleted file mode 100644 index ef1e81e..0000000 --- a/docs/adr/0013-make-the-server-session-authority.md +++ /dev/null @@ -1,9 +0,0 @@ -# Make the server the session authority - -The Ito Server is responsible for deciding whether a pilot may acquire a robot, serializing competing acquisition attempts, creating piloting sessions, and marking robots Occupied. In v1, it may reserve a robot only when the robot driver has fresh status over its WebSocket control connection and is reporting Available. Drivers send status/heartbeat every 1 second by default, and the server marks a robot Unavailable when its driver WebSocket disconnects or misses the configurable Driver Status Watchdog, defaulting to 2 seconds. Acquisition is serialized at the server: the server atomically reserves the robot and marks it Occupied before asking the driver to start its session procedure, so two near-simultaneous pilots cannot both acquire the same robot. If the driver reports success, the server completes acquisition and returns a Session Identity to the Pilot Client. If the driver reports failure or times out, the server releases the reservation and returns a failed acquisition result. V1 does not expose separate Allocated and Active session states in the protocol or glossary. Robot drivers report availability and remain responsible for robot-specific control mapping, control-loss behavior, control resumption, and driver-terminated session failures. - -Live session data can follow a directional flow: client input to robot driver, robot camera to server reconstruction, and splats back to client. Session lifecycle/control-plane messages are different: the server communicates directly with both the Pilot Client and robot driver over WebSocket. If the server ends a session, such as because reconstruction failed, it marks the session ended immediately and sends session-ended messages to both endpoints rather than relying on the client to forward lifecycle state to the driver. Server-originated session end includes a displayable Session Termination Reason, such as `session.ended.reconstruction_failed` for reconstruction failure. When the pilot chooses to stop piloting from the in-VR menu, the Pilot Client requests session end from the Ito Server over WebSocket without resuming robot-directed pilot input first, and the server uses the same session-ending path. V1 does not wait for driver acknowledgement before considering the session ended. - -The robot driver may also request session end from the Ito Server, but this is reserved for non-recoverable driver or robot conditions, or explicit local stop/override. Ordinary missing pilot input, client visual-freshness timeout, and transient network loss should use control loss and recovery behavior instead of immediate driver termination. Driver-requested session end includes a displayable Session Termination Reason using the same resource-key-or-free-text model as other display strings. - -This keeps race handling and future authorization in one place. It also prepares Ito for future RBAC, where the server will decide which robots a pilot may see or acquire. The trade-off is that a piloting session is no longer purely a robot-driver-owned resource; the server must coordinate session lifecycle carefully while avoiding robot-control safety decisions. diff --git a/docs/adr/0014-externalize-pilot-facing-ui-text.md b/docs/adr/0014-externalize-pilot-facing-ui-text.md deleted file mode 100644 index dac6552..0000000 --- a/docs/adr/0014-externalize-pilot-facing-ui-text.md +++ /dev/null @@ -1,11 +0,0 @@ -# Externalize pilot-facing UI text - -Pilot-facing UI text is loaded from JSON resource files using an i18next-style structure. UI components should reference message keys rather than hardcoding labels, status messages, menu text, or session-state copy. - -Driver- or server-provided display strings, such as availability details or Session Termination Reasons, may be either resource keys or free text. The Pilot Client first attempts to resolve such values through its localization resources and falls back to displaying the supplied text as-is. - -Robot names are proper names supplied by the driver and are displayed as-is. Enum-style labels controlled by Ito, such as Robot Type and Robot Status, are localized through resource files. - -V1 ships one default resource set. The resource structure should leave room for future localization and themed language variants, but v1 does not implement language/theme selection. - -This adds small upfront structure to the VR client, but it prevents hardcoded strings from spreading through the product surface. It keeps future localization feasible and allows later themed language variants without rewriting UI components. diff --git a/docs/adr/0016-keep-ito-protocol-boundaries-explicit.md b/docs/adr/0016-keep-ito-protocol-boundaries-explicit.md deleted file mode 100644 index c50dac1..0000000 --- a/docs/adr/0016-keep-ito-protocol-boundaries-explicit.md +++ /dev/null @@ -1,9 +0,0 @@ -# Keep Ito Protocol boundaries explicit - -Ito v1 is implemented as three cooperating programs: the Pilot Client, the Ito Server, and a Robot Driver. Their shared boundary is the Ito Protocol: message contracts, media paths, session lifecycle semantics, and displayable status/reason values. Technology-specific code such as ROS integration, WebXR input/rendering, WebRTC peer management, Spark.JS insertion, and reconstruction algorithm wrappers should sit behind that boundary rather than becoming the domain model itself. - -This keeps ROS specific to the Ito Droid driver, keeps browser/WebXR concerns out of the robot driver, and keeps server/session logic independent from any one robot or reconstruction algorithm. The goal is not heavy layering; it is one generic protocol interface between each program so future robot types, driver languages, capture modalities, and reconstruction algorithms can be swapped without rewriting the whole teleoperation loop. - -For v1, those seams are documented in `docs/protocol.md` and manually implemented in Python for the Ito Server and robot drivers, and plain JavaScript for the Pilot Client. V1 does not use TypeScript or generated protocol bindings. - -V1 uses exact Ito Protocol Version matching with the identifier `ito.v1`. It fails fast on version mismatch. diff --git a/docs/adr/0017-use-websocket-control-plane-and-webrtc-live-data.md b/docs/adr/0017-use-websocket-control-plane-and-webrtc-live-data.md deleted file mode 100644 index 9524601..0000000 --- a/docs/adr/0017-use-websocket-control-plane-and-webrtc-live-data.md +++ /dev/null @@ -1,9 +0,0 @@ -# Use WebSocket for control-plane IPC and WebRTC for live data - -Ito v1 uses WebSocket for IPC between Pilot Client, Ito Server, and Robot Driver: catalog request/response, robot-driver status/heartbeat, acquisition, session lifecycle, displayable errors/reasons, and WebRTC offer/answer signaling. The static Pilot Client can be hosted separately, such as by nginx; HTTP is not an Ito application IPC surface in v1. - -Control-plane messages use a common MessagePack-encoded envelope over binary WebSocket frames. This gives Ito one compact structure for version checks, message type routing, logging, and request/response correlation. - -Robot drivers send status/heartbeat every 1 second by default. The server marks a robot Unavailable if the driver WebSocket disconnects or if no fresh status/heartbeat arrives within 2 seconds by default. Both values are configurable, and the server evaluates liveness proactively rather than waiting for a client catalog request. - -WebRTC remains the live data/media plane: Pilot Client ↔ Robot Driver for pilot input, Robot Driver ↔ Ito Server for H.264 camera media, and Ito Server ↔ Pilot Client for Splat Batches. This keeps low-latency streams on WebRTC while giving the server an immediate reliable path to tell the driver to start/end sessions and to notify the client of lifecycle events. diff --git a/docs/adr/0018-use-messagepack-control-message-envelopes.md b/docs/adr/0018-use-messagepack-control-message-envelopes.md deleted file mode 100644 index 99b5d60..0000000 --- a/docs/adr/0018-use-messagepack-control-message-envelopes.md +++ /dev/null @@ -1,5 +0,0 @@ -# Use MessagePack control message envelopes - -Ito v1 uses a common envelope for WebSocket control-plane messages, encoded as MessagePack over binary WebSocket frames. The envelope includes protocol version, message ID, optional reply-to message ID, dot-separated v1 message type, optional robot/session scope, and payload. Protocol examples document decoded MessagePack maps as field tables, not JSON text-frame examples. - -This keeps client, server, and robot-driver IPC structured enough for routing, logging, version checks, and request/response correlation while staying compact on a live control connection. Responses correlate explicitly with `replyToMessageId` rather than relying on message order, because session events and signaling messages can interleave on the same WebSocket. Request/response payloads use a standard result shape with `ok: true` plus `value`, or `ok: false` plus a Display Reason containing a localization key and/or free text fallback. Any request expecting a response has a configurable Request Timeout; the default timeout impact is to fail the pending operation while keeping the WebSocket open, but message definitions may escalate to ending the affected session or rejecting the endpoint. V1 defines only the message types needed for v1 rather than reserving names for future features. It also avoids introducing TypeScript or generated bindings in v1; the envelope is documented docs-first and manually implemented in Python and plain JavaScript. diff --git a/docs/adr/0019-one-application-one-pilot-one-robot.md b/docs/adr/0019-one-application-one-pilot-one-robot.md new file mode 100644 index 0000000..becb7b1 --- /dev/null +++ b/docs/adr/0019-one-application-one-pilot-one-robot.md @@ -0,0 +1,22 @@ +# ADR 0019: One application, one pilot, one robot + +## Decision + +Ito is one Python application for one pilot and one configured robot. The +application hosts the WebXR client, terminates its control and live-data paths, +runs reconstruction, and controls the robot through one narrow adapter. + +The default adapter is local and in-process. An optional remote adapter connects +one lightweight robot driver when the robot cannot run Ito onboard. Placement is +configuration and doesn't alter the pilot-facing protocol. + +Multi-user support, multiple robots per application, discovery, catalogs, +browsing, stable robot identities, allocation, reservations, availability +lists, fleet heartbeats, and distributed session coordination are explicitly +out of scope. If those become real product requirements, they require a new +architecture rather than dormant compatibility layers. + +## Consequences + +Ito has fewer processes, network links, messages, identifiers, states, and +deployment units. Breaking the previous prototype protocol is intentional. diff --git a/docs/adr/0020-keep-robot-safety-local.md b/docs/adr/0020-keep-robot-safety-local.md new file mode 100644 index 0000000..4e1388c --- /dev/null +++ b/docs/adr/0020-keep-robot-safety-local.md @@ -0,0 +1,11 @@ +# ADR 0020: Keep robot safety local + +## Decision + +Pilot-input timeout, command limits, neutralization, emergency stop, and safe +resumption live in or next to the local adapter. In external mode the lightweight +driver retains the same guarantees on the robot. + +Loss of pilot input or any external connection must produce a safe local +response without waiting for another machine. Ito may pause or stop control, but +it isn't a remote prerequisite for the robot's safety behavior. diff --git a/docs/adr/0021-only-network-where-physical-placement-requires-it.md b/docs/adr/0021-only-network-where-physical-placement-requires-it.md new file mode 100644 index 0000000..07e7efa --- /dev/null +++ b/docs/adr/0021-only-network-where-physical-placement-requires-it.md @@ -0,0 +1,11 @@ +# ADR 0021: Use network transport only at physical boundaries + +## Decision + +The always-present boundary is Pilot client ↔ Ito application. Local robot input, +sensors, and reconstruction use calls and memory. External mode adds exactly one +driver boundary because robot and Ito are on different machines. + +WebSocket carries MessagePack lifecycle and non-trickle signaling. WebRTC carries +replaceable pilot input, remote camera media when necessary, and binary Splat +Batches. Local camera frames never take a fake loopback transport path. diff --git a/docs/adr/0022-client-owns-the-bounded-splat-scene.md b/docs/adr/0022-client-owns-the-bounded-splat-scene.md new file mode 100644 index 0000000..c0e9b13 --- /dev/null +++ b/docs/adr/0022-client-owns-the-bounded-splat-scene.md @@ -0,0 +1,10 @@ +# ADR 0022: The pilot client owns a bounded Splat Scene + +## Decision + +Ito sends incremental binary Splat Batches. The client applies them to its +rolling scene and evicts by age and budget. Visual-freshness timeout pauses pilot +input until fresh reconstruction returns. + +The scene exists for current control, isn't a persistent world, and isn't shared +with another pilot. diff --git a/docs/future.md b/docs/future.md index ea331f9..35df388 100644 --- a/docs/future.md +++ b/docs/future.md @@ -1,54 +1,35 @@ -# Future Design Considerations +# Future design considerations -These are potential expansions, not accepted requirements or architecture. +These are candidate improvements within the one-pilot/one-robot product, not +accepted requirements. -## End-to-end visual freshness feedback +## Reconstruction and capture -A robot driver could attach a timestamp from its own monotonic clock to each captured camera frame. The Ito Server would preserve source timing through reconstruction updates, and the client would echo the timestamp associated with the latest splats actually shown to the pilot alongside pilot input. Because the timestamp originated at the robot, the driver could compare the echoed value with its current clock and optionally reject control when displayed visual information exceeds a robot-specific safety threshold. +Evaluate onboard-capable systems such as MonoGS, MASt3R-SLAM, and Depth +Anything 3 against concrete robot hardware. Add only the camera calibration, +pose, stereo, RGB-D, or accelerator integration required by a selected model. +Keep reconstruction inside the Ito operational unit even when a native runtime +or tightly managed subprocess is needed. -Before implementing this, define how one reconstruction update represents timing when it incorporates multiple camera frames or updates only part of the scene. +## Visual age feedback -## Authentication and authorization +A robot adapter could compare source capture time with the latest scene update +acknowledged by the pilot. Define timing semantics for multi-frame +reconstruction before using this as a safety signal. -Ito v1 does not authenticate robot drivers or pilots at the application layer. A future security design should treat these as separate responsibilities: +## Transport security -- authenticate each robot driver so another machine cannot claim its catalog identity; -- authenticate pilots and use role-based access control to limit which robots they may see or pilot; -- protect transport integrity and confidentiality against interception or modification; -- authenticate piloting-session resumption so only the original pilot can recover control. +Deployment-specific TLS or private-network authentication may be needed. Any +security mechanism should protect the one Ito endpoint and, in external mode, +its one driver connection without introducing accounts, RBAC, discovery, or a +cloud control plane. -RBAC alone does not prevent a fake robot driver or a man-in-the-middle attack. Candidate mechanisms include provisioned driver keys, mutual TLS, and trusted network identity, but no mechanism has been selected. +## Pilot experience -## Multiple capture modalities +PCVR, hand tracking, gaze interaction, and richer robot-specific controls may +be evaluated after Pico 4 acceptance. The narrow adapter remains responsible +for concrete robot mapping and safety. -Robot drivers may eventually provide monocular RGB, calibrated stereo RGB, RGB-D, or other sensor configurations. The Ito Server should select reconstruction suited to the available Capture Modality rather than treating v1's monocular algorithm as universal. For example, a learned monocular prior such as MASt3R-SLAM is unnecessary when calibrated stereo or measured depth already provides geometric information. V1 implements only the available USB webcam's monocular path and should not build the general selection system yet. - -The reconstruction spike should determine the minimum metadata required for each supported algorithm. Camera intrinsics, mounting extrinsics, capture timestamps, and robot-reported camera poses are candidates, not baseline protocol requirements; omit anything the selected algorithm does not demonstrably use. - -V1 sends monocular camera input as a WebRTC media track. Future stereo can use multiple synchronized media tracks or a packed stereo video representation. Future RGB-D can use an RGB media track plus a depth representation if the selected reconstruction algorithm benefits from it. Point clouds and other non-image geometry are not a natural media-track fit; they should likely use a binary data channel or another explicit geometry transport. The transport should be selected per Capture Modality rather than assuming every sensor type is a single video track. - -V1 requires H.264 for the robot camera media track. Future versions may add camera codec negotiation, VP8/VP9/AV1, hardware-specific codec preferences, or direct ingest paths if target robots or reconstruction pipelines justify the complexity. - -V1 does not define required camera resolution or frame rate. Future algorithm-specific requirements may define recommended or required capture settings per Capture Modality. - -V1 leaves WebRTC media bitrate, congestion behavior, and related camera transport tuning to defaults. Future versions may expose media transport settings if reconstruction quality or network behavior requires explicit control. - -## PCVR and desktop WebXR - -Ito v1 targets standalone WebXR on Pico 4's built-in browser. PCVR through systems such as Virtual Desktop would be useful for higher rendering budgets and development ergonomics, but desktop WebXR support is not treated as reliable enough to be a v1 acceptance path. - -## More physical VR interaction - -Ito v1 uses controller-ray interaction for VR UI. Future versions may explore hand tracking, gaze interaction, direct-touch panels, or other more physical VR interaction models, but those are not required to prove the v1 reconstruction and teleoperation loop. - -## Robot-type control conventions - -Robots of the same Robot Type should generally feel similar to pilot. For example, a Car is likely to rely more on joystick and trigger controls than full-body pose, while humanoid types are likely to rely more on body mapping. Ito may eventually define explicit Control Conventions per Robot Type, but v1 does not introduce a formal control-profile or capability system. The robot driver remains responsible for concrete mapping and robot-specific controls. - -## Persistent robot catalog - -Ito v1 keeps the Robot Catalog in server memory and builds it from robot-driver reports. A future version may add a persistent robot registry or configured inventory so unavailable robots remain visible across server restarts, but v1 does not require that. - -## Themed UI language - -Ito v1 externalizes pilot-facing UI text into JSON resource files for future localization. The same resource mechanism could later support selected language styles or themes, such as Ghost in the Shell, Matrix, or cyberpunk-inspired terminology, without changing UI components. +Multiple pilots, multiple robots, fleets, catalogs, and allocation remain out +of scope. They require a future product redesign, not dormant abstractions in +this codebase. diff --git a/docs/local-v1.md b/docs/local-v1.md index 82787b4..69ae842 100644 --- a/docs/local-v1.md +++ b/docs/local-v1.md @@ -1,113 +1,38 @@ -# Local v1 Operation +# Local deployment -This guide covers local Ito v1 operation with Docker Compose or Podman Compose. It keeps the -same boundaries as `docs/v1.md`: the Pilot Client is static web content, the Ito -Server owns catalog/session/reconstruction authority, and robot drivers connect -outward over the Ito Protocol WebSocket control plane. +## Default onboard-shaped mode -## Compose Services +Start one application/container: -- `ito-server`: Python Ito Server on `ws://localhost:8765`. -- `pilot-client`: nginx static hosting for `client/` on - `http://localhost:8080`. -- `mock-robot`: Mock Robot driver. It starts by default and uses - `fixtures/mock-camera.mp4` when present. -- `ito-droid`: optional profile-backed physical Ito Droid ROS driver. It is - intended for robot-side use with host networking and an existing ROS camera - feed/servo command path. - -## Local Stack - -Build and run the local server, Pilot Client, and Mock Robot: - -```sh -docker compose up --build -# or, with Podman Compose: -podman compose up --build +```bash +docker compose up --build ito ``` -The Compose file uses Podman's pre-existing external `podman` network instead of -a project-created bridge network. This keeps local Podman operation independent -of `aardvark-dns`/user-bus startup. The Mock Robot reaches the Ito Server through -`host.containers.internal:8765`; Docker users can still use the same Compose file -because the server port is published to the host. +Open `http://localhost:8765/`. The application serves the client and accepts its +WebSocket at `ws://localhost:8765/ws`. No nginx or reverse proxy is required. -Bind mounts use the `Z` SELinux relabel option so rootless Podman containers can -read files from the user's home directory. Without this, nginx may return -`403 Forbidden` even though Unix file permissions look readable. +The default `ITO_ROBOT_BACKEND=local` uses the in-process adapter. Production +robot code supplies its control sink, safe-stop callback, and sensor frames. -Open `http://localhost:8080/`. The client defaults to the Ito Server control -WebSocket at `ws://:8765`, which is `ws://localhost:8765` for this -Compose setup. +## External-driver fallback -Stop and remove local containers: +Provide an H.264 sample file and start the remote profile: -```sh -docker compose down -# or: -podman compose down +```bash +ITO_ROBOT_BACKEND=remote \ +ITO_MOCK_ROBOT_CAMERA_VIDEO_HOST=/absolute/path/to/camera.mp4 \ +docker compose --profile remote up --build ``` -## Mock Robot Video - -By default, the Mock Robot mounts `fixtures/mock-camera.mp4`. To use a -different local video file, override `ITO_MOCK_ROBOT_CAMERA_VIDEO_HOST`: - -```sh -ITO_MOCK_ROBOT_CAMERA_VIDEO_HOST=/absolute/path/to/mock-camera.mp4 \ - docker compose up --build -# or: -ITO_MOCK_ROBOT_CAMERA_VIDEO_HOST=/absolute/path/to/mock-camera.mp4 \ - podman compose up --build -``` +The browser still opens `http://localhost:8765/`. Only the configured last hop +changes: Ito connects through its remote adapter to the lightweight driver. -Useful log streams while testing acquisition and pilot input: +## Checks -```sh -docker compose logs -f ito-server mock-robot -# or: -podman compose logs -f ito-server mock-robot +```bash +docker compose config +curl --fail http://localhost:8765/ ``` -The Mock Robot exercises the v1 WebSocket control plane, acquisition/session -lifecycle, and client-to-driver pilot-input WebRTC data channel. Driver-to-server -H.264 WebRTC camera publishing remains tied to TODO 23, so local Mock Robot -operation does not yet prove camera media ingestion into reconstruction. - -## Ito Droid - -Run the physical Ito Droid driver on the robot or in the robot's ROS network. -The environment must already provide the configured ROS camera topic and servo -command topic. - -```sh -ITO_SERVER_URL=ws://:8765 \ -ITO_DROID_ROS_CAMERA_TOPIC=/image_raw \ -ITO_DROID_ROS_SERVO_COMMAND_TOPIC=/ito_droid/camera_pan/command \ - docker compose --profile droid up --build ito-droid -``` - -Because the `ito-droid` service uses host networking, `ITO_SERVER_URL` must be -reachable from the robot host. The ROS topic names must match the robot-local -ROS graph; ROS setup and camera-driver bring-up are outside Ito v1. - -## Local Test Commands - -Python tests: - -```sh -python -m pip install -r server/requirements.txt -r drivers/mock-robot/requirements.txt -pytest -q -``` - -`aiortc` and PyAV are required for the WebRTC/H.264 paths. If PyAV builds from -source instead of installing a wheel, the host needs FFmpeg development -libraries available through `pkg-config`; otherwise tests that require `aiortc` -will be skipped or dependency installation will fail. - -Pilot Client tests: - -```sh -cd client -npm test -``` +The development and test workflow uses `.devcontainer/`; don't install project +tools on the host. diff --git a/docs/protocol.md b/docs/protocol.md index 2fc0bd3..9bcb7f7 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1,322 +1,78 @@ -# Ito Protocol seams +# Ito protocol -This document records the v1 IPC seams between the three Ito programs: Pilot Client, Ito Server, and Robot Driver. The goal is to make the contracts explicit before implementation details leak across program boundaries. +Ito has one always-present network boundary and one optional boundary: -V1 keeps these contracts docs-first and manually implemented. The server and v1 robot drivers are Python. The Pilot Client is plain JavaScript. TypeScript and generated protocol bindings are deferred. +1. Pilot client ↔ Ito application (all deployments). +2. Ito application ↔ remote robot driver (external deployment only). -The v1 protocol version identifier is `ito.v1`. Pilot Client, Ito Server, and Robot Driver must advertise this exact Ito Protocol Version in control-plane messages. If versions do not match, acquisition fails fast with a clear acquisition failure reason or Session Termination Reason. +Local robot integration and reconstruction use calls, queues, and memory, not +this protocol. -WebSocket control-plane messages use MessagePack over binary WebSocket frames. Protocol documentation describes decoded MessagePack maps as field tables; these examples are not JSON and should not be implemented as text frames. +## Envelope -Each message uses a common envelope: +WebSocket control messages are binary MessagePack maps: | Field | Required | Meaning | | --- | --- | --- | -| `protocolVersion` | yes | Exact Ito Protocol Version, `ito.v1` in v1. | -| `messageId` | yes | Unique message identifier for logging, tracing, and correlation. | -| `replyToMessageId` | no | The request message ID when this message is a response. | -| `type` | yes | Dot-separated v1 message type. | -| `robotId` | no | Stable robot identity when the message is robot-scoped. | -| `sessionId` | no | Piloting session identity when the message is session-scoped. | -| `payload` | yes | Message-specific decoded MessagePack map. | +| `protocolVersion` | yes | Exact value `ito.v1`. | +| `messageId` | yes | Correlation and diagnostics identifier. | +| `type` | yes | Message type below. | +| `payload` | yes | Message-specific map. | +| `replyToMessageId` | responses | Request being answered. | -`messageId` is used for logging and tracing every message. Responses set `replyToMessageId` to the request message's `messageId`; events and fire-and-forget messages leave it null or absent. `robotId` and `sessionId` may be null when the message is not scoped to a robot or session. +There is no robot or control-session identifier. One endpoint represents one +configured robot and one active-control lifecycle. -Identifier ownership: +Results use `{ok: true, value: {...}}` or +`{ok: false, reason: {code?, text?}}`. -| Identifier | Owner | Rule | -| --- | --- | --- | -| `messageId` | Message sender | Generated by the sender for every WebSocket message. Used only for tracing and correlation, not authority. | -| `sessionId` | Ito Server | Generated only by the Ito Server when it allocates a piloting session. Clients and robot drivers must not mint authoritative session IDs. | -| `robotId` | Robot Driver | Configured/provided by the Robot Driver and stable across driver restarts. Not proof of authenticity in v1. | - -Request/response messages use a standard result payload shape. Successful result payloads contain: - -| Field | Required | Meaning | -| --- | --- | --- | -| `ok` | yes | `true`. | -| `value` | yes | Message-specific decoded MessagePack map. | - -Failed result payloads contain: - -| Field | Required | Meaning | -| --- | --- | --- | -| `ok` | yes | `false`. | -| `reason` | yes | Display Reason map. | -| `reason.code` | no | Localization resource key when Ito knows one. | -| `reason.text` | no | Free text fallback. | - -`reason.code` is a localization resource key when Ito knows one. `reason.text` is a free text fallback. At least one of `code` or `text` must be present for failed results. This same Display Reason model is used for acquisition failures, availability details, and Session Termination Reasons. - -Any request message that expects a response has a configurable Request Timeout. If no response with matching `replyToMessageId` arrives before the timeout, the requester handles the timeout according to that message's definition. The default behavior is to fail the pending operation with an appropriate Display Reason while keeping the WebSocket connection open. Some message-specific timeouts may instead be treated as a total failure for the affected session or endpoint. V1 timeout values are configured through environment variables for server/driver behavior and client runtime settings where the Pilot Client owns the request. - -V1 message types use dot-separated names scoped by domain/action. The v1 protocol should define only message types required by v1. Initial v1 types are: - -- `catalog.get` -- `catalog.get.result` -- `connection.hello` -- `connection.hello.result` -- `robot.status` -- `session.acquire` -- `session.acquire.result` -- `driver.session.start` -- `driver.session.start.result` -- `session.end` -- `session.end.result` -- `session.ended` -- `webrtc.offer` -- `webrtc.answer` - -Acquisition is serialized by the Ito Server. When handling `session.acquire`, -the server atomically reserves the selected robot before asking the robot driver -to start the session. While that request is pending, the robot is Occupied and -other acquisition attempts for the same robot fail or observe it as not -Available. If the driver start procedure succeeds, `session.acquire.result` -returns the server-owned Session Identity to the Pilot Client. If it fails or -times out, the server releases the reservation and returns a failed -`session.acquire.result`. V1 does not expose separate Allocated and Active -session states in protocol messages. - -## WebSocket connection lifecycle - -Each Pilot Client and Robot Driver opens a WebSocket control connection to the Ito Server. After connecting or reconnecting, the endpoint sends `connection.hello`. - -`connection.hello` payload fields: - -| Field | Required | Meaning | -| --- | --- | --- | -| `role` | yes | `pilotClient` or `robotDriver`. | -| `robotId` | driver only | Robot Identity for a robot driver connection. | -| `sessionId` | no | Existing Session Identity when a pilot client is trying to resume a session. | - -The Ito Server replies with `connection.hello.result`. If the protocol version, role, robot identity, or claimed session cannot be accepted, the result fails with a Display Reason. - -Reconnect behavior: - -- Robot Drivers reconnect forever with backoff. -- Pilot Clients reconnect while the app is open and, during a recoverable piloting session, include the existing `sessionId` in `connection.hello`. -- The Ito Server decides whether a reconnected endpoint can resume an existing session. -- When a session resumes after WebSocket reconnect, v1 recreates the relevant WebRTC peer connections with fresh non-trickle offer/answer signaling. Ito does not try to preserve old WebRTC peer connections across reconnect/resume. -- If a Pilot Client reconnects after the session is gone or no longer resumable, the server returns a failed `connection.hello.result`; the client shows the Display Reason and returns to the Robot Catalog. -- Because v1 has no application-layer authentication, reconnect/resume is trusted-network behavior, not proof of pilot identity. - -## Programs - -- `Pilot Client`: WebXR browser application that renders the pilot view and sends Pilot Input Snapshots. -- `Ito Server`: Python application that owns catalog/session authority and runs reconstruction. -- `Robot Driver`: Python application for v1 Ito Droid that reports availability, handles driver-side session start/end procedures, receives pilot input, forwards sensor feeds, and translates input to robot-specific commands. - -## V1 seams - -### Pilot Client ↔ Ito Server - -- WebSocket: catalog request/response, acquire/session control-plane messages, session lifecycle events, displayable errors/reasons, and non-trickle WebRTC offer/answer signaling. -- WebRTC data channel: binary Splat Batches from server to client. -- Session lifecycle: server sends session-ended state and Session Termination Reason to the client. +## Pilot client protocol -### Ito Server ↔ Robot Driver +The client opens `/ws` on the same origin that served it. -- WebSocket: driver status/heartbeat, driver-side session start/end control-plane messages, displayable errors/reasons, and non-trickle WebRTC signaling. -- WebRTC media track: H.264 robot camera media from driver to server. -- Session lifecycle: server asks the driver to perform session start/end procedures; driver can request session end for non-recoverable driver/robot conditions. -- Catalog reporting: driver reports `robotId`, `name`, Robot Type, availability, and availability detail. -- `robotId` is a stable machine-readable key for catalog/session bookkeeping, not a credential. If two connected drivers report the same `robotId`, the server must not pick one; it marks the affected robot Unavailable and logs an error. -- The Ito Server applies a Driver Status Watchdog. V1 drivers send status/heartbeat every 1 second by default, and the server marks the robot Unavailable if the driver WebSocket disconnects or if no fresh status/heartbeat arrives within 2 seconds by default. Both values are configurable by environment variable. The server evaluates this proactively rather than waiting for a client catalog request. - -### Pilot Client ↔ Robot Driver - -- WebRTC data channel: Pilot Input Snapshots from client to driver. -- V1 default channel profile is unordered and unreliable; profile is session configuration supplied by the server. -- The driver uses the newest available Pilot Input Snapshot at each Driver Control Tick and owns control-loss and resumption behavior. -- Pilot Input Snapshot data-channel messages are UTF-8 JSON in v1 because the payload is small control state, not high-volume reconstruction data. Each message is a full snapshot with `protocolVersion`, `sessionId`, `sequence`, `timestampMs`, `headsetYawRad`, and `controllers`. - - -## V1 payload tables - -### `catalog.get` - -Pilot Client request to fetch the current Robot Catalog. Expects `catalog.get.result`. Request Timeout impact: fail only this catalog refresh in the Pilot Client. - -| Field | Required | Meaning | -| --- | --- | --- | -| `includeUnavailable` | no | Boolean. Defaults to `true`; v1 clients normally include unavailable known robots so operational defects stay visible. | - -### `catalog.get.result` - -Standard result payload. On success, `value` contains: - -| Field | Required | Meaning | -| --- | --- | --- | -| `robots` | yes | Array of Robot Catalog Entry maps. | - -Robot Catalog Entry fields: - -| Field | Required | Meaning | -| --- | --- | --- | -| `robotId` | yes | Stable machine-readable Robot Identity supplied by the driver. | -| `name` | yes | Pilot-facing proper robot name supplied by the driver and displayed as-is. | -| `type` | yes | Robot Type: `Mecha`, `Android Robot`, `Droid`, `Drone`, `Car`, or `Plane`. | -| `status` | yes | Robot Status: `Available`, `Occupied`, or `Unavailable`. | -| `availabilityDetail` | no | Display Reason map explaining current availability or unavailability. | - -### `connection.hello.result` - -Standard result payload. On success, `value` contains: - -| Field | Required | Meaning | +| Message | Direction | Purpose | | --- | --- | --- | -| `protocolVersion` | yes | Accepted Ito Protocol Version, `ito.v1`. | -| `role` | yes | Accepted role: `pilotClient` or `robotDriver`. | -| `sessionResumed` | no | Boolean, present for Pilot Client reconnect attempts. `true` means the server accepted the claimed session for resume. | -| `sessionConfig` | no | Session Configuration map when a piloting session is resumed. | - -### `robot.status` +| `connection.hello` | client → Ito | `{role: "pilotClient"}`. | +| `connection.hello.result` | Ito → client | Readiness, active state, and control configuration. | +| `control.start` | client → Ito | Explicitly begin control. | +| `control.start.result` | Ito → client | Confirms start and returns control configuration. | +| `control.stop` | client → Ito | Stop control with an optional Display Reason. | +| `control.stop.result` | Ito → client | Confirms the request was accepted. | +| `control.stopped` | Ito → client | Final stopped state and reason. | +| `robot.ready` | Ito → client | The configured robot became ready or unavailable. | +| `webrtc.offer` / `webrtc.answer` | either | Non-trickle signaling for a live path. | -Robot Driver heartbeat/status report. Fire-and-forget; no response expected. The Ito Server uses receipt time for the Driver Status Watchdog. +Pilot live paths are: -| Field | Required | Meaning | -| --- | --- | --- | -| `name` | yes | Pilot-facing proper robot name. | -| `type` | yes | Robot Type: `Mecha`, `Android Robot`, `Droid`, `Drone`, `Car`, or `Plane`. | -| `status` | yes | Driver-declared Robot Status: normally `Available` or `Unavailable`; the server may publish `Occupied` while reserved or in session. | -| `availabilityDetail` | no | Display Reason map or free text fallback for catalog display. | +- `pilotInput`: client-created unordered/unreliable data channel. Each UTF-8 + JSON snapshot contains `protocolVersion`, `sequence`, `timestampMs`, + `headsetYawRad`, and a `controllers` list. It has no allocation identity. +- `splatBatches`: Ito-created reliable/ordered data channel carrying the binary + Splat Batch format. -### `session.acquire` - -Pilot Client request to acquire an available robot. Expects `session.acquire.result`. Request Timeout impact: fail only this acquisition attempt in the Pilot Client; the server remains responsible for releasing any reservation it owns. - -| Field | Required | Meaning | -| --- | --- | --- | -| `robotId` | yes | Robot Identity to acquire. Also appears in the envelope `robotId`. | - -### `session.acquire.result` - -Standard result payload. On success, `value` contains: - -| Field | Required | Meaning | -| --- | --- | --- | -| `sessionId` | yes | Server-owned Session Identity allocated for this piloting session. | -| `robotId` | yes | Acquired Robot Identity. | -| `sessionConfig` | yes | Session Configuration map for WebRTC data channel profiles and client-owned runtime behavior. | - -### `driver.session.start` - -Ito Server request asking a Robot Driver to perform session-start procedures. Expects `driver.session.start.result`. Request Timeout impact: server releases the acquisition reservation and fails the pending `session.acquire` with a Display Reason. - -| Field | Required | Meaning | -| --- | --- | --- | -| `sessionId` | yes | Server-owned Session Identity. Also appears in envelope `sessionId`. | -| `sessionConfig` | yes | Session Configuration map relevant to driver setup and WebRTC paths. | - -### `driver.session.start.result` - -Standard result payload. On success, `value` contains: - -| Field | Required | Meaning | -| --- | --- | --- | -| `sessionId` | yes | Session Identity that is ready to start exchanging live data. | - -### `session.end` - -Request to end a piloting session. May be sent by the Pilot Client or Robot Driver to the Ito Server, and by the Ito Server to the Robot Driver for clean session-end procedure. Expects `session.end.result` when sent as a request. Request Timeout impact: the server still marks the session ended and fans out `session.ended` without waiting for driver acknowledgement. - -| Field | Required | Meaning | -| --- | --- | --- | -| `reason` | no | Display Reason map for why the session is ending. | -| `clean` | no | Boolean. `true` for normal pilot-requested stop; `false` or absent for failures or unknown cleanliness. | - -### `session.end.result` - -Standard result payload. On success, `value` contains: - -| Field | Required | Meaning | -| --- | --- | --- | -| `sessionId` | yes | Session Identity whose endpoint-specific end procedure accepted the request. | - -### `session.ended` - -Server event sent directly to the Pilot Client and Robot Driver after the server has marked the session ended. Fire-and-forget; no response expected. - -| Field | Required | Meaning | -| --- | --- | --- | -| `reason` | yes | Display Reason map for pilot display and driver logging. | -| `endedBy` | yes | `pilotClient`, `robotDriver`, or `server`. | -| `clean` | no | Boolean indicating normal stop when known. | - -### `webrtc.offer` - -Non-trickle WebRTC Session Description Protocol offer. Expects `webrtc.answer`. Request Timeout impact depends on the live path: during acquisition it fails acquisition; during active-session renegotiation it ends the affected session. - -| Field | Required | Meaning | -| --- | --- | --- | -| `path` | yes | Live path being negotiated: `pilotInput`, `cameraMedia`, or `splatBatches`. | -| `sdp` | yes | Complete SDP offer string after ICE gathering is complete. | - -### `webrtc.answer` - -Non-trickle WebRTC Session Description Protocol answer. Response to `webrtc.offer`. - -| Field | Required | Meaning | -| --- | --- | --- | -| `path` | yes | Live path being negotiated: `pilotInput`, `cameraMedia`, or `splatBatches`. | -| `sdp` | yes | Complete SDP answer string after ICE gathering is complete. | - -## Shared value maps - -### Session Configuration - -| Field | Required | Meaning | -| --- | --- | --- | -| `pilotInputDataChannel` | yes | Data Channel Profile for Pilot Input Snapshots. | -| `splatBatchDataChannel` | yes | Data Channel Profile for Splat Batches. | -| `pilotInputRateHz` | no | Server-provided default or accepted client runtime setting. | -| `visualFreshnessTimeoutMs` | no | Client visual-freshness timeout value for this session. | - -### Data Channel Profile - -| Field | Required | Meaning | -| --- | --- | --- | -| `ordered` | yes | Boolean WebRTC data channel ordering flag. | -| `maxRetransmits` | no | WebRTC `maxRetransmits`; absent means browser/runtime default. | -| `maxPacketLifeTime` | no | WebRTC `maxPacketLifeTime`; absent means browser/runtime default. | - -## Splat Batch binary layout - -V1 Splat Batches are binary WebRTC data-channel messages. Multi-byte values are little-endian. - -Header, 28 bytes: - -| Offset | Type | Field | -| --- | --- | --- | -| 0 | `char[8]` | Magic bytes `ITOSPLAT`. | -| 8 | `uint16` | Format version, currently `1`. | -| 10 | `uint16` | Flags, currently `0` unless a processor-specific extension is documented. | -| 12 | `uint32` | Batch sequence. | -| 16 | `uint32` | Splat count. | -| 20 | `uint16` | Record stride in bytes, currently `36`. | -| 22 | `uint8[6]` | Reserved, zero-filled. | +## Optional remote-driver protocol -Each splat record is 36 bytes: +The remote driver connects to the same `/ws` endpoint with +`{role: "remoteRobotDriver", ready: boolean}`. Only the configured remote +backend accepts this role, and only one driver may attach. -| Offset | Type | Field | +| Message | Direction | Purpose | | --- | --- | --- | -| 0 | `float32[3]` | Position xyz. | -| 12 | `float32[3]` | Scale xyz. | -| 24 | `int16[4]` | Rotation quaternion xyzw normalized to `[-32767, 32767]`. | -| 32 | `uint8[4]` | RGBA color. | +| `driver.control.start` | Ito → driver | Enter the driver's active-control state. | +| `driver.control.start.result` | driver → Ito | Success or robot-local reason for failure. | +| `driver.control.stop` | Ito → driver | Enter the safe stopped state. | +| `driver.control.stop.result` | driver → Ito | Confirms local stop handling. | +| `webrtc.offer` / `webrtc.answer` | either | `pilotInput` negotiation or driver `cameraMedia` transport. | -This layout is intentionally compact and directly typed-array friendly for the -Pilot Client. Spark.JS-specific insertion may still choose a faster internal -copy path after Pico 4 testing, but that must preserve this wire layout unless -the protocol version changes. +`cameraMedia` is the optional remote-driver-to-Ito H.264 media path. In onboard +mode this path doesn't exist; frames enter reconstruction directly. -## Contract style +## Failure behavior -- WebSocket control-plane messages should be documented as explicit MessagePack envelope and payload shapes before or alongside implementation. -- Request/response message definitions should state whether a response is expected and which configurable Request Timeout applies. -- Request/response message definitions should state timeout impact: fail only the pending operation, end the affected session, or close/reject the endpoint. -- Message `type` values should use dot-separated names and should not reserve names for future features before those features are part of the protocol. -- Control-plane messages that cross program boundaries should include `protocolVersion: "ito.v1"` unless the surrounding transport or endpoint is already explicitly version-scoped. -- Binary payloads should document their headers, version fields, units, coordinate frames, and interpretation rules. -- WebRTC media tracks should document codec and semantic meaning, not robot-specific camera setup. -- ROS concepts belong only inside the Ito Droid driver implementation and must not appear in the Ito Protocol. +- Pilot disconnect: Ito stops control and invokes local safety immediately. +- Remote driver disconnect: Ito stops control; driver safety already acts + locally without waiting for Ito. +- Stale pilot input: the adapter/driver uses its robot-specific timeout and + safe response. +- Reconstruction or live-path failure: Ito stops control and sends a + `control.stopped` reason without crashing the application. diff --git a/docs/todo.md b/docs/todo.md index 1da9bce..894f50c 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,150 +1,29 @@ -# Ito v1 TODO +# Ito work -Reference docs are canonical: `README.md`, `docs/v1.md`, `docs/protocol.md`, and `docs/adr/`. -Keep task details in those docs rather than duplicating them here. -When checking off a TODO whose task description does not fully describe the implementation, add a nested checked box with a useful note for the next agent. +## Implemented foundation -1. [x] Define the v1 protocol payload tables missing from `docs/protocol.md`. - - [x] Added payload tables for all initial v1 message types plus shared Session Configuration and Data Channel Profile maps. -2. [x] Create the shared protocol constants and MessagePack envelope helpers. - - [x] Added Python shared constants, envelope creation, validation, and MessagePack pack/unpack helpers under `server/ito/protocol.py`. -3. [x] Add protocol-version validation and Display Reason helpers. - - [x] Added exact `ito.v1` validation, standard result payload helpers, and Display Reason validation. -4. [x] Scaffold the Python Ito Server application and container. - - [x] Added `server.ito.app` entry point, package scaffolding, requirements, and `server/Dockerfile`. -5. [x] Implement server configuration from environment variables. - - [x] Added env-backed `ServerConfig` for request timeout, driver watchdog, session cleanup timeout, and data channel profiles. -6. [x] Implement server WebSocket accept, hello, routing, and request timeouts. - - [x] Added MessagePack WebSocket accept, mandatory hello handling, basic role-based routing, and error results. - - [x] Added outbound `driver.session.start` request tracking with configured timeout handling; server-sent `session.end` does not wait for acknowledgement per protocol. -7. [x] Implement robot-driver connection tracking and status watchdogs. - - [x] Added driver connection records, disconnect handling, status freshness evaluation, and proactive watchdog marking for stale drivers. -8. [x] Implement the in-memory Robot Catalog. - - [x] Added in-memory driver records that produce protocol Robot Catalog entries from latest driver status. -9. [x] Implement duplicate `robotId` detection. - - [x] Duplicate driver hellos mark the affected robot unavailable and log an operational error instead of choosing one connection. -10. [x] Implement pilot-client catalog requests. - - [x] Pilot clients can request `catalog.get` after hello and receive MessagePack `catalog.get.result` responses, with optional unavailable filtering. -11. [x] Implement server-side acquisition reservation. - - [x] Acquisition now serializes through a server lock, marks the robot Occupied before driver start, and rejects competing pilots while the reservation or session exists. -12. [x] Implement driver session-start request/result handling. - - [x] The server sends `driver.session.start`, correlates `driver.session.start.result` by `replyToMessageId`, releases reservations on failure or timeout, and validates the returned `sessionId`. -13. [x] Implement server-owned session allocation. - - [x] The server generates `session-*` identities, stores in-memory session records, and returns Session Configuration in successful acquire and resume results. -14. [x] Implement session end and `session.ended` fan-out. - - [x] Pilot/driver `session.end` requests mark the session ended, free the robot, send a driver end request when needed, and fan out `session.ended` to connected endpoints. -15. [x] Implement session cleanup for disappeared endpoints. - - [x] Disconnect bookkeeping keeps sessions resumable until `ITO_SESSION_CLEANUP_TIMEOUT_MS`, then ends stale sessions with `session.ended.endpoint_disappeared`. -16. [x] Implement reconnect hello handling for resumable sessions. - - [x] Pilot `connection.hello` with an active `sessionId` resumes the session and returns Session Configuration; unavailable sessions fail hello with `session.resume_unavailable`. Reconnected drivers are reattached to active sessions for their robot. -17. [x] Add server tests for catalog, acquisition, lifecycle, and reconnect behavior. - - [x] Added unit tests for successful acquisition, competing acquisition, start failure, start timeout, session end fan-out, disappeared-endpoint cleanup, and reconnect resume/rejection. -18. [x] Scaffold the Mock Robot driver and container. - - [x] Added `drivers/mock-robot` Python package, entrypoint, requirements, Dockerfile, and README run/build instructions. -19. [x] Implement Mock Robot status reporting. - - [x] Mock Robot sends v1 MessagePack `connection.hello` and periodic `robot.status`; it reports Unavailable until `ITO_MOCK_ROBOT_CAMERA_VIDEO` is configured. -20. [x] Implement Mock Robot acquisition and session lifecycle handling. - - [x] Handles `driver.session.start`, `session.end`, and `session.ended`, tracks one active server-owned session, opens/closes mock camera input, and sends standard result payloads. -21. [x] Implement Mock Robot pilot-input reception and logging. - - [x] Added `receive_pilot_input_snapshot()` as the driver-side receive/log sink for TODO 24's WebRTC data-channel transport; snapshots are JSON-logged to stdout and no fake robot pose is maintained. -22. [x] Add video-file-backed Mock Robot camera input. - - [x] Added `VideoFileCamera` source that validates and reads a configured video file in chunks, optionally looping; WebRTC H.264 publishing remains TODO 23. -23. [ ] Implement driver-to-server WebRTC H.264 media transport. - - [ ] Local progress: server-side camera media receive and Mock Robot video-file publishing are locally wired; Ito Droid ROS-frame-to-H.264 WebRTC publishing and physical camera verification remain incomplete. - - [x] Added server-side `cameraMedia` aiortc track consumption into session-scoped reconstruction frames, using the existing reconstruction runtime and Null processor seam until a v1 processor is selected. - - [x] Added Mock Robot video-file `cameraMedia` publishing over aiortc `MediaPlayer`, with H.264 codec preference, non-trickle offer/answer signaling, and session cleanup. - - [x] Extended the Mock Robot local e2e test to assert that `cameraMedia` delivers a decoded frame to server reconstruction when aiortc/PyAV/FFmpeg H.264 support are installed; the test skips clearly when those optional dependencies are unavailable. - - [ ] Remaining: wire Ito Droid ROS camera frames into concrete H.264 WebRTC publishing and verify against physical camera hardware. -24. [x] Implement client-to-driver WebRTC pilot-input data channel. - - [x] Added browser non-trickle Pilot Input data-channel offer creation plus driver-side JSON snapshot data-channel decoding into the existing `receive_pilot_input_snapshot()` sink. -25. [x] Implement server-to-client WebRTC Splat Batch data channel. - - [x] Added browser Splat Batch peer negotiation/receiver and server-side Splat Batch data-channel registry for sending encoded binary batches when the server-owned channel opens. -26. [x] Add non-trickle WebRTC signaling over the WebSocket control plane. - - [x] Server validates WebRTC live paths, relays `pilotInput` offers/answers between pilot and driver, and answers server-terminated `cameraMedia`/`splatBatches` offers through an injectable live-path acceptor. -27. [ ] Record representative USB-webcam reconstruction test sequences. - - [ ] Not completed locally: requires physical USB-webcam capture with representative piloting head motion/environments. -28. [ ] Spike MASt3R-SLAM on the recorded sequences. - - [ ] Blocked on TODO 27 recorded sequences and local GPU/research setup. -29. [ ] Spike MonoGS on the recorded sequences. - - [ ] Blocked on TODO 27 recorded sequences and local GPU/research setup. -30. [ ] Select the v1 monocular reconstruction path. - - [ ] Not selected: MASt3R-SLAM and MonoGS comparison is still pending. -31. [x] Define the server-internal reconstruction processor interface. - - [x] Added `server/processors/base.py` with `ReconstructionFrame`, `GaussianSplat`, `ProcessorSplatBatch`, and `ReconstructionProcessor`. -32. [ ] Integrate the selected processor under `server/processors/`. - - [ ] Added `NullReconstructionProcessor` as an integration seam only; no selected v1 algorithm has been integrated. -33. [x] Implement camera media decoding into reconstruction frames. - - [x] Added `H264CameraDecoder` that uses PyAV to decode H.264 samples into RGB `ReconstructionFrame` values for processor ingress. -34. [x] Implement reconstruction failure isolation per session. - - [x] Added `ReconstructionSessionRuntime` that catches processor exceptions, reports `session.ended.reconstruction_failed`, and prevents repeated failures from escaping the affected session. -35. [ ] Spike Spark.JS Splat Batch insertion on Pico 4. - - [ ] Not completed locally: requires Pico 4 browser/Spark.JS performance testing. -36. [x] Freeze the v1 Splat Batch binary layout. - - [x] Documented the v1 `ITOSPLAT` little-endian binary header and 36-byte splat record layout in `docs/protocol.md`. -37. [x] Implement server Splat Batch encoding. - - [x] Added `server/ito/splat.py` encoder/decoder-header helpers for the v1 binary Splat Batch format. -38. [x] Scaffold the plain-JavaScript Pilot Client. - - [x] Added a static A-Frame/WebXR client under `client/` with plain ES modules, no build step, and Node built-in tests. -39. [x] Implement the browser Enter VR launch surface. - - [x] Added a minimal non-VR launch page whose primary action calls `a-scene.enterVR()` from a user gesture. -40. [x] Implement client configuration defaults and Local Storage settings. - - [x] Added defaults for server URL, request timeout, visual-freshness timeout, Pilot Input Rate, Splat Budget, and Splat Lifetime persisted under `ito.pilotClient.settings.v1`. -41. [x] Add pilot-facing text resource loading. - - [x] Added `resources/en/default.json` with i18next-style nested keys and resource-key/free-text Display Reason fallback. -42. [x] Implement in-VR controller-ray UI foundations. - - [x] Added A-Frame laser controller raycasters, clickable VR button entities, and reusable panel/button/label helpers. -43. [x] Implement the in-VR Robot Catalog. - - [x] Added MessagePack WebSocket `connection.hello` and `catalog.get` handling with localized robot type/status labels and refresh. -44. [x] Implement acquisition and connecting states in VR. - - [x] Added `session.acquire` flow with an in-VR connecting panel, disabled duplicate controls, and Display Reason fallback on failure. -45. [x] Implement session view with Spark.JS Splat Scene ownership. - - [x] Added a client-owned `SplatSceneOwner` and `SparkJsSplatAdapter` seam. Actual Spark insertion remains intentionally isolated behind the adapter because TODO 35-37 have not frozen the Pico 4 insertion path or binary layout. -46. [x] Implement Splat Lifetime and Splat Budget eviction. - - [x] Added age-based and oldest-first budget eviction on the client-owned batch registry. -47. [x] Implement headset-yaw Pilot Input Snapshots. - - [x] Added relative headset-yaw snapshot generation with full controller button/axis state and a data-channel transport seam; actual WebRTC attachment remains covered by TODO 24-26. -48. [x] Implement client visual-freshness timeout behavior. - - [x] Added timeout tracking from the last normal splat apply path; stale visuals freeze the Splat Scene and withhold pilot input while keeping VR UI active. -49. [x] Implement in-VR menu pause and session end action. - - [x] Added controller/menu-button pause behavior that withholds pilot input, plus clean `session.end` request from the in-VR menu. -50. [x] Implement session-ended popup and return-to-catalog flow. - - [x] Added `session.ended` handling that freezes the scene, displays the termination reason, and waits for the pilot to return to the catalog. -51. [ ] Verify the client on Pico 4's built-in browser. -52. [x] Scaffold the Ito Droid ROS driver and container. - - [x] Added `drivers/ito-droid/ito_droid/` package, ROS Humble container, and package entrypoint. -53. [x] Implement Ito Droid environment-based configuration. - - [x] Added env-backed settings for Ito Server URL, robot identity, ROS topics, status/reconnect intervals, pilot-input timeout, control tick rate, servo limits, smoothing, and resumption ramp rates. -54. [x] Implement Ito Droid status reporting. - - [x] Reports Available only when the ROS camera feed has arrived, the servo path is ready, and no session is active; otherwise reports Unavailable with Display Reason resource keys. -55. [x] Consume the configured ROS camera feed. - - [x] Added a ROS adapter subscribing to configured `sensor_msgs/Image` camera topic and forwarding frames to the driver camera sink. -56. [x] Publish camera media to the server over WebRTC. - - [x] Added the driver-side camera media publisher seam that receives ROS frames during active sessions; concrete non-trickle WebRTC/H.264 transport remains covered by TODO 23 and TODO 26. -57. [x] Receive Pilot Input Snapshots from the client. - - [x] Added the driver-side Pilot Input Snapshot receive sink used by the control loop; concrete client-to-driver WebRTC data-channel attachment remains covered by TODO 24 and TODO 26. -58. [x] Implement yaw-to-camera-pan servo mapping. - - [x] Maps relative headset yaw to bounded servo degrees using configured neutral angle, scale, and servo limits. -59. [x] Implement driver control tick processing. - - [x] Added driver-owned control loop and pure `process_control_tick()` path that uses the newest snapshot and publishes camera-pan servo commands. -60. [x] Implement pilot-input timeout behavior. - - [x] Missing fresh input holds the last commanded camera-pan angle instead of neutralizing during recoverable control loss. -61. [x] Implement safe control resumption ramping. - - [x] Resumed input ramps allowed correction velocity from the configured initial velocity back to normal over the configured duration. -62. [x] Implement session-start servo neutralization. - - [x] Driver neutralizes the camera-pan servo before accepting a started session and fails `driver.session.start` if neutralization fails. -63. [x] Implement clean session-end servo neutralization. - - [x] Clean server `session.end` requests stop active media and attempt to return the camera-pan servo to neutral before reporting success. -64. [x] Add driver tests around mapping, timeout, and lifecycle behavior. - - [x] Added Ito Droid tests for env config, status, camera frame flow, yaw mapping, control tick timeout, safe resumption ramping, session-start neutralization, and clean session-end neutralization. -65. [x] Add end-to-end Mock Robot tests over WebSocket and WebRTC. - - [x] Added `tests/test_mock_robot_e2e.py`, which starts a real local Ito Server WebSocket endpoint, runs the actual Mock Robot driver against it, acquires the robot as a pilot, negotiates relayed `pilotInput` WebRTC with `aiortc`, and sends a Pilot Input Snapshot over the data channel into the mock driver's logging sink. The test is skipped when the local Python environment has not installed the documented `aiortc` dependency. - - [x] Added Mock Robot `pilotInput` WebRTC offer handling in `drivers/mock-robot/mock_robot/webrtc.py` and `drivers/mock-robot/mock_robot/driver.py`; driver-to-server H.264 camera media remains TODO 23. -66. [ ] Add end-to-end Ito Droid smoke testing on physical hardware. - - [x] Documented physical smoke-test expectations in `drivers/ito-droid/README.md` and the hardware-only acceptance checklist in `docs/acceptance-v1.md`. - - [ ] Not run locally: requires physical Ito Droid hardware, reachable Ito Server, robot-local ROS camera feed, servo command path, and Pico 4 browser. -67. [x] Document Docker Compose commands for local v1 operation. - - [x] Added `compose.yaml` with `ito-server`, `pilot-client`, optional `mock` profile, and optional `droid` profile services. - - [x] Added `docs/local-v1.md` with build/run/log/down commands, Mock Robot H.264 sample-file mounting, Ito Droid robot-side profile usage, and local test commands. -68. [ ] Run a full v1 acceptance pass against the core outcome. - - [x] Recorded the current local acceptance pass in `docs/acceptance-v1.md`, including server/protocol/client/driver unit coverage and the new Mock Robot WebSocket/WebRTC e2e path. - - [ ] Full core-outcome acceptance remains blocked by TODO 23 driver-to-server H.264 WebRTC media transport, TODO 27-30/32 reconstruction selection and integration, TODO 35/51 Pico 4 Spark/browser validation, and TODO 66 physical Ito Droid smoke testing. +- [x] One Ito application hosts the WebXR client and `/ws` endpoint. +- [x] One pilot connects directly and explicitly starts/stops control. +- [x] Catalog, browsing, allocation, reservation, stable robot IDs, fleet + heartbeats, and competing-pilot locking are removed. +- [x] Local robot adapter moves sensor frames and pilot input in-process. +- [x] Remote robot adapter and lightweight mock driver preserve the low-spec + external deployment option without changing the pilot protocol. +- [x] Pilot input and Splat Batch WebRTC paths terminate at the Ito endpoint in + onboard mode. +- [x] Robot-local timeout and neutralization behavior remains in Ito Droid. +- [x] Local adapter owns input timeout, newest-input rate limiting, neutral stop, + emergency stop, and explicit restart after an emergency stop. +- [x] Ito Droid remote fallback carries pilot input and ROS camera frames over + WebRTC while retaining control and safety on the robot. +- [x] One-container Compose deployment serves the client. + +## Next concrete work + +- [ ] Select and integrate the first onboard reconstruction algorithm against + the existing `ReconstructionProcessor` seam. +- [ ] Connect a production local adapter to Ito Droid ROS sensor and actuator + APIs so the physical reference robot uses onboard mode. +- [ ] Run Pico 4 hardware acceptance for control, visual freshness, disconnect, + and safe resumption. +- [ ] Measure end-to-end latency and avoidable copies in both placement modes. diff --git a/docs/v1.md b/docs/v1.md index bc4643d..79aa959 100644 --- a/docs/v1.md +++ b/docs/v1.md @@ -1,132 +1,72 @@ # Ito v1 -Ito v1 proves the complete teleoperation loop with one pilot and one physical reference robot. The highest-risk premise—live reconstruction for an immersive, latency-compensated view—is implemented first rather than represented by static test data. - -## Core outcome - -A pilot uses the WebXR client to acquire an available Ito Droid, perceive its surroundings through a live session-scoped reconstruction, and control at least its camera-pan servo through head movement. Temporary connection failure invokes the driver's robot-specific control-loss response while preserving the piloting session for recovery. - -## System boundary - -- Ito v1 consists of three cooperating programs: the Pilot Client, the Ito Server, and a Robot Driver. For the reference robot, that driver is the Ito Droid robot driver. -- The Ito Protocol is the boundary between those programs. Client, server, and driver implementations should depend on the protocol concepts and messages, not on each other's internal implementation details. -- Ito Protocol seams are documented in `docs/protocol.md`. V1 keeps these contracts docs-first and manually implemented rather than introducing protocol code generation. -- Ito v1 requires an exact Ito Protocol Version match. Pilot Client, Ito Server, and Robot Driver must advertise `ito.v1`; mismatches fail acquisition or session start with a clear Session Termination Reason or acquisition failure reason. -- One monolithic Ito Server program, process, and IPC API maintains the Robot Catalog, session coordination, and reconstruction. -- The Pilot Client is a static website hosted separately, such as by an nginx container. Static web hosting is not part of Ito IPC. -- The server targets an x86_64 PC with an NVIDIA GPU and runs as one container with NVIDIA Container Runtime GPU access. -- The v1 Pilot Client acceptance target is Pico 4's built-in browser, not PCVR or desktop WebXR. -- A ROS-based robot driver runs on the robot as a container built on a ROS base image and connects outward to the server. -- The Ito Droid driver consumes existing ROS sensor feeds, such as a camera topic. Producing and configuring those feeds is outside Ito v1's scope. -- ROS is specific to the Ito Droid driver implementation. ROS concepts must not leak into the Pilot Client, Ito Server, or Ito Protocol. -- Pilot Client and Robot Driver each maintain a WebSocket control connection to the Ito Server for Ito IPC: catalog requests/responses, driver status, acquisition, session lifecycle, errors/reasons, and WebRTC signaling. V1 WebSocket control messages use a common MessagePack-encoded envelope over binary WebSocket frames. -- Pilot input flows directly from client to driver; camera footage flows to server reconstruction; reconstructed Gaussian splats flow to the client. -- V1 uses WebSocket for control-plane IPC and WebRTC signaling. V1 uses three WebRTC peer connections for live session paths: Pilot Client ↔ robot driver for pilot input, robot driver ↔ Ito Server for robot camera/media transport, and Ito Server ↔ Pilot Client for Splat Batches. V1 robot camera transport uses a WebRTC media track encoded as H.264; bitrate, congestion behavior, and other media tuning are left to WebRTC defaults. The Ito Server owns decoding received camera media into frames for reconstruction inside the monolithic server boundary, using internal libraries or tools as needed. Reconstruction may be in-process or a subprocess inside the same server container, but not a separately deployed service. -- WebSocket endpoints identify themselves with `connection.hello` after connect or reconnect. Robot Drivers reconnect forever with backoff. Pilot Clients reconnect while the app is open and include the existing `sessionId` when attempting to resume a piloting session. The Ito Server decides whether the session can resume; if not, the Pilot Client shows the Display Reason and returns to the Robot Catalog. On successful resume, v1 recreates the relevant WebRTC peer connections with fresh non-trickle offer/answer signaling rather than preserving old peer connections. -- Pilot input is sent as repeated full-state Pilot Input Snapshots. The v1 default pilot-input data channel is unordered and unreliable so stale pose samples do not block newer samples. The data channel reliability/ordering profile is configurable through server container environment variables and exposed to the Pilot Client as session configuration. V1 sends Pilot Input Snapshots at 60 Hz by default, configurable through client runtime settings. -- Splat Batches are sent as binary data channel payloads. The v1 default Splat Batch data channel is reliable and ordered because batches mutate the client-owned Splat Scene. The Splat Batch data channel reliability/ordering profile is configurable through server container environment variables and exposed to the Pilot Client as session configuration. The v1 binary format should be chosen to minimize parsing, decoding, and conversion before insertion into the A-Frame + Spark.JS Splat Scene; avoid JSON for splat payloads. -- Reconstruction failure stops the affected piloting session but must not crash the Ito Server. The server sends session-ended messages directly to both the Pilot Client and robot driver with a server-generated Session Termination Reason, such as the resource key `session.ended.reconstruction_failed`. This preserves the path toward future multi-robot and multi-pilot operation. -- During recoverable reconnect, the Ito Server retains reconstruction state only if the selected reconstruction algorithm can safely continue after transport or camera discontinuity. If the algorithm cannot tolerate that discontinuity, the server may reset reconstruction state within the same piloting session and resume fresh reconstruction when camera media returns. This does not by itself end the session unless reconstruction cannot restart. -- A reconstruction reset within the same piloting session does not force the Pilot Client to clear its Splat Scene in v1. The client continues normal Splat Batch insertion, Splat Lifetime, and Splat Budget eviction so fresh splats gradually replace older splats. -- The server does not send a dedicated reconstruction-reset event in v1. Visual Freshness Timeout and session-ended handling cover pilot-visible reconstruction interruption or failure. -- The Ito Server does not make robot-control decisions. The robot driver owns robot safety behavior; the Pilot Client can trigger control loss by withholding pilot input when its local checks fail. -- The Ito Server is the session authority: it serializes acquisition attempts, reserves robots during acquisition, allocates piloting sessions, and marks robots Occupied. Robot drivers remain responsible for whether they are available and for robot-control safety. -- The Pilot Client decides reconstruction freshness using time since the last Splat Batch completed its normal receive-and-apply path. V1 does not require a separate protocol-validity, view-quality, or per-sensor visibility metric. -- When visual freshness times out, the Pilot Client freezes the last known Splat Scene, greys or fades the reconstructed view, shows that robot controls are paused because the visual feed is lost, and stops sending pilot input. V1 defaults this client-side timeout to 2 seconds, with the value configurable on the client. In-VR UI remains usable while robot controls are paused, including settings and ending the session. -- If fresh Splat Batches resume while the piloting session still exists, the Pilot Client unfreezes the Splat Scene, exits the paused visual state, and resumes sending pilot input. The robot driver is responsible for safe control resumption, including limiting and slowly ramping the velocity used to correct differences between pilot pose and robot pose. -- The system runs inside a trusted private network without application-layer authentication: initially local Wi-Fi, later a private VPN such as Tailscale when the headset, server, and robot are not on the same LAN. -- One pilot exclusively occupies one robot. Other clients may see that it is occupied but cannot enter or observe its session. -- The Ito Server marks a robot Unavailable when its driver WebSocket disconnects or does not send fresh status/heartbeat before the Driver Status Watchdog expires. V1 drivers send status/heartbeat every 1 second by default, and the watchdog expires after 2 seconds by default. Both values are configurable through environment variables. This is catalog correctness, not robot safety, and the server evaluates it proactively rather than waiting for a client catalog request. -- The Ito Server may also expire stale session records when responsible endpoints disappear. This cleanup exists for resource cleanup, not robot safety. V1 defaults this session cleanup timeout to 30 seconds. - -## Server configuration - -The Ito Server is implemented in Python and runs as a Docker container. Runtime settings, including server-owned WebSocket Request Timeouts, the driver-status watchdog, server-side session cleanup timeout, and WebRTC data channel reliability/ordering profiles, are configured through environment variables supplied by Docker Compose. V1 should prefer explicit environment variables over separate configuration services or mutable in-app configuration. - -Reconstruction algorithm implementations live under `server/processors/` inside the Ito Server boundary. Each implementation should present the same server-internal interface while declaring which Capture Modality it accepts. V1 starts with a monocular RGB path from the USB webcam feed; future algorithms may accept calibrated stereo RGB, RGB-D, or other Capture Modalities. - -## Robot-driver configuration - -The Ito Droid robot driver is implemented in Python and also runs as a Docker container configured through environment variables supplied by Docker Compose. V1 driver settings include the Ito Server URL, ROS topic names, driver status/heartbeat interval, pilot-input timeout, driver control tick rate, servo neutral angle, servo limits, smoothing, and control-resumption ramp rates. - -## Client configuration - -The Pilot Client is implemented in plain JavaScript, not TypeScript. It stores runtime settings in browser Local Storage and falls back to sane defaults defined in code. V1 client settings include visual-freshness timeout, Pilot Input Rate, Splat Budget, and Splat Lifetime. These settings are edited through an in-VR settings menu. - -The non-VR page is limited to the browser-required launch surface: the pilot opens the Ito site and clicks Enter VR because browsers require a user gesture before starting an immersive WebXR session. Catalog browsing, robot acquisition, session state, and settings belong inside VR. V1 uses controller-ray interaction for VR UI. During an active session, the pilot opens the in-VR menu with a controller button. While the menu is open, the Pilot Client enters Menu Pause: robot-directed pilot input is withheld, but UI interaction remains active. If Menu Pause lasts long enough to exceed the driver's pilot-input timeout, the driver enters its normal control-loss response. When the menu closes, the Pilot Client automatically resumes sending pilot input; the robot driver owns safe control resumption and must not instantly snap the robot to a substantially different pilot pose. - -Pilot-facing UI text is loaded from JSON resource files using an i18next-style structure. V1 should not hardcode pilot-facing labels, status messages, or menu text in UI components. Driver-provided availability details or failure reasons may be either resource keys or free text. The Pilot Client first attempts to resolve them through its localization resources and falls back to displaying the supplied text as-is. - -Robot names are proper names supplied by the driver and are displayed as-is. Enum-style labels controlled by Ito, such as Robot Type and Robot Status, are localized through resource files. - -V1 ships one default resource set. The resource structure should not block future localization or themed language variants, but v1 does not implement theme selection or multiple language packs. - -The v1 in-VR Robot Catalog is requested by the Pilot Client from the Ito Server over WebSocket. The server keeps the catalog in memory and builds it from Driver Status Reports sent over robot-driver WebSocket control connections. Each robot driver reports a stable machine-readable `robotId`, a pilot-facing `name`, a driver-declared Robot Type, Robot Status, and one short availability detail. `robotId` is used for catalog and session bookkeeping; because v1 skips credentials and authentication, it is not proof of identity. V1 Robot Status values are Available, Occupied, and Unavailable. Available robots can be acquired; Occupied robots are reserved for an in-progress acquisition or already allocated to a piloting session; Unavailable robots are known but not currently acquirable, such as when their driver has missed the Driver Status Watchdog, is not ready to accept control, or is in conflict with another driver reporting the same `robotId`. Duplicate `robotId` reports are treated as an operational defect: the server marks the affected robot Unavailable and logs an error rather than choosing one driver. Unavailable robots known to the server remain visible but unselectable because their absence would hide useful operational information. After a server restart, a robot is not shown until its driver reports again. Robot health/problem diagnosis is outside the Pilot Client's scope. Robot Type orients the pilot to the embodiment and likely control style they are about to use; it is not a formal capability contract. Ito's initial Robot Types are Mecha, Android Robot, Droid, Drone, Car, and Plane. Each Robot Type may develop a Control Convention over time, but v1 still treats concrete control mapping as robot-driver-owned. - -When a pilot selects an Available robot, the Pilot Client asks the Ito Server to acquire it. The server handles competing acquisition attempts in one place by atomically reserving the robot before asking the driver to start. The server may reserve a robot only when the robot driver is currently connected and reporting Available. Once reserved, the robot is Occupied so another near-simultaneous acquisition attempt cannot also reserve it. While acquisition is pending, the Pilot Client shows an in-VR "Connecting..." state with controls disabled. The driver performs its session-start procedure, such as neutralizing the camera-pan servo. If the driver reports success, the server completes acquisition, allocates the piloting session, returns the Session Identity to the Pilot Client, and the session can accept pilot input. If the start procedure fails, or if the `driver.session.start` request reaches its configured Request Timeout before `driver.session.start.result` arrives, the server releases the reservation and the client returns to the catalog with a human-understandable Display Reason. - -Live data paths may follow the directional loop of client input to robot driver, robot camera to server reconstruction, and splats back to client. Session lifecycle/control-plane messages do not follow that loop. The server communicates session lifecycle directly with both the Pilot Client and robot driver over WebSocket. When the server ends a session, v1 marks it ended immediately and sends session-ended messages to both endpoints without waiting for driver acknowledgement. When the pilot chooses to stop piloting from the in-VR menu, the Pilot Client requests session end from the Ito Server over WebSocket; the client does not resume robot-directed pilot input first, and the server uses the same session-ending path. - -The robot driver may request session end from the Ito Server only for non-recoverable driver or robot conditions, or explicit local stop/override. Examples include session-start failure, required hardware/control interface loss, required ROS topic loss, local physical emergency stop, or an internal driver defect. Ordinary missing pilot input, client visual-freshness timeout, or transient network loss should use control loss and recovery behavior instead of immediate driver termination. Driver- and server-requested session ends include a Session Termination Reason for client display, using the same resource-key-or-free-text model as driver availability details. - -For many non-happy-flow cases during an active session, v1 prefers a simple outcome: end the affected session and show the pilot a human-understandable Display Reason. Ito should not build a large recovery taxonomy before the happy path and core safety boundaries are proven. - -When the Pilot Client receives session-ended, it freezes the last known Splat Scene and shows an in-VR popup over it with the Session Termination Reason and a button to return to the Robot Catalog. It does not immediately replace the scene with the catalog. - -The in-VR catalog requests the Robot Catalog when opened or returned to after session end, and includes a refresh action for requesting it again. V1 does not auto-refresh or poll the catalog. - -## Physical acceptance target - -The initial Ito Droid uses available hardware: a Raspberry Pi, one USB webcam, and at least one camera-pan servo. A synchronized stereo camera and additional actuation are later Ito Droid hardware improvements, not v1 prerequisites. - -The Raspberry Pi environment must provide a ROS camera feed for the USB webcam. Ito v1 validates consuming that feed, not installing or managing the camera stack itself. - -V1 does not require a fixed camera resolution or frame rate. The robot driver, available sensor feed, and reconstruction spike determine what camera settings are practical. - -## V1 control mapping - -Ito v1 proves one embodied control mapping: pilot headset yaw controls the Ito Droid camera-pan servo. The pilot's yaw at piloting-session start defines zero yaw for that session. The robot driver maps relative yaw to the servo target and applies robot-specific limits, smoothing, and rate limits before publishing to ROS. - -For v1, Pilot Input Snapshots include the current headset yaw relative to the Pilot Frame and current controller state. Button state is represented as current state in the snapshot rather than relying on every individual button event arriving. The default Pilot Input Rate is 60 Hz. - -The robot driver runs its own control loop. On each Driver Control Tick, it uses the newest received Pilot Input Snapshot and discards older queued snapshots. Ito Droid v1 defaults this tick rate to 60 Hz, configurable through driver container environment variables. - -The Ito Droid driver treats missing fresh pilot input for 2 seconds as control loss. This timeout is driver-owned and robot-specific, even though Ito Droid's v1 default matches the client visual-freshness timeout. - -During recoverable control loss, the Ito Droid driver holds the camera-pan servo at its last commanded position. It returns the servo toward neutral only on clean session end or driver-terminated session. - -When pilot input resumes after control loss, the Ito Droid driver ramps the allowed camera-pan servo correction velocity from a low value back to normal over a configurable duration. This prevents an instant snap if pilot yaw and servo position diverged while input was paused. - -During the session-start procedure, the robot driver moves the camera-pan servo to its neutral position before accepting pilot yaw input. On clean session end, the driver attempts to return the servo to neutral again. This lifecycle behavior belongs to the robot driver, not the server or client. - -If the driver cannot move the camera-pan servo to neutral at session start, it requests session end with a Session Termination Reason and logs an error. This is treated as a robot or robot-driver defect, not as a recoverable pilot calibration problem. - -V1 does not include a recenter control. Recenter is treated as a temporary convenience that would disappear once Ito supports richer body mapping, so it is not part of the initial product surface. - -## First implementation milestone - -Record representative USB-webcam sequences containing the head motion and environments expected during piloting. Evaluate MASt3R-SLAM and MonoGS against the same inputs before selecting v1's monocular reconstruction algorithm. Compare at least tracking survival, reconstruction update rate, visual usefulness, end-to-end integration difficulty, GPU memory, frame-ingress/egress overhead, and observed behavior after short media interruptions with resumed feed from a slightly changed pose. Built-in recovery after interruption is a benefit, not a hard algorithm requirement; Ito can also reset reconstruction state within the same session if the selected algorithm needs that. - -Reconstruction must be testable with a Mock Robot that provides an input Sensor Feed from a video file. This belongs in the test harness rather than as a production replay mode inside the reconstruction module. Unit tests may inject decoded frames or an abstract feed directly. Integration tests should use a Mock Robot over the real WebRTC H.264 media path to exercise transport and decoding. - -The Mock Robot behaves like a robot driver in every relevant Ito interaction. If it is running and reports to the Ito Server, it appears in the Robot Catalog like any other robot; hiding it is done by not running it. It accepts pilot input and logs received Pilot Input Snapshots to stdout so session/control/menu/control-loss paths can be tested end-to-end. It does not maintain an internal fake robot pose unless a later test needs that state for a concrete assertion. - -Before freezing the Splat Batch wire format, spike the Spark.JS insertion path on the Pico 4 browser and choose the binary layout that requires the least client-side parsing, decoding, and conversion. - -## Explicitly deferred - -- persistent or cross-session reconstruction; -- multi-viewer and multi-pilot sessions; -- public-internet deployment, RBAC, and robot-driver authentication; -- operating TURN infrastructure; -- HTTP REST APIs for Ito IPC; -- trickle ICE unless non-trickle WebRTC signaling proves insufficient; -- a separate media ingestion/decoding service outside the monolithic Ito Server; -- a fixed requirement that reconstruction must run in the same OS process as the web/API runtime; -- reconstruction failures that crash the whole Ito Server; -- camera codec negotiation or support for non-H.264 camera codecs; -- TypeScript adoption or generated protocol bindings; -- robot-specific visual-freshness thresholds; -- hand-tracking, gaze-dwell, direct-touch, or other more physical VR UI interaction models; -- stereo-camera hardware as a requirement; -- CPU-only reconstruction support. +Ito v1 proves one pilot operating one configured robot through one Ito +application. + +## Product boundary + +The default deployment runs Ito on the robot. One Python process: + +- serves `client/`; +- exposes the pilot WebSocket/WebRTC endpoint; +- owns a local robot adapter; +- receives camera and sensor frames in-process; +- runs reconstruction in-process or in a tightly managed native subprocess; +- sends binary Splat Batches to the pilot. + +Raw local sensor frames do not cross a loopback protocol boundary. Pilot input +terminates at Ito and is passed to the local adapter directly. + +## Low-spec fallback + +If the robot cannot run Ito or reconstruction, the same Ito application runs on +an external machine with `ITO_ROBOT_BACKEND=remote`. One lightweight driver on +the robot connects outward to Ito. It forwards camera/sensor data, receives +control, and owns robot-local safety. Placement is configuration; it doesn't +change the pilot client or its URL shape. + +## Lifecycle + +```text +Ito starts +→ configured adapter becomes ready +→ pilot opens Ito's hosted page +→ client connects to /ws +→ pilot explicitly starts control +→ pilot input, robot control, sensors, and reconstruction run +→ pilot stops or disconnects +→ adapter enters its safe local state +``` + +Only one pilot connection and one control-active state exist. A second +simultaneous pilot is rejected. Disconnect stops control immediately; there is +no server-owned recoverable allocation to resume. + +## Live data + +- Pilot Input Snapshots use an unordered, unreliable WebRTC data channel so + stale pose data doesn't block newer samples. +- Local input flows from Ito's data-channel receiver to the adapter by call. +- In external mode Ito relays the pilot-input WebRTC negotiation to the one + configured driver; client behavior remains identical. +- Local sensor frames enter reconstruction by call. External camera media uses + the physically necessary driver-to-Ito WebRTC path. +- Reconstruction output uses a reliable, ordered binary Splat Batch channel. + +## Safety + +Robot safety never depends on Ito continuing to send data. The local adapter or +remote driver owns pilot-input timeout, bounded commands, neutralization, +emergency stop, and safe resumption. The client also pauses outgoing input when +visual reconstruction becomes stale. + +The generic local adapter enforces input timeout, coalesces excess input so the +newest snapshot wins, invokes the robot's neutral and emergency-stop callbacks, +and requires an explicit new `control.start` after emergency stop. Ito Droid's +driver additionally clamps servo position and velocity and ramps resumed input. + +## Explicitly out of scope + +Multiple pilots, multiple robots, discovery, catalogs, browsing, allocation, +reservations, fleet heartbeats, stable robot identities, accounts, RBAC, cloud +control planes, and service orchestration are not v1 concepts. diff --git a/drivers/ito-droid/README.md b/drivers/ito-droid/README.md index a49cff1..c72ff49 100644 --- a/drivers/ito-droid/README.md +++ b/drivers/ito-droid/README.md @@ -1,90 +1,16 @@ -# Ito Droid ROS Driver - -This is the v1 Robot Driver for the physical Ito Droid target. It adapts -between Ito Protocol control-plane messages, direct pilot input snapshots, and -robot-local ROS camera and servo topics. - -The driver is intentionally robot-side only: ROS topics and servo commands do -not leak into the Pilot Client, Ito Server, or Ito Protocol. - -## Container - -Build from the repository root: - -```bash -docker build -f drivers/ito-droid/Dockerfile -t ito-droid-driver . -``` - -Run it on the robot or in a ROS network where the configured camera and servo -topics are available: - -```bash -docker run --rm --network host \ - -e ITO_SERVER_URL=ws://ito-server.local:8765 \ - -e ITO_DROID_ROS_CAMERA_TOPIC=/image_raw \ - -e ITO_DROID_ROS_SERVO_COMMAND_TOPIC=/ito_droid/camera_pan/command \ - ito-droid-driver -``` - -## Environment - -| Variable | Default | Meaning | -| --- | --- | --- | -| `ITO_SERVER_URL` | `ws://localhost:8765` | Ito Server WebSocket control URL. | -| `ITO_DROID_ROBOT_ID` | `ito-droid-1` | Stable robot identity reported to the server. | -| `ITO_DROID_NAME` | `Ito Droid` | Pilot-facing robot name. | -| `ITO_DROID_STATUS_INTERVAL_MS` | `1000` | Driver status/heartbeat interval. | -| `ITO_DROID_RECONNECT_INITIAL_DELAY_MS` | `250` | Initial reconnect backoff. | -| `ITO_DROID_RECONNECT_MAX_DELAY_MS` | `5000` | Maximum reconnect backoff. | -| `ITO_DROID_ROS_CAMERA_TOPIC` | `/image_raw` | ROS `sensor_msgs/Image` camera feed to consume. | -| `ITO_DROID_ROS_SERVO_COMMAND_TOPIC` | `/ito_droid/camera_pan/command` | ROS `std_msgs/Float64` camera-pan command topic, in degrees. | -| `ITO_DROID_ROS_NODE_NAME` | `ito_droid_driver` | ROS node name. | -| `ITO_DROID_PILOT_INPUT_TIMEOUT_MS` | `2000` | Missing pilot-input timeout before control loss. | -| `ITO_DROID_CONTROL_TICK_HZ` | `60` | Driver-owned control loop tick rate. | -| `ITO_DROID_SERVO_NEUTRAL_DEGREES` | `90` | Camera-pan neutral angle. | -| `ITO_DROID_SERVO_MIN_DEGREES` | `15` | Camera-pan lower limit. | -| `ITO_DROID_SERVO_MAX_DEGREES` | `165` | Camera-pan upper limit. | -| `ITO_DROID_YAW_TO_SERVO_DEGREES_PER_RADIAN` | `57.29577951308232` | Relative headset-yaw to servo-angle scale. | -| `ITO_DROID_SERVO_SMOOTHING` | `0.35` | Per-tick smoothing factor from current command toward target. | -| `ITO_DROID_SERVO_MAX_VELOCITY_DEGREES_PER_SECOND` | `180` | Normal correction velocity limit. | -| `ITO_DROID_RESUMPTION_INITIAL_VELOCITY_DEGREES_PER_SECOND` | `20` | Correction velocity immediately after recoverable control loss. | -| `ITO_DROID_RESUMPTION_RAMP_DURATION_MS` | `1500` | Duration for ramping correction velocity back to normal. | - -## Current WebRTC State - -The driver has explicit seams for: - -- consuming ROS camera frames; -- handing frames to the driver-to-server camera media publisher; -- receiving Pilot Input Snapshots from the client-to-driver path. - -Concrete H.264 camera media transport to the server remains covered by TODO 23 -in `../../docs/todo.md`. - -## Physical Smoke Test Expectations - -Run this smoke pass only with the Ito Droid on a trusted private network, the -Ito Server reachable from the robot host, the configured ROS camera topic -publishing, and the configured servo command topic connected to the camera-pan -servo path. - -Expected checks: - -- The driver connects to the Ito Server and the Droid appears in the Robot - Catalog as Available only after the ROS camera feed and servo path are ready. -- Acquiring the Droid starts a server-owned piloting session and the driver - neutralizes the camera-pan servo before accepting pilot input. -- Pilot headset yaw maps to camera-pan servo motion within the configured - servo limits, smoothing, and velocity limits. -- Opening the in-VR menu or otherwise withholding pilot input for longer than - `ITO_DROID_PILOT_INPUT_TIMEOUT_MS` holds the last commanded servo position. -- Resuming pilot input after control loss ramps correction velocity according - to `ITO_DROID_RESUMPTION_INITIAL_VELOCITY_DEGREES_PER_SECOND` and - `ITO_DROID_RESUMPTION_RAMP_DURATION_MS` instead of snapping immediately. -- A clean session end attempts to return the camera-pan servo to neutral and - returns the robot to catalog availability when the driver is otherwise ready. -- Driver- or hardware-failure conditions end the affected session with a - displayable Session Termination Reason rather than crashing the Ito Server. - -Record physical results in `../../docs/acceptance-v1.md`. Keep hardware-only -acceptance unchecked until this pass is actually run. +# Ito Droid robot integration + +Ito Droid keeps ROS-specific actuation and robot-local safety at the robot. Its +driver consumes the configured camera topic, maps newest pilot yaw to the +camera-pan servo, enforces a local input timeout, bounds motion, and neutralizes +on control stop or connection loss. + +For the preferred onboard deployment, expose these camera/control seams through +a `LocalRobotAdapter` inside Ito. The lightweight external driver can also +attach to an external Ito application with `ITO_URL`; it receives pilot input +on a WebRTC data channel and publishes the configured ROS camera as H.264 +WebRTC media. + +Important configuration includes `ITO_URL`, `ITO_DROID_ROS_CAMERA_TOPIC`, +`ITO_DROID_ROS_SERVO_COMMAND_TOPIC`, `ITO_DROID_PILOT_INPUT_TIMEOUT_MS`, and the +servo limit/smoothing/resumption variables in `ito_droid/config.py`. diff --git a/drivers/ito-droid/ito_droid/config.py b/drivers/ito-droid/ito_droid/config.py index daed597..c400a86 100644 --- a/drivers/ito-droid/ito_droid/config.py +++ b/drivers/ito-droid/ito_droid/config.py @@ -34,10 +34,7 @@ def _env_float(name: str, default: float, *, minimum: float | None = None) -> fl @dataclass(frozen=True) class ItoDroidConfig: - server_url: str = "ws://localhost:8765" - robot_id: str = "ito-droid-1" - name: str = "Ito Droid" - status_interval_ms: int = 1000 + ito_url: str = "ws://localhost:8765/ws" reconnect_initial_delay_ms: int = 250 reconnect_max_delay_ms: int = 5000 ros_camera_topic: str = "/image_raw" @@ -57,14 +54,7 @@ class ItoDroidConfig: @classmethod def from_env(cls) -> "ItoDroidConfig": return cls( - server_url=os.getenv("ITO_SERVER_URL", cls.server_url), - robot_id=os.getenv("ITO_DROID_ROBOT_ID", cls.robot_id), - name=os.getenv("ITO_DROID_NAME", cls.name), - status_interval_ms=_env_int( - "ITO_DROID_STATUS_INTERVAL_MS", - cls.status_interval_ms, - minimum=1, - ), + ito_url=os.getenv("ITO_URL", cls.ito_url), reconnect_initial_delay_ms=_env_int( "ITO_DROID_RECONNECT_INITIAL_DELAY_MS", cls.reconnect_initial_delay_ms, @@ -132,4 +122,3 @@ def validated(self) -> "ItoDroidConfig": ): raise ValueError("ITO_DROID_RESUMPTION_INITIAL_VELOCITY_DEGREES_PER_SECOND must be <= max velocity") return self - diff --git a/drivers/ito-droid/ito_droid/control.py b/drivers/ito-droid/ito_droid/control.py index ec52114..2ba5399 100644 --- a/drivers/ito-droid/ito_droid/control.py +++ b/drivers/ito-droid/ito_droid/control.py @@ -35,9 +35,9 @@ def neutralize(self) -> float: return self.command_degrees def receive_snapshot(self, snapshot: Mapping[str, Any], now_seconds: float) -> PilotInputSnapshot: - yaw = snapshot.get("headsetYawRadians") + yaw = snapshot.get("headsetYawRad") if not isinstance(yaw, (int, float)): - raise ValueError("Pilot Input Snapshot requires numeric headsetYawRadians") + raise ValueError("Pilot Input Snapshot requires numeric headsetYawRad") parsed = PilotInputSnapshot(float(yaw), now_seconds) was_lost = self._control_lost self._latest_snapshot = parsed @@ -62,6 +62,7 @@ def tick(self, now_seconds: float, dt_seconds: float) -> float: if age_ms > self.config.pilot_input_timeout_ms: self._control_lost = True self._resumed_at_seconds = None + self.command_degrees = self.config.servo_neutral_degrees return self.command_degrees target = self.target_for_yaw(snapshot.headset_yaw_radians) @@ -91,4 +92,3 @@ def _allowed_velocity(self, now_seconds: float) -> float: def _clamp(value: float, lower: float, upper: float) -> float: return min(upper, max(lower, value)) - diff --git a/drivers/ito-droid/ito_droid/driver.py b/drivers/ito-droid/ito_droid/driver.py index 7a5e624..bd38fbe 100644 --- a/drivers/ito-droid/ito_droid/driver.py +++ b/drivers/ito-droid/ito_droid/driver.py @@ -1,4 +1,4 @@ -"""Ito Droid ROS driver implementation.""" +"""Ito Droid lightweight remote-driver lifecycle and robot-local safety.""" from __future__ import annotations @@ -11,19 +11,18 @@ from websockets.exceptions import ConnectionClosed from server.ito.protocol import ( - ROLE_ROBOT_DRIVER, - ROBOT_STATUS_AVAILABLE, - ROBOT_STATUS_UNAVAILABLE, - ROBOT_TYPE_DROID, + DisplayReason, + ROLE_REMOTE_ROBOT_DRIVER, TYPE_CONNECTION_HELLO, TYPE_CONNECTION_HELLO_RESULT, - TYPE_DRIVER_SESSION_START, - TYPE_DRIVER_SESSION_START_RESULT, - TYPE_ROBOT_STATUS, - TYPE_SESSION_END, - TYPE_SESSION_END_RESULT, - TYPE_SESSION_ENDED, - DisplayReason, + TYPE_DRIVER_CONTROL_START, + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_DRIVER_CONTROL_STOP, + TYPE_DRIVER_CONTROL_STOP_RESULT, + TYPE_WEBRTC_ANSWER, + TYPE_WEBRTC_OFFER, + WEBRTC_PATH_CAMERA_MEDIA, + WEBRTC_PATH_PILOT_INPUT, make_envelope, pack_envelope, result_error, @@ -35,19 +34,19 @@ from .control import CameraPanController from .media import CameraMediaPublisher from .ros_io import CameraFrame, CameraFrameSink, LoggingServoPublisher, RosBridge, ServoPublisher +from .webrtc import PilotInputWebRtcReceiver LOGGER = logging.getLogger(__name__) class ItoDroidDriver(CameraFrameSink): - """ROS-backed Ito Droid driver with testable control behavior.""" - def __init__( self, config: ItoDroidConfig, *, servo_publisher: ServoPublisher | None = None, media_publisher: CameraMediaPublisher | None = None, + pilot_input_webrtc: Any | None = None, clock: Any = time.monotonic, ) -> None: self.config = config @@ -55,33 +54,14 @@ def __init__( self.controller = CameraPanController(config) self.servo_publisher = servo_publisher or LoggingServoPublisher() self.media_publisher = media_publisher or CameraMediaPublisher() - self.session_id: str | None = None - self.session_config: dict[str, object] | None = None + self.pilot_input_webrtc = pilot_input_webrtc + self.control_active = False self.camera_ready = False self.servo_ready = True @property - def available(self) -> bool: - return self.camera_ready and self.servo_ready and self.session_id is None - - def status_payload(self) -> dict[str, object]: - if self.available: - return { - "name": self.config.name, - "type": ROBOT_TYPE_DROID, - "status": ROBOT_STATUS_AVAILABLE, - } - detail = "ito_droid.session_active" - if not self.camera_ready: - detail = "ito_droid.camera_feed_missing" - elif not self.servo_ready: - detail = "ito_droid.servo_unavailable" - return { - "name": self.config.name, - "type": ROBOT_TYPE_DROID, - "status": ROBOT_STATUS_UNAVAILABLE, - "availabilityDetail": {"code": detail}, - } + def ready(self) -> bool: + return self.camera_ready and self.servo_ready def receive_camera_frame(self, frame: CameraFrame) -> None: self.camera_ready = True @@ -91,8 +71,7 @@ def receive_pilot_input_snapshot(self, snapshot: Mapping[str, Any]) -> None: self.controller.receive_snapshot(snapshot, self.clock()) def neutralize_servo(self) -> None: - angle = self.controller.neutralize() - self.servo_publisher.publish_angle(angle) + self.servo_publisher.publish_angle(self.controller.neutralize()) def process_control_tick(self, dt_seconds: float) -> float: angle = self.controller.tick(self.clock(), dt_seconds) @@ -100,173 +79,170 @@ def process_control_tick(self, dt_seconds: float) -> float: return angle async def run_forever(self) -> None: - delay_seconds = self.config.reconnect_initial_delay_ms / 1000 - max_delay_seconds = self.config.reconnect_max_delay_ms / 1000 + delay = self.config.reconnect_initial_delay_ms / 1000 + maximum = self.config.reconnect_max_delay_ms / 1000 while True: + if not self.ready: + await asyncio.sleep(0.1) + continue try: await self.run_once() - delay_seconds = self.config.reconnect_initial_delay_ms / 1000 + delay = self.config.reconnect_initial_delay_ms / 1000 except (ConnectionClosed, OSError) as exc: - LOGGER.warning("Ito Droid control connection lost: %s", exc) - await asyncio.sleep(delay_seconds) - delay_seconds = min(max_delay_seconds, delay_seconds * 2) + LOGGER.warning("Ito Droid connection lost: %s", exc) + await asyncio.sleep(delay) + delay = min(maximum, delay * 2) async def run_once(self) -> None: - async with connect(self.config.server_url) as websocket: + async with connect(self.config.ito_url) as websocket: hello = make_envelope( TYPE_CONNECTION_HELLO, - {"role": ROLE_ROBOT_DRIVER, "robotId": self.config.robot_id}, - robot_id=self.config.robot_id, + {"role": ROLE_REMOTE_ROBOT_DRIVER, "ready": self.ready}, ) await websocket.send(pack_envelope(hello)) - result = unpack_envelope(await websocket.recv()) - if result["type"] != TYPE_CONNECTION_HELLO_RESULT or not result["payload"].get("ok"): - raise RuntimeError(f"Ito Droid hello rejected: {result['payload']}") - - tasks = [ - asyncio.create_task(self._status_loop(websocket)), - asyncio.create_task(self._control_loop()), - ] + response = unpack_envelope(await websocket.recv()) + if response["type"] != TYPE_CONNECTION_HELLO_RESULT or not response["payload"].get("ok"): + raise RuntimeError(f"Ito Droid hello rejected: {response['payload']}") + control_task = asyncio.create_task(self._control_loop()) try: async for frame in websocket: - if not isinstance(frame, bytes): - LOGGER.warning("Ignoring non-binary Ito control frame") - continue - await self.handle_message(websocket, unpack_envelope(frame)) + if isinstance(frame, bytes): + await self.handle_message(websocket, unpack_envelope(frame)) finally: - for task in tasks: - task.cancel() - self._clear_session() - - async def _status_loop(self, websocket: Any) -> None: - while True: - await self.send_status(websocket) - await asyncio.sleep(self.config.status_interval_ms / 1000) + control_task.cancel() + await self._stop_locally() async def _control_loop(self) -> None: period = 1 / self.config.control_tick_hz while True: started = self.clock() - if self.session_id is not None: + if self.control_active: self.process_control_tick(period) - elapsed = self.clock() - started - await asyncio.sleep(max(0, period - elapsed)) - - async def send_status(self, websocket: Any) -> None: - await websocket.send( - pack_envelope( - make_envelope( - TYPE_ROBOT_STATUS, - self.status_payload(), - robot_id=self.config.robot_id, - session_id=self.session_id, - ) - ) - ) + await asyncio.sleep(max(0, period - (self.clock() - started))) async def handle_message(self, websocket: Any, envelope: Mapping[str, Any]) -> None: - message_type = envelope["type"] - if message_type == TYPE_DRIVER_SESSION_START: - await self.handle_session_start(websocket, envelope) - elif message_type == TYPE_SESSION_END: - await self.handle_session_end(websocket, envelope) - elif message_type == TYPE_SESSION_ENDED: - LOGGER.info("Ito Droid session ended: %s", envelope["payload"]) - self._clear_session() - else: - LOGGER.info("Ito Droid ignoring unsupported message type %s", message_type) - - async def handle_session_start(self, websocket: Any, envelope: Mapping[str, Any]) -> None: - requested_session_id = envelope.get("sessionId") or envelope["payload"].get("sessionId") - if not isinstance(requested_session_id, str) or not requested_session_id: - await self._send_start_result( - websocket, - envelope, - result_error(DisplayReason(code="driver.session_start.invalid_session")), - ) - return - if self.session_id is not None: - await self._send_start_result( + if envelope["type"] == TYPE_DRIVER_CONTROL_START: + await self.handle_control_start(websocket, envelope) + elif envelope["type"] == TYPE_DRIVER_CONTROL_STOP: + await self.handle_control_stop(websocket, envelope) + elif envelope["type"] == TYPE_WEBRTC_OFFER: + await self.handle_webrtc_offer(websocket, envelope) + elif envelope["type"] == TYPE_WEBRTC_ANSWER: + await self.handle_webrtc_answer(envelope) + + async def handle_control_start(self, websocket: Any, envelope: Mapping[str, Any]) -> None: + if self.control_active: + await self._send_result( websocket, envelope, - result_error(DisplayReason(code="driver.session_start.already_active")), + TYPE_DRIVER_CONTROL_START_RESULT, + result_error(DisplayReason(code="control.already_active")), ) return if not self.camera_ready: - await self._send_start_result( + await self._send_result( websocket, envelope, + TYPE_DRIVER_CONTROL_START_RESULT, result_error(DisplayReason(code="ito_droid.camera_feed_missing")), ) return try: self.neutralize_servo() - except Exception as exc: # pragma: no cover - hardware adapter failure path - LOGGER.error("Ito Droid failed to neutralize camera-pan servo: %s", exc) + except Exception as exc: self.servo_ready = False - await self._send_start_result( + await self._send_result( websocket, envelope, - result_error( - DisplayReason(code="ito_droid.servo_neutralization_failed", text=str(exc)) - ), + TYPE_DRIVER_CONTROL_START_RESULT, + result_error(DisplayReason(code="ito_droid.servo_neutralization_failed", text=str(exc))), ) return + self.control_active = True + self.media_publisher.start() + await self._send_result( + websocket, envelope, TYPE_DRIVER_CONTROL_START_RESULT, result_ok() + ) + await self._start_camera_media(websocket) - self.session_id = requested_session_id - self.session_config = dict(envelope["payload"].get("sessionConfig") or {}) - self.media_publisher.start(self.session_id) - await self._send_start_result( - websocket, - envelope, - result_ok({"sessionId": self.session_id}), + async def handle_control_stop(self, websocket: Any, envelope: Mapping[str, Any]) -> None: + await self._stop_locally() + await self._send_result( + websocket, envelope, TYPE_DRIVER_CONTROL_STOP_RESULT, result_ok() ) - async def handle_session_end(self, websocket: Any, envelope: Mapping[str, Any]) -> None: - ended_session_id = envelope.get("sessionId") or self.session_id - clean = bool(envelope["payload"].get("clean")) - self._clear_session(neutralize=clean) + async def handle_webrtc_offer( + self, websocket: Any, envelope: Mapping[str, Any] + ) -> None: + path = envelope["payload"].get("path") + sdp = envelope["payload"].get("sdp") + if path != WEBRTC_PATH_PILOT_INPUT or not isinstance(sdp, str): + return + if self.pilot_input_webrtc is None: + self.pilot_input_webrtc = PilotInputWebRtcReceiver( + self.receive_pilot_input_snapshot + ) + answer = await self.pilot_input_webrtc.accept_offer(sdp=sdp) await websocket.send( pack_envelope( make_envelope( - TYPE_SESSION_END_RESULT, - result_ok({"sessionId": ended_session_id}), + TYPE_WEBRTC_ANSWER, + {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": answer}, reply_to_message_id=envelope["messageId"], - robot_id=self.config.robot_id, - session_id=ended_session_id if isinstance(ended_session_id, str) else None, ) ) ) - async def _send_start_result( + async def handle_webrtc_answer(self, envelope: Mapping[str, Any]) -> None: + path = envelope["payload"].get("path") + sdp = envelope["payload"].get("sdp") + if path == WEBRTC_PATH_CAMERA_MEDIA and isinstance(sdp, str): + await self.media_publisher.accept_answer(sdp=sdp) + + async def _start_camera_media(self, websocket: Any) -> None: + offer = await self.media_publisher.create_offer() + await websocket.send( + pack_envelope( + make_envelope( + TYPE_WEBRTC_OFFER, + {"path": WEBRTC_PATH_CAMERA_MEDIA, "sdp": offer}, + ) + ) + ) + + async def _stop_locally(self) -> None: + self.control_active = False + self.media_publisher.stop() + await self.media_publisher.close() + if self.pilot_input_webrtc is not None: + await self.pilot_input_webrtc.close() + try: + self.neutralize_servo() + except Exception as exc: + LOGGER.error("Ito Droid safe neutralization failed: %s", exc) + + async def emergency_stop(self) -> None: + """Robot-local emergency stop; it doesn't depend on Ito connectivity.""" + + await self._stop_locally() + + async def _send_result( self, websocket: Any, request: Mapping[str, Any], + message_type: str, payload: Mapping[str, Any], ) -> None: - session_id = request.get("sessionId") or request["payload"].get("sessionId") await websocket.send( pack_envelope( make_envelope( - TYPE_DRIVER_SESSION_START_RESULT, + message_type, payload, reply_to_message_id=request["messageId"], - robot_id=self.config.robot_id, - session_id=session_id if isinstance(session_id, str) else None, ) ) ) - def _clear_session(self, *, neutralize: bool = False) -> None: - self.media_publisher.stop() - self.session_id = None - self.session_config = None - if neutralize: - try: - self.neutralize_servo() - except Exception as exc: # pragma: no cover - hardware adapter failure path - LOGGER.error("Ito Droid failed clean session-end neutralization: %s", exc) - async def run(config: ItoDroidConfig | None = None) -> None: resolved_config = config or ItoDroidConfig.from_env() diff --git a/drivers/ito-droid/ito_droid/media.py b/drivers/ito-droid/ito_droid/media.py index fc91dc1..4990ce3 100644 --- a/drivers/ito-droid/ito_droid/media.py +++ b/drivers/ito-droid/ito_droid/media.py @@ -3,8 +3,10 @@ from __future__ import annotations import logging +from typing import Any from .ros_io import CameraFrame +from .webrtc import CameraMediaWebRtcPublisher LOGGER = logging.getLogger(__name__) @@ -12,22 +14,18 @@ class CameraMediaPublisher: """Receives ROS camera frames for the driver-to-server WebRTC media path. - TODO 23/26 will attach this seam to the real non-trickle WebRTC H.264 - transport. Keeping the boundary explicit lets the ROS camera consumer and - session lifecycle be tested before the shared WebRTC signaling work lands. + The newest raw ROS frame enters the WebRTC video track in memory. The + transport then encodes it for the physically necessary robot-to-Ito hop. """ - def __init__(self) -> None: - self.started_session_id: str | None = None + def __init__(self, transport: Any | None = None) -> None: + self.active = False self.frame_count = 0 self.last_frame: CameraFrame | None = None + self._transport = transport - @property - def active(self) -> bool: - return self.started_session_id is not None - - def start(self, session_id: str) -> None: - self.started_session_id = session_id + def start(self) -> None: + self.active = True self.frame_count = 0 self.last_frame = None @@ -36,8 +34,25 @@ def publish_frame(self, frame: CameraFrame) -> None: return self.frame_count += 1 self.last_frame = frame + if self._transport is not None: + self._transport.publish_frame(frame) LOGGER.debug("camera_media_frame bytes=%s encoding=%s", len(frame.data), frame.encoding) - def stop(self) -> None: - self.started_session_id = None + async def create_offer(self) -> str: + if self._transport is None: + self._transport = CameraMediaWebRtcPublisher() + if self.last_frame is not None: + self._transport.publish_frame(self.last_frame) + return await self._transport.create_offer() + + async def accept_answer(self, *, sdp: str) -> None: + if self._transport is None: + raise RuntimeError("camera media has no active offer") + await self._transport.accept_answer(sdp=sdp) + async def close(self) -> None: + if self._transport is not None: + await self._transport.close() + + def stop(self) -> None: + self.active = False diff --git a/drivers/ito-droid/ito_droid/webrtc.py b/drivers/ito-droid/ito_droid/webrtc.py index 811e7ef..e1ce3cb 100644 --- a/drivers/ito-droid/ito_droid/webrtc.py +++ b/drivers/ito-droid/ito_droid/webrtc.py @@ -2,10 +2,15 @@ from __future__ import annotations -import json +import asyncio +from fractions import Fraction import logging from typing import Any, Callable, Mapping +from server.ito.webrtc import decode_pilot_input_snapshot + +from .ros_io import CameraFrame + LOGGER = logging.getLogger(__name__) @@ -26,21 +31,156 @@ def on_message(message: str | bytes) -> None: self.receive_snapshot(snapshot) -def decode_pilot_input_snapshot(message: str | bytes) -> dict[str, Any]: - if isinstance(message, bytes): - message = message.decode("utf-8") +class PilotInputWebRtcReceiver: + """Accept the pilot data-channel offer and feed the local controller.""" + + def __init__(self, receive_snapshot: Callable[[Mapping[str, Any]], None]) -> None: + try: + from aiortc import RTCConfiguration, RTCPeerConnection, RTCSessionDescription + except ImportError as exc: # pragma: no cover - runtime dependency + raise RuntimeError("aiortc is required for Ito Droid pilot input") from exc + self._configuration_type = RTCConfiguration + self._peer_connection_type = RTCPeerConnection + self._session_description_type = RTCSessionDescription + self._receiver = PilotInputDataChannelReceiver(receive_snapshot) + self._peer_connection: Any | None = None + + async def accept_offer(self, *, sdp: str) -> str: + await self.close() + pc = self._peer_connection_type(configuration=self._configuration_type(iceServers=[])) + self._peer_connection = pc + + @pc.on("datachannel") + def on_data_channel(channel: Any) -> None: + self._receiver.attach(channel) + + await pc.setRemoteDescription(self._session_description_type(sdp=sdp, type="offer")) + await pc.setLocalDescription(await pc.createAnswer()) + await _wait_for_ice_gathering_complete(pc) + return pc.localDescription.sdp + + async def close(self) -> None: + pc = self._peer_connection + self._peer_connection = None + if pc is not None: + await pc.close() + + +class CameraMediaWebRtcPublisher: + """Publish newest ROS camera frames to Ito as one WebRTC video track.""" + + def __init__(self) -> None: + try: + from aiortc import RTCConfiguration, RTCPeerConnection, RTCSessionDescription + from aiortc import VideoStreamTrack + except ImportError as exc: # pragma: no cover - runtime dependency + raise RuntimeError("aiortc is required for Ito Droid camera media") from exc + + class RosCameraTrack(VideoStreamTrack): + def __init__(self) -> None: + super().__init__() + self.frames: asyncio.Queue[CameraFrame] = asyncio.Queue(maxsize=1) + + def publish(self, frame: CameraFrame) -> None: + if self.frames.full(): + self.frames.get_nowait() + self.frames.put_nowait(frame) + + async def recv(self) -> Any: + frame = await self.frames.get() + return _video_frame(frame) + + self._configuration_type = RTCConfiguration + self._peer_connection_type = RTCPeerConnection + self._session_description_type = RTCSessionDescription + self._track = RosCameraTrack() + self._peer_connection: Any | None = None + + def publish_frame(self, frame: CameraFrame) -> None: + self._track.publish(frame) + + async def create_offer(self) -> str: + await self.close() + pc = self._peer_connection_type(configuration=self._configuration_type(iceServers=[])) + self._peer_connection = pc + pc.addTrack(self._track) + _prefer_h264(pc) + await pc.setLocalDescription(await pc.createOffer()) + await _wait_for_ice_gathering_complete(pc) + return pc.localDescription.sdp + + async def accept_answer(self, *, sdp: str) -> None: + if self._peer_connection is None: + raise RuntimeError("no camera-media offer is active") + await self._peer_connection.setRemoteDescription( + self._session_description_type(sdp=sdp, type="answer") + ) + + async def close(self) -> None: + pc = self._peer_connection + self._peer_connection = None + if pc is not None: + await pc.close() + + +def _video_frame(frame: CameraFrame) -> Any: + try: + from av import VideoFrame + except ImportError as exc: # pragma: no cover - runtime dependency + raise RuntimeError("PyAV is required for Ito Droid camera media") from exc + if frame.width is None or frame.height is None: + raise ValueError("ROS camera frames require width and height") + formats = { + "rgb8": ("rgb24", 3), + "bgr8": ("bgr24", 3), + "rgba8": ("rgba", 4), + "bgra8": ("bgra", 4), + "mono8": ("gray", 1), + } try: - payload = json.loads(message) - except json.JSONDecodeError as exc: - raise ValueError("snapshot is not valid JSON") from exc - if not isinstance(payload, dict): - raise ValueError("snapshot must be a JSON object") - if payload.get("protocolVersion") != "ito.v1": - raise ValueError("snapshot protocolVersion must be ito.v1") - if not isinstance(payload.get("sessionId"), str): - raise ValueError("snapshot requires sessionId") - if not isinstance(payload.get("sequence"), int): - raise ValueError("snapshot requires integer sequence") - if not isinstance(payload.get("headsetYawRad"), (int, float)): - raise ValueError("snapshot requires headsetYawRad") - return payload + pixel_format, bytes_per_pixel = formats[frame.encoding or ""] + except KeyError as exc: + raise ValueError(f"unsupported ROS camera encoding: {frame.encoding}") from exc + video_frame = VideoFrame(frame.width, frame.height, pixel_format) + plane = video_frame.planes[0] + row_size = frame.width * bytes_per_pixel + if len(frame.data) != row_size * frame.height: + raise ValueError("ROS camera frame size does not match its dimensions") + if plane.line_size == row_size: + packed = frame.data + else: + padding = b"\0" * (plane.line_size - row_size) + packed = b"".join( + frame.data[offset : offset + row_size] + padding + for offset in range(0, len(frame.data), row_size) + ) + plane.update(packed) + video_frame.pts = int(frame.received_at_seconds * 90_000) + video_frame.time_base = Fraction(1, 90_000) + return video_frame + + +def _prefer_h264(peer_connection: Any) -> None: + from aiortc import RTCRtpSender + + codecs = [ + codec + for codec in RTCRtpSender.getCapabilities("video").codecs + if codec.mimeType.lower() == "video/h264" + ] + for transceiver in peer_connection.getTransceivers(): + if transceiver.kind == "video" and codecs: + transceiver.setCodecPreferences(codecs) + + +async def _wait_for_ice_gathering_complete(peer_connection: Any) -> None: + if getattr(peer_connection, "iceGatheringState", None) == "complete": + return + complete = asyncio.Event() + + @peer_connection.on("icegatheringstatechange") + def on_ice_gathering_state_change() -> None: + if peer_connection.iceGatheringState == "complete": + complete.set() + + await complete.wait() diff --git a/drivers/mock-robot/README.md b/drivers/mock-robot/README.md index 54fc1e3..7c14819 100644 --- a/drivers/mock-robot/README.md +++ b/drivers/mock-robot/README.md @@ -1,47 +1,16 @@ -# Mock Robot +# Mock remote robot driver -The Mock Robot is Ito's robot-driver test double. +This lightweight driver exercises the external-deployment fallback. It connects +to one Ito application, forwards a configured camera file over WebRTC, receives +pilot input, and logs snapshots. It has no identity, catalog status, heartbeat, +or allocation state. -It speaks the Ito v1 WebSocket control plane as a Robot Driver, reports itself -to the Robot Catalog, accepts server-owned session lifecycle requests, logs -Pilot Input Snapshots to stdout, and opens a configured video file as mock -camera input. The local end-to-end tests exercise client-to-driver pilot input -over an `aiortc` WebRTC data channel. H.264 WebRTC publishing from the mock -camera file to the server remains covered by TODO 23. +Configuration: -## Configuration +- `ITO_URL` (default `ws://localhost:8765/ws`) +- `ITO_MOCK_ROBOT_CAMERA_VIDEO` (required for readiness) +- `ITO_MOCK_ROBOT_CAMERA_LOOP` (default `true`) +- `ITO_REMOTE_DRIVER_RECONNECT_INITIAL_DELAY_MS` (default `250`) +- `ITO_REMOTE_DRIVER_RECONNECT_MAX_DELAY_MS` (default `5000`) -Environment variables: - -- `ITO_SERVER_URL`: Ito Server WebSocket URL. Default: `ws://localhost:8765`. -- `ITO_MOCK_ROBOT_ID`: stable mock robot identity. Default: `mock-robot-1`. -- `ITO_MOCK_ROBOT_NAME`: pilot-facing robot name. Default: `Mock Robot`. -- `ITO_MOCK_ROBOT_STATUS_INTERVAL_MS`: status heartbeat interval. Default: - `1000`. -- `ITO_MOCK_ROBOT_CAMERA_VIDEO`: required video file path for an Available mock - robot. -- `ITO_MOCK_ROBOT_CAMERA_CHUNK_SIZE`: file read chunk size used by the camera - source. Default: `65536`. -- `ITO_MOCK_ROBOT_CAMERA_LOOP`: whether the file source loops at EOF. Default: - `true`. - -Run locally from the repository root: - -```sh -PYTHONPATH=. python drivers/mock-robot/main.py -``` - -Build the container from the repository root: - -```sh -docker build -f drivers/mock-robot/Dockerfile -t ito-mock-robot . -``` - -Run through Docker Compose with the local Ito Server and Pilot Client: - -```sh -docker compose up --build -``` - -This uses `fixtures/mock-camera.mp4` by default. Override it with -`ITO_MOCK_ROBOT_CAMERA_VIDEO_HOST=/absolute/path/to/video.mp4` when needed. +Use the Compose `remote` profile described in `docs/local-v1.md`. diff --git a/drivers/mock-robot/mock_robot/camera.py b/drivers/mock-robot/mock_robot/camera.py index 7b790d5..418740e 100644 --- a/drivers/mock-robot/mock_robot/camera.py +++ b/drivers/mock-robot/mock_robot/camera.py @@ -49,12 +49,7 @@ def close(self) -> None: self._file = None def samples(self) -> Iterator[CameraSample]: - """Yield file chunks until EOF, looping when configured. - - TODO 23 will consume these bytes through WebRTC H.264 media transport. - This class deliberately does not decode frames or implement a production - replay mode in the reconstruction module. - """ + """Yield file chunks for diagnostics; WebRTC reads the video path directly.""" if self._file is None: self.open() @@ -68,4 +63,3 @@ def samples(self) -> Iterator[CameraSample]: if not self.loop: break self._file.seek(0) - diff --git a/drivers/mock-robot/mock_robot/config.py b/drivers/mock-robot/mock_robot/config.py index b43b6fe..64597a4 100644 --- a/drivers/mock-robot/mock_robot/config.py +++ b/drivers/mock-robot/mock_robot/config.py @@ -1,4 +1,4 @@ -"""Environment-backed Mock Robot configuration.""" +"""Environment-backed configuration for the lightweight mock remote driver.""" from __future__ import annotations @@ -33,10 +33,7 @@ def _env_bool(name: str, default: bool) -> bool: @dataclass(frozen=True) class MockRobotConfig: - server_url: str = "ws://localhost:8765" - robot_id: str = "mock-robot-1" - name: str = "Mock Robot" - status_interval_ms: int = 1000 + ito_url: str = "ws://localhost:8765/ws" reconnect_initial_delay_ms: int = 250 reconnect_max_delay_ms: int = 5000 camera_video_path: str | None = None @@ -46,30 +43,20 @@ class MockRobotConfig: @classmethod def from_env(cls) -> "MockRobotConfig": return cls( - server_url=os.getenv("ITO_SERVER_URL", cls.server_url), - robot_id=os.getenv("ITO_MOCK_ROBOT_ID", cls.robot_id), - name=os.getenv("ITO_MOCK_ROBOT_NAME", cls.name), - status_interval_ms=_env_int( - "ITO_MOCK_ROBOT_STATUS_INTERVAL_MS", - cls.status_interval_ms, - minimum=1, - ), + ito_url=os.getenv("ITO_URL", cls.ito_url), reconnect_initial_delay_ms=_env_int( - "ITO_MOCK_ROBOT_RECONNECT_INITIAL_DELAY_MS", + "ITO_REMOTE_DRIVER_RECONNECT_INITIAL_DELAY_MS", cls.reconnect_initial_delay_ms, minimum=1, ), reconnect_max_delay_ms=_env_int( - "ITO_MOCK_ROBOT_RECONNECT_MAX_DELAY_MS", + "ITO_REMOTE_DRIVER_RECONNECT_MAX_DELAY_MS", cls.reconnect_max_delay_ms, minimum=1, ), camera_video_path=os.getenv("ITO_MOCK_ROBOT_CAMERA_VIDEO"), camera_chunk_size=_env_int( - "ITO_MOCK_ROBOT_CAMERA_CHUNK_SIZE", - cls.camera_chunk_size, - minimum=1, + "ITO_MOCK_ROBOT_CAMERA_CHUNK_SIZE", cls.camera_chunk_size, minimum=1 ), camera_loop=_env_bool("ITO_MOCK_ROBOT_CAMERA_LOOP", cls.camera_loop), ) - diff --git a/drivers/mock-robot/mock_robot/driver.py b/drivers/mock-robot/mock_robot/driver.py index 835e7d6..1a60322 100644 --- a/drivers/mock-robot/mock_robot/driver.py +++ b/drivers/mock-robot/mock_robot/driver.py @@ -1,4 +1,4 @@ -"""Mock Robot driver implementation.""" +"""Lightweight mock remote robot driver.""" from __future__ import annotations @@ -11,23 +11,18 @@ from websockets.exceptions import ConnectionClosed from server.ito.protocol import ( - ROLE_ROBOT_DRIVER, - ROBOT_STATUS_AVAILABLE, - ROBOT_STATUS_UNAVAILABLE, - ROBOT_TYPE_DROID, + DisplayReason, + ROLE_REMOTE_ROBOT_DRIVER, TYPE_CONNECTION_HELLO, TYPE_CONNECTION_HELLO_RESULT, - TYPE_DRIVER_SESSION_START, - TYPE_DRIVER_SESSION_START_RESULT, - TYPE_ROBOT_STATUS, - TYPE_SESSION_END, - TYPE_SESSION_END_RESULT, - TYPE_SESSION_ENDED, + TYPE_DRIVER_CONTROL_START, + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_DRIVER_CONTROL_STOP, + TYPE_DRIVER_CONTROL_STOP_RESULT, TYPE_WEBRTC_ANSWER, TYPE_WEBRTC_OFFER, WEBRTC_PATH_CAMERA_MEDIA, WEBRTC_PATH_PILOT_INPUT, - DisplayReason, make_envelope, pack_envelope, result_error, @@ -43,12 +38,9 @@ class MockRobotDriver: - """A robot-driver test double that speaks Ito v1 control-plane messages.""" - def __init__(self, config: MockRobotConfig, *, camera_media_webrtc: Any | None = None) -> None: self.config = config - self.session_id: str | None = None - self.session_config: dict[str, object] | None = None + self.control_active = False self.camera = ( VideoFileCamera( config.camera_video_path, @@ -62,259 +54,160 @@ def __init__(self, config: MockRobotConfig, *, camera_media_webrtc: Any | None = self.camera_media_webrtc = camera_media_webrtc @property - def available(self) -> bool: + def ready(self) -> bool: return self.camera is not None - def status_payload(self) -> dict[str, object]: - if self.available: - return { - "name": self.config.name, - "type": ROBOT_TYPE_DROID, - "status": ROBOT_STATUS_AVAILABLE, - } - return { - "name": self.config.name, - "type": ROBOT_TYPE_DROID, - "status": ROBOT_STATUS_UNAVAILABLE, - "availabilityDetail": {"code": "mock_robot.camera_video_required"}, - } - async def run_forever(self) -> None: delay_seconds = self.config.reconnect_initial_delay_ms / 1000 - max_delay_seconds = self.config.reconnect_max_delay_ms / 1000 + maximum = self.config.reconnect_max_delay_ms / 1000 while True: try: await self.run_once() delay_seconds = self.config.reconnect_initial_delay_ms / 1000 except (ConnectionClosed, OSError) as exc: - LOGGER.warning("Mock Robot control connection lost: %s", exc) + LOGGER.warning("Remote robot connection lost: %s", exc) await asyncio.sleep(delay_seconds) - delay_seconds = min(max_delay_seconds, delay_seconds * 2) + delay_seconds = min(maximum, delay_seconds * 2) async def run_once(self) -> None: if self.camera is not None: self.camera.validate() - async with connect(self.config.server_url) as websocket: + async with connect(self.config.ito_url) as websocket: hello = make_envelope( TYPE_CONNECTION_HELLO, - {"role": ROLE_ROBOT_DRIVER, "robotId": self.config.robot_id}, - robot_id=self.config.robot_id, + {"role": ROLE_REMOTE_ROBOT_DRIVER, "ready": self.ready}, ) await websocket.send(pack_envelope(hello)) result = unpack_envelope(await websocket.recv()) if result["type"] != TYPE_CONNECTION_HELLO_RESULT or not result["payload"].get("ok"): - raise RuntimeError(f"Mock Robot hello rejected: {result['payload']}") - - status_task = asyncio.create_task(self._status_loop(websocket)) + raise RuntimeError(f"remote robot hello rejected: {result['payload']}") try: async for frame in websocket: - if not isinstance(frame, bytes): - LOGGER.warning("Ignoring non-binary Ito control frame") - continue - await self.handle_message(websocket, unpack_envelope(frame)) + if isinstance(frame, bytes): + await self.handle_message(websocket, unpack_envelope(frame)) finally: - status_task.cancel() - await self._clear_session() - - async def _status_loop(self, websocket: Any) -> None: - while True: - await self.send_status(websocket) - await asyncio.sleep(self.config.status_interval_ms / 1000) - - async def send_status(self, websocket: Any) -> None: - await websocket.send( - pack_envelope( - make_envelope( - TYPE_ROBOT_STATUS, - self.status_payload(), - robot_id=self.config.robot_id, - session_id=self.session_id, - ) - ) - ) + await self._clear_control() async def handle_message(self, websocket: Any, envelope: Mapping[str, Any]) -> None: message_type = envelope["type"] - if message_type == TYPE_DRIVER_SESSION_START: - await self.handle_session_start(websocket, envelope) - elif message_type == TYPE_SESSION_END: - await self.handle_session_end(websocket, envelope) - elif message_type == TYPE_SESSION_ENDED: - LOGGER.info("Mock Robot session ended: %s", envelope["payload"]) - await self._clear_session() + if message_type == TYPE_DRIVER_CONTROL_START: + await self.handle_control_start(websocket, envelope) + elif message_type == TYPE_DRIVER_CONTROL_STOP: + await self.handle_control_stop(websocket, envelope) elif message_type == TYPE_WEBRTC_OFFER: await self.handle_webrtc_offer(websocket, envelope) elif message_type == TYPE_WEBRTC_ANSWER: await self.handle_webrtc_answer(envelope) - else: - LOGGER.info("Mock Robot ignoring unsupported message type %s", message_type) - async def handle_session_start(self, websocket: Any, envelope: Mapping[str, Any]) -> None: - requested_session_id = envelope.get("sessionId") or envelope["payload"].get("sessionId") - if not isinstance(requested_session_id, str) or not requested_session_id: - await self._send_start_result( + async def handle_control_start(self, websocket: Any, envelope: Mapping[str, Any]) -> None: + if self.control_active: + await self._send_result( websocket, envelope, - result_error(DisplayReason(code="driver.session_start.invalid_session")), - ) - return - if self.session_id is not None: - await self._send_start_result( - websocket, - envelope, - result_error(DisplayReason(code="driver.session_start.already_active")), + TYPE_DRIVER_CONTROL_START_RESULT, + result_error(DisplayReason(code="control.already_active")), ) return if self.camera is None: - await self._send_start_result( + await self._send_result( websocket, envelope, + TYPE_DRIVER_CONTROL_START_RESULT, result_error(DisplayReason(code="mock_robot.camera_video_required")), ) return try: self.camera.open() except (OSError, ValueError) as exc: - LOGGER.error("Mock Robot camera input failed: %s", exc) - await self._send_start_result( + await self._send_result( websocket, envelope, - result_error( - DisplayReason( - code="mock_robot.camera_video_unavailable", - text=str(exc), - ) - ), + TYPE_DRIVER_CONTROL_START_RESULT, + result_error(DisplayReason(code="mock_robot.camera_video_unavailable", text=str(exc))), ) return - - self.session_id = requested_session_id - self.session_config = dict(envelope["payload"].get("sessionConfig") or {}) - LOGGER.info("Mock Robot session started: %s", self.session_id) - await self._send_start_result( - websocket, - envelope, - result_ok({"sessionId": self.session_id}), + self.control_active = True + await self._send_result( + websocket, envelope, TYPE_DRIVER_CONTROL_START_RESULT, result_ok() ) await self._start_camera_media(websocket) - async def handle_session_end(self, websocket: Any, envelope: Mapping[str, Any]) -> None: - ended_session_id = envelope.get("sessionId") or self.session_id - await self._clear_session() - await websocket.send( - pack_envelope( - make_envelope( - TYPE_SESSION_END_RESULT, - result_ok({"sessionId": ended_session_id}), - reply_to_message_id=envelope["messageId"], - robot_id=self.config.robot_id, - session_id=ended_session_id, - ) - ) + async def handle_control_stop(self, websocket: Any, envelope: Mapping[str, Any]) -> None: + await self._clear_control() + await self._send_result( + websocket, envelope, TYPE_DRIVER_CONTROL_STOP_RESULT, result_ok() ) async def handle_webrtc_offer(self, websocket: Any, envelope: Mapping[str, Any]) -> None: - if envelope["payload"].get("path") != WEBRTC_PATH_PILOT_INPUT: - LOGGER.info("Mock Robot ignoring unsupported WebRTC path %s", envelope["payload"].get("path")) - return - session_id = envelope.get("sessionId") or self.session_id sdp = envelope["payload"].get("sdp") - if not isinstance(session_id, str) or session_id != self.session_id or not isinstance(sdp, str): - LOGGER.warning("Mock Robot ignoring invalid pilot-input WebRTC offer") + if envelope["payload"].get("path") != WEBRTC_PATH_PILOT_INPUT or not isinstance(sdp, str): return if self.pilot_input_webrtc is None: self.pilot_input_webrtc = PilotInputWebRtcReceiver(self.receive_pilot_input_snapshot) - answer_sdp = await self.pilot_input_webrtc.accept_offer(session_id=session_id, sdp=sdp) + answer = await self.pilot_input_webrtc.accept_offer(sdp=sdp) await websocket.send( pack_envelope( make_envelope( TYPE_WEBRTC_ANSWER, - {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": answer_sdp}, + {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": answer}, reply_to_message_id=envelope["messageId"], - robot_id=self.config.robot_id, - session_id=session_id, ) ) ) async def handle_webrtc_answer(self, envelope: Mapping[str, Any]) -> None: - if envelope["payload"].get("path") != WEBRTC_PATH_CAMERA_MEDIA: - LOGGER.info("Mock Robot ignoring unsupported WebRTC answer path %s", envelope["payload"].get("path")) - return - session_id = envelope.get("sessionId") or self.session_id sdp = envelope["payload"].get("sdp") - if not isinstance(session_id, str) or not isinstance(sdp, str): - LOGGER.warning("Mock Robot ignoring invalid camera-media WebRTC answer") - return - if self.camera_media_webrtc is None: - LOGGER.warning("Mock Robot received camera-media answer without an active publisher") + if envelope["payload"].get("path") != WEBRTC_PATH_CAMERA_MEDIA or not isinstance(sdp, str): return - await self.camera_media_webrtc.accept_answer(session_id=session_id, sdp=sdp) + if self.camera_media_webrtc is not None: + await self.camera_media_webrtc.accept_answer(sdp=sdp) async def _start_camera_media(self, websocket: Any) -> None: - if self.session_id is None or self.camera is None: - return - try: - if self.camera_media_webrtc is None: - self.camera_media_webrtc = CameraMediaWebRtcPublisher() - sdp = await self.camera_media_webrtc.create_offer( - session_id=self.session_id, - video_path=self.camera.path, - loop=self.camera.loop, - ) - except Exception as exc: - LOGGER.error("Mock Robot failed to start cameraMedia WebRTC: %s", exc) + if not self.control_active or self.camera is None: return + if self.camera_media_webrtc is None: + self.camera_media_webrtc = CameraMediaWebRtcPublisher() + sdp = await self.camera_media_webrtc.create_offer( + video_path=self.camera.path, + loop=self.camera.loop, + ) await websocket.send( pack_envelope( make_envelope( TYPE_WEBRTC_OFFER, {"path": WEBRTC_PATH_CAMERA_MEDIA, "sdp": sdp}, - robot_id=self.config.robot_id, - session_id=self.session_id, ) ) ) def receive_pilot_input_snapshot(self, snapshot: Mapping[str, Any]) -> None: - """Receive and log a Pilot Input Snapshot. - - The mock keeps no fake robot pose; stdout logging is the observable - behavior requested for end-to-end session/control testing. - """ - - payload = dict(snapshot) - LOGGER.info("pilot_input_snapshot %s", json.dumps(payload, sort_keys=True)) + LOGGER.info("pilot_input_snapshot %s", json.dumps(dict(snapshot), sort_keys=True)) - async def _send_start_result( + async def _send_result( self, websocket: Any, request: Mapping[str, Any], + message_type: str, payload: Mapping[str, Any], ) -> None: - session_id = request.get("sessionId") or request["payload"].get("sessionId") await websocket.send( pack_envelope( make_envelope( - TYPE_DRIVER_SESSION_START_RESULT, + message_type, payload, reply_to_message_id=request["messageId"], - robot_id=self.config.robot_id, - session_id=session_id if isinstance(session_id, str) else None, ) ) ) - async def _clear_session(self) -> None: - session_id = self.session_id + async def _clear_control(self) -> None: + self.control_active = False if self.camera is not None: self.camera.close() if self.pilot_input_webrtc is not None: - await self.pilot_input_webrtc.close_session(session_id) + await self.pilot_input_webrtc.close() if self.camera_media_webrtc is not None: - await self.camera_media_webrtc.close_session(session_id) - self.session_id = None - self.session_config = None + await self.camera_media_webrtc.close() async def run(config: MockRobotConfig | None = None) -> None: diff --git a/drivers/mock-robot/mock_robot/webrtc.py b/drivers/mock-robot/mock_robot/webrtc.py index ae013be..88a756c 100644 --- a/drivers/mock-robot/mock_robot/webrtc.py +++ b/drivers/mock-robot/mock_robot/webrtc.py @@ -23,11 +23,12 @@ def __init__(self, receive_snapshot: Callable[[Mapping[str, Any]], None]) -> Non self._peer_connection_type = RTCPeerConnection self._session_description_type = RTCSessionDescription self._receive_snapshot = receive_snapshot - self._peer_connections: dict[str, Any] = {} + self._peer_connection: Any | None = None - async def accept_offer(self, *, session_id: str, sdp: str) -> str: + async def accept_offer(self, *, sdp: str) -> str: + await self.close() pc = self._peer_connection_type(configuration=self._configuration_type(iceServers=[])) - self._peer_connections[session_id] = pc + self._peer_connection = pc @pc.on("datachannel") def on_data_channel(channel: Any) -> None: @@ -45,18 +46,12 @@ def on_message(message: str | bytes) -> None: await _wait_for_ice_gathering_complete(pc) return pc.localDescription.sdp - async def close_session(self, session_id: str | None) -> None: - if session_id is None: - return - pc = self._peer_connections.pop(session_id, None) + async def close(self) -> None: + pc = self._peer_connection + self._peer_connection = None if pc is not None: await pc.close() - async def close_all(self) -> None: - peer_connections = list(self._peer_connections.values()) - self._peer_connections.clear() - await asyncio.gather(*(pc.close() for pc in peer_connections), return_exceptions=True) - class CameraMediaWebRtcPublisher: """Publishes a video file to the server over the `cameraMedia` WebRTC path.""" @@ -71,10 +66,11 @@ def __init__(self) -> None: self._peer_connection_type = RTCPeerConnection self._session_description_type = RTCSessionDescription self._media_player_type = MediaPlayer - self._peer_connections: dict[str, Any] = {} - self._players: dict[str, Any] = {} + self._peer_connection: Any | None = None + self._player: Any | None = None - async def create_offer(self, *, session_id: str, video_path: str | Path, loop: bool) -> str: + async def create_offer(self, *, video_path: str | Path, loop: bool) -> str: + await self.close() pc = self._peer_connection_type(configuration=self._configuration_type(iceServers=[])) player = self._media_player_type(str(video_path), loop=loop) if player.video is None: @@ -82,39 +78,29 @@ async def create_offer(self, *, session_id: str, video_path: str | Path, loop: b raise RuntimeError(f"mock camera video has no video stream: {video_path}") pc.addTrack(player.video) self._prefer_h264(pc) - self._peer_connections[session_id] = pc - self._players[session_id] = player + self._peer_connection = pc + self._player = player offer = await pc.createOffer() await pc.setLocalDescription(offer) await _wait_for_ice_gathering_complete(pc) return pc.localDescription.sdp - async def accept_answer(self, *, session_id: str, sdp: str) -> None: - pc = self._peer_connections.get(session_id) + async def accept_answer(self, *, sdp: str) -> None: + pc = self._peer_connection if pc is None: - raise RuntimeError(f"no cameraMedia peer connection for session {session_id}") + raise RuntimeError("no cameraMedia peer connection for active control") await pc.setRemoteDescription(self._session_description_type(sdp=sdp, type="answer")) - async def close_session(self, session_id: str | None) -> None: - if session_id is None: - return - pc = self._peer_connections.pop(session_id, None) - player = self._players.pop(session_id, None) + async def close(self) -> None: + pc = self._peer_connection + player = self._player + self._peer_connection = None + self._player = None if player is not None and player.video is not None: player.video.stop() if pc is not None: await pc.close() - async def close_all(self) -> None: - peer_connections = list(self._peer_connections.values()) - players = list(self._players.values()) - self._peer_connections.clear() - self._players.clear() - for player in players: - if player.video is not None: - player.video.stop() - await asyncio.gather(*(pc.close() for pc in peer_connections), return_exceptions=True) - def _prefer_h264(self, pc: Any) -> None: try: from aiortc import RTCRtpSender @@ -143,15 +129,13 @@ def decode_pilot_input_snapshot(message: str | bytes) -> dict[str, Any]: raise ValueError("snapshot must be a JSON object") if payload.get("protocolVersion") != "ito.v1": raise ValueError("snapshot protocolVersion must be ito.v1") - if not isinstance(payload.get("sessionId"), str): - raise ValueError("snapshot requires sessionId") if not isinstance(payload.get("sequence"), int): raise ValueError("snapshot requires integer sequence") if not isinstance(payload.get("timestampMs"), (int, float)): raise ValueError("snapshot requires timestampMs") if not isinstance(payload.get("headsetYawRad"), (int, float)): raise ValueError("snapshot requires headsetYawRad") - if not isinstance(payload.get("controllers"), dict): + if not isinstance(payload.get("controllers"), list): raise ValueError("snapshot requires controllers") return payload diff --git a/server/Dockerfile b/server/Dockerfile index e7ce88a..e6bdc63 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -4,5 +4,7 @@ WORKDIR /app COPY server/requirements.txt /app/server/requirements.txt RUN pip install --no-cache-dir -r /app/server/requirements.txt COPY server /app/server +COPY client /app/client ENV PYTHONPATH=/app +EXPOSE 8765 CMD ["python", "-m", "server.ito.app"] diff --git a/server/README.md b/server/README.md index 631b3f0..2e9accc 100644 --- a/server/README.md +++ b/server/README.md @@ -1,23 +1,22 @@ -# Ito Server - -The Ito Server coordinates Ito's shared server-side state and reconstruction. - -## Dependencies - -The server uses `msgpack` and `websockets` for the Ito control plane. WebRTC and -H.264 camera decoding use `aiortc` and `av`/PyAV. - -Install local Python dependencies from the repository root: - -```sh -python -m pip install -r server/requirements.txt -``` - -Run tests from the repository root: - -```sh -pytest -q -``` - -Docker Compose commands for running the Ito Server with the static Pilot Client -and optional Mock Robot are documented in `../docs/local-v1.md`. +# Ito application + +`server/ito/` is the single Python application boundary. It hosts `client/`, +accepts the pilot endpoint, runs reconstruction, and owns one robot adapter. + +Configuration: + +| Variable | Default | Purpose | +| --- | --- | --- | +| `ITO_HOST` | `0.0.0.0` | HTTP/WebSocket bind address. | +| `ITO_PORT` | `8765` | Shared client and `/ws` port. | +| `ITO_ROBOT_BACKEND` | `local` | `local` or `remote`. | +| `ITO_CLIENT_DIR` | repository `client/` | Hosted WebXR assets. | +| `ITO_REQUEST_TIMEOUT_MS` | `5000` | Remote-driver request timeout. | + +`LocalRobotAdapter` is the default in-process boundary. `RemoteRobotAdapter` is +the one-driver fallback for an external Ito deployment. A production local +integration supplies control, neutral-stop, emergency-stop, and safe-resumption +callbacks; the adapter enforces input timeout and newest-input rate limiting +inside the Ito process. It also supplies the reconstruction processor factory. +The stock image intentionally reports not-ready until a hardware integration is +wired instead of pretending to actuate a robot. diff --git a/server/ito/__init__.py b/server/ito/__init__.py index 76e6ef3..9b9d82a 100644 --- a/server/ito/__init__.py +++ b/server/ito/__init__.py @@ -1 +1 @@ -"""Ito Server Python package.""" +"""Ito application Python package.""" diff --git a/server/ito/app.py b/server/ito/app.py index ef3b37e..beb655f 100644 --- a/server/ito/app.py +++ b/server/ito/app.py @@ -1,44 +1,39 @@ -"""Ito Server WebSocket control-plane implementation.""" +"""The single Ito application: web host, pilot endpoint, and robot control.""" from __future__ import annotations import asyncio -from dataclasses import dataclass, field +from collections.abc import Callable +from dataclasses import dataclass import logging -from time import monotonic +import mimetypes from typing import Any -from uuid import uuid4 +from urllib.parse import unquote, urlsplit from websockets.asyncio.server import ServerConnection, serve +from websockets.datastructures import Headers from websockets.exceptions import ConnectionClosed +from websockets.http11 import Response -from .config import ServerConfig -from .media import AiortcCameraTrackReceiver +from .config import ItoConfig from .protocol import ( DisplayReason, PROTOCOL_VERSION, ProtocolError, ROLE_PILOT_CLIENT, - ROLE_ROBOT_DRIVER, - ROBOT_STATUS_AVAILABLE, - ROBOT_STATUS_OCCUPIED, - ROBOT_STATUS_UNAVAILABLE, - ROBOT_STATUSES, - ROBOT_TYPES, - TYPE_CATALOG_GET, - TYPE_CATALOG_GET_RESULT, - TYPE_DRIVER_SESSION_START, - TYPE_DRIVER_SESSION_START_RESULT, - TYPE_SESSION_ACQUIRE, - TYPE_SESSION_ACQUIRE_RESULT, - TYPE_SESSION_END, - TYPE_SESSION_END_RESULT, - TYPE_SESSION_ENDED, - TYPE_WEBRTC_ANSWER, - TYPE_WEBRTC_OFFER, + ROLE_REMOTE_ROBOT_DRIVER, TYPE_CONNECTION_HELLO, TYPE_CONNECTION_HELLO_RESULT, - TYPE_ROBOT_STATUS, + TYPE_CONTROL_START, + TYPE_CONTROL_START_RESULT, + TYPE_CONTROL_STOP, + TYPE_CONTROL_STOP_RESULT, + TYPE_CONTROL_STOPPED, + TYPE_ROBOT_READY, + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_DRIVER_CONTROL_STOP_RESULT, + TYPE_WEBRTC_ANSWER, + TYPE_WEBRTC_OFFER, WEBRTC_PATH_CAMERA_MEDIA, WEBRTC_PATH_PILOT_INPUT, WEBRTC_PATH_SPLAT_BATCHES, @@ -48,480 +43,397 @@ result_ok, unpack_envelope, ) -from .reconstruction import ReconstructionSessionRuntime -from .webrtc import MissingWebRtcStack, ServerLivePathAcceptor, SplatBatchChannelRegistry +from .reconstruction import ReconstructionRuntime +from .media import AiortcCameraTrackReceiver +from .robot import LocalRobotAdapter, RemoteRobotAdapter +from .webrtc import MissingWebRtcStack, ServerLivePathAcceptor, SplatBatchChannel from server.processors.null import NullReconstructionProcessor +from server.processors.base import ReconstructionProcessor LOGGER = logging.getLogger(__name__) -SESSION_STATE_STARTING = "starting" -SESSION_STATE_ACTIVE = "active" -SESSION_STATE_ENDED = "ended" - @dataclass(eq=False) class ConnectionState: websocket: ServerConnection role: str | None = None - robot_id: str | None = None - session_id: str | None = None - - -@dataclass -class DriverRecord: - robot_id: str - connection: ConnectionState | None = None - name: str | None = None - robot_type: str | None = None - driver_status: str = ROBOT_STATUS_UNAVAILABLE - availability_detail: dict[str, str] | None = None - last_status_at: float | None = None - duplicate: bool = False - occupied: bool = False - - def catalog_entry(self, now: float, watchdog_seconds: float) -> dict[str, Any]: - status = self.effective_status(now, watchdog_seconds) - entry: dict[str, Any] = { - "robotId": self.robot_id, - "name": self.name or self.robot_id, - "type": self.robot_type or "Droid", - "status": status, - } - if self.availability_detail: - entry["availabilityDetail"] = self.availability_detail - elif status == ROBOT_STATUS_UNAVAILABLE: - entry["availabilityDetail"] = {"code": "robot.unavailable"} - return entry - - def effective_status(self, now: float, watchdog_seconds: float) -> str: - if self.duplicate: - return ROBOT_STATUS_UNAVAILABLE - if self.occupied: - return ROBOT_STATUS_OCCUPIED - if self.connection is None or self.last_status_at is None: - return ROBOT_STATUS_UNAVAILABLE - if now - self.last_status_at > watchdog_seconds: - return ROBOT_STATUS_UNAVAILABLE - return self.driver_status - - -@dataclass -class SessionRecord: - session_id: str - robot_id: str - pilot_connection: ConnectionState | None - driver_connection: ConnectionState | None - session_config: dict[str, object] - state: str = SESSION_STATE_STARTING - created_at: float = field(default_factory=monotonic) - endpoint_missing_since: float | None = None - ended_reason: dict[str, str] | None = None - ended_by: str | None = None - clean: bool = False - - def note_endpoint_missing(self) -> None: - if self.endpoint_missing_since is None: - self.endpoint_missing_since = monotonic() - - def note_endpoint_present(self) -> None: - if self.pilot_connection is not None and self.driver_connection is not None: - self.endpoint_missing_since = None - - -class ItoServer: - def __init__(self, config: ServerConfig | None = None) -> None: - self.config = config or ServerConfig.from_env() - self.connections: set[ConnectionState] = set() - self.drivers: dict[str, DriverRecord] = {} - self.sessions: dict[str, SessionRecord] = {} - self._pending_requests: dict[str, asyncio.Future[dict[str, Any]]] = {} - self._pending_webrtc_routes: dict[str, tuple[ConnectionState, str]] = {} - self.splat_channels = SplatBatchChannelRegistry() - self.live_paths: ServerLivePathAcceptor = MissingWebRtcStack() - self.reconstruction_runtimes: dict[str, ReconstructionSessionRuntime] = {} - self._acquisition_lock = asyncio.Lock() - self._watchdog_task: asyncio.Task[None] | None = None - self._cleanup_task: asyncio.Task[None] | None = None - self._install_default_live_paths() - @property - def watchdog_seconds(self) -> float: - return self.config.driver_status_watchdog_ms / 1000 - @property - def request_timeout_seconds(self) -> float: - return self.config.request_timeout_ms / 1000 - - @property - def session_cleanup_seconds(self) -> float: - return self.config.session_cleanup_timeout_ms / 1000 +class ItoApplication: + def __init__( + self, + config: ItoConfig | None = None, + *, + adapter: LocalRobotAdapter | RemoteRobotAdapter | None = None, + processor_factory: Callable[[], ReconstructionProcessor] | None = None, + ) -> None: + self.config = config or ItoConfig.from_env() + if adapter is None: + adapter = ( + RemoteRobotAdapter(request_timeout_ms=self.config.request_timeout_ms) + if self.config.robot_backend == "remote" + else LocalRobotAdapter(ready=False) + ) + self.adapter = adapter + self.processor_factory = processor_factory or NullReconstructionProcessor + self.pilot: ConnectionState | None = None + self.control_active = False + self.splat_channels = SplatBatchChannel() + self.live_paths: ServerLivePathAcceptor = MissingWebRtcStack() + self.reconstruction_runtime: ReconstructionRuntime | None = None + if isinstance(self.adapter, LocalRobotAdapter): + self.adapter.set_sensor_sink(self._process_sensor_frame) + self._install_default_live_paths() def _install_default_live_paths(self) -> None: try: from .webrtc import AiortcServerLivePaths - except ImportError: # pragma: no cover - module is local - return - try: + self.live_paths = AiortcServerLivePaths( on_camera_track=self._accept_camera_track, + on_pilot_input=( + self.adapter.receive_pilot_input + if isinstance(self.adapter, LocalRobotAdapter) + else None + ), splat_channels=self.splat_channels, ) except RuntimeError: self.live_paths = MissingWebRtcStack() - def _accept_camera_track(self, track: object, session_id: str) -> None: - if getattr(track, "kind", None) != "video": - return - runtime = self._reconstruction_runtime(session_id) - receiver = AiortcCameraTrackReceiver(runtime.process_frame) - asyncio.create_task(receiver.consume(track)) - - def _reconstruction_runtime(self, session_id: str) -> ReconstructionSessionRuntime: - runtime = self.reconstruction_runtimes.get(session_id) - if runtime is not None: - return runtime - runtime = ReconstructionSessionRuntime( - session_id, - NullReconstructionProcessor(), - send_splat_batch=lambda payload: self.splat_channels.send(session_id, payload), - fail_session=lambda reason: asyncio.create_task( - self._fail_session_from_reconstruction(session_id, reason) - ), - ) - runtime.start() - self.reconstruction_runtimes[session_id] = runtime - return runtime - - async def _fail_session_from_reconstruction(self, session_id: str, reason: dict[str, str]) -> None: - session = self.sessions.get(session_id) - if session is None or session.state == SESSION_STATE_ENDED: - return - await self._end_session( - session, - reason=reason, - ended_by="server", - clean=False, - request_driver_end=True, + async def serve_forever(self) -> None: + LOGGER.info( + "Starting Ito application on http://%s:%s with %s robot adapter", + self.config.host, + self.config.port, + self.config.robot_backend, ) + async with serve( + self._handle_connection, + self.config.host, + self.config.port, + process_request=self.process_http_request, + ): + await asyncio.Future() - async def serve_forever(self) -> None: - LOGGER.info("Starting Ito Server on %s:%s", self.config.host, self.config.port) - self._watchdog_task = asyncio.create_task(self._watchdog_loop()) - self._cleanup_task = asyncio.create_task(self._session_cleanup_loop()) + async def process_http_request(self, _connection: object, request: object) -> Response | None: + request_path = urlsplit(request.path).path + if request_path == "/ws": + return None + relative_path = unquote(request_path).lstrip("/") or "index.html" + client_dir = self.config.client_dir.resolve() + candidate = (client_dir / relative_path).resolve() + if client_dir not in candidate.parents and candidate != client_dir: + return self._http_response(404, b"Not found", "text/plain; charset=utf-8") + if candidate.is_dir(): + candidate = candidate / "index.html" try: - async with serve(self._handle_connection, self.config.host, self.config.port): - await asyncio.Future() - finally: - self._watchdog_task.cancel() - self._cleanup_task.cancel() + body = candidate.read_bytes() + except (FileNotFoundError, IsADirectoryError, PermissionError): + return self._http_response(404, b"Not found", "text/plain; charset=utf-8") + content_type = mimetypes.guess_type(candidate.name)[0] or "application/octet-stream" + if content_type.startswith("text/") or content_type in {"application/javascript", "application/json"}: + content_type += "; charset=utf-8" + return self._http_response(200, body, content_type) + + @staticmethod + def _http_response(status: int, body: bytes, content_type: str) -> Response: + reason = "OK" if status == 200 else "Not Found" + return Response( + status, + reason, + Headers( + { + "Content-Type": content_type, + "Content-Length": str(len(body)), + "Cache-Control": "no-cache", + } + ), + body, + ) async def _handle_connection(self, websocket: ServerConnection) -> None: - state = ConnectionState(websocket=websocket) - self.connections.add(state) + state = ConnectionState(websocket) try: async for frame in websocket: await self._handle_frame(state, frame) except ConnectionClosed: pass finally: - self.connections.discard(state) - if state.role == ROLE_ROBOT_DRIVER and state.robot_id: - record = self.drivers.get(state.robot_id) - if record and record.connection is state: - record.connection = None - record.driver_status = ROBOT_STATUS_UNAVAILABLE - record.availability_detail = {"code": "robot.unavailable.driver_disconnected"} - self._mark_connection_disappeared(state) + await self._disconnect(state) + + async def _disconnect(self, state: ConnectionState) -> None: + if self.pilot is state: + await self._stop_control( + {"code": "control.stopped.pilot_disconnected"}, notify=False + ) + self.pilot = None + if ( + state.role == ROLE_REMOTE_ROBOT_DRIVER + and isinstance(self.adapter, RemoteRobotAdapter) + and self.adapter.connection is state.websocket + ): + self.adapter.detach(state.websocket) + await self._notify_robot_ready(False) + await self._stop_control( + {"code": "control.stopped.robot_driver_disconnected"} + ) async def _handle_frame(self, state: ConnectionState, frame: bytes | str) -> None: if not isinstance(frame, bytes): - await self._send_error(state, TYPE_CONNECTION_HELLO_RESULT, None, "protocol.invalid_frame") + await self._send_error( + state, TYPE_CONNECTION_HELLO_RESULT, None, "protocol.invalid_frame" + ) return try: envelope = unpack_envelope(frame) - except ProtocolError as exc: - LOGGER.warning("Rejecting invalid Ito envelope: %s", exc) - await self._send_error(state, TYPE_CONNECTION_HELLO_RESULT, None, "protocol.invalid_message") + except ProtocolError: + await self._send_error( + state, TYPE_CONNECTION_HELLO_RESULT, None, "protocol.invalid_message" + ) return - if state.role is None and envelope["type"] != TYPE_CONNECTION_HELLO: - await self._send_error(state, TYPE_CONNECTION_HELLO_RESULT, envelope["messageId"], "connection.hello_required") + await self._send_error( + state, + TYPE_CONNECTION_HELLO_RESULT, + envelope["messageId"], + "connection.hello_required", + ) return - if envelope["type"] == TYPE_CONNECTION_HELLO: + message_type = envelope["type"] + if message_type == TYPE_CONNECTION_HELLO: await self._handle_hello(state, envelope) - elif envelope["type"] in {TYPE_DRIVER_SESSION_START_RESULT, TYPE_SESSION_END_RESULT}: - self._handle_response(envelope) - elif envelope["type"] == TYPE_ROBOT_STATUS: - self._handle_robot_status(state, envelope) - elif envelope["type"] == TYPE_CATALOG_GET: - await self._handle_catalog_get(state, envelope) - elif envelope["type"] == TYPE_SESSION_ACQUIRE: - await self._handle_session_acquire(state, envelope) - elif envelope["type"] == TYPE_SESSION_END: - await self._handle_session_end(state, envelope) - elif envelope["type"] == TYPE_WEBRTC_OFFER: + elif message_type == TYPE_CONTROL_START: + await self._handle_control_start(state, envelope) + elif message_type == TYPE_CONTROL_STOP: + await self._handle_control_stop(state, envelope) + elif ( + message_type in { + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_DRIVER_CONTROL_STOP_RESULT, + TYPE_WEBRTC_ANSWER, + } + and state.role == ROLE_REMOTE_ROBOT_DRIVER + and isinstance(self.adapter, RemoteRobotAdapter) + and self.adapter.connection is state.websocket + ): + self.adapter.handle_response(envelope) + elif message_type == TYPE_WEBRTC_OFFER: await self._handle_webrtc_offer(state, envelope) - elif envelope["type"] == TYPE_WEBRTC_ANSWER: - await self._handle_webrtc_answer(state, envelope) else: - result_type = { - TYPE_SESSION_ACQUIRE: TYPE_SESSION_ACQUIRE_RESULT, - TYPE_SESSION_END: TYPE_SESSION_END_RESULT, - }.get(envelope["type"], TYPE_CONNECTION_HELLO_RESULT) - await self._send_result( + await self._send_error( state, - result_type, + TYPE_CONNECTION_HELLO_RESULT, envelope["messageId"], - result_error(DisplayReason(code="protocol.unsupported_message")), + "protocol.unsupported_message", ) async def _handle_hello(self, state: ConnectionState, envelope: dict[str, Any]) -> None: - payload = envelope["payload"] - role = payload.get("role") - if role == ROLE_ROBOT_DRIVER: - robot_id = payload.get("robotId") or envelope.get("robotId") - if not isinstance(robot_id, str) or not robot_id: - await self._send_error(state, TYPE_CONNECTION_HELLO_RESULT, envelope["messageId"], "connection.robot_id_required") - return - state.role = role - state.robot_id = robot_id - record = self.drivers.setdefault(robot_id, DriverRecord(robot_id=robot_id)) - if record.connection is not None and record.connection is not state: - record.duplicate = True - record.connection = None - LOGGER.error("Duplicate robotId reported: %s", robot_id) - elif not record.duplicate: - record.connection = state - session = self._active_session_for_robot(robot_id) - if session and session.driver_connection is None: - session.driver_connection = state - state.session_id = session.session_id - session.note_endpoint_present() - await self._send_result(state, TYPE_CONNECTION_HELLO_RESULT, envelope["messageId"], result_ok({"protocolVersion": PROTOCOL_VERSION, "role": role})) + role = envelope["payload"].get("role") + if role == ROLE_REMOTE_ROBOT_DRIVER: + await self._handle_remote_driver_hello(state, envelope) return - if role == ROLE_PILOT_CLIENT: - requested_session_id = payload.get("sessionId") - if requested_session_id is not None: - if not isinstance(requested_session_id, str): - await self._send_error(state, TYPE_CONNECTION_HELLO_RESULT, envelope["messageId"], "connection.invalid_session") - return - session = self.sessions.get(requested_session_id) - if session is None or session.state != SESSION_STATE_ACTIVE: - await self._send_error(state, TYPE_CONNECTION_HELLO_RESULT, envelope["messageId"], "session.resume_unavailable") - return - state.role = role - state.session_id = requested_session_id - session.pilot_connection = state - session.note_endpoint_present() - await self._send_result( - state, - TYPE_CONNECTION_HELLO_RESULT, - envelope["messageId"], - result_ok( - { - "protocolVersion": PROTOCOL_VERSION, - "role": role, - "sessionResumed": True, - "sessionConfig": session.session_config, - } - ), - ) - return - state.role = role - await self._send_result(state, TYPE_CONNECTION_HELLO_RESULT, envelope["messageId"], result_ok({"protocolVersion": PROTOCOL_VERSION, "role": role})) - return - await self._send_error(state, TYPE_CONNECTION_HELLO_RESULT, envelope["messageId"], "connection.invalid_role") - - def _handle_response(self, envelope: dict[str, Any]) -> None: - reply_to = envelope.get("replyToMessageId") - if not isinstance(reply_to, str): - LOGGER.warning("Ignoring response without replyToMessageId: %s", envelope["type"]) + if role != ROLE_PILOT_CLIENT: + await self._send_error( + state, + TYPE_CONNECTION_HELLO_RESULT, + envelope["messageId"], + "connection.invalid_role", + ) return - pending = self._pending_requests.get(reply_to) - if pending is None or pending.done(): - LOGGER.warning("Ignoring response for unknown request: %s", reply_to) + if self.pilot is not None and self.pilot is not state: + await self._send_error( + state, + TYPE_CONNECTION_HELLO_RESULT, + envelope["messageId"], + "connection.pilot_already_connected", + ) return - pending.set_result(envelope) + state.role = ROLE_PILOT_CLIENT + self.pilot = state + await self._send_result( + state, + TYPE_CONNECTION_HELLO_RESULT, + envelope["messageId"], + result_ok( + { + "protocolVersion": PROTOCOL_VERSION, + "robotReady": self.adapter.ready, + "controlActive": self.control_active, + "controlConfig": self.config.control_config_payload(), + } + ), + ) - def _handle_robot_status(self, state: ConnectionState, envelope: dict[str, Any]) -> None: - if state.role != ROLE_ROBOT_DRIVER or not state.robot_id: - LOGGER.warning("Ignoring robot.status from non-driver connection") - return - payload = envelope["payload"] - if payload.get("status") not in ROBOT_STATUSES or payload.get("type") not in ROBOT_TYPES or not isinstance(payload.get("name"), str): - LOGGER.warning("Ignoring invalid robot.status from %s", state.robot_id) - return - record = self.drivers.setdefault(state.robot_id, DriverRecord(robot_id=state.robot_id)) - if record.duplicate: - return - record.connection = state - record.name = payload["name"] - record.robot_type = payload["type"] - record.driver_status = payload["status"] - detail = payload.get("availabilityDetail") - record.availability_detail = detail if isinstance(detail, dict) else None - record.last_status_at = monotonic() - - async def _handle_catalog_get(self, state: ConnectionState, envelope: dict[str, Any]) -> None: - if state.role != ROLE_PILOT_CLIENT: - await self._send_error(state, TYPE_CATALOG_GET_RESULT, envelope["messageId"], "catalog.pilot_client_required") + async def _handle_remote_driver_hello( + self, state: ConnectionState, envelope: dict[str, Any] + ) -> None: + if not isinstance(self.adapter, RemoteRobotAdapter): + await self._send_error( + state, + TYPE_CONNECTION_HELLO_RESULT, + envelope["messageId"], + "connection.remote_driver_not_configured", + ) return - include_unavailable = envelope["payload"].get("includeUnavailable", True) - now = monotonic() - robots = [r.catalog_entry(now, self.watchdog_seconds) for r in self.drivers.values()] - if not include_unavailable: - robots = [r for r in robots if r["status"] != ROBOT_STATUS_UNAVAILABLE] - await self._send_result(state, TYPE_CATALOG_GET_RESULT, envelope["messageId"], result_ok({"robots": robots})) - - async def _handle_session_acquire(self, state: ConnectionState, envelope: dict[str, Any]) -> None: - if state.role != ROLE_PILOT_CLIENT: - await self._send_error(state, TYPE_SESSION_ACQUIRE_RESULT, envelope["messageId"], "session.acquire.pilot_client_required") + if self.adapter.connection not in {None, state.websocket}: + await self._send_error( + state, + TYPE_CONNECTION_HELLO_RESULT, + envelope["messageId"], + "connection.remote_driver_already_connected", + ) return - robot_id = envelope["payload"].get("robotId") or envelope.get("robotId") - if not isinstance(robot_id, str) or not robot_id: - await self._send_error(state, TYPE_SESSION_ACQUIRE_RESULT, envelope["messageId"], "session.acquire.robot_id_required") + ready = envelope["payload"].get("ready") + if not isinstance(ready, bool): + await self._send_error( + state, + TYPE_CONNECTION_HELLO_RESULT, + envelope["messageId"], + "connection.remote_driver_ready_required", + ) return + state.role = ROLE_REMOTE_ROBOT_DRIVER + self.adapter.attach(state.websocket, ready=ready) + await self._send_result( + state, + TYPE_CONNECTION_HELLO_RESULT, + envelope["messageId"], + result_ok({"protocolVersion": PROTOCOL_VERSION}), + ) + await self._notify_robot_ready(ready) - async with self._acquisition_lock: - record = self.drivers.get(robot_id) - now = monotonic() - if record is None or record.effective_status(now, self.watchdog_seconds) != ROBOT_STATUS_AVAILABLE or record.connection is None: - await self._send_error(state, TYPE_SESSION_ACQUIRE_RESULT, envelope["messageId"], "session.acquire.robot_unavailable") - return - - record.occupied = True - session_id = self._make_session_id() - session_config = self.config.session_config_payload() - session = SessionRecord( - session_id=session_id, - robot_id=robot_id, - pilot_connection=state, - driver_connection=record.connection, - session_config=session_config, + async def _handle_control_start( + self, state: ConnectionState, envelope: dict[str, Any] + ) -> None: + if state is not self.pilot: + await self._send_error( + state, + TYPE_CONTROL_START_RESULT, + envelope["messageId"], + "control.pilot_required", ) - self.sessions[session_id] = session - - start_result = await self._request_driver_session_start(record.connection, robot_id, session_id, session_config) - if not start_result["payload"].get("ok"): - self._release_failed_acquisition(session) - await self._send_result(state, TYPE_SESSION_ACQUIRE_RESULT, envelope["messageId"], start_result["payload"]) - return - - value = start_result["payload"].get("value", {}) - if value.get("sessionId") != session_id: - self._release_failed_acquisition(session) - await self._send_error(state, TYPE_SESSION_ACQUIRE_RESULT, envelope["messageId"], "driver.session_start.invalid_session") - return - - session.state = SESSION_STATE_ACTIVE - state.session_id = session_id - record.connection.session_id = session_id - await self._send_result( + return + if not self.adapter.ready: + await self._send_error( state, - TYPE_SESSION_ACQUIRE_RESULT, + TYPE_CONTROL_START_RESULT, envelope["messageId"], - result_ok( - { - "sessionId": session_id, - "robotId": robot_id, - "sessionConfig": session_config, - } - ), + "robot.not_ready", ) - - async def _request_driver_session_start( - self, - driver: ConnectionState, - robot_id: str, - session_id: str, - session_config: dict[str, object], - ) -> dict[str, Any]: - request = make_envelope( - TYPE_DRIVER_SESSION_START, - {"sessionId": session_id, "sessionConfig": session_config}, - robot_id=robot_id, - session_id=session_id, + return + if not self.control_active: + self._start_reconstruction() + try: + await _maybe_await(self.adapter.start_control()) + except Exception as exc: + LOGGER.warning("Robot adapter refused control start: %s", exc) + try: + await _maybe_await(self.adapter.stop_control()) + except Exception: + LOGGER.exception("Robot adapter cleanup failed after start refusal") + runtime = self.reconstruction_runtime + self.reconstruction_runtime = None + if runtime is not None: + runtime.close() + await self._send_result( + state, + TYPE_CONTROL_START_RESULT, + envelope["messageId"], + result_error( + DisplayReason(code="control.start_failed", text=str(exc)) + ), + ) + return + self.control_active = True + await self._send_result( + state, + TYPE_CONTROL_START_RESULT, + envelope["messageId"], + result_ok({"controlConfig": self.config.control_config_payload()}), ) - future: asyncio.Future[dict[str, Any]] = asyncio.get_running_loop().create_future() - self._pending_requests[request["messageId"]] = future - try: - await driver.websocket.send(pack_envelope(request)) - return await asyncio.wait_for(future, timeout=self.request_timeout_seconds) - except TimeoutError: - return make_envelope( - TYPE_DRIVER_SESSION_START_RESULT, - result_error(DisplayReason(code="request.timeout")), - reply_to_message_id=request["messageId"], - robot_id=robot_id, - session_id=session_id, + + async def _handle_control_stop( + self, state: ConnectionState, envelope: dict[str, Any] + ) -> None: + if state is not self.pilot: + await self._send_error( + state, + TYPE_CONTROL_STOP_RESULT, + envelope["messageId"], + "control.pilot_required", ) - finally: - self._pending_requests.pop(request["messageId"], None) - - def _release_failed_acquisition(self, session: SessionRecord) -> None: - self.sessions.pop(session.session_id, None) - record = self.drivers.get(session.robot_id) - if record: - record.occupied = False - if session.pilot_connection and session.pilot_connection.session_id == session.session_id: - session.pilot_connection.session_id = None - if session.driver_connection and session.driver_connection.session_id == session.session_id: - session.driver_connection.session_id = None - - async def _handle_session_end(self, state: ConnectionState, envelope: dict[str, Any]) -> None: - session_id = envelope.get("sessionId") or state.session_id - if not isinstance(session_id, str) or session_id not in self.sessions: - await self._send_error(state, TYPE_SESSION_END_RESULT, envelope["messageId"], "session.end.unknown_session") - return - session = self.sessions[session_id] - if session.state == SESSION_STATE_ENDED: - await self._send_result(state, TYPE_SESSION_END_RESULT, envelope["messageId"], result_ok({"sessionId": session_id})) - return - if state not in {session.pilot_connection, session.driver_connection}: - await self._send_error(state, TYPE_SESSION_END_RESULT, envelope["messageId"], "session.end.endpoint_required") return - reason = envelope["payload"].get("reason") if not isinstance(reason, dict): - reason = {"code": "session.ended.requested"} - clean = bool(envelope["payload"].get("clean", False)) - ended_by = state.role or "server" - - await self._send_result(state, TYPE_SESSION_END_RESULT, envelope["messageId"], result_ok({"sessionId": session_id})) - await self._end_session(session, reason=reason, ended_by=ended_by, clean=clean, request_driver_end=state is not session.driver_connection) + reason = {"code": "control.stopped.pilot_requested"} + await self._send_result( + state, + TYPE_CONTROL_STOP_RESULT, + envelope["messageId"], + result_ok(), + ) + await self._stop_control(reason) - async def _handle_webrtc_offer(self, state: ConnectionState, envelope: dict[str, Any]) -> None: - session = self._session_for_live_path(state, envelope) - if session is None: - LOGGER.warning("Ignoring WebRTC offer for unknown or inactive session") + def _start_reconstruction(self) -> None: + if self.reconstruction_runtime is not None: return - path = envelope["payload"]["path"] - if path == WEBRTC_PATH_PILOT_INPUT: - await self._relay_pilot_input_offer(state, session, envelope) + runtime = ReconstructionRuntime( + self.processor_factory(), + send_splat_batch=self.splat_channels.send, + fail_control=lambda reason: asyncio.create_task( + self._stop_control(reason) + ), + ) + runtime.start() + self.reconstruction_runtime = runtime + + def _process_sensor_frame(self, frame: object) -> None: + if self.control_active and self.reconstruction_runtime is not None: + self.reconstruction_runtime.process_frame(frame) + + def _accept_camera_track(self, track: object) -> None: + if getattr(track, "kind", None) != "video": return - if path not in {WEBRTC_PATH_CAMERA_MEDIA, WEBRTC_PATH_SPLAT_BATCHES}: - LOGGER.warning("Ignoring unsupported WebRTC path %s", path) + receiver = AiortcCameraTrackReceiver(self._process_sensor_frame) + asyncio.create_task(receiver.consume(track)) + + async def _handle_webrtc_offer( + self, state: ConnectionState, envelope: dict[str, Any] + ) -> None: + if not self.control_active: return - if (path == WEBRTC_PATH_CAMERA_MEDIA and state is not session.driver_connection) or ( - path == WEBRTC_PATH_SPLAT_BATCHES and state is not session.pilot_connection - ): - LOGGER.warning("Ignoring WebRTC %s offer from wrong endpoint", path) + path = envelope["payload"].get("path") + sdp = envelope["payload"].get("sdp") + if not isinstance(sdp, str): return try: - answer_sdp = await self.live_paths.accept_offer( - path=path, - session_id=session.session_id, - sdp=envelope["payload"]["sdp"], - ) + if ( + path == WEBRTC_PATH_PILOT_INPUT + and state is self.pilot + and isinstance(self.adapter, RemoteRobotAdapter) + ): + answer_sdp = await self.adapter.accept_pilot_input_offer(sdp) + elif path in {WEBRTC_PATH_PILOT_INPUT, WEBRTC_PATH_SPLAT_BATCHES} and state is self.pilot: + answer_sdp = await self.live_paths.accept_offer( + path=path, sdp=sdp + ) + elif ( + path == WEBRTC_PATH_CAMERA_MEDIA + and state.role == ROLE_REMOTE_ROBOT_DRIVER + and isinstance(self.adapter, RemoteRobotAdapter) + and self.adapter.connection is state.websocket + ): + answer_sdp = await self.live_paths.accept_offer( + path=path, sdp=sdp + ) + else: + return except Exception as exc: - LOGGER.exception("WebRTC %s negotiation failed for session %s", path, session.session_id) - await self._end_session( - session, - reason={"code": "session.ended.reconstruction_failed", "text": str(exc)}, - ended_by="server", - clean=False, - request_driver_end=True, + LOGGER.exception("WebRTC negotiation failed for %s", path) + await self._stop_control( + {"code": "control.stopped.transport_failed", "text": str(exc)} ) return await state.websocket.send( @@ -530,193 +442,76 @@ async def _handle_webrtc_offer(self, state: ConnectionState, envelope: dict[str, TYPE_WEBRTC_ANSWER, {"path": path, "sdp": answer_sdp}, reply_to_message_id=envelope["messageId"], - robot_id=session.robot_id, - session_id=session.session_id, ) ) ) - async def _relay_pilot_input_offer( - self, - state: ConnectionState, - session: SessionRecord, - envelope: dict[str, Any], + async def _stop_control( + self, reason: dict[str, str], *, notify: bool = True ) -> None: - if state is not session.pilot_connection or session.driver_connection is None: - LOGGER.warning("Ignoring pilot-input WebRTC offer without pilot and driver endpoints") - return - forwarded = make_envelope( - TYPE_WEBRTC_OFFER, - envelope["payload"], - robot_id=session.robot_id, - session_id=session.session_id, - ) - self._pending_webrtc_routes[forwarded["messageId"]] = (state, envelope["messageId"]) - await session.driver_connection.websocket.send(pack_envelope(forwarded)) - - async def _handle_webrtc_answer(self, state: ConnectionState, envelope: dict[str, Any]) -> None: - reply_to = envelope.get("replyToMessageId") - if not isinstance(reply_to, str): - LOGGER.warning("Ignoring WebRTC answer without replyToMessageId") - return - route = self._pending_webrtc_routes.pop(reply_to, None) - if route is None: - LOGGER.warning("Ignoring WebRTC answer for unknown offer %s", reply_to) - return - destination, original_message_id = route - session = self._session_for_live_path(destination, envelope) - if session is None: - return - await destination.websocket.send( - pack_envelope( - make_envelope( - TYPE_WEBRTC_ANSWER, - envelope["payload"], - reply_to_message_id=original_message_id, - robot_id=session.robot_id, - session_id=session.session_id, + was_active = self.control_active + self.control_active = False + if was_active: + await _maybe_await(self.adapter.stop_control()) + runtime = self.reconstruction_runtime + self.reconstruction_runtime = None + if runtime is not None: + runtime.close() + close_control = getattr(self.live_paths, "close_control", None) + if close_control is not None: + await close_control() + if notify and self.pilot is not None: + await self.pilot.websocket.send( + pack_envelope( + make_envelope(TYPE_CONTROL_STOPPED, {"reason": reason}) ) ) - ) - def _session_for_live_path( - self, state: ConnectionState, envelope: dict[str, Any] - ) -> SessionRecord | None: - session_id = envelope.get("sessionId") or state.session_id - if not isinstance(session_id, str): - return None - session = self.sessions.get(session_id) - if session is None or session.state != SESSION_STATE_ACTIVE: - return None - if state not in {session.pilot_connection, session.driver_connection}: - return None - return session + async def _notify_robot_ready(self, ready: bool) -> None: + if self.pilot is None: + return + await self.pilot.websocket.send( + pack_envelope(make_envelope(TYPE_ROBOT_READY, {"ready": ready})) + ) - async def _end_session( + async def _send_error( self, - session: SessionRecord, - *, - reason: dict[str, str], - ended_by: str, - clean: bool, - request_driver_end: bool = True, + state: ConnectionState, + message_type: str, + reply_to: str | None, + code: str, ) -> None: - if session.state == SESSION_STATE_ENDED: - return - session.state = SESSION_STATE_ENDED - session.ended_reason = reason - session.ended_by = ended_by - session.clean = clean - record = self.drivers.get(session.robot_id) - if record: - record.occupied = False - if session.pilot_connection and session.pilot_connection.session_id == session.session_id: - session.pilot_connection.session_id = None - if session.driver_connection and session.driver_connection.session_id == session.session_id: - session.driver_connection.session_id = None - runtime = self.reconstruction_runtimes.pop(session.session_id, None) - if runtime is not None: - runtime.close() - close_session = getattr(self.live_paths, "close_session", None) - if close_session is not None: - await close_session(session.session_id) - - if request_driver_end and session.driver_connection is not None: - await self._send_driver_session_end(session, reason, clean) - - ended_payload = {"reason": reason, "endedBy": ended_by, "clean": clean} - await self._send_session_ended(session.pilot_connection, session, ended_payload) - await self._send_session_ended(session.driver_connection, session, ended_payload) - - async def _send_driver_session_end(self, session: SessionRecord, reason: dict[str, str], clean: bool) -> None: - if session.driver_connection is None: - return - await session.driver_connection.websocket.send( - pack_envelope( - make_envelope( - TYPE_SESSION_END, - {"reason": reason, "clean": clean}, - robot_id=session.robot_id, - session_id=session.session_id, - ) - ) + await self._send_result( + state, + message_type, + reply_to, + result_error(DisplayReason(code=code)), ) - async def _send_session_ended( - self, state: ConnectionState | None, session: SessionRecord, payload: dict[str, Any] + async def _send_result( + self, + state: ConnectionState, + message_type: str, + reply_to: str | None, + payload: dict[str, Any], ) -> None: - if state is None: - return await state.websocket.send( pack_envelope( make_envelope( - TYPE_SESSION_ENDED, - payload, - robot_id=session.robot_id, - session_id=session.session_id, + message_type, payload, reply_to_message_id=reply_to ) ) ) - async def _send_error(self, state: ConnectionState, message_type: str, reply_to: str | None, code: str) -> None: - await self._send_result(state, message_type, reply_to, result_error(DisplayReason(code=code))) - - async def _send_result(self, state: ConnectionState, message_type: str, reply_to: str | None, payload: dict[str, Any]) -> None: - await state.websocket.send(pack_envelope(make_envelope(message_type, payload, reply_to_message_id=reply_to, robot_id=state.robot_id, session_id=state.session_id))) - - def _make_session_id(self) -> str: - return f"session-{uuid4()}" - - def _active_session_for_robot(self, robot_id: str) -> SessionRecord | None: - for session in self.sessions.values(): - if session.robot_id == robot_id and session.state != SESSION_STATE_ENDED: - return session - return None - - def _mark_connection_disappeared(self, state: ConnectionState) -> None: - for session in self.sessions.values(): - changed = False - if session.pilot_connection is state: - session.pilot_connection = None - changed = True - if session.driver_connection is state: - session.driver_connection = None - changed = True - if changed and session.state != SESSION_STATE_ENDED: - session.note_endpoint_missing() - - async def _watchdog_loop(self) -> None: - while True: - await asyncio.sleep(self.watchdog_seconds / 2) - now = monotonic() - for record in self.drivers.values(): - if record.effective_status(now, self.watchdog_seconds) == ROBOT_STATUS_UNAVAILABLE and record.connection is not None and record.last_status_at is not None and now - record.last_status_at > self.watchdog_seconds: - record.driver_status = ROBOT_STATUS_UNAVAILABLE - record.availability_detail = {"code": "robot.unavailable.driver_status_timeout"} - - async def _session_cleanup_loop(self) -> None: - while True: - await asyncio.sleep(self.session_cleanup_seconds / 2) - await self._cleanup_disappeared_endpoint_sessions() - - async def _cleanup_disappeared_endpoint_sessions(self) -> None: - now = monotonic() - for session in list(self.sessions.values()): - if session.state == SESSION_STATE_ENDED or session.endpoint_missing_since is None: - continue - if now - session.endpoint_missing_since >= self.session_cleanup_seconds: - await self._end_session( - session, - reason={"code": "session.ended.endpoint_disappeared"}, - ended_by="server", - clean=False, - request_driver_end=True, - ) - self.sessions.pop(session.session_id, None) + +async def _maybe_await(result: object) -> object: + if asyncio.iscoroutine(result): + return await result + return result -async def run(config: ServerConfig | None = None) -> None: - await ItoServer(config).serve_forever() +async def run(config: ItoConfig | None = None) -> None: + await ItoApplication(config).serve_forever() def main() -> None: diff --git a/server/ito/config.py b/server/ito/config.py index 6385ac6..a06d5df 100644 --- a/server/ito/config.py +++ b/server/ito/config.py @@ -1,9 +1,10 @@ -"""Environment-backed Ito Server configuration.""" +"""Environment-backed configuration for the Ito application.""" from __future__ import annotations from dataclasses import dataclass import os +from pathlib import Path def _env_int(name: str, default: int, *, minimum: int = 0) -> int: @@ -33,108 +34,55 @@ def _env_bool(name: str, default: bool) -> bool: @dataclass(frozen=True) class DataChannelConfig: - """WebRTC data channel reliability/ordering profile exposed in sessions.""" - ordered: bool max_retransmits: int | None = None - max_packet_lifetime_ms: int | None = None def to_protocol(self) -> dict[str, int | bool]: payload: dict[str, int | bool] = {"ordered": self.ordered} if self.max_retransmits is not None: payload["maxRetransmits"] = self.max_retransmits - if self.max_packet_lifetime_ms is not None: - payload["maxPacketLifeTime"] = self.max_packet_lifetime_ms return payload @dataclass(frozen=True) -class ServerConfig: +class ItoConfig: host: str = "0.0.0.0" port: int = 8765 request_timeout_ms: int = 5000 - driver_status_watchdog_ms: int = 2000 - session_cleanup_timeout_ms: int = 30000 + robot_backend: str = "local" + client_dir: Path = Path(__file__).resolve().parents[2] / "client" pilot_input_data_channel: DataChannelConfig = DataChannelConfig( ordered=False, max_retransmits=0 ) splat_batch_data_channel: DataChannelConfig = DataChannelConfig(ordered=True) + def __post_init__(self) -> None: + if self.robot_backend not in {"local", "remote"}: + raise ValueError("robot_backend must be local or remote") + @classmethod - def from_env(cls) -> "ServerConfig": + def from_env(cls) -> "ItoConfig": return cls( - host=os.getenv("ITO_SERVER_HOST", cls.host), - port=_env_int("ITO_SERVER_PORT", cls.port, minimum=1), + host=os.getenv("ITO_HOST", cls.host), + port=_env_int("ITO_PORT", cls.port, minimum=1), request_timeout_ms=_env_int( "ITO_REQUEST_TIMEOUT_MS", cls.request_timeout_ms, minimum=1 ), - driver_status_watchdog_ms=_env_int( - "ITO_DRIVER_STATUS_WATCHDOG_MS", - cls.driver_status_watchdog_ms, - minimum=1, - ), - session_cleanup_timeout_ms=_env_int( - "ITO_SESSION_CLEANUP_TIMEOUT_MS", - cls.session_cleanup_timeout_ms, - minimum=1, + robot_backend=os.getenv("ITO_ROBOT_BACKEND", cls.robot_backend), + client_dir=Path(os.getenv("ITO_CLIENT_DIR", str(cls.client_dir))), + pilot_input_data_channel=DataChannelConfig( + ordered=_env_bool("ITO_PILOT_INPUT_ORDERED", False), + max_retransmits=_env_int( + "ITO_PILOT_INPUT_MAX_RETRANSMITS", 0, minimum=0 + ), ), - pilot_input_data_channel=_data_channel_config_from_env( - ordered_name="ITO_PILOT_INPUT_ORDERED", - ordered_default=False, - max_retransmits_name="ITO_PILOT_INPUT_MAX_RETRANSMITS", - max_retransmits_default=0, - max_packet_lifetime_name="ITO_PILOT_INPUT_MAX_PACKET_LIFETIME_MS", - ), - splat_batch_data_channel=_data_channel_config_from_env( - ordered_name="ITO_SPLAT_BATCH_ORDERED", - ordered_default=True, - max_retransmits_name="ITO_SPLAT_BATCH_MAX_RETRANSMITS", - max_retransmits_default=None, - max_packet_lifetime_name="ITO_SPLAT_BATCH_MAX_PACKET_LIFETIME_MS", + splat_batch_data_channel=DataChannelConfig( + ordered=_env_bool("ITO_SPLAT_BATCH_ORDERED", True) ), ) - def session_config_payload(self) -> dict[str, object]: + def control_config_payload(self) -> dict[str, object]: return { "pilotInputDataChannel": self.pilot_input_data_channel.to_protocol(), "splatBatchDataChannel": self.splat_batch_data_channel.to_protocol(), } - - -def _optional_int(name: str, default: int | None) -> int | None: - raw = os.getenv(name) - if raw is None or raw == "": - return default - return _env_int(name, 0, minimum=0) - - -def _env_is_set(name: str) -> bool: - raw = os.getenv(name) - return raw is not None and raw != "" - - -def _data_channel_config_from_env( - *, - ordered_name: str, - ordered_default: bool, - max_retransmits_name: str, - max_retransmits_default: int | None, - max_packet_lifetime_name: str, -) -> DataChannelConfig: - has_max_retransmits = _env_is_set(max_retransmits_name) - has_max_packet_lifetime = _env_is_set(max_packet_lifetime_name) - if has_max_retransmits and has_max_packet_lifetime: - raise ValueError( - f"{max_retransmits_name} and {max_packet_lifetime_name} are mutually exclusive" - ) - - max_packet_lifetime_ms = _optional_int(max_packet_lifetime_name, None) - max_retransmits = _optional_int( - max_retransmits_name, - None if has_max_packet_lifetime else max_retransmits_default, - ) - return DataChannelConfig( - ordered=_env_bool(ordered_name, ordered_default), - max_retransmits=max_retransmits, - max_packet_lifetime_ms=max_packet_lifetime_ms, - ) diff --git a/server/ito/protocol.py b/server/ito/protocol.py index bdbdf50..90ed01e 100644 --- a/server/ito/protocol.py +++ b/server/ito/protocol.py @@ -1,4 +1,4 @@ -"""Shared Ito v1 protocol constants and MessagePack envelope helpers.""" +"""Network messages for one Ito application, one pilot, and one robot.""" from __future__ import annotations @@ -10,18 +10,18 @@ PROTOCOL_VERSION = "ito.v1" -TYPE_CATALOG_GET = "catalog.get" -TYPE_CATALOG_GET_RESULT = "catalog.get.result" TYPE_CONNECTION_HELLO = "connection.hello" TYPE_CONNECTION_HELLO_RESULT = "connection.hello.result" -TYPE_ROBOT_STATUS = "robot.status" -TYPE_SESSION_ACQUIRE = "session.acquire" -TYPE_SESSION_ACQUIRE_RESULT = "session.acquire.result" -TYPE_DRIVER_SESSION_START = "driver.session.start" -TYPE_DRIVER_SESSION_START_RESULT = "driver.session.start.result" -TYPE_SESSION_END = "session.end" -TYPE_SESSION_END_RESULT = "session.end.result" -TYPE_SESSION_ENDED = "session.ended" +TYPE_CONTROL_START = "control.start" +TYPE_CONTROL_START_RESULT = "control.start.result" +TYPE_CONTROL_STOP = "control.stop" +TYPE_CONTROL_STOP_RESULT = "control.stop.result" +TYPE_CONTROL_STOPPED = "control.stopped" +TYPE_ROBOT_READY = "robot.ready" +TYPE_DRIVER_CONTROL_START = "driver.control.start" +TYPE_DRIVER_CONTROL_START_RESULT = "driver.control.start.result" +TYPE_DRIVER_CONTROL_STOP = "driver.control.stop" +TYPE_DRIVER_CONTROL_STOP_RESULT = "driver.control.stop.result" TYPE_WEBRTC_OFFER = "webrtc.offer" TYPE_WEBRTC_ANSWER = "webrtc.answer" @@ -34,70 +34,40 @@ MESSAGE_TYPES = frozenset( { - TYPE_CATALOG_GET, - TYPE_CATALOG_GET_RESULT, TYPE_CONNECTION_HELLO, TYPE_CONNECTION_HELLO_RESULT, - TYPE_ROBOT_STATUS, - TYPE_SESSION_ACQUIRE, - TYPE_SESSION_ACQUIRE_RESULT, - TYPE_DRIVER_SESSION_START, - TYPE_DRIVER_SESSION_START_RESULT, - TYPE_SESSION_END, - TYPE_SESSION_END_RESULT, - TYPE_SESSION_ENDED, + TYPE_CONTROL_START, + TYPE_CONTROL_START_RESULT, + TYPE_CONTROL_STOP, + TYPE_CONTROL_STOP_RESULT, + TYPE_CONTROL_STOPPED, + TYPE_ROBOT_READY, + TYPE_DRIVER_CONTROL_START, + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_DRIVER_CONTROL_STOP, + TYPE_DRIVER_CONTROL_STOP_RESULT, TYPE_WEBRTC_OFFER, TYPE_WEBRTC_ANSWER, } ) ROLE_PILOT_CLIENT = "pilotClient" -ROLE_ROBOT_DRIVER = "robotDriver" -ROLES = frozenset({ROLE_PILOT_CLIENT, ROLE_ROBOT_DRIVER}) - -ROBOT_STATUS_AVAILABLE = "Available" -ROBOT_STATUS_OCCUPIED = "Occupied" -ROBOT_STATUS_UNAVAILABLE = "Unavailable" -ROBOT_STATUSES = frozenset( - {ROBOT_STATUS_AVAILABLE, ROBOT_STATUS_OCCUPIED, ROBOT_STATUS_UNAVAILABLE} -) - -ROBOT_TYPE_MECHA = "Mecha" -ROBOT_TYPE_ANDROID_ROBOT = "Android Robot" -ROBOT_TYPE_DROID = "Droid" -ROBOT_TYPE_DRONE = "Drone" -ROBOT_TYPE_CAR = "Car" -ROBOT_TYPE_PLANE = "Plane" -ROBOT_TYPES = frozenset( - { - ROBOT_TYPE_MECHA, - ROBOT_TYPE_ANDROID_ROBOT, - ROBOT_TYPE_DROID, - ROBOT_TYPE_DRONE, - ROBOT_TYPE_CAR, - ROBOT_TYPE_PLANE, - } -) - -REASON_PROTOCOL_VERSION_MISMATCH = "protocol.version_mismatch" -REASON_INVALID_MESSAGE = "protocol.invalid_message" -REASON_REQUEST_TIMEOUT = "request.timeout" +ROLE_REMOTE_ROBOT_DRIVER = "remoteRobotDriver" +ROLES = frozenset({ROLE_PILOT_CLIENT, ROLE_REMOTE_ROBOT_DRIVER}) class ProtocolError(ValueError): - """Raised when a decoded control-plane message violates Ito Protocol.""" + pass @dataclass(frozen=True) class DisplayReason: - """Pilot-displayable reason represented by resource key and/or free text.""" - code: str | None = None text: str | None = None def __post_init__(self) -> None: if not self.code and not self.text: - raise ValueError("DisplayReason requires code, text, or both") + raise ValueError("display reason requires code or text") def to_payload(self) -> dict[str, str]: payload: dict[str, str] = {} @@ -108,116 +78,65 @@ def to_payload(self) -> dict[str, str]: return payload -def result_ok(value: Mapping[str, Any] | None = None) -> dict[str, Any]: - """Build the standard successful request/result payload.""" - - return {"ok": True, "value": dict(value or {})} - - -def result_error(reason: DisplayReason | Mapping[str, str]) -> dict[str, Any]: - """Build the standard failed request/result payload.""" - - reason_payload = reason.to_payload() if isinstance(reason, DisplayReason) else dict(reason) - validate_display_reason(reason_payload) - return {"ok": False, "reason": reason_payload} - - -def make_message_id() -> str: - """Generate a sender-owned unique message identifier.""" - - return str(uuid4()) - - def make_envelope( message_type: str, payload: Mapping[str, Any] | None = None, *, message_id: str | None = None, reply_to_message_id: str | None = None, - robot_id: str | None = None, - session_id: str | None = None, - protocol_version: str = PROTOCOL_VERSION, ) -> dict[str, Any]: - """Create a decoded Ito control-plane envelope.""" - if message_type not in MESSAGE_TYPES: - raise ProtocolError(f"unknown Ito message type: {message_type}") - + raise ProtocolError(f"unsupported message type: {message_type}") envelope: dict[str, Any] = { - "protocolVersion": protocol_version, - "messageId": message_id or make_message_id(), + "protocolVersion": PROTOCOL_VERSION, + "messageId": message_id or str(uuid4()), "type": message_type, "payload": dict(payload or {}), } if reply_to_message_id is not None: envelope["replyToMessageId"] = reply_to_message_id - if robot_id is not None: - envelope["robotId"] = robot_id - if session_id is not None: - envelope["sessionId"] = session_id - validate_envelope(envelope) return envelope -def validate_protocol_version(protocol_version: str) -> None: - if protocol_version != PROTOCOL_VERSION: - raise ProtocolError( - f"unsupported Ito protocol version {protocol_version!r}; expected {PROTOCOL_VERSION!r}" - ) +def pack_envelope(envelope: Mapping[str, Any]) -> bytes: + validate_envelope(envelope) + return msgpack.packb(dict(envelope), use_bin_type=True) -def validate_display_reason(reason: Mapping[str, Any]) -> None: - code = reason.get("code") - text = reason.get("text") - if not isinstance(reason, Mapping) or (not code and not text): - raise ProtocolError("Display Reason requires code, text, or both") - if code is not None and not isinstance(code, str): - raise ProtocolError("Display Reason code must be a string") - if text is not None and not isinstance(text, str): - raise ProtocolError("Display Reason text must be a string") +def unpack_envelope(frame: bytes) -> dict[str, Any]: + try: + envelope = msgpack.unpackb(frame, raw=False) + except (ValueError, msgpack.ExtraData) as exc: + raise ProtocolError("invalid MessagePack envelope") from exc + validate_envelope(envelope) + return dict(envelope) def validate_envelope(envelope: Mapping[str, Any]) -> None: if not isinstance(envelope, Mapping): - raise ProtocolError("Ito envelope must be a map") + raise ProtocolError("envelope must be a map") validate_protocol_version(envelope.get("protocolVersion")) if not isinstance(envelope.get("messageId"), str) or not envelope["messageId"]: - raise ProtocolError("Ito envelope requires non-empty string messageId") - if envelope.get("replyToMessageId") is not None and not isinstance( - envelope.get("replyToMessageId"), str - ): - raise ProtocolError("replyToMessageId must be a string when present") + raise ProtocolError("messageId must be a non-empty string") if envelope.get("type") not in MESSAGE_TYPES: - raise ProtocolError(f"unknown Ito message type: {envelope.get('type')!r}") - if "payload" not in envelope or not isinstance(envelope.get("payload"), Mapping): - raise ProtocolError("Ito envelope requires payload map") - for field in ("robotId", "sessionId"): - if envelope.get(field) is not None and not isinstance(envelope.get(field), str): - raise ProtocolError(f"{field} must be a string when present") - if envelope.get("type") in {TYPE_WEBRTC_OFFER, TYPE_WEBRTC_ANSWER}: - validate_webrtc_signal_payload(envelope["payload"]) + raise ProtocolError("unknown message type") + if not isinstance(envelope.get("payload"), Mapping): + raise ProtocolError("payload must be a map") + reply_to = envelope.get("replyToMessageId") + if reply_to is not None and not isinstance(reply_to, str): + raise ProtocolError("replyToMessageId must be a string") + if "robotId" in envelope or "sessionId" in envelope: + raise ProtocolError("robot and session identifiers aren't part of this protocol") -def validate_webrtc_signal_payload(payload: Mapping[str, Any]) -> None: - if payload.get("path") not in WEBRTC_PATHS: - raise ProtocolError(f"unknown WebRTC live path: {payload.get('path')!r}") - if not isinstance(payload.get("sdp"), str) or not payload["sdp"]: - raise ProtocolError("WebRTC signaling payload requires non-empty SDP") +def validate_protocol_version(version: object) -> None: + if version != PROTOCOL_VERSION: + raise ProtocolError(f"protocolVersion must be {PROTOCOL_VERSION}") -def pack_envelope(envelope: Mapping[str, Any]) -> bytes: - """Validate and encode an envelope as a MessagePack binary WebSocket frame.""" - - validate_envelope(envelope) - return msgpack.packb(dict(envelope), use_bin_type=True) +def result_ok(value: Mapping[str, Any] | None = None) -> dict[str, Any]: + return {"ok": True, "value": dict(value or {})} -def unpack_envelope(frame: bytes) -> dict[str, Any]: - """Decode and validate a MessagePack binary WebSocket frame.""" - - try: - envelope = msgpack.unpackb(frame, raw=False, strict_map_key=False) - except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError) as exc: - raise ProtocolError("invalid MessagePack Ito envelope") from exc - validate_envelope(envelope) - return dict(envelope) +def result_error(reason: DisplayReason) -> dict[str, Any]: + return {"ok": False, "reason": reason.to_payload()} diff --git a/server/ito/reconstruction.py b/server/ito/reconstruction.py index 5ce1777..b1add1b 100644 --- a/server/ito/reconstruction.py +++ b/server/ito/reconstruction.py @@ -1,4 +1,4 @@ -"""Session-scoped reconstruction runtime and failure isolation.""" +"""Control-lifecycle reconstruction runtime and failure isolation.""" from __future__ import annotations @@ -12,25 +12,23 @@ LOGGER = logging.getLogger(__name__) -class ReconstructionSessionRuntime: - """Owns one processor instance for one piloting session.""" +class ReconstructionRuntime: + """Owns the processor while control is active.""" def __init__( self, - session_id: str, processor: ReconstructionProcessor, *, send_splat_batch: Callable[[bytes], None], - fail_session: Callable[[dict[str, str]], None], + fail_control: Callable[[dict[str, str]], None], ) -> None: - self.session_id = session_id self.processor = processor self.send_splat_batch = send_splat_batch - self.fail_session = fail_session + self.fail_control = fail_control self.failed = False def start(self) -> None: - self.processor.start(self.session_id) + self.processor.start() def process_frame(self, frame: ReconstructionFrame) -> None: if self.failed: @@ -39,12 +37,12 @@ def process_frame(self, frame: ReconstructionFrame) -> None: for batch in self.processor.process_frame(frame): self.send_splat_batch(encode_splat_batch(batch)) except Exception: - LOGGER.exception("Reconstruction failed for session %s", self.session_id) + LOGGER.exception("Reconstruction failed while control is active") self.failed = True - self.fail_session({"code": "session.ended.reconstruction_failed"}) + self.fail_control({"code": "control.stopped.reconstruction_failed"}) def close(self) -> None: try: self.processor.close() except Exception: - LOGGER.exception("Reconstruction processor close failed for session %s", self.session_id) + LOGGER.exception("Reconstruction processor close failed") diff --git a/server/ito/robot.py b/server/ito/robot.py new file mode 100644 index 0000000..bafa98c --- /dev/null +++ b/server/ito/robot.py @@ -0,0 +1,252 @@ +"""Narrow robot integration seam for the Ito application.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +import asyncio +import time +from typing import Any + +from .protocol import ( + DisplayReason, + TYPE_DRIVER_CONTROL_START, + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_DRIVER_CONTROL_STOP, + TYPE_DRIVER_CONTROL_STOP_RESULT, + TYPE_WEBRTC_ANSWER, + TYPE_WEBRTC_OFFER, + WEBRTC_PATH_PILOT_INPUT, + make_envelope, + pack_envelope, +) +from server.processors.base import ReconstructionFrame + + +class LocalRobotAdapter: + """In-process robot integration used by the normal onboard deployment.""" + + def __init__( + self, + *, + control_sink: Callable[[Mapping[str, Any]], None] | None = None, + safe_stop: Callable[[], None] | None = None, + emergency_stop: Callable[[], None] | None = None, + safe_resume: Callable[[], None] | None = None, + ready: bool = True, + pilot_input_timeout_ms: int = 500, + max_control_rate_hz: float = 60.0, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if pilot_input_timeout_ms <= 0: + raise ValueError("pilot_input_timeout_ms must be positive") + if max_control_rate_hz <= 0: + raise ValueError("max_control_rate_hz must be positive") + self.control_active = False + self.ready = ready + self.input_timed_out = False + self._control_sink = control_sink + self._safe_stop = safe_stop + self._emergency_stop = emergency_stop or safe_stop + self._safe_resume = safe_resume + self._sensor_sink: Callable[[ReconstructionFrame], None] | None = None + self._pilot_input_timeout_seconds = pilot_input_timeout_ms / 1000 + self._control_interval_seconds = 1 / max_control_rate_hz + self._clock = clock + self._latest_snapshot: Mapping[str, Any] | None = None + self._last_forwarded_at: float | None = None + self._input_timeout_handle: asyncio.TimerHandle | None = None + self._rate_limit_handle: asyncio.TimerHandle | None = None + + def set_sensor_sink(self, sink: Callable[[ReconstructionFrame], None]) -> None: + self._sensor_sink = sink + + def publish_sensor_frame(self, frame: ReconstructionFrame) -> None: + if self._sensor_sink is not None: + self._sensor_sink(frame) + + def start_control(self) -> None: + self._cancel_timers() + self.control_active = True + self.input_timed_out = False + self._latest_snapshot = None + self._last_forwarded_at = None + + def receive_pilot_input(self, snapshot: Mapping[str, Any]) -> None: + if not self.control_active: + return + self._latest_snapshot = dict(snapshot) + self._arm_input_timeout() + now = self._clock() + if ( + self._last_forwarded_at is None + or now - self._last_forwarded_at >= self._control_interval_seconds + ): + self._flush_latest_input() + return + if self._rate_limit_handle is None: + delay = self._control_interval_seconds - (now - self._last_forwarded_at) + try: + loop = asyncio.get_running_loop() + except RuntimeError: + self._flush_latest_input() + return + self._rate_limit_handle = loop.call_later( + max(delay, 0), self._flush_latest_input + ) + + def stop_control(self) -> None: + was_active = self.control_active + self.control_active = False + self._cancel_timers() + self._latest_snapshot = None + if was_active and self._safe_stop is not None: + self._safe_stop() + + def emergency_stop(self) -> None: + """Latch a robot-local stop until an explicit control start.""" + + was_active = self.control_active + self.control_active = False + self.input_timed_out = True + self._cancel_timers() + self._latest_snapshot = None + if was_active and self._emergency_stop is not None: + self._emergency_stop() + + def _arm_input_timeout(self) -> None: + if self._input_timeout_handle is not None: + self._input_timeout_handle.cancel() + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + self._input_timeout_handle = loop.call_later( + self._pilot_input_timeout_seconds, self._handle_input_timeout + ) + + def _handle_input_timeout(self) -> None: + self._input_timeout_handle = None + if not self.control_active or self.input_timed_out: + return + self.input_timed_out = True + self._latest_snapshot = None + if self._rate_limit_handle is not None: + self._rate_limit_handle.cancel() + self._rate_limit_handle = None + if self._safe_stop is not None: + self._safe_stop() + + def _flush_latest_input(self) -> None: + self._rate_limit_handle = None + snapshot = self._latest_snapshot + self._latest_snapshot = None + if not self.control_active or snapshot is None: + return + if self.input_timed_out: + if self._safe_resume is not None: + self._safe_resume() + self.input_timed_out = False + self._last_forwarded_at = self._clock() + if self._control_sink is not None: + self._control_sink(snapshot) + + def _cancel_timers(self) -> None: + for handle in (self._input_timeout_handle, self._rate_limit_handle): + if handle is not None: + handle.cancel() + self._input_timeout_handle = None + self._rate_limit_handle = None + + +class RemoteRobotAdapter: + """Proxy for the single lightweight driver used by external Ito deployments.""" + + def __init__(self, *, request_timeout_ms: int = 5000) -> None: + self.request_timeout_ms = request_timeout_ms + self.connection: object | None = None + self.ready = False + self.control_active = False + self._pending: dict[str, asyncio.Future[dict[str, Any]]] = {} + + def attach(self, connection: object, *, ready: bool) -> None: + self.connection = connection + self.ready = ready + + def detach(self, connection: object) -> None: + if self.connection is not connection: + return + self.connection = None + self.ready = False + self.control_active = False + for future in self._pending.values(): + if not future.done(): + future.set_exception(RuntimeError("remote robot driver disconnected")) + self._pending.clear() + + async def start_control(self) -> None: + result = await self._request( + TYPE_DRIVER_CONTROL_START, + {}, + TYPE_DRIVER_CONTROL_START_RESULT, + ) + self._require_ok(result) + self.control_active = True + + async def stop_control(self) -> None: + if self.connection is not None and self.control_active: + result = await self._request( + TYPE_DRIVER_CONTROL_STOP, + {"reason": {"code": "control.stopped.ito_requested"}}, + TYPE_DRIVER_CONTROL_STOP_RESULT, + ) + self._require_ok(result) + self.control_active = False + + async def accept_pilot_input_offer(self, sdp: str) -> str: + result = await self._request( + TYPE_WEBRTC_OFFER, + {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": sdp}, + TYPE_WEBRTC_ANSWER, + ) + answer = result.get("payload", {}).get("sdp") + if not isinstance(answer, str): + raise RuntimeError("remote robot driver returned an invalid WebRTC answer") + return answer + + def handle_response(self, envelope: Mapping[str, Any]) -> bool: + reply_to = envelope.get("replyToMessageId") + if not isinstance(reply_to, str): + return False + future = self._pending.pop(reply_to, None) + if future is None or future.done(): + return False + future.set_result(dict(envelope)) + return True + + async def _request( + self, message_type: str, payload: Mapping[str, Any], expected_type: str + ) -> dict[str, Any]: + if self.connection is None: + raise RuntimeError("remote robot driver isn't connected") + request = make_envelope(message_type, payload) + future = asyncio.get_running_loop().create_future() + self._pending[request["messageId"]] = future + try: + await self.connection.send(pack_envelope(request)) + response = await asyncio.wait_for( + future, timeout=self.request_timeout_ms / 1000 + ) + finally: + self._pending.pop(request["messageId"], None) + if response.get("type") != expected_type: + raise RuntimeError("remote robot driver returned an unexpected response") + return response + + @staticmethod + def _require_ok(envelope: Mapping[str, Any]) -> None: + payload = envelope.get("payload", {}) + if payload.get("ok"): + return + reason = payload.get("reason", {}) + text = reason.get("text") or reason.get("code") or "remote robot request failed" + raise RuntimeError(text) diff --git a/server/ito/webrtc.py b/server/ito/webrtc.py index c974b29..f387229 100644 --- a/server/ito/webrtc.py +++ b/server/ito/webrtc.py @@ -3,36 +3,45 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Awaitable, Callable, Protocol +import json +import logging +from typing import Any, Awaitable, Callable, Mapping, Protocol -from .protocol import WEBRTC_PATH_CAMERA_MEDIA, WEBRTC_PATH_SPLAT_BATCHES +from .protocol import ( + PROTOCOL_VERSION, + WEBRTC_PATH_CAMERA_MEDIA, + WEBRTC_PATH_PILOT_INPUT, + WEBRTC_PATH_SPLAT_BATCHES, +) + +LOGGER = logging.getLogger(__name__) class ServerLivePathAcceptor(Protocol): - async def accept_offer(self, *, path: str, session_id: str, sdp: str) -> str: + async def accept_offer(self, *, path: str, sdp: str) -> str: ... class MissingWebRtcStack: - async def accept_offer(self, *, path: str, session_id: str, sdp: str) -> str: + async def accept_offer(self, *, path: str, sdp: str) -> str: raise RuntimeError("aiortc is required for server-terminated WebRTC live paths") -class SplatBatchChannelRegistry: - """Tracks open server-to-client Splat Batch data channels by session.""" +class SplatBatchChannel: + """Tracks the one open Ito-to-client Splat Batch data channel.""" def __init__(self) -> None: - self.channels: dict[str, object] = {} + self.channel: object | None = None - def attach(self, session_id: str, data_channel: object) -> None: - self.channels[session_id] = data_channel + def attach(self, data_channel: object) -> None: + self.channel = data_channel - def detach(self, session_id: str, data_channel: object | None = None) -> None: - if data_channel is None or self.channels.get(session_id) is data_channel: - self.channels.pop(session_id, None) + def detach(self, data_channel: object | None = None) -> None: + if data_channel is None or self.channel is data_channel: + self.channel = None - def send(self, session_id: str, payload: bytes) -> bool: - channel = self.channels.get(session_id) + def send(self, payload: bytes) -> bool: + channel = self.channel if channel is None or getattr(channel, "readyState", None) != "open": return False channel.send(payload) @@ -47,9 +56,10 @@ class AiortcServerLivePaths: reconstruction integration attaches track/data-channel handlers here. """ - on_camera_track: Callable[[object, str], Awaitable[None] | None] | None = None - on_splat_channel: Callable[[object, str], Awaitable[None] | None] | None = None - splat_channels: SplatBatchChannelRegistry | None = None + on_camera_track: Callable[[object], Awaitable[None] | None] | None = None + on_pilot_input: Callable[[Mapping[str, Any]], None] | None = None + on_splat_channel: Callable[[object], Awaitable[None] | None] | None = None + splat_channels: SplatBatchChannel | None = None def __post_init__(self) -> None: try: @@ -59,33 +69,50 @@ def __post_init__(self) -> None: self._configuration_type = RTCConfiguration self._peer_connection_type = RTCPeerConnection self._session_description_type = RTCSessionDescription - self.peer_connections: dict[tuple[str, str], object] = {} - - async def accept_offer(self, *, path: str, session_id: str, sdp: str) -> str: - if path not in {WEBRTC_PATH_CAMERA_MEDIA, WEBRTC_PATH_SPLAT_BATCHES}: + self.peer_connections: dict[str, object] = {} + + async def accept_offer(self, *, path: str, sdp: str) -> str: + if path not in { + WEBRTC_PATH_CAMERA_MEDIA, + WEBRTC_PATH_PILOT_INPUT, + WEBRTC_PATH_SPLAT_BATCHES, + }: raise ValueError(f"server cannot terminate WebRTC path {path}") pc = self._peer_connection_type(configuration=self._configuration_type(iceServers=[])) - self.peer_connections[(session_id, path)] = pc + previous = self.peer_connections.pop(path, None) + if previous is not None: + await previous.close() + self.peer_connections[path] = pc if path == WEBRTC_PATH_CAMERA_MEDIA and self.on_camera_track is not None: @pc.on("track") async def on_track(track: object) -> None: - result = self.on_camera_track(track, session_id) + result = self.on_camera_track(track) if result is not None: await result + if path == WEBRTC_PATH_PILOT_INPUT and self.on_pilot_input is not None: + @pc.on("datachannel") + def on_data_channel(channel: object) -> None: + @channel.on("message") + def on_message(message: str | bytes) -> None: + try: + self.on_pilot_input(decode_pilot_input_snapshot(message)) + except ValueError as exc: + LOGGER.warning("Ignoring invalid pilot input: %s", exc) + if path == WEBRTC_PATH_SPLAT_BATCHES: channel = pc.createDataChannel("ito.splatBatches", ordered=True) if self.splat_channels is not None: @channel.on("open") def on_open() -> None: - self.splat_channels.attach(session_id, channel) + self.splat_channels.attach(channel) @channel.on("close") def on_close() -> None: - self.splat_channels.detach(session_id, channel) + self.splat_channels.detach(channel) if self.on_splat_channel is not None: - result = self.on_splat_channel(channel, session_id) + result = self.on_splat_channel(channel) if result is not None: await result @@ -96,16 +123,13 @@ def on_close() -> None: await _wait_for_ice_gathering_complete(pc) return pc.localDescription.sdp - async def close_session(self, session_id: str) -> None: + async def close_control(self) -> None: import asyncio - peers = [ - self.peer_connections.pop(key) - for key in list(self.peer_connections) - if key[0] == session_id - ] + peers = list(self.peer_connections.values()) + self.peer_connections.clear() if self.splat_channels is not None: - self.splat_channels.detach(session_id) + self.splat_channels.detach() await asyncio.gather(*(pc.close() for pc in peers), return_exceptions=True) @@ -122,3 +146,25 @@ def on_ice_gathering_state_change() -> None: complete.set() await complete.wait() + + +def decode_pilot_input_snapshot(message: str | bytes) -> dict[str, Any]: + if isinstance(message, bytes): + message = message.decode("utf-8") + try: + payload = json.loads(message) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("pilot input isn't valid JSON") from exc + if not isinstance(payload, dict): + raise ValueError("pilot input must be an object") + if payload.get("protocolVersion") != PROTOCOL_VERSION: + raise ValueError(f"pilot input protocolVersion must be {PROTOCOL_VERSION}") + if not isinstance(payload.get("sequence"), int): + raise ValueError("pilot input requires an integer sequence") + if not isinstance(payload.get("timestampMs"), (int, float)): + raise ValueError("pilot input requires timestampMs") + if not isinstance(payload.get("headsetYawRad"), (int, float)): + raise ValueError("pilot input requires headsetYawRad") + if not isinstance(payload.get("controllers"), list): + raise ValueError("pilot input requires a controllers list") + return payload diff --git a/server/processors/README.md b/server/processors/README.md index 18a2d98..d5e786b 100644 --- a/server/processors/README.md +++ b/server/processors/README.md @@ -1,11 +1,10 @@ -# Server Processors +# Reconstruction processors -This directory contains server-internal reconstruction algorithm modules. +These modules run inside the Ito application boundary. A processor starts when +control starts, accepts `ReconstructionFrame` values directly from local sensor +ingress or decoded remote media, and yields binary-ready `ProcessorSplatBatch` +values. -These modules are part of the Ito Server codebase, not separately deployed Ito -programs. - -All processors implement the interface in `base.py`: start a session, accept -decoded `ReconstructionFrame` values, and yield `ProcessorSplatBatch` values. -`null.py` is only an integration seam used before MASt3R-SLAM or MonoGS is -selected; it is not the v1 reconstruction algorithm. +`null.py` is only an integration seam; it is not the selected reconstruction +algorithm. Native libraries, GPU runtimes, or a tightly managed model subprocess +may implement this interface without becoming another deployed Ito service. diff --git a/server/processors/base.py b/server/processors/base.py index 2d2aaeb..a0be806 100644 --- a/server/processors/base.py +++ b/server/processors/base.py @@ -1,4 +1,4 @@ -"""Server-internal reconstruction processor interface.""" +"""Ito application reconstruction processor interface.""" from __future__ import annotations @@ -11,7 +11,7 @@ @dataclass(frozen=True) class ReconstructionFrame: - """Decoded camera frame passed from server media ingress to reconstruction.""" + """Decoded camera frame passed from sensor ingress to reconstruction.""" data: bytes timestamp_ms: int @@ -36,11 +36,11 @@ class ProcessorSplatBatch: class ReconstructionProcessor(Protocol): - """Common interface for algorithms under server/processors/.""" + """Common interface for reconstruction algorithms.""" capture_modality: str - def start(self, session_id: str) -> None: + def start(self) -> None: ... def process_frame(self, frame: ReconstructionFrame) -> Iterable[ProcessorSplatBatch]: diff --git a/server/processors/null.py b/server/processors/null.py index c174ad3..dc9bd60 100644 --- a/server/processors/null.py +++ b/server/processors/null.py @@ -15,18 +15,18 @@ class NullReconstructionProcessor: """Consumes frames and emits no splats. This is a local integration seam, not the selected v1 algorithm. It lets the - server exercise media ingress, session failure handling, and Splat Batch + application exercise sensor ingress, failure handling, and Splat Batch encoding without claiming MASt3R-SLAM or MonoGS have been selected. """ capture_modality = CAPTURE_MODALITY_MONOCULAR_RGB def __init__(self) -> None: - self.session_id: str | None = None + self.active = False self.frame_count = 0 - def start(self, session_id: str) -> None: - self.session_id = session_id + def start(self) -> None: + self.active = True self.frame_count = 0 def process_frame(self, frame: ReconstructionFrame) -> Iterable[ProcessorSplatBatch]: @@ -37,4 +37,4 @@ def reset(self) -> None: self.frame_count = 0 def close(self) -> None: - self.session_id = None + self.active = False diff --git a/tests/test_application.py b/tests/test_application.py new file mode 100644 index 0000000..65cc965 --- /dev/null +++ b/tests/test_application.py @@ -0,0 +1,271 @@ +import asyncio +from pathlib import Path +from types import SimpleNamespace + +from server.ito.app import ConnectionState, ItoApplication +from server.ito.config import ItoConfig +from server.ito.protocol import ( + ROLE_PILOT_CLIENT, + ROLE_REMOTE_ROBOT_DRIVER, + TYPE_CONNECTION_HELLO, + TYPE_CONTROL_START, + TYPE_CONTROL_START_RESULT, + TYPE_CONTROL_STOP, + TYPE_CONTROL_STOPPED, + TYPE_ROBOT_READY, + make_envelope, + pack_envelope, + unpack_envelope, +) +from server.ito.robot import LocalRobotAdapter +from server.ito.robot import RemoteRobotAdapter + + +class FakeWebSocket: + def __init__(self): + self.sent = [] + + async def send(self, frame): + self.sent.append(unpack_envelope(frame)) + + +def state(): + return ConnectionState(FakeWebSocket()) + + +async def send(application, connection, message_type, payload, message_id): + await application._handle_frame( + connection, + pack_envelope(make_envelope(message_type, payload, message_id=message_id)), + ) + return connection.websocket.sent[-1] + + +async def hello(application, connection): + return await send( + application, + connection, + TYPE_CONNECTION_HELLO, + {"role": ROLE_PILOT_CLIENT}, + "hello-1", + ) + + +def test_pilot_connects_directly_to_the_one_configured_robot(): + async def scenario(): + application = ItoApplication(adapter=LocalRobotAdapter()) + pilot = state() + + response = await hello(application, pilot) + + assert response["payload"] == { + "ok": True, + "value": { + "protocolVersion": "ito.v1", + "robotReady": True, + "controlActive": False, + "controlConfig": application.config.control_config_payload(), + }, + } + assert "robotId" not in response + assert "sessionId" not in response + + asyncio.run(scenario()) + + +def test_unwired_default_local_backend_does_not_claim_robot_readiness(): + async def scenario(): + application = ItoApplication() + pilot = state() + + response = await hello(application, pilot) + + assert response["payload"]["value"]["robotReady"] is False + + asyncio.run(scenario()) + + +def test_application_runs_the_configured_reconstruction_processor(): + class Processor: + capture_modality = "monocularRgb" + + def start(self): + self.started = True + + def process_frame(self, frame): + return [] + + def reset(self): + pass + + def close(self): + pass + + async def scenario(): + processor = Processor() + application = ItoApplication( + adapter=LocalRobotAdapter(), + processor_factory=lambda: processor, + ) + pilot = state() + await hello(application, pilot) + + await send(application, pilot, TYPE_CONTROL_START, {}, "start") + + assert processor.started is True + assert application.reconstruction_runtime.processor is processor + + asyncio.run(scenario()) + + +def test_start_and_stop_control_call_the_local_adapter_without_allocation(): + async def scenario(): + safe_stops = [] + adapter = LocalRobotAdapter(safe_stop=lambda: safe_stops.append(True)) + application = ItoApplication(adapter=adapter) + pilot = state() + await hello(application, pilot) + + started = await send(application, pilot, TYPE_CONTROL_START, {}, "start-1") + stopped = await send( + application, + pilot, + TYPE_CONTROL_STOP, + {"reason": {"code": "control.stopped.pilot_requested"}}, + "stop-1", + ) + + assert started["type"] == TYPE_CONTROL_START_RESULT + assert started["payload"]["ok"] is True + assert stopped["type"] == TYPE_CONTROL_STOPPED + assert application.control_active is False + assert adapter.control_active is False + assert safe_stops == [True] + + asyncio.run(scenario()) + + +def test_second_simultaneous_pilot_is_rejected(): + async def scenario(): + application = ItoApplication(adapter=LocalRobotAdapter()) + first = state() + second = state() + await hello(application, first) + + response = await hello(application, second) + + assert response["payload"] == { + "ok": False, + "reason": {"code": "connection.pilot_already_connected"}, + } + + asyncio.run(scenario()) + + +def test_pilot_disconnect_stops_control_locally(): + async def scenario(): + safe_stops = [] + adapter = LocalRobotAdapter(safe_stop=lambda: safe_stops.append(True)) + application = ItoApplication(adapter=adapter) + pilot = state() + await hello(application, pilot) + await send(application, pilot, TYPE_CONTROL_START, {}, "start-1") + + await application._disconnect(pilot) + + assert application.control_active is False + assert safe_stops == [True] + + asyncio.run(scenario()) + + +def test_application_serves_the_webxr_client(tmp_path): + client_dir = tmp_path / "client" + client_dir.mkdir() + (client_dir / "index.html").write_text("

Ito Pilot

") + application = ItoApplication( + config=ItoConfig(client_dir=client_dir), + adapter=LocalRobotAdapter(), + ) + + response = asyncio.run( + application.process_http_request(None, SimpleNamespace(path="/")) + ) + + assert response.status_code == 200 + assert response.body == b"

Ito Pilot

" + assert response.headers["Content-Type"].startswith("text/html") + + +def test_remote_driver_is_configuration_not_a_client_visible_product_choice(): + async def scenario(): + adapter = RemoteRobotAdapter(request_timeout_ms=1000) + application = ItoApplication( + config=ItoConfig(robot_backend="remote"), adapter=adapter + ) + driver = state() + pilot = state() + + driver_response = await send( + application, + driver, + TYPE_CONNECTION_HELLO, + {"role": ROLE_REMOTE_ROBOT_DRIVER, "ready": True}, + "driver-hello", + ) + pilot_response = await hello(application, pilot) + + assert driver_response["payload"]["ok"] is True + assert pilot_response["payload"]["value"]["robotReady"] is True + assert "backend" not in pilot_response["payload"]["value"] + assert "robotId" not in pilot_response + + asyncio.run(scenario()) + + +def test_pilot_is_notified_when_the_remote_driver_becomes_ready(): + async def scenario(): + adapter = RemoteRobotAdapter(request_timeout_ms=1000) + application = ItoApplication( + config=ItoConfig(robot_backend="remote"), adapter=adapter + ) + pilot = state() + driver = state() + await hello(application, pilot) + + await send( + application, + driver, + TYPE_CONNECTION_HELLO, + {"role": ROLE_REMOTE_ROBOT_DRIVER, "ready": True}, + "driver-hello", + ) + + ready_event = pilot.websocket.sent[-1] + assert ready_event["type"] == TYPE_ROBOT_READY + assert ready_event["payload"] == {"ready": True} + + asyncio.run(scenario()) + + +def test_remote_start_failure_returns_an_error_and_closes_reconstruction(): + class FailingAdapter(LocalRobotAdapter): + async def start_control(self): + raise RuntimeError("robot refused control") + + async def scenario(): + application = ItoApplication(adapter=FailingAdapter()) + pilot = state() + await hello(application, pilot) + + response = await send(application, pilot, TYPE_CONTROL_START, {}, "start") + + assert response["type"] == TYPE_CONTROL_START_RESULT + assert response["payload"] == { + "ok": False, + "reason": {"code": "control.start_failed", "text": "robot refused control"}, + } + assert application.control_active is False + assert application.reconstruction_runtime is None + + asyncio.run(scenario()) diff --git a/tests/test_config.py b/tests/test_config.py index 72ae95f..44a34bd 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,66 +1,32 @@ -from server.ito.config import ServerConfig +from pathlib import Path +import pytest -def test_server_config_defaults(monkeypatch): - for name in [ - "ITO_SERVER_HOST", - "ITO_SERVER_PORT", - "ITO_REQUEST_TIMEOUT_MS", - "ITO_DRIVER_STATUS_WATCHDOG_MS", - "ITO_SESSION_CLEANUP_TIMEOUT_MS", - "ITO_PILOT_INPUT_MAX_RETRANSMITS", - "ITO_PILOT_INPUT_MAX_PACKET_LIFETIME_MS", - ]: +from server.ito.config import ItoConfig + + +def test_ito_config_defaults_to_onboard_local_adapter(monkeypatch): + for name in ["ITO_HOST", "ITO_PORT", "ITO_ROBOT_BACKEND", "ITO_CLIENT_DIR"]: monkeypatch.delenv(name, raising=False) - config = ServerConfig.from_env() + config = ItoConfig.from_env() assert config.host == "0.0.0.0" assert config.port == 8765 - assert config.request_timeout_ms == 5000 - assert config.driver_status_watchdog_ms == 2000 - assert config.session_cleanup_timeout_ms == 30000 - assert config.session_config_payload()["pilotInputDataChannel"] == { - "ordered": False, - "maxRetransmits": 0, - } - - -def test_server_config_reads_environment(monkeypatch): - monkeypatch.setenv("ITO_SERVER_HOST", "127.0.0.1") - monkeypatch.setenv("ITO_SERVER_PORT", "9000") - monkeypatch.setenv("ITO_SPLAT_BATCH_ORDERED", "false") - - config = ServerConfig.from_env() - - assert config.host == "127.0.0.1" - assert config.port == 9000 - assert config.session_config_payload()["splatBatchDataChannel"] == {"ordered": False} - + assert config.robot_backend == "local" + assert config.client_dir.name == "client" -def test_pilot_input_packet_lifetime_omits_default_retransmits(monkeypatch): - monkeypatch.delenv("ITO_PILOT_INPUT_MAX_RETRANSMITS", raising=False) - monkeypatch.setenv("ITO_PILOT_INPUT_MAX_PACKET_LIFETIME_MS", "250") - config = ServerConfig.from_env() +def test_remote_driver_placement_is_one_configuration_value(monkeypatch, tmp_path): + monkeypatch.setenv("ITO_ROBOT_BACKEND", "remote") + monkeypatch.setenv("ITO_CLIENT_DIR", str(tmp_path)) - assert config.session_config_payload()["pilotInputDataChannel"] == { - "ordered": False, - "maxPacketLifeTime": 250, - } + config = ItoConfig.from_env() + assert config.robot_backend == "remote" + assert config.client_dir == Path(tmp_path) -def test_data_channel_reliability_caps_are_mutually_exclusive(monkeypatch): - monkeypatch.setenv("ITO_PILOT_INPUT_MAX_RETRANSMITS", "3") - monkeypatch.setenv("ITO_PILOT_INPUT_MAX_PACKET_LIFETIME_MS", "250") - try: - ServerConfig.from_env() - except ValueError as exc: - assert ( - str(exc) - == "ITO_PILOT_INPUT_MAX_RETRANSMITS and " - "ITO_PILOT_INPUT_MAX_PACKET_LIFETIME_MS are mutually exclusive" - ) - else: - raise AssertionError("expected mutually exclusive reliability caps to fail") +def test_unknown_backend_is_rejected(): + with pytest.raises(ValueError): + ItoConfig(robot_backend="fleet") diff --git a/tests/test_ito_droid.py b/tests/test_ito_droid.py index 51a1cd7..5cf45c3 100644 --- a/tests/test_ito_droid.py +++ b/tests/test_ito_droid.py @@ -3,42 +3,42 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -DROID_DRIVER_ROOT = ROOT / "drivers" / "ito-droid" -sys.path.insert(0, str(DROID_DRIVER_ROOT)) +sys.path.insert(0, str(ROOT / "drivers" / "ito-droid")) from ito_droid.config import ItoDroidConfig from ito_droid.control import CameraPanController from ito_droid.driver import ItoDroidDriver from ito_droid.media import CameraMediaPublisher from ito_droid.ros_io import CameraFrame -from ito_droid.webrtc import PilotInputDataChannelReceiver, decode_pilot_input_snapshot +from ito_droid.webrtc import decode_pilot_input_snapshot from server.ito.protocol import ( - TYPE_DRIVER_SESSION_START, - TYPE_DRIVER_SESSION_START_RESULT, - TYPE_SESSION_END, - TYPE_SESSION_END_RESULT, + TYPE_DRIVER_CONTROL_START, + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_DRIVER_CONTROL_STOP, + TYPE_DRIVER_CONTROL_STOP_RESULT, + TYPE_WEBRTC_ANSWER, + TYPE_WEBRTC_OFFER, + WEBRTC_PATH_CAMERA_MEDIA, + WEBRTC_PATH_PILOT_INPUT, make_envelope, unpack_envelope, ) class FakeClock: - def __init__(self, now=0.0): - self.now = now + def __init__(self): + self.now = 0.0 def __call__(self): return self.now - def advance(self, seconds): - self.now += seconds - class RecordingServo: def __init__(self): self.angles = [] - def publish_angle(self, angle_degrees): - self.angles.append(angle_degrees) + def publish_angle(self, angle): + self.angles.append(angle) class FakeWebSocket: @@ -49,238 +49,142 @@ async def send(self, frame): self.sent.append(unpack_envelope(frame)) -def test_config_reads_environment(monkeypatch): - monkeypatch.setenv("ITO_SERVER_URL", "ws://server.example/ws") - monkeypatch.setenv("ITO_DROID_ROBOT_ID", "droid-a") - monkeypatch.setenv("ITO_DROID_ROS_CAMERA_TOPIC", "/camera/image") - monkeypatch.setenv("ITO_DROID_CONTROL_TICK_HZ", "30") - monkeypatch.setenv("ITO_DROID_SERVO_MIN_DEGREES", "10") - - config = ItoDroidConfig.from_env() - - assert config.server_url == "ws://server.example/ws" - assert config.robot_id == "droid-a" - assert config.ros_camera_topic == "/camera/image" - assert config.control_tick_hz == 30 - assert config.servo_min_degrees == 10 - - -def test_yaw_to_camera_pan_mapping_clamps_to_servo_limits(): - config = ItoDroidConfig( - servo_neutral_degrees=90, - servo_min_degrees=60, - servo_max_degrees=120, - yaw_to_servo_degrees_per_radian=30, +def test_yaw_mapping_clamps_to_robot_limits(): + controller = CameraPanController( + ItoDroidConfig( + servo_neutral_degrees=90, + servo_min_degrees=60, + servo_max_degrees=120, + yaw_to_servo_degrees_per_radian=30, + ) ) - controller = CameraPanController(config) - assert controller.target_for_yaw(0) == 90 assert controller.target_for_yaw(1) == 120 assert controller.target_for_yaw(-2) == 60 -def test_control_tick_uses_newest_snapshot_and_holds_on_timeout(): - config = ItoDroidConfig( - pilot_input_timeout_ms=100, - servo_smoothing=1, - servo_max_velocity_degrees_per_second=1000, - ) - clock = FakeClock() - controller = CameraPanController(config) - - controller.receive_snapshot({"headsetYawRadians": 0.5}, clock()) - angle = controller.tick(clock(), 1 / 60) - assert angle > config.servo_neutral_degrees - - held_angle = angle - clock.advance(0.101) - assert controller.tick(clock(), 1 / 60) == held_angle - - -def test_safe_resumption_ramps_correction_velocity_after_timeout(): - config = ItoDroidConfig( - pilot_input_timeout_ms=100, - servo_smoothing=1, - servo_max_velocity_degrees_per_second=100, - resumption_initial_velocity_degrees_per_second=10, - resumption_ramp_duration_ms=1000, - ) +def test_driver_uses_newest_input_and_times_out_locally(): clock = FakeClock() - controller = CameraPanController(config) - - controller.receive_snapshot({"headsetYawRadians": 0}, clock()) - assert controller.tick(clock(), 0.1) == config.servo_neutral_degrees - - clock.advance(0.101) - assert controller.tick(clock(), 0.1) == config.servo_neutral_degrees - - controller.receive_snapshot({"headsetYawRadians": 1}, clock()) - resumed_angle = controller.tick(clock(), 0.1) - assert resumed_angle == config.servo_neutral_degrees + 1 - - clock.advance(1.0) - controller.receive_snapshot({"headsetYawRadians": 1}, clock()) - later_angle = controller.tick(clock(), 0.1) - assert later_angle > resumed_angle + 1 - - -def test_status_reports_unavailable_until_camera_feed_arrives(): - driver = ItoDroidDriver(ItoDroidConfig()) - - assert driver.status_payload() == { - "name": "Ito Droid", - "type": "Droid", - "status": "Unavailable", - "availabilityDetail": {"code": "ito_droid.camera_feed_missing"}, - } - - driver.receive_camera_frame(CameraFrame(b"rgb", 1.0, encoding="rgb8", width=1, height=1)) - - assert driver.status_payload() == { - "name": "Ito Droid", - "type": "Droid", - "status": "Available", - } - - -def test_ros_camera_frames_flow_to_camera_media_publisher(): - publisher = CameraMediaPublisher() - driver = ItoDroidDriver(ItoDroidConfig(), media_publisher=publisher) - - publisher.start("session-1") - frame = CameraFrame(b"frame", 1.0, encoding="rgb8", width=2, height=2) - driver.receive_camera_frame(frame) - - assert publisher.frame_count == 1 - assert publisher.last_frame == frame - - -def test_session_start_neutralizes_servo_and_starts_media(): - asyncio.run(_session_start_neutralizes_servo_and_starts_media()) - - -async def _session_start_neutralizes_servo_and_starts_media(): - servo = RecordingServo() - media = CameraMediaPublisher() - driver = ItoDroidDriver(ItoDroidConfig(), servo_publisher=servo, media_publisher=media) - driver.camera_ready = True - websocket = FakeWebSocket() - - await driver.handle_session_start( - websocket, - make_envelope( - TYPE_DRIVER_SESSION_START, - {"sessionId": "session-1", "sessionConfig": {"cameraMedia": {"codec": "H264"}}}, - message_id="start-1", - robot_id="ito-droid-1", - session_id="session-1", - ), + driver = ItoDroidDriver( + ItoDroidConfig(pilot_input_timeout_ms=100, servo_smoothing=1), clock=clock ) - - assert driver.session_id == "session-1" - assert servo.angles == [driver.config.servo_neutral_degrees] - assert media.started_session_id == "session-1" - assert websocket.sent[-1]["type"] == TYPE_DRIVER_SESSION_START_RESULT - assert websocket.sent[-1]["replyToMessageId"] == "start-1" - assert websocket.sent[-1]["payload"] == {"ok": True, "value": {"sessionId": "session-1"}} - - -def test_session_start_fails_without_camera_feed(): - asyncio.run(_session_start_fails_without_camera_feed()) - - -async def _session_start_fails_without_camera_feed(): - driver = ItoDroidDriver(ItoDroidConfig()) - websocket = FakeWebSocket() - - await driver.handle_session_start( - websocket, - make_envelope( - TYPE_DRIVER_SESSION_START, - {"sessionId": "session-1", "sessionConfig": {}}, - message_id="start-1", - robot_id="ito-droid-1", - session_id="session-1", - ), + driver.receive_pilot_input_snapshot({"headsetYawRad": 0.5}) + angle = driver.process_control_tick(1 / 60) + clock.now = 0.101 + + assert angle != driver.config.servo_neutral_degrees + assert driver.process_control_tick(1 / 60) == driver.config.servo_neutral_degrees + + +def test_control_lifecycle_neutralizes_on_start_and_stop(): + async def scenario(): + class LifecycleMedia(CameraMediaPublisher): + async def create_offer(self): + return "camera offer" + + async def close(self): + pass + + servo = RecordingServo() + media = LifecycleMedia() + driver = ItoDroidDriver( + ItoDroidConfig(), servo_publisher=servo, media_publisher=media + ) + driver.receive_camera_frame( + CameraFrame(b"rgb", 1.0, encoding="rgb8", width=1, height=1) + ) + websocket = FakeWebSocket() + + await driver.handle_control_start( + websocket, + make_envelope(TYPE_DRIVER_CONTROL_START, {}, message_id="start"), + ) + assert driver.control_active is True + assert media.active is True + assert websocket.sent[-2]["type"] == TYPE_DRIVER_CONTROL_START_RESULT + assert websocket.sent[-1]["type"] == TYPE_WEBRTC_OFFER + + await driver.handle_control_stop( + websocket, + make_envelope(TYPE_DRIVER_CONTROL_STOP, {}, message_id="stop"), + ) + assert driver.control_active is False + assert media.active is False + assert servo.angles[-1] == driver.config.servo_neutral_degrees + assert websocket.sent[-1]["type"] == TYPE_DRIVER_CONTROL_STOP_RESULT + + asyncio.run(scenario()) + + +def test_pilot_input_has_no_session_identity(): + snapshot = decode_pilot_input_snapshot( + '{"protocolVersion":"ito.v1","sequence":2,"timestampMs":1,' + '"headsetYawRad":0,"controllers":[]}' ) - assert websocket.sent[-1]["type"] == TYPE_DRIVER_SESSION_START_RESULT - assert websocket.sent[-1]["payload"] == { - "ok": False, - "reason": {"code": "ito_droid.camera_feed_missing"}, - } - - -def test_clean_session_end_neutralizes_servo_and_stops_media(): - asyncio.run(_clean_session_end_neutralizes_servo_and_stops_media()) - - -def test_pilot_input_data_channel_receiver_decodes_snapshot_json(): - received = [] - - class FakeDataChannel: - def on(self, event): - assert event == "message" - - def register(callback): - self.callback = callback - return callback - - return register - - channel = FakeDataChannel() - receiver = PilotInputDataChannelReceiver(received.append) - receiver.attach(channel) - channel.callback( - b'{"protocolVersion":"ito.v1","sessionId":"session-1","sequence":1,"headsetYawRad":0.25}' - ) + assert snapshot["sequence"] == 2 + assert "sessionId" not in snapshot + + +def test_driver_negotiates_pilot_input_and_camera_media(): + class FakePilotInputReceiver: + async def accept_offer(self, *, sdp): + assert sdp == "pilot offer" + return "pilot answer" + + async def close(self): + pass + + class FakeMediaPublisher(CameraMediaPublisher): + async def create_offer(self): + return "camera offer" + + async def accept_answer(self, *, sdp): + assert sdp == "camera answer" + + async def close(self): + pass + + async def scenario(): + media = FakeMediaPublisher() + driver = ItoDroidDriver( + ItoDroidConfig(), + media_publisher=media, + pilot_input_webrtc=FakePilotInputReceiver(), + ) + websocket = FakeWebSocket() + driver.camera_ready = True + + await driver.handle_control_start( + websocket, + make_envelope(TYPE_DRIVER_CONTROL_START, {}, message_id="start"), + ) + assert websocket.sent[-1]["type"] == TYPE_WEBRTC_OFFER + assert websocket.sent[-1]["payload"] == { + "path": WEBRTC_PATH_CAMERA_MEDIA, + "sdp": "camera offer", + } - assert received == [ - { - "protocolVersion": "ito.v1", - "sessionId": "session-1", - "sequence": 1, - "headsetYawRad": 0.25, + await driver.handle_message( + websocket, + make_envelope( + TYPE_WEBRTC_OFFER, + {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": "pilot offer"}, + message_id="pilot", + ), + ) + assert websocket.sent[-1]["type"] == TYPE_WEBRTC_ANSWER + assert websocket.sent[-1]["payload"] == { + "path": WEBRTC_PATH_PILOT_INPUT, + "sdp": "pilot answer", } - ] - assert decode_pilot_input_snapshot( - '{"protocolVersion":"ito.v1","sessionId":"session-1","sequence":2,"headsetYawRad":0}' - )["sequence"] == 2 - - -async def _clean_session_end_neutralizes_servo_and_stops_media(): - servo = RecordingServo() - media = CameraMediaPublisher() - driver = ItoDroidDriver(ItoDroidConfig(), servo_publisher=servo, media_publisher=media) - driver.camera_ready = True - websocket = FakeWebSocket() - - await driver.handle_session_start( - websocket, - make_envelope( - TYPE_DRIVER_SESSION_START, - {"sessionId": "session-1", "sessionConfig": {}}, - message_id="start-1", - robot_id="ito-droid-1", - session_id="session-1", - ), - ) - driver.receive_pilot_input_snapshot({"headsetYawRadians": 0.5}) - driver.process_control_tick(1 / 60) - - await driver.handle_session_end( - websocket, - make_envelope( - TYPE_SESSION_END, - {"reason": {"code": "session.ended.pilot_requested"}, "clean": True}, - message_id="end-1", - robot_id="ito-droid-1", - session_id="session-1", - ), - ) - assert driver.session_id is None - assert media.started_session_id is None - assert servo.angles[-1] == driver.config.servo_neutral_degrees - assert websocket.sent[-1]["type"] == TYPE_SESSION_END_RESULT - assert websocket.sent[-1]["payload"] == {"ok": True, "value": {"sessionId": "session-1"}} + await driver.handle_message( + websocket, + make_envelope( + TYPE_WEBRTC_ANSWER, + {"path": WEBRTC_PATH_CAMERA_MEDIA, "sdp": "camera answer"}, + ), + ) + + asyncio.run(scenario()) diff --git a/tests/test_mock_robot.py b/tests/test_mock_robot.py index 2c04f8d..0a7ae58 100644 --- a/tests/test_mock_robot.py +++ b/tests/test_mock_robot.py @@ -1,23 +1,19 @@ import asyncio -import importlib.util -import logging import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -MOCK_DRIVER_ROOT = ROOT / "drivers" / "mock-robot" -sys.path.insert(0, str(MOCK_DRIVER_ROOT)) +sys.path.insert(0, str(ROOT / "drivers" / "mock-robot")) from mock_robot.camera import VideoFileCamera from mock_robot.config import MockRobotConfig from mock_robot.driver import MockRobotDriver from server.ito.protocol import ( - TYPE_DRIVER_SESSION_START, - TYPE_DRIVER_SESSION_START_RESULT, - TYPE_SESSION_END, - TYPE_SESSION_END_RESULT, + TYPE_DRIVER_CONTROL_START, + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_DRIVER_CONTROL_STOP, + TYPE_DRIVER_CONTROL_STOP_RESULT, TYPE_WEBRTC_OFFER, - WEBRTC_PATH_CAMERA_MEDIA, make_envelope, unpack_envelope, ) @@ -33,141 +29,51 @@ async def send(self, frame): class FakeCameraMediaPublisher: def __init__(self): - self.offers = [] - self.answers = [] self.closed = [] - async def create_offer(self, *, session_id, video_path, loop): - self.offers.append({"sessionId": session_id, "videoPath": str(video_path), "loop": loop}) - return "fake-camera-offer" + async def create_offer(self, *, video_path, loop): + return "camera offer" - async def accept_answer(self, *, session_id, sdp): - self.answers.append({"sessionId": session_id, "sdp": sdp}) - - async def close_session(self, session_id): - self.closed.append(session_id) - - -def test_mock_robot_imports_without_websocket_side_effects(): - spec = importlib.util.spec_from_file_location( - "mock_robot_main", - MOCK_DRIVER_ROOT / "main.py", - ) - assert spec is not None - - -def test_status_requires_camera_video_for_availability(): - driver = MockRobotDriver(MockRobotConfig(camera_video_path=None)) - - assert driver.status_payload() == { - "name": "Mock Robot", - "type": "Droid", - "status": "Unavailable", - "availabilityDetail": {"code": "mock_robot.camera_video_required"}, - } + async def close(self): + self.closed.append(True) def test_video_file_camera_reads_samples(tmp_path): video = tmp_path / "camera.h264" video.write_bytes(b"abcdef") - camera = VideoFileCamera(video, chunk_size=4, loop=False) - samples = list(camera.samples()) + samples = list(VideoFileCamera(video, chunk_size=4, loop=False).samples()) assert [sample.data for sample in samples] == [b"abcd", b"ef"] - assert [sample.offset for sample in samples] == [0, 4] - -def test_session_lifecycle_opens_and_closes_camera(tmp_path): - asyncio.run(_session_lifecycle_opens_and_closes_camera(tmp_path)) +def test_remote_driver_starts_and_stops_without_robot_or_session_identity(tmp_path): + async def scenario(): + video = tmp_path / "camera.h264" + video.write_bytes(b"frame") + publisher = FakeCameraMediaPublisher() + driver = MockRobotDriver( + MockRobotConfig(camera_video_path=str(video)), + camera_media_webrtc=publisher, + ) + websocket = FakeWebSocket() -async def _session_lifecycle_opens_and_closes_camera(tmp_path): - video = tmp_path / "camera.h264" - video.write_bytes(b"frame-data") - publisher = FakeCameraMediaPublisher() - driver = MockRobotDriver( - MockRobotConfig(camera_video_path=str(video), camera_loop=False), - camera_media_webrtc=publisher, - ) - websocket = FakeWebSocket() - - await driver.handle_session_start( - websocket, - make_envelope( - TYPE_DRIVER_SESSION_START, - {"sessionId": "session-1", "sessionConfig": {"pilotInputDataChannel": {"ordered": False}}}, - message_id="start-1", - robot_id="mock-robot-1", - session_id="session-1", - ), - ) - - assert driver.session_id == "session-1" - assert driver.camera is not None - assert driver.camera.is_open - assert websocket.sent[-2]["type"] == TYPE_DRIVER_SESSION_START_RESULT - assert websocket.sent[-2]["replyToMessageId"] == "start-1" - assert websocket.sent[-2]["payload"] == {"ok": True, "value": {"sessionId": "session-1"}} - assert websocket.sent[-1]["type"] == TYPE_WEBRTC_OFFER - assert websocket.sent[-1]["payload"]["path"] == WEBRTC_PATH_CAMERA_MEDIA - assert websocket.sent[-1]["payload"]["sdp"] == "fake-camera-offer" - assert publisher.offers == [{"sessionId": "session-1", "videoPath": str(video), "loop": False}] - - await driver.handle_session_end( - websocket, - make_envelope( - TYPE_SESSION_END, - {"reason": {"code": "session.ended.pilot_requested"}, "clean": True}, - message_id="end-1", - robot_id="mock-robot-1", - session_id="session-1", - ), - ) - - assert driver.session_id is None - assert not driver.camera.is_open - assert websocket.sent[-1]["type"] == TYPE_SESSION_END_RESULT - assert websocket.sent[-1]["payload"] == {"ok": True, "value": {"sessionId": "session-1"}} - assert publisher.closed == ["session-1"] - - -def test_session_start_fails_without_camera_video(): - asyncio.run(_session_start_fails_without_camera_video()) - - -async def _session_start_fails_without_camera_video(): - driver = MockRobotDriver(MockRobotConfig(camera_video_path=None)) - websocket = FakeWebSocket() - - await driver.handle_session_start( - websocket, - make_envelope( - TYPE_DRIVER_SESSION_START, - {"sessionId": "session-1", "sessionConfig": {}}, - message_id="start-1", - robot_id="mock-robot-1", - session_id="session-1", - ), - ) - - assert websocket.sent[-1]["type"] == TYPE_DRIVER_SESSION_START_RESULT - assert websocket.sent[-1]["payload"] == { - "ok": False, - "reason": {"code": "mock_robot.camera_video_required"}, - } - - -def test_pilot_input_snapshot_is_logged(caplog): - driver = MockRobotDriver(MockRobotConfig(camera_video_path=None)) - - with caplog.at_level(logging.INFO): - driver.receive_pilot_input_snapshot( - { - "headsetYawRadians": 0.25, - "controllers": {"right": {"triggerPressed": True}}, - } + await driver.handle_control_start( + websocket, + make_envelope(TYPE_DRIVER_CONTROL_START, {}, message_id="start-1"), + ) + assert driver.control_active is True + assert websocket.sent[-2]["type"] == TYPE_DRIVER_CONTROL_START_RESULT + assert websocket.sent[-1]["type"] == TYPE_WEBRTC_OFFER + assert "robotId" not in websocket.sent[-1] + assert "sessionId" not in websocket.sent[-1] + + await driver.handle_control_stop( + websocket, + make_envelope(TYPE_DRIVER_CONTROL_STOP, {}, message_id="stop-1"), ) + assert driver.control_active is False + assert websocket.sent[-1]["type"] == TYPE_DRIVER_CONTROL_STOP_RESULT + assert publisher.closed == [True] - assert "pilot_input_snapshot" in caplog.text - assert '"headsetYawRadians": 0.25' in caplog.text + asyncio.run(scenario()) diff --git a/tests/test_mock_robot_e2e.py b/tests/test_mock_robot_e2e.py index 75f9bb1..9e9b906 100644 --- a/tests/test_mock_robot_e2e.py +++ b/tests/test_mock_robot_e2e.py @@ -1,246 +1,118 @@ import asyncio -import json -import logging -import sys from contextlib import suppress +import sys from pathlib import Path -import pytest from websockets.asyncio.client import connect from websockets.asyncio.server import serve -aiortc = pytest.importorskip("aiortc") -av = pytest.importorskip("av") -RTCConfiguration = aiortc.RTCConfiguration -RTCPeerConnection = aiortc.RTCPeerConnection -RTCSessionDescription = aiortc.RTCSessionDescription - ROOT = Path(__file__).resolve().parents[1] -MOCK_DRIVER_ROOT = ROOT / "drivers" / "mock-robot" -sys.path.insert(0, str(MOCK_DRIVER_ROOT)) +sys.path.insert(0, str(ROOT / "drivers" / "mock-robot")) from mock_robot.config import MockRobotConfig from mock_robot.driver import MockRobotDriver -from server.ito.app import ItoServer -from server.ito.config import ServerConfig +from server.ito.app import ItoApplication +from server.ito.config import ItoConfig from server.ito.protocol import ( ROLE_PILOT_CLIENT, - TYPE_CATALOG_GET, - TYPE_CATALOG_GET_RESULT, TYPE_CONNECTION_HELLO, - TYPE_CONNECTION_HELLO_RESULT, - TYPE_SESSION_ACQUIRE, - TYPE_SESSION_ACQUIRE_RESULT, - TYPE_SESSION_END, - TYPE_WEBRTC_ANSWER, - TYPE_WEBRTC_OFFER, - WEBRTC_PATH_PILOT_INPUT, + TYPE_CONTROL_START, + TYPE_CONTROL_START_RESULT, + TYPE_CONTROL_STOP, + TYPE_CONTROL_STOP_RESULT, make_envelope, pack_envelope, unpack_envelope, ) +from server.ito.robot import RemoteRobotAdapter -def test_mock_robot_e2e_acquire_and_pilot_input_over_websocket_and_webrtc(tmp_path, caplog): - asyncio.run(_mock_robot_e2e_acquire_and_pilot_input_over_websocket_and_webrtc(tmp_path, caplog)) +class FakeCameraPublisher: + async def create_offer(self, **_kwargs): + return "camera offer" + async def accept_answer(self, **_kwargs): + pass -async def _mock_robot_e2e_acquire_and_pilot_input_over_websocket_and_webrtc(tmp_path, caplog): - video = tmp_path / "camera.mp4" - _write_h264_sample_video(video) - server = ItoServer( - ServerConfig( - host="127.0.0.1", - port=0, - request_timeout_ms=3000, - driver_status_watchdog_ms=1000, - session_cleanup_timeout_ms=1000, - ) + async def close(self): + pass + + +class FakeLivePaths: + async def accept_offer(self, *, path, sdp): + assert path == "cameraMedia" + assert sdp == "camera offer" + return "camera answer" + + async def close_control(self): + pass + + +def test_external_mode_keeps_the_same_direct_pilot_lifecycle(tmp_path): + asyncio.run(_external_mode_keeps_the_same_direct_pilot_lifecycle(tmp_path)) + + +async def _external_mode_keeps_the_same_direct_pilot_lifecycle(tmp_path): + video = tmp_path / "camera.h264" + video.write_bytes(b"camera") + adapter = RemoteRobotAdapter(request_timeout_ms=1000) + application = ItoApplication( + ItoConfig(host="127.0.0.1", port=1, robot_backend="remote"), + adapter=adapter, ) + application.live_paths = FakeLivePaths() - async with serve(server._handle_connection, "127.0.0.1", 0) as websocket_server: + async with serve(application._handle_connection, "127.0.0.1", 0) as websocket_server: port = websocket_server.sockets[0].getsockname()[1] - server_url = f"ws://127.0.0.1:{port}" driver = MockRobotDriver( - MockRobotConfig( - server_url=server_url, - robot_id="mock-robot-1", - status_interval_ms=50, - camera_video_path=str(video), - camera_loop=False, - ) + MockRobotConfig(ito_url=f"ws://127.0.0.1:{port}", camera_video_path=str(video)), + camera_media_webrtc=FakeCameraPublisher(), ) driver_task = asyncio.create_task(driver.run_once()) - peer_connection = RTCPeerConnection(configuration=RTCConfiguration(iceServers=[])) try: - with caplog.at_level(logging.INFO, logger="mock_robot.driver"): - async with connect(server_url) as pilot_ws: - await _send( - pilot_ws, + for _ in range(20): + if adapter.ready: + break + await asyncio.sleep(0.01) + assert adapter.ready + + async with connect(f"ws://127.0.0.1:{port}") as pilot: + await pilot.send( + pack_envelope( make_envelope( TYPE_CONNECTION_HELLO, {"role": ROLE_PILOT_CLIENT}, - message_id="pilot-hello", - ), + message_id="hello", + ) ) - hello = await _recv_type(pilot_ws, TYPE_CONNECTION_HELLO_RESULT, "pilot-hello") - assert hello["payload"]["ok"] is True + ) + hello = unpack_envelope(await pilot.recv()) + assert hello["payload"]["value"]["robotReady"] is True - await _wait_for_mock_robot_available(pilot_ws) - await _send( - pilot_ws, - make_envelope( - TYPE_SESSION_ACQUIRE, - {"robotId": "mock-robot-1"}, - message_id="acquire-mock", - robot_id="mock-robot-1", - ), + await pilot.send( + pack_envelope( + make_envelope(TYPE_CONTROL_START, {}, message_id="start") ) - acquired = await _recv_type(pilot_ws, TYPE_SESSION_ACQUIRE_RESULT, "acquire-mock") - assert acquired["payload"]["ok"] is True - session_id = acquired["payload"]["value"]["sessionId"] - await _wait_for_camera_frame(server, session_id) - - data_channel = peer_connection.createDataChannel( - "ito.pilotInput", - ordered=False, - maxRetransmits=0, - ) - offer = await peer_connection.createOffer() - await peer_connection.setLocalDescription(offer) - await _wait_for_ice_gathering_complete(peer_connection) - await _send( - pilot_ws, - make_envelope( - TYPE_WEBRTC_OFFER, - {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": peer_connection.localDescription.sdp}, - message_id="pilot-input-offer", - robot_id="mock-robot-1", - session_id=session_id, - ), - ) - answer = await _recv_type(pilot_ws, TYPE_WEBRTC_ANSWER, "pilot-input-offer") - assert answer["payload"]["path"] == WEBRTC_PATH_PILOT_INPUT - await peer_connection.setRemoteDescription( - RTCSessionDescription(sdp=answer["payload"]["sdp"], type="answer") - ) - await _wait_for_data_channel_open(data_channel) - - snapshot = { - "protocolVersion": "ito.v1", - "sessionId": session_id, - "sequence": 1, - "timestampMs": 12345, - "headsetYawRad": 0.42, - "controllers": {"left": {}, "right": {"triggerPressed": True}}, - } - data_channel.send(json.dumps(snapshot)) - await _wait_for_log(caplog, '"headsetYawRad": 0.42') - - await _send( - pilot_ws, - make_envelope( - TYPE_SESSION_END, - {"reason": {"code": "session.ended.pilot_requested"}, "clean": True}, - message_id="end-mock", - robot_id="mock-robot-1", - session_id=session_id, - ), + ) + started = await _receive_reply(pilot, TYPE_CONTROL_START_RESULT, "start") + assert started["payload"]["ok"] is True + + await pilot.send( + pack_envelope( + make_envelope(TYPE_CONTROL_STOP, {}, message_id="stop") ) - await _recv_type(pilot_ws, "session.end.result", "end-mock") + ) + stopped = await _receive_reply(pilot, TYPE_CONTROL_STOP_RESULT, "stop") + assert stopped["payload"]["ok"] is True finally: - await peer_connection.close() driver_task.cancel() with suppress(asyncio.CancelledError): await driver_task -async def _wait_for_mock_robot_available(pilot_ws): - for attempt in range(20): - message_id = f"catalog-{attempt}" - await _send( - pilot_ws, - make_envelope(TYPE_CATALOG_GET, {"includeUnavailable": True}, message_id=message_id), - ) - catalog = await _recv_type(pilot_ws, TYPE_CATALOG_GET_RESULT, message_id) - robots = catalog["payload"]["value"]["robots"] - if robots and robots[0]["robotId"] == "mock-robot-1" and robots[0]["status"] == "Available": - return - await asyncio.sleep(0.05) - raise AssertionError("Mock Robot did not become available in the catalog") - - -async def _send(websocket, envelope): - await websocket.send(pack_envelope(envelope)) - - -async def _recv_type(websocket, message_type, reply_to): - for _ in range(20): - envelope = unpack_envelope(await asyncio.wait_for(websocket.recv(), timeout=3)) +async def _receive_reply(websocket, message_type, reply_to): + for _ in range(10): + envelope = unpack_envelope(await asyncio.wait_for(websocket.recv(), timeout=2)) if envelope["type"] == message_type and envelope.get("replyToMessageId") == reply_to: return envelope - raise AssertionError(f"Did not receive {message_type} replying to {reply_to}") - - -async def _wait_for_data_channel_open(data_channel): - if data_channel.readyState == "open": - return - opened = asyncio.Event() - - @data_channel.on("open") - def on_open(): - opened.set() - - await asyncio.wait_for(opened.wait(), timeout=5) - - -async def _wait_for_ice_gathering_complete(peer_connection): - if peer_connection.iceGatheringState == "complete": - return - complete = asyncio.Event() - - @peer_connection.on("icegatheringstatechange") - def on_ice_gathering_state_change(): - if peer_connection.iceGatheringState == "complete": - complete.set() - - await asyncio.wait_for(complete.wait(), timeout=5) - - -async def _wait_for_log(caplog, text): - for _ in range(50): - if text in caplog.text: - return - await asyncio.sleep(0.05) - raise AssertionError(f"Did not find log text: {text}") - - -def _write_h264_sample_video(path): - try: - container = av.open(str(path), mode="w") - stream = container.add_stream("libx264", rate=5) - stream.width = 16 - stream.height = 16 - stream.pix_fmt = "yuv420p" - for index in range(3): - frame = av.VideoFrame(16, 16, "yuv420p") - frame.planes[0].update(bytes([32 + index * 20]) * frame.planes[0].buffer_size) - frame.planes[1].update(bytes([128]) * frame.planes[1].buffer_size) - frame.planes[2].update(bytes([128]) * frame.planes[2].buffer_size) - for packet in stream.encode(frame): - container.mux(packet) - for packet in stream.encode(): - container.mux(packet) - container.close() - except Exception as exc: - pytest.skip(f"local PyAV/FFmpeg cannot create an H.264 sample video: {exc}") - - -async def _wait_for_camera_frame(server, session_id): - for _ in range(80): - runtime = server.reconstruction_runtimes.get(session_id) - processor = getattr(runtime, "processor", None) - if getattr(processor, "frame_count", 0) > 0: - return - await asyncio.sleep(0.05) - raise AssertionError("cameraMedia did not deliver a decoded frame to reconstruction") + raise AssertionError(f"missing {message_type}") diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 8e487df..87c61e7 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -1,46 +1,41 @@ import pytest from server.ito.protocol import ( - DisplayReason, + MESSAGE_TYPES, PROTOCOL_VERSION, - ProtocolError, TYPE_CONNECTION_HELLO, + TYPE_CONTROL_START, + TYPE_CONTROL_STOP, + ProtocolError, make_envelope, pack_envelope, - result_error, - result_ok, unpack_envelope, - validate_protocol_version, ) -def test_pack_round_trip_envelope(): +def test_protocol_has_direct_control_without_catalog_or_allocation_messages(): + assert TYPE_CONTROL_START in MESSAGE_TYPES + assert TYPE_CONTROL_STOP in MESSAGE_TYPES + assert not any("catalog" in message_type for message_type in MESSAGE_TYPES) + assert not any("acquire" in message_type for message_type in MESSAGE_TYPES) + assert not any("reservation" in message_type for message_type in MESSAGE_TYPES) + + +def test_envelopes_have_no_robot_or_session_identity(): envelope = make_envelope( TYPE_CONNECTION_HELLO, {"role": "pilotClient"}, message_id="msg-1", - session_id="session-1", ) - assert unpack_envelope(pack_envelope(envelope)) == envelope - - -def test_rejects_protocol_version_mismatch(): - with pytest.raises(ProtocolError): - validate_protocol_version("ito.v0") - - -def test_result_helpers_build_standard_payloads(): - assert result_ok({"protocolVersion": PROTOCOL_VERSION}) == { - "ok": True, - "value": {"protocolVersion": PROTOCOL_VERSION}, - } - assert result_error(DisplayReason(code="request.timeout")) == { - "ok": False, - "reason": {"code": "request.timeout"}, + assert unpack_envelope(pack_envelope(envelope)) == { + "protocolVersion": PROTOCOL_VERSION, + "messageId": "msg-1", + "type": TYPE_CONNECTION_HELLO, + "payload": {"role": "pilotClient"}, } -def test_display_reason_requires_code_or_text(): - with pytest.raises(ValueError): - DisplayReason() +def test_unknown_message_type_is_rejected(): + with pytest.raises(ProtocolError): + make_envelope("catalog.get", {}) diff --git a/tests/test_remote_robot_adapter.py b/tests/test_remote_robot_adapter.py new file mode 100644 index 0000000..551bf60 --- /dev/null +++ b/tests/test_remote_robot_adapter.py @@ -0,0 +1,72 @@ +import asyncio + +from server.ito.protocol import ( + TYPE_DRIVER_CONTROL_START, + TYPE_DRIVER_CONTROL_START_RESULT, + TYPE_WEBRTC_ANSWER, + TYPE_WEBRTC_OFFER, + make_envelope, + unpack_envelope, +) +from server.ito.robot import RemoteRobotAdapter + + +class FakeWebSocket: + def __init__(self): + self.sent = [] + + async def send(self, frame): + self.sent.append(unpack_envelope(frame)) + + +def test_remote_adapter_starts_the_one_attached_driver(): + async def scenario(): + websocket = FakeWebSocket() + adapter = RemoteRobotAdapter(request_timeout_ms=1000) + adapter.attach(websocket, ready=True) + + starting = asyncio.create_task(adapter.start_control()) + await asyncio.sleep(0) + request = websocket.sent[-1] + assert request["type"] == TYPE_DRIVER_CONTROL_START + adapter.handle_response( + make_envelope( + TYPE_DRIVER_CONTROL_START_RESULT, + {"ok": True, "value": {}}, + reply_to_message_id=request["messageId"], + ) + ) + await starting + + assert adapter.control_active is True + assert adapter.ready is True + + asyncio.run(scenario()) + + +def test_remote_adapter_relays_pilot_input_negotiation_without_identity(): + async def scenario(): + websocket = FakeWebSocket() + adapter = RemoteRobotAdapter(request_timeout_ms=1000) + adapter.attach(websocket, ready=True) + + negotiating = asyncio.create_task( + adapter.accept_pilot_input_offer("pilot offer") + ) + await asyncio.sleep(0) + offer = websocket.sent[-1] + assert offer["type"] == TYPE_WEBRTC_OFFER + assert offer["payload"] == {"path": "pilotInput", "sdp": "pilot offer"} + assert "robotId" not in offer + assert "sessionId" not in offer + adapter.handle_response( + make_envelope( + TYPE_WEBRTC_ANSWER, + {"path": "pilotInput", "sdp": "driver answer"}, + reply_to_message_id=offer["messageId"], + ) + ) + + assert await negotiating == "driver answer" + + asyncio.run(scenario()) diff --git a/tests/test_robot_adapter.py b/tests/test_robot_adapter.py new file mode 100644 index 0000000..893c206 --- /dev/null +++ b/tests/test_robot_adapter.py @@ -0,0 +1,97 @@ +import asyncio + +from server.ito.robot import LocalRobotAdapter +from server.processors.base import ReconstructionFrame + + +def test_local_robot_adapter_moves_sensor_and_control_data_in_process(): + frames = [] + controls = [] + safe_stops = [] + adapter = LocalRobotAdapter( + control_sink=controls.append, + safe_stop=lambda: safe_stops.append(True), + ) + adapter.set_sensor_sink(frames.append) + + adapter.start_control() + frame = ReconstructionFrame(b"rgb", 1, 1, 1) + adapter.publish_sensor_frame(frame) + adapter.receive_pilot_input({"sequence": 1, "headsetYawRad": 0.25}) + adapter.stop_control() + + assert frames == [frame] + assert controls == [{"sequence": 1, "headsetYawRad": 0.25}] + assert safe_stops == [True] + assert adapter.control_active is False + + +def test_local_robot_adapter_ignores_control_when_paused(): + controls = [] + adapter = LocalRobotAdapter(control_sink=controls.append) + + adapter.receive_pilot_input({"sequence": 1}) + adapter.start_control() + adapter.stop_control() + adapter.receive_pilot_input({"sequence": 2}) + + assert controls == [] + + +def test_local_robot_adapter_neutralizes_when_pilot_input_times_out(): + async def scenario(): + controls = [] + safe_stops = [] + adapter = LocalRobotAdapter( + control_sink=controls.append, + safe_stop=lambda: safe_stops.append(True), + pilot_input_timeout_ms=10, + ) + + adapter.start_control() + adapter.receive_pilot_input({"sequence": 1}) + await asyncio.sleep(0.02) + + assert controls == [{"sequence": 1}] + assert safe_stops == [True] + assert adapter.input_timed_out is True + + asyncio.run(scenario()) + + +def test_local_robot_adapter_rate_limits_and_keeps_the_newest_input(): + async def scenario(): + controls = [] + adapter = LocalRobotAdapter( + control_sink=controls.append, + pilot_input_timeout_ms=1000, + max_control_rate_hz=20, + ) + + adapter.start_control() + adapter.receive_pilot_input({"sequence": 1}) + adapter.receive_pilot_input({"sequence": 2}) + adapter.receive_pilot_input({"sequence": 3}) + await asyncio.sleep(0.06) + adapter.stop_control() + + assert controls == [{"sequence": 1}, {"sequence": 3}] + + asyncio.run(scenario()) + + +def test_local_robot_adapter_emergency_stop_is_latched_until_control_restarts(): + controls = [] + emergency_stops = [] + adapter = LocalRobotAdapter( + control_sink=controls.append, + emergency_stop=lambda: emergency_stops.append(True), + ) + + adapter.start_control() + adapter.emergency_stop() + adapter.receive_pilot_input({"sequence": 1}) + + assert emergency_stops == [True] + assert controls == [] + assert adapter.control_active is False diff --git a/tests/test_server_app.py b/tests/test_server_app.py deleted file mode 100644 index 6b1b8e5..0000000 --- a/tests/test_server_app.py +++ /dev/null @@ -1,451 +0,0 @@ -import asyncio - -from server.ito.app import ItoServer, ConnectionState -from server.ito.config import ServerConfig -from server.ito.protocol import ( - ROLE_PILOT_CLIENT, - ROLE_ROBOT_DRIVER, - ROBOT_STATUS_AVAILABLE, - ROBOT_STATUS_UNAVAILABLE, - ROBOT_TYPE_DROID, - TYPE_CATALOG_GET, - TYPE_CATALOG_GET_RESULT, - TYPE_CONNECTION_HELLO, - TYPE_CONNECTION_HELLO_RESULT, - TYPE_DRIVER_SESSION_START, - TYPE_DRIVER_SESSION_START_RESULT, - TYPE_ROBOT_STATUS, - TYPE_SESSION_ACQUIRE, - TYPE_SESSION_ACQUIRE_RESULT, - TYPE_SESSION_END, - TYPE_SESSION_END_RESULT, - TYPE_SESSION_ENDED, - make_envelope, - pack_envelope, - result_error, - result_ok, - unpack_envelope, -) - - -class FakeWebSocket: - def __init__(self): - self.sent = [] - - async def send(self, frame): - self.sent.append(unpack_envelope(frame)) - - -def state(): - return ConnectionState(websocket=FakeWebSocket()) - - -def test_pilot_hello_and_empty_catalog(): - asyncio.run(_pilot_hello_and_empty_catalog()) - - -async def _pilot_hello_and_empty_catalog(): - server = ItoServer(ServerConfig(driver_status_watchdog_ms=1000)) - pilot = state() - - await server._handle_frame( - pilot, - pack_envelope( - make_envelope(TYPE_CONNECTION_HELLO, {"role": ROLE_PILOT_CLIENT}, message_id="hello-1") - ), - ) - await server._handle_frame( - pilot, - pack_envelope(make_envelope(TYPE_CATALOG_GET, {"includeUnavailable": True}, message_id="cat-1")), - ) - - hello = pilot.websocket.sent[0] - catalog = pilot.websocket.sent[1] - assert hello["type"] == TYPE_CONNECTION_HELLO_RESULT - assert hello["payload"]["ok"] is True - assert catalog["type"] == TYPE_CATALOG_GET_RESULT - assert catalog["replyToMessageId"] == "cat-1" - assert catalog["payload"] == {"ok": True, "value": {"robots": []}} - - -def test_driver_status_populates_catalog(): - asyncio.run(_driver_status_populates_catalog()) - - -async def _driver_status_populates_catalog(): - server = ItoServer(ServerConfig(driver_status_watchdog_ms=1000)) - driver = state() - pilot = state() - - await server._handle_frame( - driver, - pack_envelope( - make_envelope( - TYPE_CONNECTION_HELLO, - {"role": ROLE_ROBOT_DRIVER, "robotId": "droid-1"}, - message_id="driver-hello", - ) - ), - ) - await server._handle_frame( - driver, - pack_envelope( - make_envelope( - TYPE_ROBOT_STATUS, - {"name": "Dory", "type": ROBOT_TYPE_DROID, "status": ROBOT_STATUS_AVAILABLE}, - robot_id="droid-1", - ) - ), - ) - await server._handle_frame( - pilot, - pack_envelope(make_envelope(TYPE_CONNECTION_HELLO, {"role": ROLE_PILOT_CLIENT})), - ) - await server._handle_frame(pilot, pack_envelope(make_envelope(TYPE_CATALOG_GET))) - - robots = pilot.websocket.sent[-1]["payload"]["value"]["robots"] - assert robots == [ - { - "robotId": "droid-1", - "name": "Dory", - "type": ROBOT_TYPE_DROID, - "status": ROBOT_STATUS_AVAILABLE, - } - ] - - -def test_duplicate_robot_id_is_cataloged_unavailable(): - asyncio.run(_duplicate_robot_id_is_cataloged_unavailable()) - - -async def _duplicate_robot_id_is_cataloged_unavailable(): - server = ItoServer(ServerConfig(driver_status_watchdog_ms=1000)) - first = state() - second = state() - pilot = state() - - for conn in (first, second): - await server._handle_frame( - conn, - pack_envelope( - make_envelope( - TYPE_CONNECTION_HELLO, - {"role": ROLE_ROBOT_DRIVER, "robotId": "droid-1"}, - ) - ), - ) - await server._handle_frame(pilot, pack_envelope(make_envelope(TYPE_CONNECTION_HELLO, {"role": ROLE_PILOT_CLIENT}))) - await server._handle_frame(pilot, pack_envelope(make_envelope(TYPE_CATALOG_GET))) - - robots = pilot.websocket.sent[-1]["payload"]["value"]["robots"] - assert robots[0]["robotId"] == "droid-1" - assert robots[0]["status"] == ROBOT_STATUS_UNAVAILABLE - - -async def hello_pilot(server, pilot, session_id=None): - payload = {"role": ROLE_PILOT_CLIENT} - if session_id: - payload["sessionId"] = session_id - await server._handle_frame(pilot, pack_envelope(make_envelope(TYPE_CONNECTION_HELLO, payload))) - - -async def hello_available_driver(server, driver, robot_id="droid-1"): - await server._handle_frame( - driver, - pack_envelope( - make_envelope( - TYPE_CONNECTION_HELLO, - {"role": ROLE_ROBOT_DRIVER, "robotId": robot_id}, - robot_id=robot_id, - ) - ), - ) - await server._handle_frame( - driver, - pack_envelope( - make_envelope( - TYPE_ROBOT_STATUS, - {"name": "Dory", "type": ROBOT_TYPE_DROID, "status": ROBOT_STATUS_AVAILABLE}, - robot_id=robot_id, - ) - ), - ) - - -async def acquire_task(server, pilot, robot_id="droid-1", message_id="acquire-1"): - return asyncio.create_task( - server._handle_frame( - pilot, - pack_envelope( - make_envelope( - TYPE_SESSION_ACQUIRE, - {"robotId": robot_id}, - message_id=message_id, - robot_id=robot_id, - ) - ), - ) - ) - - -async def answer_driver_start(server, driver, ok=True): - start = driver.websocket.sent[-1] - assert start["type"] == TYPE_DRIVER_SESSION_START - session_id = start["sessionId"] - payload = result_ok({"sessionId": session_id}) if ok else result_error({"code": "driver.start_failed"}) - await server._handle_frame( - driver, - pack_envelope( - make_envelope( - TYPE_DRIVER_SESSION_START_RESULT, - payload, - reply_to_message_id=start["messageId"], - robot_id="droid-1", - session_id=session_id, - ) - ), - ) - return session_id - - -def test_acquire_reserves_robot_starts_driver_and_allocates_session(): - asyncio.run(_acquire_reserves_robot_starts_driver_and_allocates_session()) - - -async def _acquire_reserves_robot_starts_driver_and_allocates_session(): - server = ItoServer(ServerConfig(request_timeout_ms=1000, driver_status_watchdog_ms=1000)) - driver = state() - pilot = state() - await hello_available_driver(server, driver) - await hello_pilot(server, pilot) - - task = await acquire_task(server, pilot) - await asyncio.sleep(0) - - assert server.drivers["droid-1"].occupied is True - session_id = await answer_driver_start(server, driver) - await task - - acquire = pilot.websocket.sent[-1] - assert acquire["type"] == TYPE_SESSION_ACQUIRE_RESULT - assert acquire["replyToMessageId"] == "acquire-1" - assert acquire["payload"]["ok"] is True - assert acquire["payload"]["value"]["sessionId"] == session_id - assert acquire["payload"]["value"]["robotId"] == "droid-1" - assert acquire["payload"]["value"]["sessionConfig"] == server.config.session_config_payload() - assert server.sessions[session_id].state == "active" - assert server.drivers["droid-1"].occupied is True - - -def test_acquisition_reservation_blocks_competing_pilot(): - asyncio.run(_acquisition_reservation_blocks_competing_pilot()) - - -async def _acquisition_reservation_blocks_competing_pilot(): - server = ItoServer(ServerConfig(request_timeout_ms=1000, driver_status_watchdog_ms=1000)) - driver = state() - first = state() - second = state() - await hello_available_driver(server, driver) - await hello_pilot(server, first) - await hello_pilot(server, second) - - first_task = await acquire_task(server, first, message_id="acquire-1") - await asyncio.sleep(0) - second_task = await acquire_task(server, second, message_id="acquire-2") - await asyncio.sleep(0) - - assert len([msg for msg in driver.websocket.sent if msg["type"] == TYPE_DRIVER_SESSION_START]) == 1 - await answer_driver_start(server, driver) - await first_task - await second_task - - assert first.websocket.sent[-1]["payload"]["ok"] is True - assert second.websocket.sent[-1]["type"] == TYPE_SESSION_ACQUIRE_RESULT - assert second.websocket.sent[-1]["payload"] == { - "ok": False, - "reason": {"code": "session.acquire.robot_unavailable"}, - } - - -def test_driver_start_failure_releases_reservation(): - asyncio.run(_driver_start_failure_releases_reservation()) - - -async def _driver_start_failure_releases_reservation(): - server = ItoServer(ServerConfig(request_timeout_ms=1000, driver_status_watchdog_ms=1000)) - driver = state() - pilot = state() - await hello_available_driver(server, driver) - await hello_pilot(server, pilot) - - task = await acquire_task(server, pilot) - await asyncio.sleep(0) - session_id = await answer_driver_start(server, driver, ok=False) - await task - - assert session_id not in server.sessions - assert server.drivers["droid-1"].occupied is False - assert pilot.websocket.sent[-1]["payload"] == { - "ok": False, - "reason": {"code": "driver.start_failed"}, - } - - -def test_driver_start_timeout_releases_reservation(): - asyncio.run(_driver_start_timeout_releases_reservation()) - - -async def _driver_start_timeout_releases_reservation(): - server = ItoServer(ServerConfig(request_timeout_ms=1, driver_status_watchdog_ms=1000)) - driver = state() - pilot = state() - await hello_available_driver(server, driver) - await hello_pilot(server, pilot) - - task = await acquire_task(server, pilot) - await task - - assert server.sessions == {} - assert server.drivers["droid-1"].occupied is False - assert pilot.websocket.sent[-1]["type"] == TYPE_SESSION_ACQUIRE_RESULT - assert pilot.websocket.sent[-1]["payload"] == { - "ok": False, - "reason": {"code": "request.timeout"}, - } - - -def test_session_end_marks_ended_and_fans_out(): - asyncio.run(_session_end_marks_ended_and_fans_out()) - - -async def _session_end_marks_ended_and_fans_out(): - server = ItoServer(ServerConfig(request_timeout_ms=1000, driver_status_watchdog_ms=1000)) - driver = state() - pilot = state() - await hello_available_driver(server, driver) - await hello_pilot(server, pilot) - task = await acquire_task(server, pilot) - await asyncio.sleep(0) - session_id = await answer_driver_start(server, driver) - await task - - await server._handle_frame( - pilot, - pack_envelope( - make_envelope( - TYPE_SESSION_END, - {"reason": {"code": "session.ended.pilot_requested"}, "clean": True}, - message_id="end-1", - session_id=session_id, - ) - ), - ) - - assert pilot.websocket.sent[-2]["type"] == TYPE_SESSION_END_RESULT - assert pilot.websocket.sent[-2]["payload"] == {"ok": True, "value": {"sessionId": session_id}} - assert driver.websocket.sent[-2]["type"] == TYPE_SESSION_END - assert pilot.websocket.sent[-1]["type"] == TYPE_SESSION_ENDED - assert driver.websocket.sent[-1]["type"] == TYPE_SESSION_ENDED - assert pilot.websocket.sent[-1]["payload"] == { - "reason": {"code": "session.ended.pilot_requested"}, - "endedBy": ROLE_PILOT_CLIENT, - "clean": True, - } - driver_sent_count = len(driver.websocket.sent) - await server._handle_frame( - driver, - pack_envelope( - make_envelope( - TYPE_SESSION_END_RESULT, - result_ok({"sessionId": session_id}), - reply_to_message_id=driver.websocket.sent[-2]["messageId"], - session_id=session_id, - ) - ), - ) - assert len(driver.websocket.sent) == driver_sent_count - assert server.sessions[session_id].state == "ended" - assert server.drivers["droid-1"].occupied is False - - -def test_cleanup_ends_session_after_disappeared_endpoint_timeout(): - asyncio.run(_cleanup_ends_session_after_disappeared_endpoint_timeout()) - - -async def _cleanup_ends_session_after_disappeared_endpoint_timeout(): - server = ItoServer( - ServerConfig( - request_timeout_ms=1000, - driver_status_watchdog_ms=1000, - session_cleanup_timeout_ms=1, - ) - ) - driver = state() - pilot = state() - await hello_available_driver(server, driver) - await hello_pilot(server, pilot) - task = await acquire_task(server, pilot) - await asyncio.sleep(0) - session_id = await answer_driver_start(server, driver) - await task - - server._mark_connection_disappeared(pilot) - await asyncio.sleep(0.002) - await server._cleanup_disappeared_endpoint_sessions() - - assert session_id not in server.sessions - assert driver.websocket.sent[-2]["type"] == TYPE_SESSION_END - assert driver.websocket.sent[-1]["type"] == TYPE_SESSION_ENDED - assert driver.websocket.sent[-1]["payload"]["reason"] == {"code": "session.ended.endpoint_disappeared"} - - -def test_pilot_reconnect_hello_resumes_active_session(): - asyncio.run(_pilot_reconnect_hello_resumes_active_session()) - - -async def _pilot_reconnect_hello_resumes_active_session(): - server = ItoServer(ServerConfig(request_timeout_ms=1000, driver_status_watchdog_ms=1000)) - driver = state() - pilot = state() - reconnected = state() - await hello_available_driver(server, driver) - await hello_pilot(server, pilot) - task = await acquire_task(server, pilot) - await asyncio.sleep(0) - session_id = await answer_driver_start(server, driver) - await task - - server._mark_connection_disappeared(pilot) - await hello_pilot(server, reconnected, session_id=session_id) - - resumed = reconnected.websocket.sent[-1] - assert resumed["type"] == TYPE_CONNECTION_HELLO_RESULT - assert resumed["payload"] == { - "ok": True, - "value": { - "protocolVersion": "ito.v1", - "role": ROLE_PILOT_CLIENT, - "sessionResumed": True, - "sessionConfig": server.config.session_config_payload(), - }, - } - assert server.sessions[session_id].pilot_connection is reconnected - assert server.sessions[session_id].endpoint_missing_since is None - - -def test_pilot_reconnect_hello_rejects_missing_session(): - asyncio.run(_pilot_reconnect_hello_rejects_missing_session()) - - -async def _pilot_reconnect_hello_rejects_missing_session(): - server = ItoServer(ServerConfig()) - pilot = state() - - await hello_pilot(server, pilot, session_id="session-missing") - - assert pilot.websocket.sent[-1]["type"] == TYPE_CONNECTION_HELLO_RESULT - assert pilot.websocket.sent[-1]["payload"] == { - "ok": False, - "reason": {"code": "session.resume_unavailable"}, - } diff --git a/tests/test_webrtc_and_reconstruction.py b/tests/test_webrtc_and_reconstruction.py index f36182b..7108303 100644 --- a/tests/test_webrtc_and_reconstruction.py +++ b/tests/test_webrtc_and_reconstruction.py @@ -1,148 +1,18 @@ -import asyncio - -from server.ito.app import ItoServer -from server.ito.config import ServerConfig -from server.ito.protocol import ( - ROLE_PILOT_CLIENT, - TYPE_WEBRTC_ANSWER, - TYPE_WEBRTC_OFFER, - WEBRTC_PATH_CAMERA_MEDIA, - WEBRTC_PATH_PILOT_INPUT, - WEBRTC_PATH_SPLAT_BATCHES, - make_envelope, - pack_envelope, - unpack_envelope, -) -from server.ito.reconstruction import ReconstructionSessionRuntime from server.ito.media import AiortcCameraTrackReceiver +from server.ito.reconstruction import ReconstructionRuntime from server.ito.splat import decode_splat_batch_header, encode_splat_batch -from server.ito.webrtc import SplatBatchChannelRegistry +from server.ito.webrtc import SplatBatchChannel, decode_pilot_input_snapshot from server.processors.base import GaussianSplat, ProcessorSplatBatch, ReconstructionFrame -from tests.test_server_app import ( - acquire_task, - answer_driver_start, - hello_available_driver, - hello_pilot, - state, -) - - -class FakeLivePaths: - def __init__(self): - self.offers = [] - - async def accept_offer(self, *, path, session_id, sdp): - self.offers.append({"path": path, "sessionId": session_id, "sdp": sdp}) - return f"answer for {path}" - - -def test_pilot_input_webrtc_offer_is_relayed_and_answer_is_routed_back(): - asyncio.run(_pilot_input_webrtc_offer_is_relayed_and_answer_is_routed_back()) - - -async def _pilot_input_webrtc_offer_is_relayed_and_answer_is_routed_back(): - server = ItoServer(ServerConfig(request_timeout_ms=1000, driver_status_watchdog_ms=1000)) - driver = state() - pilot = state() - await hello_available_driver(server, driver) - await hello_pilot(server, pilot) - task = await acquire_task(server, pilot) - await asyncio.sleep(0) - session_id = await answer_driver_start(server, driver) - await task - - await server._handle_frame( - pilot, - pack_envelope( - make_envelope( - TYPE_WEBRTC_OFFER, - {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": "pilot offer"}, - message_id="pilot-offer", - robot_id="droid-1", - session_id=session_id, - ) - ), - ) - forwarded = driver.websocket.sent[-1] - assert forwarded["type"] == TYPE_WEBRTC_OFFER - assert forwarded["payload"] == {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": "pilot offer"} - - await server._handle_frame( - driver, - pack_envelope( - make_envelope( - TYPE_WEBRTC_ANSWER, - {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": "driver answer"}, - reply_to_message_id=forwarded["messageId"], - robot_id="droid-1", - session_id=session_id, - ) - ), +def test_pilot_input_decoder_has_no_session_identity(): + snapshot = decode_pilot_input_snapshot( + b'{"protocolVersion":"ito.v1","sequence":1,"timestampMs":12,' + b'"headsetYawRad":0.25,"controllers":[]}' ) - answer = pilot.websocket.sent[-1] - assert answer["type"] == TYPE_WEBRTC_ANSWER - assert answer["replyToMessageId"] == "pilot-offer" - assert answer["payload"] == {"path": WEBRTC_PATH_PILOT_INPUT, "sdp": "driver answer"} - - -def test_server_terminated_webrtc_offer_returns_non_trickle_answer(): - asyncio.run(_server_terminated_webrtc_offer_returns_non_trickle_answer()) - - -async def _server_terminated_webrtc_offer_returns_non_trickle_answer(): - server = ItoServer(ServerConfig(request_timeout_ms=1000, driver_status_watchdog_ms=1000)) - server.live_paths = FakeLivePaths() - driver = state() - pilot = state() - await hello_available_driver(server, driver) - await hello_pilot(server, pilot) - task = await acquire_task(server, pilot) - await asyncio.sleep(0) - session_id = await answer_driver_start(server, driver) - await task - - await server._handle_frame( - pilot, - pack_envelope( - make_envelope( - TYPE_WEBRTC_OFFER, - {"path": WEBRTC_PATH_SPLAT_BATCHES, "sdp": "splat offer"}, - message_id="splat-offer", - session_id=session_id, - ) - ), - ) - - answer = pilot.websocket.sent[-1] - assert answer["type"] == TYPE_WEBRTC_ANSWER - assert answer["replyToMessageId"] == "splat-offer" - assert answer["payload"] == {"path": WEBRTC_PATH_SPLAT_BATCHES, "sdp": "answer for splatBatches"} - - await server._handle_frame( - driver, - pack_envelope( - make_envelope( - TYPE_WEBRTC_OFFER, - {"path": WEBRTC_PATH_CAMERA_MEDIA, "sdp": "camera offer"}, - message_id="camera-offer", - robot_id="droid-1", - session_id=session_id, - ) - ), - ) - - assert server.live_paths.offers[-1] == { - "path": WEBRTC_PATH_CAMERA_MEDIA, - "sessionId": session_id, - "sdp": "camera offer", - } - assert driver.websocket.sent[-1]["payload"] == { - "path": WEBRTC_PATH_CAMERA_MEDIA, - "sdp": "answer for cameraMedia", - } + assert snapshot["sequence"] == 1 + assert "sessionId" not in snapshot def test_splat_batch_encoder_header_and_size(): @@ -161,7 +31,6 @@ def test_splat_batch_encoder_header_and_size(): payload = encode_splat_batch(batch) header = decode_splat_batch_header(payload) - assert header.version == 1 assert header.sequence == 7 assert header.splat_count == 1 assert len(payload) == 28 + header.record_stride @@ -170,8 +39,8 @@ def test_splat_batch_encoder_header_and_size(): class FailingProcessor: capture_modality = "monocularRgb" - def start(self, session_id): - self.session_id = session_id + def start(self): + pass def process_frame(self, frame): raise RuntimeError("boom") @@ -185,21 +54,20 @@ def close(self): def test_reconstruction_failure_is_reported_without_raising(): failures = [] - runtime = ReconstructionSessionRuntime( - "session-1", + runtime = ReconstructionRuntime( FailingProcessor(), send_splat_batch=lambda payload: None, - fail_session=failures.append, + fail_control=failures.append, ) runtime.start() runtime.process_frame(ReconstructionFrame(b"rgb", 1, 1, 1)) runtime.process_frame(ReconstructionFrame(b"rgb", 2, 1, 1)) - assert failures == [{"code": "session.ended.reconstruction_failed"}] + assert failures == [{"code": "control.stopped.reconstruction_failed"}] -def test_aiortc_camera_track_receiver_converts_video_frames_to_reconstruction_frames(): +def test_camera_track_receiver_converts_video_frames_for_reconstruction(): class Plane: def __bytes__(self): return b"rgb" @@ -214,20 +82,13 @@ class Frame: def to_rgb(self): return self - frames = [] - receiver = AiortcCameraTrackReceiver(frames.append) - - frame = receiver._reconstruction_frame(Frame()) + frame = AiortcCameraTrackReceiver(lambda value: None)._reconstruction_frame(Frame()) assert frame.data == b"rgb" assert frame.timestamp_ms == 1000 - assert frame.width == 1 - assert frame.height == 1 - assert frame.pixel_format == "rgb24" - assert frame.sequence == 1 -def test_splat_batch_channel_registry_sends_only_when_open(): +def test_splat_channel_sends_only_when_open(): class Channel: readyState = "open" @@ -237,12 +98,9 @@ def __init__(self): def send(self, payload): self.sent.append(payload) - registry = SplatBatchChannelRegistry() + registry = SplatBatchChannel() channel = Channel() + registry.attach(channel) - assert registry.send("session-1", b"batch") is False - registry.attach("session-1", channel) - assert registry.send("session-1", b"batch") is True + assert registry.send(b"batch") is True assert channel.sent == [b"batch"] - registry.detach("session-1", channel) - assert registry.send("session-1", b"batch") is False