Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `register_agent` now writes supplied `metadata` or `persona` instead of silently discarding it when returning a cached token, and its new `verify_metadata` option reports whether the write actually persisted.
- Spawned agents now launch Agent Relay MCP through an installed local `agent-relay` executable instead of cold `npx` resolution.
- Spawn now fails before start with an actionable error when no usable Agent Relay MCP executable is available.
- Fleet `spawn:<harness>` actions resolve from the spawn's own verified result instead of bare worker-registry presence, so a node that registers a worker whose process dies during startup now returns `spawn_failed: <detail>` with the startup exit status and worker log path rather than `spawned: true`.
- A fleet node whose connection to the engine goes dead now reconnects on its own instead of disappearing from `agent-relay fleet nodes` until the broker is restarted.

## [11.6.1] - 2026-08-13
Expand Down
278 changes: 215 additions & 63 deletions crates/broker/src/runtime/fleet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1128,7 +1128,7 @@ impl BrokerRuntime {
let action_control_dedup_key =
relaycast_spawn_control_dedup_key(workspace_id.as_str(), name.as_str());

super::relaycast_events::spawn_worker_from_request(
let spawn_result = super::relaycast_events::spawn_worker_from_request(
name.clone(),
cli,
task,
Expand Down Expand Up @@ -1161,73 +1161,86 @@ impl BrokerRuntime {

let verify_ready = super::relaycast_events::relaycast_spawn_verifies_ready(&ws_value);

// A verified spawn keeps the action open until the harness itself emits
// worker_ready. Process creation alone is not proof that the persona is
// usable; worker_events resolves this pending entry, while maintenance
// fails it after an early exit/readiness timeout and performs cleanup.
if self.workers.workers.contains_key(&name) {
if verify_ready {
if self
.workers
.workers
.get(&name)
.is_some_and(|worker| worker.ready_at.is_some())
{
self.send_fleet_action_result(verified_spawn_ready_result(
invoke.invocation_id,
&name,
))
.await;
} else {
let generation = self
.workers
.workers
.get(&name)
.expect("verified spawn worker must still exist")
.generation;
self.pending_verified_spawns.insert(
name,
PendingVerifiedSpawn {
invocation_id: invoke.invocation_id,
deadline: Instant::now() + VERIFIED_SPAWN_READY_TIMEOUT,
generation,
},
);
let spawn_outcome =
fleet_spawn_outcome(spawn_result, &name, self.workers.is_worker_live(&name));

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1: When the worker exits after the stability probe but before this decision, this guard can still emit spawned: true: Unix kill(pid, 0) treats an unreaped zombie as existing, and non-Unix builds unconditionally report the worker live. Probe the child with try_wait on every supported platform before resolving a successful spawn.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. At crates/broker/src/runtime/fleet.rs, line 1165:

<comment>When the worker exits after the stability probe but before this decision, this guard can still emit `spawned: true`: Unix `kill(pid, 0)` treats an unreaped zombie as existing, and non-Unix builds unconditionally report the worker live. Probe the child with `try_wait` on every supported platform before resolving a successful spawn.</comment>

<file context>
@@ -1161,18 +1161,8 @@ impl BrokerRuntime {
-            other => other,
-        };
+        let spawn_outcome =
+            fleet_spawn_outcome(spawn_result, &name, self.workers.is_worker_live(&name));
 
         match spawn_outcome {
</file context>


match spawn_outcome {
Ok(()) => {
// A verified spawn keeps the action open until the harness itself
// emits worker_ready. Process creation alone is not proof that the
// persona is usable; worker_events resolves this pending entry,
// while maintenance fails it after an early exit/readiness timeout
// and performs cleanup.
if verify_ready {
let (already_ready, generation) = {
let worker = self
.workers
.workers
.get(&name)
.expect("verified spawn worker must still exist");
(worker.ready_at.is_some(), worker.generation)
};
if already_ready {
self.send_fleet_action_result(verified_spawn_ready_result(
invoke.invocation_id,
&name,
))
.await;
} else {
self.pending_verified_spawns.insert(
name,
PendingVerifiedSpawn {
invocation_id: invoke.invocation_id,
deadline: Instant::now() + VERIFIED_SPAWN_READY_TIMEOUT,
generation,
},
);
}
return;
}
return;
self.send_fleet_action_result(fleet_spawn_action_result(
&invoke.invocation_id,
&name,
Ok(()),
))
.await;
}
self.reply_action_output(
&invoke.invocation_id,
json!({ "spawned": true, "name": name.as_str() }),
)
.await;
} else {
// A registration can succeed before process creation fails. Undo
// that authoritative identity before reporting the failed launch.
match deregister_fleet_agent(&self.fleet_control_tx, &self.fleet_delivery_book, &name)
Err(error) => {
// A registration can succeed before process creation fails. Undo
// that authoritative identity before reporting the failed launch.
match deregister_fleet_agent(
&self.fleet_control_tx,
&self.fleet_delivery_book,
&name,
)
.await
{
Ok(_) => {
prune_fleet_agent_state(
&self.fleet_control_tx,
&mut self.fleet_inventory,
&mut self.fleet_delivery_book,
&name,
)
.await
}
Err(error) => {
tracing::warn!(worker = %name, %error, "retaining fleet identity after failed spawn cleanup");
prune_fleet_inventory_entry(
&self.fleet_control_tx,
&mut self.fleet_inventory,
&name,
)
.await;
{
Ok(_) => {
prune_fleet_agent_state(
&self.fleet_control_tx,
&mut self.fleet_inventory,
&mut self.fleet_delivery_book,
&name,
)
.await
}
Err(cleanup_error) => {
tracing::warn!(worker = %name, error = %cleanup_error, "retaining fleet identity after failed spawn cleanup");
prune_fleet_inventory_entry(
&self.fleet_control_tx,
&mut self.fleet_inventory,
&name,
)
.await;
}
}
}
self.reply_action_error(&invoke.invocation_id, "spawn_failed")
self.send_fleet_action_result(fleet_spawn_action_result(
&invoke.invocation_id,
&name,
Err(error),
))
.await;
}
}
}

Expand Down Expand Up @@ -1369,6 +1382,53 @@ impl BrokerRuntime {
}
}

/// Decide a fleet spawn action from the spawn's own verified outcome.
///
/// `spawn_worker_from_request` returns only after the process-stability probe,
/// so an `Err` already carries the real startup exit status and worker log
/// path. Liveness is still required on the success path: the child can exit
/// between that probe and this decision, and registry presence would survive
/// that — reporting `spawned: true` for a dead worker is precisely the failure
/// this action exists to prevent, so the guard asks whether the process is
/// alive rather than whether a map entry exists.
///
/// Split out from `handle_fleet_action_spawn` so the guard is testable without
/// a whole `BrokerRuntime`; an untestable guard is how the weaker
/// registry-presence check survived here in the first place.
fn fleet_spawn_outcome(
spawn_result: Result<()>,
name: &WorkerName,
worker_is_live: bool,
) -> Result<()> {
match spawn_result {
Ok(()) if !worker_is_live => Err(anyhow::anyhow!(
"agent '{name}' has no live worker process after spawn"
)),
other => other,
}
}

fn fleet_spawn_action_result(
invocation_id: &str,
name: &WorkerName,
spawn_result: Result<()>,
) -> ActionResult {
let result = match spawn_result {
Ok(()) => ActionResultPayload::Output(ActionResultOutput {
output: json!({ "spawned": true, "name": name.as_str() }),
}),
Err(error) => ActionResultPayload::Error(ActionResultError {
error: format!("spawn_failed: {error}"),
}),
};
ActionResult {
v: FLEET_WIRE_VERSION,
id: None,
invocation_id: invocation_id.to_string(),
result,
}
}

#[derive(Debug, Default, PartialEq, Eq)]
pub(super) struct FlushPendingRelayResult {
pub(super) flushed: usize,
Expand Down Expand Up @@ -2056,6 +2116,98 @@ mod tests {
use super::*;
use crate::protocol::PtyHarnessConfig;

#[cfg(unix)]
#[tokio::test]
async fn fleet_spawn_result_uses_verified_failure_not_registry_presence() {
let temp = tempfile::tempdir().expect("test tempdir");
let (event_tx, _event_rx) = mpsc::channel::<WorkerEvent>(4);
let mut workers = WorkerRegistry::new(
event_tx,
Vec::new(),
temp.path().join("worker-logs"),
Instant::now(),
);
let mut child = tokio::process::Command::new("sh")
.args(["-c", "exit 19"])
.stdin(Stdio::piped())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("repro child should spawn");
let _stdin = child.stdin.take().expect("repro child stdin");
child.wait().await.expect("repro child should exit");
let name = WorkerName::from("fleet-spawn-repro-1430");
let mut spec = test_agent_spec(None, None);
spec.name = name.clone();
let (command_tx, _command_rx) = mpsc::channel(4);
workers.workers.insert(
name.clone(),
WorkerHandle {
generation: Uuid::new_v4(),
spec,
parent: Some("Relaycast".to_string()),
workspace_id: None,
child,
command_tx,
harness_pid: None,
spawned_at: Instant::now(),
ready_at: None,
last_activity_at: Instant::now(),
context_budget_pct: None,
state: crate::worker::AgentWorkState::Working,
exit_reason: None,
},
);

// The registered-but-dead worker: present in the map, process gone.
// These two lines are the whole point — a guard keyed on presence sees
// a healthy worker here, and a guard keyed on liveness does not.
assert!(workers.workers.contains_key(&name));
assert!(!workers.is_worker_live(&name));

// MUST-FIRE: a successful `spawn_worker_from_request` is not enough if
// the process died between its stability probe and this decision.
let outcome = fleet_spawn_outcome(Ok(()), &name, workers.is_worker_live(&name));
let error =
outcome.expect_err("a dead worker must not resolve the spawn action as success");
assert!(
error.to_string().contains("no live worker process"),
"{error}"
);

// MUST-NOT-FIRE: a live worker with a successful spawn stays successful,
// so the guard above is not simply rejecting everything.
fleet_spawn_outcome(Ok(()), &name, true).expect("a live worker must resolve as success");

// A real launch failure keeps its detail rather than being replaced by
// the liveness message.
let propagated = fleet_spawn_outcome(Err(anyhow::anyhow!("exit status: 19")), &name, false)
.expect_err("a failed spawn must stay failed");
assert!(
propagated.to_string().contains("exit status: 19"),
"{propagated}"
);

let result = fleet_spawn_action_result(
"inv-failed-1430",
&name,
Err(anyhow::anyhow!(
"agent '{name}' process exited during startup (exit status: 19); see worker log /tmp/{name}.log"
)),
);

let ActionResultPayload::Error(error) = result.result else {
panic!("a verified spawn failure must not produce spawned:true");
};
assert_eq!(result.invocation_id, "inv-failed-1430");
assert_eq!(
error.error,
format!(
"spawn_failed: agent '{name}' process exited during startup (exit status: 19); see worker log /tmp/{name}.log"
)
);
}

fn test_agent_spec(session_id: Option<&str>, harness_session_id: Option<&str>) -> AgentSpec {
AgentSpec {
name: WorkerName::from("agent-a"),
Expand Down
Loading
Loading