From 783d3c929f5d3da1b5f35602e6f21977988a1d55 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 14 Sep 2026 14:21:48 +0000 Subject: [PATCH 1/3] feat(heroi): repo commands, image attach, and latest CLI models Make Heroi a fuller agentic control plane for DAN-70: spawn repo commands with cwd/permission mapping and command+output in the DAN-61 tool group, paste/attach vision images through Claude/Codex/Cursor, and surface Claude Fable 5.1 (claude-fable-5-1) plus GPT-6 Astra (gpt-6-astra) from live CLI catalogs. Co-authored-by: Daniels --- README.md | 9 +- ROADMAP.md | 9 + TASKS.md | 5 + crates/strand-tauri/src/heroi.rs | 469 ++++++++++++++++-- crates/strand-tauri/src/heroi/models.rs | 104 +++- docs/learnings.md | 14 +- ui/src/lib/i18n.ts | 6 + ui/src/lib/types.ts | 7 + ui/src/plugins/builtins/heroi/HeroiView.tsx | 172 ++++++- .../builtins/heroi/TurnPanels.test.tsx | 23 + .../builtins/heroi/attachments.test.ts | 55 ++ ui/src/plugins/builtins/heroi/attachments.ts | 85 ++++ ui/src/styles/features.css | 45 ++ website/docs/custom-view.md | 11 +- website/docs/keyboard-and-palette.md | 3 +- 15 files changed, 964 insertions(+), 53 deletions(-) create mode 100644 ui/src/plugins/builtins/heroi/attachments.test.ts create mode 100644 ui/src/plugins/builtins/heroi/attachments.ts diff --git a/README.md b/README.md index 1a17a92a..0d7be2be 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,12 @@ the resolved app appearance automatically. come from the selected provider. Multiple threads can run at once; `@` searches repository files, `/` searches installed/project skills, and files can be dragged from a Files pane into the composer with live drop feedback. - Assistant replies render as Markdown; each turn lists files it added, changed, - or deleted; and tool calls collapse into one grouped control (expand a row for - bounded output). Its compact thread rail and bottom command deck keep chat + Paste or attach PNG/JPEG/GIF/WebP images for vision-capable providers; they + show as thumbnails in the thread. Assistant replies render as Markdown; each + turn lists files it added, changed, or deleted; and tool calls collapse into + one grouped control (expand a row for bounded command and output). Model + pickers follow the provider CLI, including Claude Fable 5.1 and GPT-6 Astra + when advertised. Its compact thread rail and bottom command deck keep chat primary, with **Open review** routing to Strand's Review surface. Declarative plugins render from validated manifests; third-party JavaScript does not execute in the privileged webview. diff --git a/ROADMAP.md b/ROADMAP.md index 76b42593..0a99c527 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2732,6 +2732,15 @@ added/changed/deleted paths from that turn's mutating tool/activity payloads and opens a clicked path in Work's Changes view. Tool calls for a turn live in one collapsible group so the transcript stays chat-first. +**Heroi fuller agentic UX shipped (2026-09-14):** Agent command runs stay +rooted in the active repository (Codex `--cd`) and stream command+output into +the existing grouped tool block, including Claude Bash `tool_result` and +Cursor shell calls. The composer can paste or attach PNG/JPEG/GIF/WebP images; +Claude gets stream-json image blocks, Codex/Cursor get `--image` plus paths. +The model picker adds Claude Fable 5.1 (`claude-fable-5-1`, CLI ≥ 2.1.257) and +surfaces GPT-6 Astra (`gpt-6-astra`) from Codex live lists / confirmed +fallback, or Cursor when ACP advertises it. + **Strand 1.5.0 release kick (2026-08-30):** The five lockstep desktop app manifests and Cargo lockfile are synchronized at 1.5.0. The signed desktop release and Microsoft Store submission workflows are the remaining promotion diff --git a/TASKS.md b/TASKS.md index be7c1abb..a345ab4c 100644 --- a/TASKS.md +++ b/TASKS.md @@ -1421,6 +1421,11 @@ community plugins, performance and platform certification from Git feature gaps. - ☑ Heroi readable turns: Markdown replies, per-turn added/changed/deleted file list, and grouped tool calls (`MessageMarkdown`, `turnArtifacts`, `TurnPanels`, `HEROI_OPEN_FILE_EVENT` — DAN-61). +- ☑ Heroi fuller agentic UX: repo-cwd command execution with command+output + in the grouped tool block, composer paste/attach images for vision + providers, and catalog/probe for Claude Fable 5.1 (`claude-fable-5-1`) + plus GPT-6 Astra (`gpt-6-astra`) (`heroi.rs`, `heroi/models.rs`, + `HeroiView`, `attachments.ts` — DAN-70). - ☑ Plugin-creation guide for AI/manifest authors (`docs/plugin-creation.md`). - ☐ Run native workspace-scoped Workbench persistence and live-terminal continuity E2E on macOS, Windows, and Linux builds (browser QA covers layout, focus, resizing, diff --git a/crates/strand-tauri/src/heroi.rs b/crates/strand-tauri/src/heroi.rs index 4344de9b..7a5e8bef 100644 --- a/crates/strand-tauri/src/heroi.rs +++ b/crates/strand-tauri/src/heroi.rs @@ -1,9 +1,11 @@ +use std::collections::HashMap; use std::path::Path; use std::{collections::BTreeMap, fs, path::PathBuf}; use std::time::Duration; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use serde::{Deserialize, Serialize}; -use serde_json::Value; +use serde_json::{json, Value}; use tauri::ipc::Channel; use crate::ai::bin::{ @@ -88,6 +90,8 @@ mod skill_tests { const AGENT_TIMEOUT: Duration = Duration::from_secs(60 * 60); const MAX_PROMPT_BYTES: usize = 128 * 1024; +const MAX_HEROI_IMAGES: usize = 4; +const MAX_HEROI_IMAGE_BYTES: usize = 4 * 1024 * 1024; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] @@ -119,6 +123,16 @@ pub struct HeroiAgentRequest { pub agent_mode: String, pub permission_mode: String, pub cli_path: Option, + #[serde(default)] + pub images: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HeroiImageAttachment { + pub name: String, + pub mime_type: String, + pub data_base64: String, } #[derive(Debug, Serialize)] @@ -152,6 +166,25 @@ pub enum HeroiAgentEvent { struct ParseState { session_id: Option, emitted_text: bool, + pending_tools: HashMap, +} + +struct PendingTool { + label: String, + detail: Option, +} + +struct MaterializedImages { + dir: Option, + paths: Vec, +} + +impl Drop for MaterializedImages { + fn drop(&mut self) { + if let Some(dir) = &self.dir { + let _ = fs::remove_dir_all(dir); + } + } } pub fn run_agent( @@ -177,7 +210,9 @@ pub fn run_agent( request.provider.label() ) })?; - let args = build_args(&request); + let images = materialize_images(&request.images)?; + let stdin = stdin_payload(&request, &images.paths); + let args = build_args(&request, &images.paths); let mut parsed = ParseState::default(); let _ = on_event.send(HeroiAgentEvent::Status { message: format!("Starting {}", request.provider.label()), @@ -187,7 +222,7 @@ pub fn run_agent( &program, &args, Path::new(&request.path), - &request.prompt, + &stdin, AGENT_TIMEOUT, cancel, |line| parse_line(request.provider, line, &mut parsed, &on_event), @@ -202,7 +237,7 @@ pub fn run_agent( } fn validate_request(request: &HeroiAgentRequest) -> Result<(), String> { - if request.prompt.trim().is_empty() { + if request.prompt.trim().is_empty() && request.images.is_empty() { return Err("Write a message before sending.".into()); } if request.prompt.len() > MAX_PROMPT_BYTES { @@ -221,9 +256,58 @@ fn validate_request(request: &HeroiAgentRequest) -> Result<(), String> { { return Err("The selected agent settings are invalid.".into()); } + validate_images(&request.images)?; Ok(()) } +fn validate_images(images: &[HeroiImageAttachment]) -> Result<(), String> { + if images.len() > MAX_HEROI_IMAGES { + return Err("Heroi accepts up to 4 images per message.".into()); + } + for image in images { + if invalid_image_name(&image.name) { + return Err("An attached image has an invalid file name.".into()); + } + if normalize_image_mime(&image.mime_type).is_none() { + return Err("Heroi can only attach PNG, JPEG, GIF, or WebP images.".into()); + } + let bytes = BASE64 + .decode(image.data_base64.trim().as_bytes()) + .map_err(|_| "Heroi could not read an attached image.".to_string())?; + if bytes.is_empty() || bytes.len() > MAX_HEROI_IMAGE_BYTES { + return Err("Each attached image must be 4 MiB or smaller.".into()); + } + } + Ok(()) +} + +fn invalid_image_name(name: &str) -> bool { + let trimmed = name.trim(); + trimmed.is_empty() + || trimmed.len() > 128 + || trimmed.contains(['/', '\\', '\0', '\n', '\r']) + || trimmed.contains("..") +} + +fn normalize_image_mime(mime: &str) -> Option<&'static str> { + match mime.trim().to_ascii_lowercase().as_str() { + "image/png" => Some("image/png"), + "image/jpeg" | "image/jpg" => Some("image/jpeg"), + "image/gif" => Some("image/gif"), + "image/webp" => Some("image/webp"), + _ => None, + } +} + +fn image_extension(mime: &str) -> &'static str { + match mime { + "image/jpeg" => "jpg", + "image/gif" => "gif", + "image/webp" => "webp", + _ => "png", + } +} + fn valid_setting(value: Option<&str>, max_len: usize) -> bool { match value { None => true, @@ -246,11 +330,11 @@ fn selected(value: Option<&str>) -> Option<&str> { value.filter(|value| !value.trim().is_empty() && !value.eq_ignore_ascii_case("default")) } -fn build_args(request: &HeroiAgentRequest) -> Vec { +fn build_args(request: &HeroiAgentRequest, image_paths: &[PathBuf]) -> Vec { match request.provider { HeroiProvider::Claude => claude_args(request), - HeroiProvider::Codex => codex_args(request), - HeroiProvider::Cursor => cursor_args(request), + HeroiProvider::Codex => codex_args(request, image_paths), + HeroiProvider::Cursor => cursor_args(request, image_paths), } } @@ -261,6 +345,9 @@ fn claude_args(request: &HeroiAgentRequest) -> Vec { "stream-json".into(), "--verbose".into(), ]; + if !request.images.is_empty() { + args.extend(["--input-format".into(), "stream-json".into()]); + } if let Some(session_id) = request.session_id.as_deref() { args.extend(["--resume".into(), session_id.into()]); } @@ -285,8 +372,8 @@ fn claude_args(request: &HeroiAgentRequest) -> Vec { args } -fn codex_args(request: &HeroiAgentRequest) -> Vec { - let mut args = vec!["exec".into()]; +fn codex_args(request: &HeroiAgentRequest, image_paths: &[PathBuf]) -> Vec { + let mut args = vec!["exec".into(), "--cd".into(), request.path.clone()]; if let Some(session_id) = request.session_id.as_deref() { add_codex_access(&mut args, request); args.extend([ @@ -295,6 +382,7 @@ fn codex_args(request: &HeroiAgentRequest) -> Vec { "--skip-git-repo-check".into(), ]); add_codex_options(&mut args, request); + add_image_flags(&mut args, image_paths); args.extend([session_id.into(), "-".into()]); return args; } @@ -306,6 +394,7 @@ fn codex_args(request: &HeroiAgentRequest) -> Vec { ]); add_codex_options(&mut args, request); add_codex_access(&mut args, request); + add_image_flags(&mut args, image_paths); args.push("-".into()); args } @@ -343,7 +432,7 @@ fn add_codex_options(args: &mut Vec, request: &HeroiAgentRequest) { } } -fn cursor_args(request: &HeroiAgentRequest) -> Vec { +fn cursor_args(request: &HeroiAgentRequest, image_paths: &[PathBuf]) -> Vec { let mut args = vec![ "--print".into(), "--output-format".into(), @@ -361,9 +450,90 @@ fn cursor_args(request: &HeroiAgentRequest) -> Vec { } else if request.permission_mode == "full" { args.push("--force".into()); } + add_image_flags(&mut args, image_paths); args } +fn add_image_flags(args: &mut Vec, image_paths: &[PathBuf]) { + for path in image_paths { + args.extend(["--image".into(), path.to_string_lossy().into_owned()]); + } +} + +fn materialize_images(images: &[HeroiImageAttachment]) -> Result { + if images.is_empty() { + return Ok(MaterializedImages { + dir: None, + paths: Vec::new(), + }); + } + let dir = std::env::temp_dir().join(format!("strand-heroi-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&dir).map_err(|_| "Heroi could not store attached images.".to_string())?; + let mut paths = Vec::with_capacity(images.len()); + for (index, image) in images.iter().enumerate() { + let mime = normalize_image_mime(&image.mime_type).unwrap_or("image/png"); + let bytes = BASE64 + .decode(image.data_base64.trim().as_bytes()) + .map_err(|_| "Heroi could not read an attached image.".to_string())?; + let path = dir.join(format!("image-{index}.{}", image_extension(mime))); + fs::write(&path, bytes).map_err(|_| "Heroi could not store attached images.".to_string())?; + paths.push(path); + } + Ok(MaterializedImages { + dir: Some(dir), + paths, + }) +} + +fn stdin_payload(request: &HeroiAgentRequest, image_paths: &[PathBuf]) -> String { + match request.provider { + HeroiProvider::Claude if !request.images.is_empty() => claude_image_stdin(request), + _ => { + let mut text = request.prompt.clone(); + if request.provider != HeroiProvider::Claude && !image_paths.is_empty() { + let list = image_paths + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect::>() + .join("\n"); + if !text.trim().is_empty() { + text.push_str("\n\n"); + } + text.push_str("Attached images:\n"); + text.push_str(&list); + } + text + } + } +} + +fn claude_image_stdin(request: &HeroiAgentRequest) -> String { + let mut content = Vec::new(); + let text = request.prompt.trim(); + if !text.is_empty() { + content.push(json!({ "type": "text", "text": text })); + } + for image in &request.images { + let mime = normalize_image_mime(&image.mime_type).unwrap_or("image/png"); + content.push(json!({ + "type": "image", + "source": { + "type": "base64", + "media_type": mime, + "data": image.data_base64.trim(), + } + })); + } + format!( + "{}\n", + json!({ + "type": "user", + "message": { "role": "user", "content": content }, + "parent_tool_use_id": Value::Null, + }) + ) +} + fn parse_line( provider: HeroiProvider, line: &str, @@ -381,7 +551,7 @@ fn parse_line( }); } } - if let Some(activity) = activity_data(provider, &value) { + if let Some(activity) = activity_data(provider, &value, state) { let _ = on_event.send(HeroiAgentEvent::Activity { id: activity.id, label: activity.label, @@ -451,7 +621,7 @@ struct ActivityData { done: bool, } -fn activity_data(provider: HeroiProvider, value: &Value) -> Option { +fn activity_data(provider: HeroiProvider, value: &Value, state: &mut ParseState) -> Option { let event_type = value.get("type").and_then(Value::as_str)?; if provider == HeroiProvider::Codex && matches!(event_type, "item.started" | "item.completed") @@ -497,7 +667,7 @@ fn activity_data(provider: HeroiProvider, value: &Value) -> Option let kind = call.keys().next()?.as_str(); let label = if kind.to_ascii_lowercase().contains("write") { "Editing files" - } else if kind.to_ascii_lowercase().contains("terminal") { + } else if is_command_tool(kind) { "Running a command" } else { "Using a tool" @@ -510,7 +680,7 @@ fn activity_data(provider: HeroiProvider, value: &Value) -> Option .unwrap_or(kind) .to_owned(), label: label.into(), - detail: Some(format_activity_value(call_value)), + detail: Some(cursor_activity_detail(call_value)), done: value .get("status") .and_then(Value::as_str) @@ -525,20 +695,151 @@ fn activity_data(provider: HeroiProvider, value: &Value) -> Option .iter() .find(|block| block.get("type").and_then(Value::as_str) == Some("tool_use"))?; let name = block.get("name")?.as_str()?; + let id = block + .get("id") + .and_then(Value::as_str) + .unwrap_or(name) + .to_owned(); + let label = if is_command_tool(name) { + "Running a command".into() + } else { + format!("Using {name}") + }; + let detail = command_from_value(block.get("input")) + .or_else(|| block.get("input").map(format_activity_value)); + state.pending_tools.insert( + id.clone(), + PendingTool { + label: label.clone(), + detail: detail.clone(), + }, + ); return Some(ActivityData { - id: block - .get("id") - .and_then(Value::as_str) - .unwrap_or(name) - .to_owned(), - label: format!("Using {name}"), - detail: block.get("input").map(format_activity_value), + id, + label, + detail, done: false, }); } + if event_type == "user" { + let block = value + .get("message")? + .get("content")? + .as_array()? + .iter() + .find(|block| block.get("type").and_then(Value::as_str) == Some("tool_result"))?; + let id = block + .get("tool_use_id") + .and_then(Value::as_str) + .unwrap_or("tool") + .to_owned(); + let output = tool_result_text(block).filter(|text| !text.is_empty()); + let pending = state.pending_tools.remove(&id); + let label = pending + .as_ref() + .map(|tool| tool.label.clone()) + .unwrap_or_else(|| "Using a tool".into()); + let detail = match (pending.and_then(|tool| tool.detail), output) { + (Some(command), Some(output)) if !command.is_empty() && !output.is_empty() => { + Some(format!("{command}\n\n{output}")) + } + (Some(command), Some(output)) if command.is_empty() => Some(output), + (Some(command), Some(_)) => Some(command), + (Some(command), None) => Some(command), + (None, Some(output)) => Some(output), + (None, None) => None, + }; + return Some(ActivityData { + id, + label, + detail, + done: true, + }); + } None } +fn is_command_tool(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + lower.contains("bash") + || lower.contains("shell") + || lower.contains("terminal") + || lower.contains("powershell") + || lower == "command" + || lower.contains("command_execution") +} + +fn command_from_value(value: Option<&Value>) -> Option { + let value = value?; + if let Some(command) = value.get("command").and_then(Value::as_str) { + return Some(command.to_string()); + } + if let Some(command) = value.get("cmd").and_then(Value::as_str) { + return Some(command.to_string()); + } + find_string_field(value, &["command", "cmd", "shellCommand"]) +} + +fn cursor_activity_detail(value: &Value) -> String { + let command = command_from_value(Some(value)); + let output = find_string_field(value, &["aggregated_output", "output", "stdout", "result"]) + .filter(|text| !text.is_empty()); + match (command, output) { + (Some(command), Some(output)) => format!("{command}\n\n{output}"), + (Some(command), None) => command, + (None, Some(output)) => output, + (None, None) => format_activity_value(value), + } +} + +fn find_string_field(value: &Value, keys: &[&str]) -> Option { + fn walk(value: &Value, keys: &[&str], depth: usize) -> Option { + if depth > 6 { + return None; + } + match value { + Value::Object(map) => { + for key in keys { + if let Some(Value::String(text)) = map.get(*key) { + if !text.is_empty() { + return Some(text.clone()); + } + } + } + for nested in map.values() { + if let Some(found) = walk(nested, keys, depth + 1) { + return Some(found); + } + } + None + } + Value::Array(items) => items.iter().find_map(|item| walk(item, keys, depth + 1)), + _ => None, + } + } + walk(value, keys, 0) +} + +fn tool_result_text(block: &Value) -> Option { + let content = block.get("content")?; + if let Some(text) = content.as_str() { + return Some(text.to_string()); + } + let parts = content.as_array()?; + let text = parts + .iter() + .filter_map(|part| { + part.as_str().map(str::to_string).or_else(|| { + (part.get("type").and_then(Value::as_str) == Some("text")) + .then(|| part.get("text").and_then(Value::as_str).map(str::to_string)) + .flatten() + }) + }) + .collect::>() + .join("\n"); + (!text.is_empty()).then_some(text) +} + fn format_activity_value(value: &Value) -> String { const MAX_DETAIL_BYTES: usize = 16 * 1024; let text = value @@ -598,6 +899,15 @@ mod tests { agent_mode: "build".into(), permission_mode: "build".into(), cli_path: None, + images: Vec::new(), + } + } + + fn png_attachment() -> HeroiImageAttachment { + HeroiImageAttachment { + name: "shot.png".into(), + mime_type: "image/png".into(), + data_base64: BASE64.encode([0x89, b'P', b'N', b'G', 0x0D, 0x0A]), } } @@ -606,19 +916,20 @@ mod tests { let mut input = request(HeroiProvider::Codex); input.session_id = Some("thread-1".into()); input.permission_mode = "read".into(); - let args = build_args(&input); - assert_eq!(&args[..3], ["exec", "--sandbox", "read-only"]); + let args = build_args(&input, &[]); + assert_eq!(&args[..5], ["exec", "--cd", ".", "--sandbox", "read-only"]); assert!(args.windows(2).any(|pair| pair == ["resume", "--json"])); assert!(args.ends_with(&["thread-1".into(), "-".into()])); } #[test] fn claude_build_mode_accepts_edits_without_full_bypass() { - let args = build_args(&request(HeroiProvider::Claude)); + let args = build_args(&request(HeroiProvider::Claude), &[]); assert!(args .windows(2) .any(|pair| pair == ["--permission-mode", "acceptEdits"])); assert!(!args.contains(&"--dangerously-skip-permissions".into())); + assert!(!args.contains(&"--input-format".into())); } #[test] @@ -626,14 +937,14 @@ mod tests { let mut input = request(HeroiProvider::Claude); input.model = Some("opus".into()); input.thinking = Some("ultrathink".into()); - let args = build_args(&input); + let args = build_args(&input, &[]); assert!(args .windows(2) .any(|pair| pair == ["--model", "claude-opus-5"])); assert!(!args.windows(2).any(|pair| pair == ["--effort", "ultrathink"])); input.thinking = Some("xhigh".into()); - let args = build_args(&input); + let args = build_args(&input, &[]); assert!(args.windows(2).any(|pair| pair == ["--effort", "xhigh"])); } @@ -649,7 +960,7 @@ mod tests { fn cursor_plan_mode_is_explicitly_read_only() { let mut input = request(HeroiProvider::Cursor); input.agent_mode = "plan".into(); - let args = build_args(&input); + let args = build_args(&input, &[]); assert!(args.windows(2).any(|pair| pair == ["--mode", "plan"])); assert!(!args.contains(&"--force".into())); } @@ -672,12 +983,14 @@ mod tests { #[test] fn captures_expandable_codex_command_and_output() { + let mut state = ParseState::default(); let started = serde_json::json!({ "type": "item.started", "item": { "id": "cmd-1", "type": "command_execution", "command": "cargo check" } }); - let activity = activity_data(HeroiProvider::Codex, &started).unwrap(); + let activity = activity_data(HeroiProvider::Codex, &started, &mut state).unwrap(); assert_eq!(activity.id, "cmd-1"); + assert_eq!(activity.label, "Running a command"); assert_eq!(activity.detail.as_deref(), Some("cargo check")); assert!(!activity.done); @@ -690,11 +1003,109 @@ mod tests { "aggregated_output": "Finished successfully" } }); - let activity = activity_data(HeroiProvider::Codex, &completed).unwrap(); + let activity = activity_data(HeroiProvider::Codex, &completed, &mut state).unwrap(); assert_eq!(activity.detail.as_deref(), Some("cargo check\n\nFinished successfully")); assert!(activity.done); } + #[test] + fn captures_claude_bash_command_and_tool_result() { + let mut state = ParseState::default(); + let started = serde_json::json!({ + "type": "assistant", + "message": { + "content": [{ + "type": "tool_use", + "id": "toolu-1", + "name": "Bash", + "input": { "command": "pnpm test" } + }] + } + }); + let activity = activity_data(HeroiProvider::Claude, &started, &mut state).unwrap(); + assert_eq!(activity.id, "toolu-1"); + assert_eq!(activity.label, "Running a command"); + assert_eq!(activity.detail.as_deref(), Some("pnpm test")); + assert!(!activity.done); + + let completed = serde_json::json!({ + "type": "user", + "message": { + "content": [{ + "type": "tool_result", + "tool_use_id": "toolu-1", + "content": "Tests passed" + }] + } + }); + let activity = activity_data(HeroiProvider::Claude, &completed, &mut state).unwrap(); + assert_eq!(activity.detail.as_deref(), Some("pnpm test\n\nTests passed")); + assert!(activity.done); + } + + #[test] + fn captures_cursor_terminal_command_and_output() { + let mut state = ParseState::default(); + let event = serde_json::json!({ + "type": "tool_call", + "id": "term-1", + "status": "completed", + "tool_call": { + "shellToolCall": { + "args": { "command": "cargo check" }, + "result": { "output": "Finished `dev` profile" } + } + } + }); + let activity = activity_data(HeroiProvider::Cursor, &event, &mut state).unwrap(); + assert_eq!(activity.label, "Running a command"); + assert_eq!( + activity.detail.as_deref(), + Some("cargo check\n\nFinished `dev` profile") + ); + assert!(activity.done); + } + + #[test] + fn image_only_messages_are_valid_and_reach_providers() { + let mut input = request(HeroiProvider::Claude); + input.prompt = String::new(); + input.images = vec![png_attachment()]; + assert!(validate_request(&input).is_ok()); + + let args = build_args(&input, &[]); + assert!(args + .windows(2) + .any(|pair| pair == ["--input-format", "stream-json"])); + let stdin = stdin_payload(&input, &[]); + assert!(stdin.contains("\"type\":\"image\"")); + assert!(stdin.contains("image/png")); + + let image_path = PathBuf::from("/tmp/shot.png"); + let mut codex = request(HeroiProvider::Codex); + codex.images = vec![png_attachment()]; + let args = build_args(&codex, std::slice::from_ref(&image_path)); + assert!(args.windows(2).any(|pair| pair == ["--cd", "."])); + assert!(args.windows(2).any(|pair| pair == ["--image", "/tmp/shot.png"])); + let stdin = stdin_payload(&codex, std::slice::from_ref(&image_path)); + assert!(stdin.contains("Attached images:")); + assert!(stdin.contains("/tmp/shot.png")); + } + + #[test] + fn rejects_non_image_attachments() { + let mut input = request(HeroiProvider::Claude); + input.images = vec![HeroiImageAttachment { + name: "notes.txt".into(), + mime_type: "text/plain".into(), + data_base64: BASE64.encode(b"hello"), + }]; + assert_eq!( + validate_request(&input).unwrap_err(), + "Heroi can only attach PNG, JPEG, GIF, or WebP images." + ); + } + #[test] fn vendor_transcripts_are_not_returned_as_errors() { let error = friendly_error( diff --git a/crates/strand-tauri/src/heroi/models.rs b/crates/strand-tauri/src/heroi/models.rs index 535c9626..35760fc4 100644 --- a/crates/strand-tauri/src/heroi/models.rs +++ b/crates/strand-tauri/src/heroi/models.rs @@ -13,11 +13,12 @@ const CODEX_PROBE_TIMEOUT: Duration = Duration::from_secs(8); const CURSOR_PROBE_TIMEOUT: Duration = Duration::from_secs(15); const MINIMUM_CLAUDE_OPUS_5: (u32, u32, u32) = (2, 1, 219); +const MINIMUM_CLAUDE_FABLE_5_1: (u32, u32, u32) = (2, 1, 257); const MINIMUM_CLAUDE_FABLE_5: (u32, u32, u32) = (2, 1, 169); const MINIMUM_CLAUDE_OPUS_4_8: (u32, u32, u32) = (2, 1, 154); const MINIMUM_CLAUDE_OPUS_4_7: (u32, u32, u32) = (2, 1, 111); -const PREFERRED_CODEX_DEFAULTS: &[&str] = &["gpt-5.6-sol", "gpt-5.6-terra"]; +const PREFERRED_CODEX_DEFAULTS: &[&str] = &["gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra"]; #[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] @@ -76,7 +77,13 @@ pub fn normalize_claude_cli_effort(effort: &str, model: Option<&str>) -> Option< "xhigh" if !matches!( model, - Some("claude-fable-5" | "claude-opus-5" | "claude-opus-4-8" | "claude-sonnet-5") + Some( + "claude-fable-5-1" + | "claude-fable-5" + | "claude-opus-5" + | "claude-opus-4-8" + | "claude-sonnet-5" + ) ) => { Some("max".into()) @@ -157,6 +164,14 @@ fn claude_catalog(version: Option<(u32, u32, u32)>) -> Vec { ]); let mut models = Vec::new(); + if version_at_least(version, MINIMUM_CLAUDE_FABLE_5_1) { + models.push(model( + "claude-fable-5-1", + "Claude Fable 5.1", + false, + opus_effort.clone(), + )); + } if version_at_least(version, MINIMUM_CLAUDE_FABLE_5) { models.push(model( "claude-fable-5", @@ -497,6 +512,7 @@ fn codex_fallback() -> Vec { ("xhigh", "Extra High", false), ]); vec![ + model("gpt-6-astra", "GPT-6-Astra", false, reasoning.clone()), model("gpt-5.6-sol", "GPT-5.6-Sol", true, reasoning.clone()), model("gpt-5.6-terra", "GPT-5.6-Terra", false, reasoning.clone()), model("gpt-5.4", "GPT-5.4", false, reasoning), @@ -607,17 +623,23 @@ const CLAUDE_ALIASES: &[(&str, &str)] = &[ ("sonnet", "claude-sonnet-5"), ("sonnet-5", "claude-sonnet-5"), ("haiku", "claude-haiku-4-5"), + ("fable-5.1", "claude-fable-5-1"), + ("claude-fable-5.1", "claude-fable-5-1"), ]; const CODEX_ALIASES: &[(&str, &str)] = &[ ("gpt-5-codex", "gpt-5.4"), ("5.4", "gpt-5.4"), ("gpt-5.6-codex", "gpt-5.6-sol"), + ("astra", "gpt-6-astra"), + ("gpt-6", "gpt-6-astra"), ]; const CURSOR_ALIASES: &[(&str, &str)] = &[ ("composer", "composer-2"), ("default", "auto"), + ("astra", "gpt-6-astra"), + ("gpt-6", "gpt-6-astra"), ]; #[cfg(test)] @@ -628,11 +650,24 @@ mod tests { fn claude_catalog_gates_opus_5_on_cli_version() { let unknown = claude_catalog(None); assert!(!unknown.iter().any(|model| model.slug == "claude-opus-5")); + assert!(!unknown.iter().any(|model| model.slug == "claude-fable-5-1")); assert!(unknown.iter().any(|model| model.slug == "claude-sonnet-5")); let current = claude_catalog(Some((2, 1, 219))); assert!(current.iter().any(|model| model.slug == "claude-opus-5")); assert!(current.iter().any(|model| model.slug == "claude-fable-5")); + assert!(!current.iter().any(|model| model.slug == "claude-fable-5-1")); + + let fable_51 = claude_catalog(Some((2, 1, 257))); + assert!(fable_51.iter().any(|model| model.slug == "claude-fable-5-1")); + assert_eq!( + fable_51 + .iter() + .find(|model| model.slug == "claude-fable-5-1") + .map(|model| model.name.as_str()), + Some("Claude Fable 5.1") + ); + assert!(fable_51.iter().any(|model| model.slug == "claude-fable-5")); let sonnet = current .iter() .find(|model| model.slug == "claude-sonnet-5") @@ -649,7 +684,7 @@ mod tests { } #[test] - fn parses_codex_model_list_and_prefers_sol() { + fn parses_codex_model_list_and_prefers_astra() { let response = json!({ "data": [ { @@ -675,6 +710,19 @@ mod tests { { "reasoningEffort": "low" }, { "reasoningEffort": "high" } ] + }, + { + "model": "gpt-6-astra", + "displayName": "gpt-6-astra", + "hidden": false, + "isDefault": false, + "defaultReasoningEffort": "high", + "supportedReasoningEfforts": [ + { "reasoningEffort": "low" }, + { "reasoningEffort": "high" }, + { "reasoningEffort": "xhigh" }, + { "reasoningEffort": "max" } + ] } ] }); @@ -682,7 +730,10 @@ mod tests { assert_eq!(models[0].name, "GPT-5.4"); assert!(!models[0].is_default); assert_eq!(models[1].slug, "gpt-5.6-sol"); - assert!(models[1].is_default); + assert!(!models[1].is_default); + assert_eq!(models[2].slug, "gpt-6-astra"); + assert_eq!(models[2].name, "GPT-6-Astra"); + assert!(models[2].is_default); assert_eq!( models[1] .reasoning @@ -691,6 +742,10 @@ mod tests { .map(|option| option.id.as_str()), Some("high") ); + assert!(models[2] + .reasoning + .iter() + .any(|option| option.id == "max")); assert_eq!( models[0] .reasoning @@ -710,6 +765,11 @@ mod tests { "name": "Auto", "configOptions": [] }, + { + "value": "gpt-6-astra", + "name": "GPT-6 Astra", + "configOptions": [] + }, { "value": "composer-2", "name": "Composer 2", @@ -733,16 +793,18 @@ mod tests { let models = parse_cursor_models(&response); assert_eq!(models[0].slug, "auto"); assert!(models[0].reasoning.is_empty()); - assert_eq!(models[1].slug, "composer-2"); + assert_eq!(models[1].slug, "gpt-6-astra"); + assert_eq!(models[1].name, "GPT-6 Astra"); + assert_eq!(models[2].slug, "composer-2"); assert_eq!( - models[1] + models[2] .reasoning .iter() .find(|option| option.is_default) .map(|option| option.id.as_str()), Some("high") ); - assert!(models[1] + assert!(models[2] .reasoning .iter() .any(|option| option.id == "xhigh")); @@ -763,6 +825,10 @@ mod tests { normalize_claude_cli_effort("xhigh", Some("claude-opus-5")).as_deref(), Some("xhigh") ); + assert_eq!( + normalize_claude_cli_effort("xhigh", Some("claude-fable-5-1")).as_deref(), + Some("xhigh") + ); assert_eq!( apply_claude_prompt_effort("fix the tests", Some("ultrathink")), "Ultrathink:\nfix the tests" @@ -777,7 +843,31 @@ mod tests { fn aliases_expand_legacy_picker_values() { assert_eq!(canonicalize_model(HeroiProvider::Claude, "opus"), "claude-opus-5"); assert_eq!(canonicalize_model(HeroiProvider::Claude, "sonnet"), "claude-sonnet-5"); + assert_eq!( + canonicalize_model(HeroiProvider::Claude, "claude-fable-5.1"), + "claude-fable-5-1" + ); assert_eq!(canonicalize_model(HeroiProvider::Codex, "gpt-5.6-codex"), "gpt-5.6-sol"); + assert_eq!(canonicalize_model(HeroiProvider::Codex, "astra"), "gpt-6-astra"); assert_eq!(canonicalize_model(HeroiProvider::Cursor, "composer"), "composer-2"); + assert_eq!(canonicalize_model(HeroiProvider::Cursor, "gpt-6"), "gpt-6-astra"); + } + + #[test] + fn codex_fallback_includes_confirmed_astra_slug() { + let models = codex_fallback(); + assert!(models.iter().any(|model| model.slug == "gpt-6-astra")); + assert_eq!( + models.iter().find(|model| model.is_default).map(|model| model.slug.as_str()), + Some("gpt-5.6-sol") + ); + assert!(!models.iter().any(|model| model.slug == "gpt-6-astra" && model.is_default)); + } + + #[test] + fn cursor_fallback_stays_auto_until_the_cli_advertises_astra() { + let models = cursor_fallback(); + assert_eq!(models.len(), 1); + assert_eq!(models[0].slug, "auto"); } } diff --git a/docs/learnings.md b/docs/learnings.md index 76805193..a4cd413b 100644 --- a/docs/learnings.md +++ b/docs/learnings.md @@ -2523,9 +2523,17 @@ repository-relative paths. The `/` picker discovers the selected provider's user and project skill roots but inserts `$skill-name`, matching the native CLI prompt syntax. Files-tree drops must become mentions and must not also trigger the tree's move/open behavior. Report drag-hover entry/exit separately from the -drop so the composer can acknowledge a valid target before release. Tool calls for a turn belong in one grouped disclosure; individual rows expand -when provider detail exists. Retain bounded command/tool arguments and output, -never unbounded vendor transcripts or stderr. +drop so the composer can acknowledge a valid target before release. Paste and +attach PNG/JPEG/GIF/WebP images in the composer; persist thumbnails as data +URLs on the user turn and send bytes through `HeroiAgentRequest.images`. Claude +uses stream-json image blocks; Codex and Cursor Agent get `--image` plus +absolute temp paths. Never execute remote HTML for those thumbnails. Tool calls +for a turn belong in one grouped disclosure; command tools (`Bash`, Codex +`command_execution`, Cursor shell/terminal) use the **Running a command** label +and keep command plus bounded output in the expandable row. Spawn cwd is the +active repository; Codex also passes `--cd`. Model IDs are CLI slugs: Claude +Fable 5.1 is `claude-fable-5-1` (CLI ≥ 2.1.257), GPT-6 Astra is `gpt-6-astra` +from Codex `model/list` or Cursor ACP — do not invent dead slugs. --- diff --git a/ui/src/lib/i18n.ts b/ui/src/lib/i18n.ts index fdd4522a..9d4223cd 100644 --- a/ui/src/lib/i18n.ts +++ b/ui/src/lib/i18n.ts @@ -149,6 +149,12 @@ export const en = { 'plugins.heroi.tools.failed': '{count} failed', 'plugins.heroi.tools.done': 'done', 'plugins.heroi.composerPlaceholder': '{mode}… ({shortcut}+Enter to send · Shift+Tab to switch mode)', + 'plugins.heroi.attachImage': 'Attach image', + 'plugins.heroi.removeImage': 'Remove {name}', + 'plugins.heroi.imageLimit': 'Heroi accepts up to 4 images per message.', + 'plugins.heroi.imageType': 'Heroi can only attach PNG, JPEG, GIF, or WebP images.', + 'plugins.heroi.imageSize': 'Each attached image must be 4 MiB or smaller.', + 'plugins.heroi.imageReadFailed': 'Heroi could not read an attached image.', 'files.createEntry': 'New file or folder', 'files.newFile': 'New file', 'files.newFolder': 'New folder', diff --git a/ui/src/lib/types.ts b/ui/src/lib/types.ts index ce722f82..bf99062a 100644 --- a/ui/src/lib/types.ts +++ b/ui/src/lib/types.ts @@ -917,6 +917,13 @@ export interface HeroiAgentRequest { agentMode: 'plan' | 'build'; permissionMode: 'read' | 'build' | 'full'; cliPath: string | null; + images?: HeroiAgentImage[]; +} + +export interface HeroiAgentImage { + name: string; + mimeType: string; + dataBase64: string; } export type HeroiAgentEvent = diff --git a/ui/src/plugins/builtins/heroi/HeroiView.tsx b/ui/src/plugins/builtins/heroi/HeroiView.tsx index 7da2200e..b2a33889 100644 --- a/ui/src/plugins/builtins/heroi/HeroiView.tsx +++ b/ui/src/plugins/builtins/heroi/HeroiView.tsx @@ -7,6 +7,8 @@ import { useState, type KeyboardEvent, type ChangeEvent, + type ClipboardEvent, + type DragEvent, } from 'react'; import { Icon } from '../../../components/Icon'; @@ -39,6 +41,14 @@ import { replaceComposerTrigger, type HeroiComposerSuggestion, } from './composer'; +import { + clipboardImageFiles, + composerImageReject, + droppedImageFiles, + toImagePayload, + type ComposerImageReject, + type HeroiImageDraft, +} from './attachments'; export type HeroiProvider = 'claude' | 'codex' | 'cursor'; type AgentMode = 'plan' | 'build'; @@ -61,6 +71,7 @@ interface HeroiMessage { createdAt: number; state?: MessageState; activities?: HeroiActivity[]; + images?: HeroiImageDraft[]; } const EMPTY_ACTIVITIES: HeroiActivity[] = []; @@ -88,7 +99,18 @@ const MessageRow = memo(function MessageRow({ message, provider, projectPath, ac projectPath={projectPath} toolsExpanded={toolsExpanded} onToggleGroup={() => onToggleGroup(message.id, toolsExpanded)} expandedActivities={expandedActivities} onToggleActivity={onToggleActivity} onOpenPath={onOpenPath} - /> : message.text ? : null} + /> : ( + <> + {message.images && message.images.length > 0 && ( +
+ {message.images.map((image) => ( + {image.name} + ))} +
+ )} + {message.text ? : null} + + )} ; }); @@ -131,15 +153,21 @@ const MODEL_ALIASES: Record> = { opus: 'claude-opus-5', sonnet: 'claude-sonnet-5', haiku: 'claude-haiku-4-5', + 'fable-5.1': 'claude-fable-5-1', + 'claude-fable-5.1': 'claude-fable-5-1', }, codex: { default: 'gpt-5.6-sol', 'gpt-5.6-codex': 'gpt-5.6-sol', 'gpt-5-codex': 'gpt-5.4', + astra: 'gpt-6-astra', + 'gpt-6': 'gpt-6-astra', }, cursor: { default: 'auto', composer: 'composer-2', + astra: 'gpt-6-astra', + 'gpt-6': 'gpt-6-astra', }, }; @@ -179,8 +207,31 @@ function pathLeaf(path: string): string { return parts[parts.length - 1] || path; } -function titleFromText(text: string): string { - return text.trim().split('\n')[0]?.slice(0, 52) || t('plugins.heroi.untitled'); +function titleFromText(text: string, images?: readonly HeroiImageDraft[]): string { + return text.trim().split('\n')[0]?.slice(0, 52) + || images?.[0]?.name + || t('plugins.heroi.untitled'); +} + +function imageRejectMessage(reason: ComposerImageReject): string { + switch (reason) { + case 'limit': return t('plugins.heroi.imageLimit'); + case 'size': return t('plugins.heroi.imageSize'); + case 'type': return t('plugins.heroi.imageType'); + default: { + const _exhaustive: never = reason; + return _exhaustive; + } + } +} + +function readFileAsDataUrl(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result ?? '')); + reader.onerror = () => reject(reader.error ?? new Error('read failed')); + reader.readAsDataURL(file); + }); } function relativeTime(timestamp: number): string { @@ -225,6 +276,7 @@ export function HeroiView({ const [activeConversationId, setActiveConversationId] = useState(null); const [composingNew, setComposingNew] = useState(false); const [composerText, setComposerText] = useState(''); + const [composerImages, setComposerImages] = useState([]); const [draftProvider, setDraftProvider] = useState('claude'); const [draftModel, setDraftModel] = useState('default'); const [draftThinking, setDraftThinking] = useState('default'); @@ -247,6 +299,7 @@ export function HeroiView({ const rootRef = useRef(null); const composerRef = useRef(null); + const imageInputRef = useRef(null); const messagesRef = useRef(null); const stateKey = pluginStateKey('daniels.heroi', request.instanceId); const restored = restoredKey === stateKey; @@ -364,6 +417,7 @@ export function HeroiView({ setComposingNew(true); setActiveConversationId(null); setComposerText(''); + setComposerImages([]); setError(null); queueMicrotask(() => composerRef.current?.focus()); }; @@ -550,7 +604,13 @@ export function HeroiView({ const sendMessage = useCallback(async () => { const text = composerText.trim(); - if (!text || !activePath || activeRun) return; + const images = composerImages; + if ((!text && images.length === 0) || !activePath || activeRun) return; + const payloads = images.map(toImagePayload); + if (payloads.some((payload) => payload == null)) { + setError(t('plugins.heroi.imageReadFailed')); + return; + } try { broker.require('repository.read'); broker.require('ai.invoke'); @@ -571,6 +631,7 @@ export function HeroiView({ const runId = mintId('heroi-run'); const userMessage: HeroiMessage = { id: mintId('m'), role: 'user', text, createdAt: now, state: 'complete', + images: images.length > 0 ? images : undefined, }; const assistantMessage: HeroiMessage = { id: assistantMessageId, role: 'assistant', text: '', createdAt: now + 1, state: 'running', @@ -578,7 +639,7 @@ export function HeroiView({ const conversation = activeConversation ?? { id: conversationId, projectPath: activePath, - title: titleFromText(text), + title: titleFromText(text, images), provider, model: selectedModel, thinking: selectedThinking, @@ -605,6 +666,7 @@ export function HeroiView({ setComposingNew(false); } setComposerText(''); + setComposerImages([]); setError(null); setActiveRuns((current) => ({ ...current, @@ -626,6 +688,7 @@ export function HeroiView({ agentMode, permissionMode, cliPath: cliOverride(provider, openaiCli, anthropicCli), + images: payloads.filter((payload): payload is NonNullable => payload != null), }; try { const outcome = await tauri.heroiAgentSend( @@ -688,6 +751,7 @@ export function HeroiView({ agentMode, anthropicCli, broker, + composerImages, composerText, handleAgentEvent, meta?.branch, @@ -700,6 +764,40 @@ export function HeroiView({ updateConversation, ]); + const addComposerImages = useCallback(async (files: readonly File[]) => { + if (files.length === 0) return; + const next: HeroiImageDraft[] = []; + let reject: ComposerImageReject | null = null; + for (const file of files) { + const reason = composerImageReject(file, composerImages.length + next.length); + if (reason) { + reject = reason; + continue; + } + try { + const dataUrl = await readFileAsDataUrl(file); + if (!dataUrl.startsWith('data:image/')) { + reject = 'type'; + continue; + } + next.push({ + id: mintId('img'), + name: file.name || 'image', + mimeType: file.type || 'image/png', + dataUrl, + }); + } catch { + setError(t('plugins.heroi.imageReadFailed')); + return; + } + } + if (next.length > 0) { + setComposerImages((current) => [...current, ...next]); + } + if (reject) setError(imageRejectMessage(reject)); + else if (next.length > 0) setError(null); + }, [composerImages.length]); + const stopRun = useCallback(() => { if (!activeRun) return; setActiveRuns((current) => ({ @@ -755,6 +853,21 @@ export function HeroiView({ setSuggestionIndex(0); }, []); + const onComposerPaste = useCallback((event: ClipboardEvent) => { + const files = clipboardImageFiles(event.clipboardData); + if (files.length === 0) return; + event.preventDefault(); + void addComposerImages(files); + }, [addComposerImages]); + + const onComposerImageDrop = useCallback((event: DragEvent) => { + const files = droppedImageFiles(event.dataTransfer); + if (files.length === 0) return; + event.preventDefault(); + event.stopPropagation(); + void addComposerImages(files); + }, [addComposerImages]); + const onComposerKeyDown = useCallback((event: KeyboardEvent) => { if (suggestions.length > 0 && (event.key === 'ArrowDown' || event.key === 'ArrowUp')) { event.preventDefault(); @@ -819,6 +932,7 @@ export function HeroiView({ setComposingNew(true); setActiveConversationId(null); setComposerText(''); + setComposerImages([]); setError(null); queueMicrotask(() => composerRef.current?.focus()); }} @@ -944,6 +1058,12 @@ export function HeroiView({
{ + if (droppedImageFiles(event.dataTransfer).length === 0) return; + event.preventDefault(); + event.dataTransfer.dropEffect = 'copy'; + }} + onDrop={onComposerImageDrop} > {fileDropActive && (
@@ -995,6 +1115,23 @@ export function HeroiView({ ))}
)} + {composerImages.length > 0 && ( +
+ {composerImages.map((image) => ( +
+ {image.name} + +
+ ))} +
+ )}