fix(computer-use): discover and target exact application windows

docs/architecture/computer-use-control.md

@@ -149,6 +149,17 @@ they do not require a user to select the next app. Background AT-SPI startup
 does not open a portal picker. Linux portal capture and foreground seat input
 still require compositor-owned consent.
 
+`list_windows` is read-only discovery independent of the active capture and AX
+tree. macOS returns WindowServer top-level windows including off-screen entries;
+Windows returns visible, non-minimized titled top-level windows. Titles and owner
+PIDs reveal multi-window and helper-process UIs that an application list cannot.
+Hosts advertising `window-target-v1` accept `app: {pid, window_id}` to select an
+exact native window. Both IDs must be positive, and native capture validates the
+window's ownership before binding it. Omitted window IDs retain an existing
+binding. Older hosts and Linux reject exact-window requests explicitly; Linux
+keeps its existing AT-SPI and portal selection contracts. No remote-workspace
+request falls back to the controller's desktop.
+
 The optional manual-sharing API uses `awaiting_share`, opaque process-bound
 choices, observe-first capture and a separate revocable input grant. Its scope
 does not apply to normal Agent tasks. Existing remembered preferences and manual
@@ -364,6 +375,13 @@ or screen-capture-permission failures, marked `capture_preparation_error`,
 create no binding and attach no prior target's pixels. Lock, stop, ownership,
 generation, revocation and unknown errors cannot use that fallback.
 
+An exact-window preparation failure cannot fall back to another window's AX
+tree, including a previously bound window of the same PID. App-only preparation
+failures suggest native window discovery instead of asserting that the task is
+absent. `describe_screen` keeps its text-only default and honors explicit
+`include_image: true` for vision-capable models, with the selected image's
+reference and geometry in the result. Text-only models receive no image bytes.
+
 A text-only screenshot request performs a real text observation. Tool results
 preserve structured identities, AX/OCR facts, errors and geometry in the model's
 actual result channel; a short summary is not a replacement. Image attachments
@@ -403,8 +421,8 @@ operation or prove all transient or later application behavior.
 ### macOS
 
 Retain the selected content window across observations. A sharing-indicator
-window appearing earlier in WindowServer order must not replace it. Conflicting
-explicit window selection fails; validate a replacement before releasing a live
+window appearing earlier in WindowServer order must not replace it. An explicit
+window ID can select another window, including within the same PID; validate a replacement before releasing a live
 capture. AX focused-window observation resolves the actual bound window, not an
 unrelated Sharing dialog. Missing exact identity is an error.
 

scripts/fixtures/computer-use-roundtrip.m

@@ -16,7 +16,6 @@ - (BOOL)canBecomeKeyWindow { return YES; }
 @interface OBFRoundtripEditor : NSView
 @end
 @implementation OBFRoundtripEditor
-- (BOOL)acceptsFirstMouse:(NSEvent *)event { return YES; }
 - (BOOL)acceptsFirstResponder { return YES; }
 - (BOOL)isAccessibilityElement { return YES; }
 - (NSString *)accessibilityRole { return NSAccessibilityTextAreaRole; }
@@ -62,6 +61,13 @@ int main(int argc, const char **argv) { @autoreleasepool {
     [window orderFront:nil];
     [window makeKeyWindow];
     [window makeFirstResponder:editor];
+    // A second, larger window deliberately wins legacy PID-only capture.
+    // The tool must discover and explicitly select the smaller task window.
+    NSWindow *mainWindow = [[NSWindow alloc] initWithContentRect:NSMakeRect(700,100,760,500) styleMask:NSWindowStyleMaskTitled backing:NSBackingStoreBuffered defer:NO];
+    mainWindow.title = @"Computer Use Large Main Fixture";
+    mainWindow.releasedWhenClosed = NO;
+    mainWindow.backgroundColor = NSColor.purpleColor;
+    [mainWindow orderFront:nil];
     publish();
     double y = CGDisplayBounds(CGMainDisplayID()).size.height - 170;
     printf("READY %d 170 %.0f\n",getpid(),y); fflush(stdout);

src/apps/desktop/src/computer_use/control_session.rs

@@ -165,6 +165,7 @@ fn projected(r: &Resource) -> ControlSnapshot {
     s.capabilities.extend(
         [
             "window-capture",
+            "window-target-v1",
             "background-ax-semantic-input",
             "background-directed-keyboard",
             "background-directed-scroll",
@@ -177,6 +178,7 @@ fn projected(r: &Resource) -> ControlSnapshot {
     s.capabilities.extend(
         [
             "window-capture",
+            "window-target-v1",
             "background-uia-semantic-input",
             "background-window-messages",
             "foreground-seat-input",

src/apps/desktop/src/computer_use/desktop_host/mod.rs

@@ -18,12 +18,13 @@ use async_trait::async_trait;
 use log::debug;
 use openbitfun_core::agentic::tools::computer_use_host::{
     ActionRecord, AppInfo, AppSelector, AppShortcutsSnapshot, AppStateSnapshot, AppWaitPredicate,
-    ComputerScreenshot, ComputerUseDisplayInfo, ComputerUseHost, ComputerUseInteractionState,
-    ComputerUseLastMutationKind, ComputerUsePermissionSnapshot, ComputerUseScreenshotParams,
-    ComputerUseScreenshotRefinement, ComputerUseSessionSnapshot, InteractiveActionResult,
-    InteractiveClickParams, InteractiveScrollParams, InteractiveTypeTextParams, InteractiveView,
-    InteractiveViewOpts, LoopDetectionResult, UiElementLocateQuery, UiElementLocateResult,
-    VisualActionResult, VisualClickParams, VisualMarkView, VisualMarkViewOpts,
+    AppWindowInfo, ComputerScreenshot, ComputerUseDisplayInfo, ComputerUseHost,
+    ComputerUseInteractionState, ComputerUseLastMutationKind, ComputerUsePermissionSnapshot,
+    ComputerUseScreenshotParams, ComputerUseScreenshotRefinement, ComputerUseSessionSnapshot,
+    InteractiveActionResult, InteractiveClickParams, InteractiveScrollParams,
+    InteractiveTypeTextParams, InteractiveView, InteractiveViewOpts, LoopDetectionResult,
+    UiElementLocateQuery, UiElementLocateResult, VisualActionResult, VisualClickParams,
+    VisualMarkView, VisualMarkViewOpts,
 };
 #[cfg(any(target_os = "macos", target_os = "windows"))]
 use openbitfun_core::agentic::tools::computer_use_host::{
@@ -928,6 +929,8 @@ impl ComputerUseHost for DesktopComputerUseHost {
     }
 
     async fn prepare_control_target(&self, mut app: AppSelector) -> OpenBitFunResult<()> {
+        app.validate_window_target()
+            .map_err(OpenBitFunError::tool)?;
         if app.is_empty() {
             if let Some(pid) = crate::computer_use::control_session::snapshot()
                 .target
@@ -942,8 +945,13 @@ impl ComputerUseHost for DesktopComputerUseHost {
         #[cfg(target_os = "macos")]
         {
             let pid = resolve_pid_macos(self, &app).await?;
+            let window_id = app.window_id.map(u32::try_from).transpose().map_err(|_| {
+                OpenBitFunError::tool(
+                    "[INVALID_PARAMS] window_id exceeds the macOS window ID range",
+                )
+            })?;
             crate::computer_use::control_session::spawn_blocking(move || {
-                crate::computer_use::macos_capture::ensure_capture(pid, None)
+                crate::computer_use::macos_capture::ensure_capture(pid, window_id)
             })
             .await
             .map_err(|e| OpenBitFunError::tool(e.to_string()))?
@@ -966,6 +974,9 @@ impl ComputerUseHost for DesktopComputerUseHost {
         }
         #[cfg(target_os = "linux")]
         {
+            if app.window_id.is_some() {
+                return Err(OpenBitFunError::tool("[WINDOW_TARGET_UNSUPPORTED] This host uses AT-SPI app selectors and portal-selected capture, not native window IDs"));
+            }
             if !app.is_empty() {
                 crate::computer_use::linux_control_ax::bind_app_selector(&app).await?;
             }
@@ -1655,6 +1666,44 @@ impl ComputerUseHost for DesktopComputerUseHost {
         }
     }
 
+    async fn list_windows(&self, app: Option<AppSelector>) -> OpenBitFunResult<Vec<AppWindowInfo>> {
+        #[cfg(any(target_os = "macos", target_os = "windows"))]
+        {
+            let filter = if let Some(app) = app.filter(|app| !app.is_empty()) {
+                app.validate_window_target()
+                    .map_err(OpenBitFunError::tool)?;
+                Some((resolve_pid(self, &app).await?, app.window_id))
+            } else {
+                None
+            };
+            let windows = tokio::task::spawn_blocking(|| {
+                #[cfg(target_os = "macos")]
+                {
+                    crate::computer_use::macos_list_apps::list_windows()
+                }
+                #[cfg(target_os = "windows")]
+                {
+                    Ok::<_, OpenBitFunError>(crate::computer_use::windows_list_apps::list_windows())
+                }
+            })
+            .await
+            .map_err(|e| OpenBitFunError::tool(e.to_string()))??;
+            Ok(windows
+                .into_iter()
+                .filter(|window| {
+                    filter.is_none_or(|(pid, id)| {
+                        window.pid == pid && id.is_none_or(|id| window.window_id == id)
+                    })
+                })
+                .collect())
+        }
+        #[cfg(not(any(target_os = "macos", target_os = "windows")))]
+        {
+            let _ = app;
+            Err(OpenBitFunError::tool("[WINDOW_DISCOVERY_UNSUPPORTED] This host exposes AT-SPI applications and portal-selected surfaces; use list_apps and get_app_state"))
+        }
+    }
+
     async fn list_apps(&self, include_hidden: bool) -> OpenBitFunResult<Vec<AppInfo>> {
         #[cfg(target_os = "macos")]
         {
@@ -1798,7 +1847,7 @@ pub(crate) const LINUX_LEGACY_AX_UNAVAILABLE: &str = "[CONTROL_CAPABILITY_UNSUPP
 
 #[cfg(target_os = "windows")]
 fn app_selector_is_unspecified(app: &AppSelector) -> bool {
-    app.pid.is_none() && app.name.is_none() && app.bundle_id.is_none()
+    app.is_empty()
 }
 
 #[cfg(any(target_os = "macos", target_os = "windows"))]

src/apps/desktop/src/computer_use/desktop_host/native_control_roundtrip_tests.rs

@@ -260,12 +260,76 @@ pub(crate) async fn run() {
         apps.to_string().contains(&pid.to_string()),
         "fixture missing from real application discovery"
     );
-    let observation = call(
+    let before_discovery = host.control_snapshot();
+    let windows = call(&tool, &context, json!({"action":"list_windows"})).await;
+    assert_eq!(
+        host.control_snapshot().generation,
+        before_discovery.generation
+    );
+    assert_eq!(host.control_snapshot().target, before_discovery.target);
+    let window = windows["windows"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|window| window["pid"] == pid && window["title"] == "Computer Use Roundtrip Fixture")
+        .expect("small task window must be discoverable independently of AX and default capture");
+    let window_id = window["window_id"].as_u64().unwrap();
+    let main = call(
         &tool,
         &context,
         json!({"action":"get_app_state", "app":{"pid":pid}}),
     )
     .await;
+    assert_eq!(
+        main["app_state"]["window_title"],
+        "Computer Use Large Main Fixture"
+    );
+    let previous_revision = host.control_snapshot().target_revision;
+    let observation = call(
+        &tool,
+        &context,
+        json!({"action":"get_app_state", "app":{"pid":pid,"window_id":window_id}}),
+    )
+    .await;
+    assert_eq!(
+        observation["app_state"]["window_title"],
+        "Computer Use Roundtrip Fixture"
+    );
+    assert!(host.control_snapshot().target_revision > previous_revision);
+    assert_eq!(
+        host.control_snapshot().target,
+        Some(format!("pid:{pid}/window:{window_id}"))
+    );
+    // A stale/wrong-owner window request must fail without attaching the old
+    // window's AX or pixels, and without destroying its healthy capture.
+    let retained = host.control_snapshot();
+    let wrong_owner = tool.call_impl(&json!({"action":"get_app_state","app":{"pid":std::process::id(),"window_id":window_id}}), &context).await.unwrap_err();
+    assert!(
+        wrong_owner
+            .to_string()
+            .contains("TARGET_WINDOW_UNAVAILABLE"),
+        "{wrong_owner}"
+    );
+    assert_eq!(host.control_snapshot().target, retained.target);
+    assert_eq!(
+        host.control_snapshot().target_revision,
+        retained.target_revision
+    );
+    let described = call_observation(
+        &tool,
+        &context,
+        json!({"action":"describe_screen","include_image":true}),
+    )
+    .await;
+    assert_eq!(described.image_count, 1);
+    assert_eq!(
+        described.data["window_title"],
+        "Computer Use Roundtrip Fixture"
+    );
+    assert_eq!(described.data["image_bytes"], true);
+    assert!(described.data["screenshot_meta"]["screenshot_id"]
+        .as_str()
+        .is_some());
     let control = host.control_snapshot();
     assert_eq!(control.state, "active");
     assert!(control.sharing.is_none());

src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs

@@ -435,16 +435,35 @@ fn flash_click_highlight_cg(gx: f64, gy: f64) {
 impl DesktopComputerUseHost {
     #[cfg(target_os = "windows")]
     pub(super) async fn windows_target(&self, app: &AppSelector) -> OpenBitFunResult<(i32, isize)> {
+        app.validate_window_target()
+            .map_err(OpenBitFunError::tool)?;
         let pid = super::resolve_pid(self, app).await?;
         // A sharing indicator or newly opened utility window must not replace
         // the window already owned by the control session.
         let prefix = format!("pid:{pid}/window:");
-        if let Some(window) = crate::computer_use::control_session::snapshot()
+        let control = crate::computer_use::control_session::snapshot();
+        let bound = control
             .target
             .as_deref()
             .and_then(|target| target.strip_prefix(&prefix))
-            .and_then(|window| window.parse::<isize>().ok())
+            .and_then(|window| window.parse::<isize>().ok());
+        let requested = app
+            .window_id
+            .map(isize::try_from)
+            .transpose()
+            .map_err(|_| {
+                OpenBitFunError::tool(
+                    "[INVALID_PARAMS] window_id exceeds the native window handle range",
+                )
+            })?;
+        if control.sharing.is_some()
+            && (bound.is_none() || requested.is_some_and(|id| Some(id) != bound))
         {
+            return Err(OpenBitFunError::tool(
+                "[SHARED_TARGET_ONLY] Window was not shared by the user",
+            ));
+        }
+        if let Some(window) = requested.or(bound) {
             use windows::Win32::Foundation::HWND;
             use windows::Win32::UI::WindowsAndMessaging::{GetWindowThreadProcessId, IsWindow};
             let hwnd = HWND(window as *mut std::ffi::c_void);
@@ -453,7 +472,7 @@ impl DesktopComputerUseHost {
                 GetWindowThreadProcessId(hwnd, Some(&mut actual_pid));
             }
             if !unsafe { IsWindow(Some(hwnd)) }.as_bool() || actual_pid != pid as u32 {
-                return Err(OpenBitFunError::tool("[TARGET_WINDOW_UNAVAILABLE] The bound window closed; select a new target explicitly"));
+                return Err(OpenBitFunError::tool("[TARGET_WINDOW_UNAVAILABLE] The requested window no longer belongs to this process; refresh list_windows and select the intended window"));
             }
             return Ok((pid, window));
         }

src/apps/desktop/src/computer_use/macos_capture.rs

@@ -191,8 +191,7 @@ pub(super) fn ensure_capture(pid: i32, mut window_id: Option<u32>) -> Result<(),
                 "CAPTURE_GENERATION_CHANGED: Previous capture cleanup is still pending".into(),
             );
         }
-        if session.pid == pid {
-            validate_bound_window(session.window_id, window_id)?;
+        if session.pid == pid && reuse_bound_window(session.window_id, window_id) {
             return session.check();
         }
         // A bad target request must not destroy a healthy stream. Resolve the
@@ -224,12 +223,10 @@ pub(super) fn ensure_capture(pid: i32, mut window_id: Option<u32>) -> Result<(),
     Ok(())
 }
 // The sharing indicator itself may be reported as this application's front
-// window. Once selected, a window stays bound until the session is stopped.
-fn validate_bound_window(bound: u32, requested: Option<u32>) -> Result<(), String> {
-    if requested.is_some_and(|id| id != bound) {
-        return Err("TARGET_WINDOW_CHANGED: Existing capture remains active; use the bound window or start a new control session to select another window".into());
-    }
-    Ok(())
+// window. Omitted selection keeps the binding; only an explicit window ID may
+// request a replacement, validated above before the old stream is released.
+fn reuse_bound_window(bound: u32, requested: Option<u32>) -> bool {
+    requested.is_none_or(|id| id == bound)
 }
 
 pub(super) fn bound_window_id(pid: i32) -> Result<u32, String> {
@@ -340,14 +337,12 @@ extern "C" fn obf_control_native_stopped(
 
 #[cfg(test)]
 mod tests {
-    use super::validate_bound_window;
+    use super::reuse_bound_window;
 
     #[test]
     fn sharing_badge_cannot_retarget_an_existing_window_capture() {
-        assert!(validate_bound_window(100, None).is_ok());
-        assert!(validate_bound_window(100, Some(100)).is_ok());
-        assert!(validate_bound_window(100, Some(101))
-            .unwrap_err()
-            .starts_with("TARGET_WINDOW_CHANGED:"));
+        assert!(reuse_bound_window(100, None));
+        assert!(reuse_bound_window(100, Some(100)));
+        assert!(!reuse_bound_window(100, Some(101)));
     }
 }