From 05110cba76e4be09efd5ed68b7dddff72eb9f4ff Mon Sep 17 00:00:00 2001 From: Karn Date: Sun, 27 Sep 2026 03:32:42 +0530 Subject: [PATCH 1/6] feat(protocol): jev_observe / jev_act schemas, both tier full Co-Authored-By: Claude Opus 5.5 --- packages/protocol/src/index.ts | 1 + packages/protocol/src/jev.test.ts | 37 ++++++++++++++++ packages/protocol/src/jev.ts | 64 ++++++++++++++++++++++++++++ packages/protocol/src/policy.test.ts | 4 +- packages/protocol/src/policy.ts | 4 ++ 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 packages/protocol/src/jev.test.ts create mode 100644 packages/protocol/src/jev.ts diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 3459b12..5fc7dac 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -1,5 +1,6 @@ export * from "./bridge.js"; export * from "./cdp.js"; export * from "./frames.js"; +export * from "./jev.js"; export * from "./policy.js"; export * from "./ports.js"; diff --git a/packages/protocol/src/jev.test.ts b/packages/protocol/src/jev.test.ts new file mode 100644 index 0000000..0830e87 --- /dev/null +++ b/packages/protocol/src/jev.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { JevActParams, JevActResult, JevObservation } from "./jev.js"; + +describe("jev schemas", () => { + it("parses an observation", () => { + const obs = { + url: "https://x.com/", + title: "X", + text: "hello", + visible: true, + actions: [ + { id: "e1", kind: "fill", node: 3, role: "textbox", label: "Where from?", value: "" }, + { id: "wait", kind: "wait", label: "Wait for the page to update" }, + ], + }; + expect(JevObservation.parse(obs)).toEqual(obs); + }); + + it("act params carry typed text under `text` (audit-redacted key)", () => { + expect(JevActParams.parse({ op: "type", node: 3, text: "Zurich" }).text).toBe("Zurich"); + expect(() => JevActParams.parse({ op: "drag" })).toThrow(); + }); + + it("act params carry an optional label for the audit trail", () => { + expect(JevActParams.parse({ op: "click", node: 3, label: "Search" }).label).toBe("Search"); + expect(JevActParams.parse({ op: "click", node: 3 }).label).toBeUndefined(); + }); + + it("act result is ok or stale", () => { + expect(JevActResult.parse({ ok: true })).toEqual({ ok: true }); + expect(JevActResult.parse({ stale: true, reason: "gone" })).toEqual({ + stale: true, + reason: "gone", + }); + expect(() => JevActResult.parse({})).toThrow(); + }); +}); diff --git a/packages/protocol/src/jev.ts b/packages/protocol/src/jev.ts new file mode 100644 index 0000000..91154bf --- /dev/null +++ b/packages/protocol/src/jev.ts @@ -0,0 +1,64 @@ +import { z } from "zod"; + +const tabId = z.number().optional(); +const browserId = z.string().optional(); + +/** One thing Jev may choose: an operation on an observed element, or a page-level control. */ +export const JevActionKind = z.enum(["click", "fill", "select", "scroll", "wait"]); +export type JevActionKind = z.infer; + +export const JevAction = z.object({ + id: z.string(), + kind: JevActionKind, + /** Page-side identity of the element (absent for scroll/wait). */ + node: z.number().optional(), + role: z.string().optional(), + label: z.string(), + value: z.string().optional(), + current_value: z.string().optional(), + checked: z.string().optional(), + selected: z.string().optional(), + expanded: z.string().optional(), + delta: z.number().optional(), +}); +export type JevAction = z.infer; + +export const JevDialog = z.object({ type: z.string(), message: z.string() }); +export type JevDialog = z.infer; + +export const JevObservation = z.object({ + url: z.string(), + title: z.string(), + /** Visible viewport text, capped (~6,000 chars). */ + text: z.string(), + /** document.visibilityState === "visible". */ + visible: z.boolean(), + actions: z.array(JevAction), + /** Set when a JS alert/confirm/prompt blocks the page. */ + dialog: JevDialog.optional(), +}); +export type JevObservation = z.infer; + +export const JevObserveParams = z.object({ browserId, tabId }); +export type JevObserveParams = z.infer; + +export const JevActParams = z.object({ + browserId, + tabId, + op: z.enum(["click", "type", "select", "scroll", "wait"]), + node: z.number().optional(), + /** The chosen element's label, for the audit trail; the extension ignores it. */ + label: z.string().optional(), + /** Typed text. Named `text` so the audit trail redacts it. */ + text: z.string().optional(), + /** Native the way a user's pick would. */ +export function jevSelect(node: number, value: string): string { + const cache = (window as unknown as Record)[ + Symbol.for("reins.jev") + ]; + const el = cache?.nodes.get(node); + if (!(el instanceof HTMLSelectElement) || !el.isConnected) return "the dropdown is gone"; + const option = [...el.options].find( + (o) => o.value === value && !o.disabled && !o.closest("optgroup[disabled]"), + ); + if (!option) return "that option is gone"; + el.value = value; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + return "ok"; +} + +/** + * After input: wait two animation frames (≤ 50 ms), or for a typed combobox + * up to 200 ms until a visible option appears — so the next read sees the + * suggestions instead of a half-open popup. + */ +export function jevSettle(node: number | null, typed: boolean): Promise { + const cache = (window as unknown as Record)[ + Symbol.for("reins.jev") + ]; + const field = node === null ? undefined : cache?.nodes.get(node); + const combobox = typed && field?.getAttribute("role") === "combobox"; + return new Promise((resolve) => { + let frames = 0; + let stopped = false; + const finish = () => { + stopped = true; + resolve(); + }; + setTimeout(finish, combobox ? 200 : 50); + const ready = () => { + if (stopped) return; + const ids = (field?.getAttribute("aria-controls") || field?.getAttribute("aria-owns") || "") + .split(/\s+/) + .filter(Boolean); + const roots: ParentNode[] = ids.length + ? ids.map((id) => document.getElementById(id)).filter((e): e is HTMLElement => e !== null) + : [document]; + const options = roots.flatMap((root) => [...root.querySelectorAll('[role="option"]')]); + frames += 1; + const shown = options.some((e) => { + const r = e.getBoundingClientRect(); + return ( + r.width > 0 && + r.height > 0 && + r.bottom > 0 && + r.top < innerHeight && + e.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }) + ); + }); + if (frames >= 2 && (!combobox || shown)) finish(); + else requestAnimationFrame(ready); + }; + requestAnimationFrame(ready); + }); +} diff --git a/packages/extension/src/lib/pointer.browser.test.ts b/packages/extension/src/lib/pointer.browser.test.ts index ccdb0bb..e865d4a 100644 --- a/packages/extension/src/lib/pointer.browser.test.ts +++ b/packages/extension/src/lib/pointer.browser.test.ts @@ -20,6 +20,7 @@ vi.mock("./monitor.js", () => ({ isMonitored: () => false })); import { autofillGuard } from "./autofill-guard.js"; import { __resetDebugSessions, cdpClick, cdpType } from "./cdp.js"; +import { jevSnapshot } from "./jev-snapshot.js"; import { handleDialog, hover, pressKey } from "./page-actions.js"; const CHROME = [ @@ -71,6 +72,31 @@ const LOGIN_FIXTURE = ` `; +const JEV_FIXTURE = ` +

Flights

+
+ + + + + + +
+ + +
+ + +`; + let chromeProc: ChildProcess | undefined; const injected: string[] = []; let server: http.Server | undefined; @@ -100,8 +126,8 @@ async function evaluate(expression: string): Promise { } /** Fresh fixture per test; resolves once the page has loaded. */ -async function load(): Promise { - await cdp("Page.navigate", { url }); +async function load(path = "/"): Promise { + await cdp("Page.navigate", { url: new URL(path, url).href }); for (let i = 0; i < 100; i++) { if ((await evaluate("document.readyState").catch(() => "")) === "complete") return; await new Promise((r) => setTimeout(r, 20)); @@ -116,7 +142,7 @@ describe.skipIf(!CHROME)("pointer + key input in a real browser", () => { server = http.createServer((q, s) => { s.setHeader("content-type", "text/html"); // A login-style page whose password-manager host exists from first parse. - s.end(q.url === "/login" ? LOGIN_FIXTURE : FIXTURE); + s.end(q.url === "/login" ? LOGIN_FIXTURE : q.url === "/jev" ? JEV_FIXTURE : FIXTURE); }); await new Promise((r) => server?.listen(0, "127.0.0.1", r)); url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/`; @@ -390,4 +416,29 @@ describe.skipIf(!CHROME)("pointer + key input in a real browser", () => { await pressKey({ tabId: 1, key: "q" }); expect(await evaluate("document.getElementById('field').value")).toBe("q"); }); + + type Snap = NonNullable>; + const snap = () => evaluate(`(${jevSnapshot})()`); + const nodeOf = (s: Snap, label: string) => s.actions.find((a) => a.label === label)?.node; + + it("jev: lists visible controls, including a fixed-position dialog, and skips the rest", async () => { + await load("/jev"); + const s = await snap(); + const labels = s.actions.map((a) => `${a.kind}:${a.label}`); + expect(labels).toContain("fill:Where from?"); + expect(labels).toContain("fill:Where to?"); + expect(labels).toContain("click:Accept cookies"); + expect(labels).toContain("select:Cabin → Business"); + expect(labels.join()).not.toMatch(/Password|Disabled|Hidden|secret/); + expect(s.actions.find((a) => a.label === "Where from?")?.value).toBe("San Francisco"); + expect(s.text).toContain("Flights"); + }); + + it("jev: a node keeps its id across snapshots", async () => { + await load("/jev"); + const a = nodeOf(await snap(), "Search"); + const b = nodeOf(await snap(), "Search"); + expect(a).toBeDefined(); + expect(a).toBe(b); + }); }); From 638dfca8ada34cc87b4333860de80897e680d902 Mon Sep 17 00:00:00 2001 From: Karn Date: Sun, 27 Sep 2026 03:50:30 +0530 Subject: [PATCH 4/6] feat(extension): jev_observe and jev_act with dialog tracking Co-Authored-By: Claude Opus 5.5 --- packages/extension/src/lib/dispatch.test.ts | 23 +++ packages/extension/src/lib/dispatch.ts | 5 + packages/extension/src/lib/jev.test.ts | 44 +++++ packages/extension/src/lib/jev.ts | 151 ++++++++++++++++++ .../extension/src/lib/pointer.browser.test.ts | 55 ++++++- 5 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 packages/extension/src/lib/jev.test.ts create mode 100644 packages/extension/src/lib/jev.ts diff --git a/packages/extension/src/lib/dispatch.test.ts b/packages/extension/src/lib/dispatch.test.ts index 01ba86a..b5b5af3 100644 --- a/packages/extension/src/lib/dispatch.test.ts +++ b/packages/extension/src/lib/dispatch.test.ts @@ -22,6 +22,17 @@ vi.mock("./cdp.js", () => ({ cdpWaitFor: vi.fn(async () => ({ ok: true })), })); +vi.mock("./jev.js", () => ({ + jevObserve: vi.fn(async () => ({ + url: "https://x.com/", + title: "", + text: "", + visible: true, + actions: [], + })), + jevAct: vi.fn(async () => ({ ok: true })), +})); + vi.mock("./reload.js", () => ({ reloadExtension: vi.fn(async () => ({ reloading: true, version: "9.9.9" })), })); @@ -54,6 +65,7 @@ vi.mock("./policy.js", async (importOriginal) => { import { cdpClick } from "./cdp.js"; import { dispatchMethod, dispatchWithMeta } from "./dispatch.js"; +import { jevAct } from "./jev.js"; import { ensureAllowed, PolicyDenied, policy, tightenPolicy } from "./policy.js"; /** chrome stub with enough tabs API for the gate (tabs.get → host). */ @@ -163,6 +175,17 @@ describe("dispatchMethod routing (CDP)", () => { stubTabs(); expect(await dispatchMethod("wait_for", { selector: "#btn" })).toEqual({ ok: true }); }); + it("routes jev_observe to the handler", async () => { + stubTabs(); + expect(await dispatchMethod("jev_observe", {})).toMatchObject({ url: "https://x.com/" }); + }); + it("routes jev_act to the handler with the tab pinned", async () => { + stubTabs(); + const out = await dispatchWithMeta("jev_act", { op: "click", node: 3 }); + expect(out.result).toEqual({ ok: true }); + expect(jevAct).toHaveBeenCalledWith({ op: "click", node: 3, tabId: 1 }); + expect(out.meta).toMatchObject({ tabId: 1, host: "x.com" }); + }); }); describe("dispatchMethod routing (chrome.tabs)", () => { diff --git a/packages/extension/src/lib/dispatch.ts b/packages/extension/src/lib/dispatch.ts index c61b24e..cc56f12 100644 --- a/packages/extension/src/lib/dispatch.ts +++ b/packages/extension/src/lib/dispatch.ts @@ -20,6 +20,7 @@ import { cdpWaitFor, resolveTabId, } from "./cdp.js"; +import { jevAct, jevObserve } from "./jev.js"; import { readConsole, readNetwork } from "./monitor.js"; import { cdpRaw, @@ -196,6 +197,10 @@ async function runHandler(method: GatedMethod, gated: Record): return handleDialog(gated as Parameters[0]); case "cdp": return cdpRaw(gated as Parameters[0]); + case "jev_observe": + return jevObserve(gated as Parameters[0]); + case "jev_act": + return jevAct(gated as Parameters[0]); default: throw new Error(`unknown method: ${method}`); } diff --git a/packages/extension/src/lib/jev.test.ts b/packages/extension/src/lib/jev.test.ts new file mode 100644 index 0000000..ab841c9 --- /dev/null +++ b/packages/extension/src/lib/jev.test.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("./monitor.js", () => ({ isMonitored: () => false })); + +import { initDialogTracking, jevObserve, openDialog } from "./jev.js"; + +afterEach(() => vi.unstubAllGlobals()); + +function stubChrome() { + let onEvent: ((s: { tabId?: number }, m: string, p?: unknown) => void) | undefined; + const sendCommand = vi.fn(); + vi.stubGlobal("chrome", { + debugger: { + attach: vi.fn(), + detach: vi.fn(), + sendCommand, + onDetach: { addListener: () => {} }, + onEvent: { addListener: (fn: typeof onEvent) => (onEvent = fn) }, + }, + }); + initDialogTracking(); + return { + fire: (tabId: number, m: string, p?: unknown) => onEvent?.({ tabId }, m, p), + sendCommand, + }; +} + +describe("dialog tracking", () => { + it("remembers an open dialog until it closes", () => { + const { fire } = stubChrome(); + fire(4, "Page.javascriptDialogOpening", { type: "confirm", message: "Leave?" }); + expect(openDialog(4)).toEqual({ type: "confirm", message: "Leave?" }); + fire(4, "Page.javascriptDialogClosed", {}); + expect(openDialog(4)).toBeUndefined(); + }); + + it("observe reports the dialog without touching the blocked page", async () => { + const { fire, sendCommand } = stubChrome(); + fire(9, "Page.javascriptDialogOpening", { type: "alert", message: "Hi" }); + const obs = await jevObserve({ tabId: 9 }); + expect(obs.dialog).toEqual({ type: "alert", message: "Hi" }); + expect(sendCommand).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/extension/src/lib/jev.ts b/packages/extension/src/lib/jev.ts new file mode 100644 index 0000000..b4c65d5 --- /dev/null +++ b/packages/extension/src/lib/jev.ts @@ -0,0 +1,151 @@ +import type { + JevActParams, + JevActResult, + JevDialog, + JevObservation, + JevObserveParams, +} from "@reins/protocol"; +import { actionablePoint, drivePage, ensureVisible, pressAt, resolveTabId, send } from "./cdp.js"; +import { jevCheck, jevSelect, jevSettle, jevSnapshot } from "./jev-snapshot.js"; + +/** reins do waits this long for a covered or moving target, then re-reads the page. */ +export const JEV_ACTION_TIMEOUT_MS = 500; + +const DIALOGS = new Map(); + +/** + * Track JS dialogs per tab. An open alert/confirm/prompt blocks every + * Runtime.evaluate, so observe must know about it *before* touching the page. + * Page events flow on sessions armed by the autofill guard (drivePage). + */ +export function initDialogTracking(): void { + chrome.debugger.onEvent.addListener((source, method, params) => { + const tabId = source.tabId; + if (tabId === undefined) return; + if (method === "Page.javascriptDialogOpening") { + const p = (params ?? {}) as { type?: string; message?: string }; + DIALOGS.set(tabId, { type: p.type ?? "dialog", message: p.message ?? "" }); + } else if (method === "Page.javascriptDialogClosed") { + DIALOGS.delete(tabId); + } + }); + chrome.debugger.onDetach.addListener((source) => { + if (source.tabId !== undefined) DIALOGS.delete(source.tabId); + }); +} + +try { + initDialogTracking(); +} catch { + // `chrome` not available yet (unit tests stub it per case). +} + +export function openDialog(tabId: number): JevDialog | undefined { + return DIALOGS.get(tabId); +} + +async function evaluate(tabId: number, expression: string, awaitPromise = false): Promise { + const res = await send<{ result: { value: T }; exceptionDetails?: { text?: string } }>( + tabId, + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise }, + ); + if (res.exceptionDetails) { + throw new Error(`page script failed: ${res.exceptionDetails.text ?? "exception"}`); + } + return res.result.value; +} + +export async function jevObserve(params: JevObserveParams): Promise { + const tabId = await resolveTabId(params.tabId); + const dialog = openDialog(tabId); + if (dialog) return { url: "", title: "", text: "", visible: true, actions: [], dialog }; + return drivePage(tabId, async () => { + // A navigating document has no body yet: give it up to ~0.5 s. + for (let i = 0; i < 10; i++) { + const snap = await evaluate | null>( + tabId, + `(${jevSnapshot})()`, + ).catch(() => null); + if (snap) return snap; + await new Promise((r) => setTimeout(r, 50)); + } + throw new Error("the page did not finish loading"); + }); +} + +export async function jevAct(params: JevActParams): Promise { + const tabId = await resolveTabId(params.tabId); + if (params.op === "wait") { + await new Promise((r) => setTimeout(r, 250)); + return { ok: true }; + } + return drivePage(tabId, async () => { + await ensureVisible(tabId); + if (params.op === "scroll") { + const { w, h } = await evaluate<{ w: number; h: number }>( + tabId, + "({ w: innerWidth, h: innerHeight })", + ); + await send(tabId, "Input.dispatchMouseEvent", { + type: "mouseWheel", + x: Math.round(w / 2), + y: Math.round(h / 2), + deltaX: 0, + deltaY: params.delta ?? 560, + }); + await evaluate(tabId, `(${jevSettle})(null, false)`, true).catch(() => {}); + return { ok: true }; + } + const node = params.node; + if (node === undefined) throw new Error(`jev_act ${params.op} needs a node`); + const check = await evaluate(tabId, `(${jevCheck})(${node})`); + if (check !== "ok") return { stale: true, reason: check }; + + if (params.op === "select") { + const r = await evaluate( + tabId, + `(${jevSelect})(${node}, ${JSON.stringify(params.value ?? "")})`, + ); + // A select is a mutation of uncertain outcome if it fails midway: never retry it. + if (r !== "ok") throw new Error(`select failed: ${r} — check the page before retrying`); + await evaluate(tabId, `(${jevSettle})(${node}, false)`, true).catch(() => {}); + return { ok: true }; + } + + let point: { x: number; y: number }; + try { + point = await actionablePoint( + tabId, + `the element reins do chose (node ${node})`, + params.op === "type" ? "type into" : "click", + true, + { node, timeoutMs: JEV_ACTION_TIMEOUT_MS }, + ); + } catch (err) { + // Covered, moving or gone: nothing happened yet, so re-reading is safe. + return { stale: true, reason: err instanceof Error ? err.message : String(err) }; + } + await pressAt(tabId, point.x, point.y, `node ${node}`); + if (params.op === "type") { + const modifiers = /Mac/i.test(navigator.platform) ? 4 : 2; // Meta on macOS, Ctrl elsewhere + await send(tabId, "Input.dispatchKeyEvent", { + type: "keyDown", + key: "a", + code: "KeyA", + modifiers, + commands: ["selectAll"], + }); + await send(tabId, "Input.dispatchKeyEvent", { + type: "keyUp", + key: "a", + code: "KeyA", + modifiers, + }); + await send(tabId, "Input.insertText", { text: params.text ?? "" }); + } + // Settling is read-only; a navigation mid-settle is fine. + await evaluate(tabId, `(${jevSettle})(${node}, ${params.op === "type"})`, true).catch(() => {}); + return { ok: true }; + }); +} diff --git a/packages/extension/src/lib/pointer.browser.test.ts b/packages/extension/src/lib/pointer.browser.test.ts index e865d4a..d44f1c1 100644 --- a/packages/extension/src/lib/pointer.browser.test.ts +++ b/packages/extension/src/lib/pointer.browser.test.ts @@ -20,6 +20,7 @@ vi.mock("./monitor.js", () => ({ isMonitored: () => false })); import { autofillGuard } from "./autofill-guard.js"; import { __resetDebugSessions, cdpClick, cdpType } from "./cdp.js"; +import { jevAct, jevObserve } from "./jev.js"; import { jevSnapshot } from "./jev-snapshot.js"; import { handleDialog, hover, pressKey } from "./page-actions.js"; @@ -81,7 +82,7 @@ const JEV_FIXTURE = `