Skip to content

Commit d5e1dea

Browse files
committed
fix(browser): keep the agent tab composited when no panel shows it
A tab view that is in no window has no compositor surface, so CDP mouse input on it never acknowledges and screenshots never capture. The agent's tab was left that way whenever the user viewed another tab or had not opened the browser panel: clicks failed or stalled five seconds and screenshots failed. Each chat's agent tab is now parked invisibly in the main window, adopted in place when the panel shows it, and released on every tab teardown path.
1 parent f6035de commit d5e1dea

3 files changed

Lines changed: 164 additions & 34 deletions

File tree

‎apps/desktop/src/main/browser-agent/panel.ts‎

Lines changed: 91 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,14 @@ interface OccludablePanelFrame {
103103
}
104104
/** Geometry of the painted frame that is currently allowed to replace the view. */
105105
let occludableFrame: OccludablePanelFrame | null = null
106+
/**
107+
* Each chat's agent tab, kept in a window even while no panel shows it. A view that is in no
108+
* window has no compositor surface: CDP input on it never acknowledges and captures never
109+
* complete. Parked invisibly, it renders exactly like the hidden panel view does.
110+
*/
111+
const agentViews = new Map<string, WebContentsView>()
112+
/** Agent views parked invisibly, with the window each one is parked in. */
113+
const parkedViews = new Map<WebContentsView, BrowserWindow>()
106114
/** The host window whose `resize` currently drives {@link layout}, if any. */
107115
let resizeBoundWindow: BrowserWindow | null = null
108116
/** Captures nothing, so one instance serves every window it is bound to. */
@@ -114,6 +122,8 @@ export function initPanel(panelHost: PanelHost): void {
114122
// by the next session: a stale owner window that rejects legitimate panel
115123
// updates, a lease timer polling for a panel that no longer exists, a
116124
// `lastApplied*` value that dedupes away the first layout of the new one.
125+
agentViews.clear()
126+
for (const view of [...parkedViews.keys()]) unparkView(view)
117127
detachAttachedView()
118128
resetOcclusion()
119129
if (leaseTimer !== null) {
@@ -299,6 +309,11 @@ function detachAttachedView(): void {
299309
if (!view || !win) return
300310
try {
301311
if (win.isDestroyed() || view.webContents.isDestroyed()) return
312+
if (isAgentView(view)) {
313+
view.setVisible(false)
314+
parkedViews.set(view, win)
315+
return
316+
}
302317
win.contentView.removeChildView(view)
303318
} catch (error) {
304319
logger.warn('Could not detach embedded browser view', {
@@ -342,9 +357,67 @@ function hideAttachedView(): void {
342357
* tab must not pull the visible tab out of the window.
343358
*/
344359
export function detachIfAttached(view: WebContentsView): void {
360+
for (const [scopeId, agentView] of agentViews) {
361+
if (agentView === view) agentViews.delete(scopeId)
362+
}
345363
if (attachedView === view) {
346364
detachAttachedView()
347365
}
366+
unparkView(view)
367+
}
368+
369+
/**
370+
* Registers the tab a chat's agent drives, so it stays composited while the panel shows
371+
* another tab or no panel is open. Null releases the chat's previous agent tab.
372+
*/
373+
export function setAgentView(scopeId: string, view: WebContentsView | null): void {
374+
const previous = agentViews.get(scopeId)
375+
if (view) agentViews.set(scopeId, view)
376+
else agentViews.delete(scopeId)
377+
if (previous && previous !== view && !isAgentView(previous)) unparkView(previous)
378+
parkAgentViews()
379+
}
380+
381+
function isAgentView(view: WebContentsView): boolean {
382+
for (const agentView of agentViews.values()) {
383+
if (agentView === view) return true
384+
}
385+
return false
386+
}
387+
388+
/** Parks every agent view that no window holds, keeping renderer focus where it was. */
389+
function parkAgentViews(): void {
390+
const win = host.getMainWindow()
391+
if (!win || win.isDestroyed()) return
392+
for (const view of agentViews.values()) {
393+
if (view === attachedView || view.webContents.isDestroyed()) continue
394+
const parkedIn = parkedViews.get(view)
395+
if (parkedIn && !parkedIn.isDestroyed()) continue
396+
const rendererHadFocus = !win.webContents.isDestroyed() && win.webContents.isFocused()
397+
try {
398+
view.setVisible(false)
399+
win.contentView.addChildView(view)
400+
parkedViews.set(view, win)
401+
} catch (error) {
402+
logger.warn('Could not park the agent browser view', {
403+
error: getErrorMessage(error, 'unknown'),
404+
})
405+
}
406+
if (rendererHadFocus) win.webContents.focus()
407+
}
408+
}
409+
410+
function unparkView(view: WebContentsView): void {
411+
const win = parkedViews.get(view)
412+
if (!win) return
413+
parkedViews.delete(view)
414+
try {
415+
if (!win.isDestroyed() && !view.webContents.isDestroyed()) win.contentView.removeChildView(view)
416+
} catch (error) {
417+
logger.warn('Could not unpark the agent browser view', {
418+
error: getErrorMessage(error, 'unknown'),
419+
})
420+
}
348421
}
349422

350423
/**
@@ -393,22 +466,31 @@ export function layout(): void {
393466
}
394467
if (!showing || !active || !win || panelBounds === null) {
395468
hideAttachedView()
469+
parkAgentViews()
396470
return
397471
}
398472

399473
if (attachedView !== active.view) {
400-
// addChildView hands keyboard focus to the newly attached WebContentsView.
401-
// Agent-driven attaches happen while the user may be typing in the chat
402-
// composer, so if the renderer held focus before the attach, give it back —
403-
// automation drives the page over CDP and never needs OS focus.
404-
const rendererHadFocus = !win.webContents.isDestroyed() && win.webContents.isFocused()
405-
win.contentView.addChildView(active.view)
474+
// A parked agent view already sits in this window; adopting it in place avoids the
475+
// blank repaint a remove-and-add costs.
476+
const parkedIn = parkedViews.get(active.view)
477+
parkedViews.delete(active.view)
478+
if (parkedIn !== win) {
479+
if (parkedIn && !parkedIn.isDestroyed()) parkedIn.contentView.removeChildView(active.view)
480+
// addChildView hands keyboard focus to the newly attached WebContentsView.
481+
// Agent-driven attaches happen while the user may be typing in the chat
482+
// composer, so if the renderer held focus before the attach, give it back —
483+
// automation drives the page over CDP and never needs OS focus.
484+
const rendererHadFocus = !win.webContents.isDestroyed() && win.webContents.isFocused()
485+
win.contentView.addChildView(active.view)
486+
if (rendererHadFocus) {
487+
win.webContents.focus()
488+
}
489+
}
406490
hostedWindow = win
407491
attachedView = active.view
408-
if (rendererHadFocus) {
409-
win.webContents.focus()
410-
}
411492
}
493+
parkAgentViews()
412494
bindHostResize(win)
413495
const zoom = win.webContents.getZoomFactor()
414496
const [contentWidth, contentHeight] = win.getContentSize()

‎apps/desktop/src/main/browser-agent/session.test.ts‎

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2398,14 +2398,18 @@ describe('browser-agent session', () => {
23982398

23992399
it('gives background automation a viewport without taking panel ownership', () => {
24002400
const tab = session.withBrowserScope('background-chat', () => session.ensureTab())
2401+
const view = tab.view as unknown as MockView
24012402

24022403
expect(tab.view.setBounds).toHaveBeenCalledWith({
24032404
x: 0,
24042405
y: 0,
24052406
width: 1180,
24062407
height: 850,
24072408
})
2408-
expect(win.contentView.addChildView).not.toHaveBeenCalledWith(tab.view)
2409+
// Parked invisibly so input and captures on it complete; it never shows.
2410+
expect(win.contentView.addChildView).toHaveBeenCalledWith(tab.view)
2411+
expect(view.setVisible).toHaveBeenCalledWith(false)
2412+
expect(view.setVisible).not.toHaveBeenCalledWith(true)
24092413
expect(session.getActiveBrowserScopeId()).toBe('chat-test')
24102414
})
24112415

@@ -2422,11 +2426,16 @@ describe('browser-agent session', () => {
24222426
const content = (win as unknown as { contentView: { addChildView: ReturnType<typeof vi.fn> } })
24232427
.contentView
24242428

2425-
// No bounds yet: the view is not attached to the window.
2426-
expect(content.addChildView).not.toHaveBeenCalledWith(tab.view)
2429+
// No bounds yet: the agent's view is parked invisibly, never shown.
2430+
expect(content.addChildView).toHaveBeenCalledWith(tab.view)
2431+
expect(view.setVisible).toHaveBeenCalledWith(false)
2432+
expect(view.setVisible).not.toHaveBeenCalledWith(true)
24272433

2434+
// Bounds arrive: the parked view is adopted in place, not re-added.
2435+
content.addChildView.mockClear()
24282436
panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 })
2429-
expect(content.addChildView).toHaveBeenCalledWith(tab.view)
2437+
expect(content.addChildView).not.toHaveBeenCalled()
2438+
expect(view.setVisible).toHaveBeenLastCalledWith(true)
24302439
expect(view.setBounds).toHaveBeenCalledWith({ x: 100, y: 50, width: 800, height: 600 })
24312440

24322441
// Panel hidden: the view stops painting but stays attached. Detaching
@@ -2467,11 +2476,48 @@ describe('browser-agent session', () => {
24672476

24682477
const second = session.addTab()
24692478

2470-
// Hiding keeps a view attached, but a tab switch still has to detach:
2471-
// two native views stacked in the window would composite over each other.
2472-
expect(content.removeChildView).toHaveBeenCalledWith(first.view)
2479+
// A tab switch hides the previous view so two native views never composite over
2480+
// each other; the agent still drives the first tab, so it stays parked, not removed.
2481+
expect(content.removeChildView).not.toHaveBeenCalledWith(first.view)
2482+
expect(first.view.setVisible).toHaveBeenLastCalledWith(false)
24732483
expect(content.addChildView).toHaveBeenCalledWith(second.view)
24742484
expect(second.view.webContents.invalidate).toHaveBeenCalledOnce()
2485+
2486+
// A tab the agent does not drive is removed when another tab takes over.
2487+
content.removeChildView.mockClear()
2488+
session.switchTab(first.id)
2489+
expect(content.removeChildView).toHaveBeenCalledWith(second.view)
2490+
})
2491+
2492+
it('keeps the agent tab composited while the user views another tab, and releases it on close', () => {
2493+
const agentTab = session.ensureTab()
2494+
panel.setPanelBounds({ x: 0, y: 0, width: 800, height: 600 })
2495+
const content = (
2496+
win as unknown as {
2497+
contentView: {
2498+
addChildView: ReturnType<typeof vi.fn>
2499+
removeChildView: ReturnType<typeof vi.fn>
2500+
}
2501+
}
2502+
).contentView
2503+
const userTab = session.addTab()
2504+
const agentView = agentTab.view as unknown as MockView
2505+
2506+
// The user's tab is visible; the agent's tab is parked, still in the window.
2507+
expect(content.removeChildView).not.toHaveBeenCalledWith(agentTab.view)
2508+
expect(agentView.setVisible).toHaveBeenLastCalledWith(false)
2509+
2510+
// Switching back adopts the parked view in place.
2511+
content.addChildView.mockClear()
2512+
session.switchTab(agentTab.id)
2513+
expect(content.addChildView).not.toHaveBeenCalledWith(agentTab.view)
2514+
expect(agentView.setVisible).toHaveBeenLastCalledWith(true)
2515+
2516+
// Closing the agent tab while the user views another tab removes its parked view.
2517+
session.switchTab(userTab.id)
2518+
content.removeChildView.mockClear()
2519+
session.closeTab(agentTab.id)
2520+
expect(content.removeChildView).toHaveBeenCalledWith(agentTab.view)
24752521
})
24762522

24772523
// The measured report is the sole writer of bounds. A main-process

‎apps/desktop/src/main/browser-agent/session.ts‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ import {
6262
migratePanelScope,
6363
panelUpdateAllowed,
6464
panelWindow,
65+
setAgentView,
6566
} from '@/main/browser-agent/panel'
6667
import {
6768
agentAppOrigin,
@@ -2155,7 +2156,7 @@ function createFreshTabView(appSession: BrowserAppSession | undefined): WebConte
21552156
preload: join(__dirname, 'browser-preload.cjs'),
21562157
// Throttled by default: a hidden tab should idle. The one exception is
21572158
// the active tab while a tool waits on it, applied explicitly by
2158-
// applyActiveTabThrottling — never blanket across every tab.
2159+
// applyAutomationTabPolicy — never blanket across every tab.
21592160
backgroundThrottling: true,
21602161
spellcheck: false,
21612162
// The default every origin this tab visits starts at; a per-origin zoom
@@ -2476,7 +2477,7 @@ export function hasSession(): boolean {
24762477
export function setAutomationActive(active: boolean): void {
24772478
if (currentScope.automationActive === active) return
24782479
currentScope.automationActive = active
2479-
applyActiveTabThrottling()
2480+
applyAutomationTabPolicy()
24802481
events?.onTabsChanged()
24812482
}
24822483

@@ -2486,26 +2487,27 @@ export function setAutomationNeedsAttention(needsAttention: boolean): void {
24862487
events?.onTabsChanged()
24872488
}
24882489

2489-
/**
2490-
* Unthrottles the active tab while automation is active, and throttles every
2491-
* other tab. Call after anything that changes which tab is active, so the
2492-
* exemption follows the active tab rather than being stranded on the old one.
2493-
*/
24942490
/**
24952491
* Re-applies the tab throttling policy after a caller temporarily suspended it
24962492
* (the panel's reveal pulse). Exempts the automation-active tab exactly as the
24972493
* internal policy does.
24982494
*/
24992495
export function reassertTabThrottling(): void {
2500-
applyActiveTabThrottling()
2496+
applyAutomationTabPolicy()
25012497
}
25022498

2503-
function applyActiveTabThrottling(): void {
2499+
/**
2500+
* Unthrottles the automation tab while automation is active, throttles every other tab, and
2501+
* keeps the automation tab composited while no panel shows it. Call after anything that changes
2502+
* which tab the agent drives, so neither follows a stale tab.
2503+
*/
2504+
function applyAutomationTabPolicy(): void {
25042505
for (const tab of tabs) {
25052506
if (tab.view.webContents.isDestroyed()) continue
25062507
const exempt = currentScope.automationActive && tab.id === currentScope.automationTabId
25072508
tab.view.webContents.setBackgroundThrottling(!exempt)
25082509
}
2510+
setAgentView(getBrowserScopeId(), automationTab()?.view ?? null)
25092511
}
25102512

25112513
/** A closed target must not transfer its activity marker to a replacement tab. */
@@ -2643,7 +2645,7 @@ function addTabInternal({
26432645
revokeTabMediaPermissions(previousActiveTab, false)
26442646
}
26452647
currentScope.activeTabId = tab.id
2646-
applyActiveTabThrottling()
2648+
applyAutomationTabPolicy()
26472649
if (!currentScope.restoring) layout()
26482650
if (transferBrowserFocus) currentScope.focusedBrowserTabId = tab.id
26492651
if (notify && !currentScope.restoring) events?.onActiveTabChanged(tab.view.webContents)
@@ -2923,7 +2925,7 @@ export function tabForNavigation(
29232925
detachIfAttached(tab.view)
29242926
tab.view = view
29252927
contents.close()
2926-
applyActiveTabThrottling()
2928+
applyAutomationTabPolicy()
29272929
layout()
29282930
events?.onActiveTabChanged(view.webContents)
29292931
return view.webContents
@@ -3058,13 +3060,13 @@ export function restoreBrowserSession(): void {
30583060
state.lastPersistedSnapshot = previousState.lastPersistedSnapshot
30593061
if (previousDownloads) browserDownloadsByScope.set(scopeId, previousDownloads)
30603062
else browserDownloadsByScope.delete(scopeId)
3061-
applyActiveTabThrottling()
3063+
applyAutomationTabPolicy()
30623064
throw error
30633065
} finally {
30643066
state.restoring = false
30653067
}
30663068

3067-
applyActiveTabThrottling()
3069+
applyAutomationTabPolicy()
30683070
const restoredActive = restoredLoads.find(({ tab }) => tab.id === state.activeTabId)
30693071
if (restoredActive) {
30703072
pendingForegroundTabRestores.push(
@@ -3099,7 +3101,7 @@ export function addAutomationTab(url?: string, popup?: PopupWindowOptions): Agen
30993101
restoreBrowserSession()
31003102
const tab = addTabInternal({ activate: false, notify: false, url, popup })
31013103
currentScope.automationTabId = tab.id
3102-
applyActiveTabThrottling()
3104+
applyAutomationTabPolicy()
31033105
persistBrowserSession()
31043106
events?.onTabsChanged()
31053107
return tab
@@ -3113,7 +3115,7 @@ export function ensureAutomationTab(): AgentTab {
31133115
tab = activeTab()
31143116
if (tab) {
31153117
currentScope.automationTabId = tab.id
3116-
applyActiveTabThrottling()
3118+
applyAutomationTabPolicy()
31173119
events?.onTabsChanged()
31183120
return tab
31193121
}
@@ -3174,7 +3176,7 @@ export function switchTab(tabId: string, { claim = true }: { claim?: boolean } =
31743176
promotePendingTabRestore(tab)
31753177
// Visible selection does not move the automation exemption; the user may
31763178
// inspect another page while a tool continues in its background tab.
3177-
applyActiveTabThrottling()
3179+
applyAutomationTabPolicy()
31783180
layout()
31793181
if (transferBrowserFocus) currentScope.focusedBrowserTabId = tab.id
31803182
persistBrowserSession()
@@ -3189,7 +3191,7 @@ export function switchAutomationTab(tabId: string): AgentTab {
31893191
const tab = tabs.find((entry) => entry.id === tabId)
31903192
if (!tab) throw new SessionError(`No tab with id ${tabId} — call browser_list_tabs.`)
31913193
currentScope.automationTabId = tab.id
3192-
applyActiveTabThrottling()
3194+
applyAutomationTabPolicy()
31933195
events?.onTabsChanged()
31943196
return tab
31953197
}
@@ -3271,7 +3273,7 @@ function removeTab(
32713273
const opener = tabs.find((entry) => entry.id === tab.openerTabId)
32723274
currentScope.automationTabId =
32733275
opener?.id ?? (adoptNeighborForAgent ? ((tabs[index] ?? tabs[index - 1])?.id ?? null) : null)
3274-
applyActiveTabThrottling()
3276+
applyAutomationTabPolicy()
32753277
}
32763278
if (transferBrowserFocus) currentScope.focusedBrowserTabId = currentScope.activeTabId
32773279
persistBrowserSession()

0 commit comments

Comments
 (0)