diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml index 4feb7b91318..7632b0bea66 100644 --- a/.github/workflows/desktop-e2e.yml +++ b/.github/workflows/desktop-e2e.yml @@ -17,7 +17,7 @@ concurrency: jobs: e2e: name: E2E (${{ matrix.electron }}) - runs-on: macos-14 + runs-on: macos-26 strategy: fail-fast: false matrix: @@ -58,7 +58,7 @@ jobs: package-smoke: name: Unsigned package smoke - runs-on: macos-14 + runs-on: macos-26 steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index d8a9e650d89..17d7e298a5d 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -49,7 +49,7 @@ permissions: jobs: build-sign-notarize: name: Build, Sign, Notarize - runs-on: macos-14 + runs-on: macos-26 steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -167,8 +167,8 @@ jobs: if: ${{ inputs.sign }} run: | DMG="$(ls apps/desktop/release/*.dmg | head -1)" - xcrun stapler validate "$DMG" hdiutil attach "$DMG" -mountpoint /tmp/sim-dmg -nobrowse -quiet + xcrun stapler validate /tmp/sim-dmg/*.app spctl --assess --type execute --verbose /tmp/sim-dmg/*.app codesign --verify --deep --strict /tmp/sim-dmg/*.app hdiutil detach /tmp/sim-dmg -quiet diff --git a/apps/desktop/build/dmg-background.png b/apps/desktop/build/dmg-background.png new file mode 100644 index 00000000000..a79f4414535 Binary files /dev/null and b/apps/desktop/build/dmg-background.png differ diff --git a/apps/desktop/build/dmg-background@2x.png b/apps/desktop/build/dmg-background@2x.png new file mode 100644 index 00000000000..2b0f9170246 Binary files /dev/null and b/apps/desktop/build/dmg-background@2x.png differ diff --git a/apps/desktop/build/entitlements.mac.plist b/apps/desktop/build/entitlements.mac.plist index 446fe171da8..152179c4392 100644 --- a/apps/desktop/build/entitlements.mac.plist +++ b/apps/desktop/build/entitlements.mac.plist @@ -4,5 +4,10 @@ com.apple.security.cs.allow-jit + + com.apple.security.device.audio-input + diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index c13ceaaa30c..b7e130d82de 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -54,12 +54,32 @@ mac: mergeASARs: false hardenedRuntime: true gatekeeperAssess: false + # macOS refuses to show the microphone prompt at all — it kills the process — + # unless the bundle declares why it wants the device. + extendInfo: + NSMicrophoneUsageDescription: Sim uses your microphone for voice input in Chat. entitlements: build/entitlements.mac.plist entitlementsInherit: build/entitlements.mac.plist notarize: true dmg: sign: false + title: ${productName} + # Finder uses the image dimensions as the installer window dimensions. The + # @2x companion is detected automatically and keeps the mountain artwork and + # install arrow sharp on Retina displays. + background: build/dmg-background.png + iconSize: 96 + iconTextSize: 13 + # Keep both icon centers inside the compact 660x420 Finder canvas. + contents: + - x: 165 + y: 210 + type: file + - x: 495 + y: 210 + type: link + path: /Applications # node-pty ships pure N-API prebuilds, which are ABI-stable across Node and # Electron versions, so there is nothing to rebuild against Electron's ABI. diff --git a/apps/desktop/src/main/browser-agent/cdp.ts b/apps/desktop/src/main/browser-agent/cdp.ts index 02f4fe944d0..4a7088e1b24 100644 --- a/apps/desktop/src/main/browser-agent/cdp.ts +++ b/apps/desktop/src/main/browser-agent/cdp.ts @@ -36,6 +36,8 @@ export interface CdpCallbacks { const callbacksByContents = new WeakMap() /** Contents already instrumented (attach survives for the tab's lifetime). */ const instrumented = new WeakSet() +/** Tracks trusted input currently being dispatched by the agent itself. */ +const agentInputDepthByContents = new WeakMap() /** Flattened CDP child-target sessions keyed by their protocol frame/target id. */ const childSessionsByContents = new WeakMap>() const FRAME_WORLD_NAME = 'sim-browser-agent' @@ -62,6 +64,7 @@ async function sendInput( method: string, params: Record ): Promise { + agentInputDepthByContents.set(contents, (agentInputDepthByContents.get(contents) ?? 0) + 1) let timer: NodeJS.Timeout | undefined const timeout = new Promise((_resolve, reject) => { timer = setTimeout( @@ -73,9 +76,17 @@ async function sendInput( await Promise.race([send(contents, method, params), timeout]) } finally { clearTimeout(timer) + const nextDepth = (agentInputDepthByContents.get(contents) ?? 1) - 1 + if (nextDepth > 0) agentInputDepthByContents.set(contents, nextDepth) + else agentInputDepthByContents.delete(contents) } } +/** Distinguishes user input from CDP input when Electron mirrors it as an event. */ +export function isDispatchingAgentInput(contents: WebContents): boolean { + return (agentInputDepthByContents.get(contents) ?? 0) > 0 +} + /** Idempotently instruments a tab's WebContents. */ export async function ensureInstrumented(contents: WebContents, cb: CdpCallbacks): Promise { callbacksByContents.set(contents, cb) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 33875151bf8..f98083e1add 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -144,6 +144,234 @@ describe('executeTool', () => { expect(second.result).toMatchObject({ tabs: [] }) }) + it('keeps a takeover pending when the clock advances beyond twelve hours', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + const now = vi.spyOn(Date, 'now').mockReturnValue(0) + try { + let settled = false + const takeover = driver + .executeTool('chat-test', 'browser_request_takeover', { + reason: 'Please finish in the browser', + }) + .then((result) => { + settled = true + return result + }) + + await vi.advanceTimersByTimeAsync(0) + now.mockReturnValue(13 * 60 * 60 * 1000) + await vi.advanceTimersByTimeAsync(1_500) + expect(settled).toBe(false) + + await driver.handlePanelAction('chat-test', { action: 'takeover-done' }) + await vi.advanceTimersByTimeAsync(1_500) + await expect(takeover).resolves.toMatchObject({ + ok: true, + result: { completed: true }, + }) + } finally { + now.mockRestore() + vi.useRealTimers() + } + }) + + it('cancels the exact takeover and clears its attention state immediately', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + try { + const takeover = driver.executeTool( + 'chat-test', + 'browser_request_takeover', + { reason: 'Please finish in the browser' }, + 'tool-takeover' + ) + await vi.advanceTimersByTimeAsync(0) + expect(session.getTabsState().automationNeedsAttention).toBe(true) + + expect(driver.cancelTool('chat-test', 'tool-takeover')).toBe(true) + expect(session.getTabsState().automationNeedsAttention).toBe(false) + await vi.advanceTimersByTimeAsync(1_500) + + await expect(takeover).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + await expect( + driver.executeTool('chat-test', 'browser_list_tabs', {}, 'tool-takeover') + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + } finally { + vi.useRealTimers() + } + }) + + it('does not let cancelled takeover cleanup clear a newer takeover', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + try { + const firstTakeover = driver.executeTool( + 'chat-test', + 'browser_request_takeover', + { reason: 'First handoff' }, + 'tool-takeover-first' + ) + await vi.advanceTimersByTimeAsync(0) + + expect(driver.cancelTool('chat-test', 'tool-takeover-first')).toBe(true) + await expect(firstTakeover).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + + const secondTakeover = driver.executeTool( + 'chat-test', + 'browser_request_takeover', + { reason: 'Second handoff' }, + 'tool-takeover-second' + ) + await vi.advanceTimersByTimeAsync(0) + expect(session.getTabsState().automationNeedsAttention).toBe(true) + + // Let the detached first takeover observe cancellation and run finally. + await vi.advanceTimersByTimeAsync(1_500) + expect(session.getTabsState().automationNeedsAttention).toBe(true) + + await driver.handlePanelAction('chat-test', { action: 'takeover-done' }) + await vi.advanceTimersByTimeAsync(1_500) + await expect(secondTakeover).resolves.toMatchObject({ + ok: true, + result: { completed: true }, + }) + } finally { + vi.useRealTimers() + } + }) + + it('honors cancellation that arrives before the authorized tool invocation', async () => { + expect(driver.cancelTool('chat-test', 'tool-before-authorization')).toBe(true) + + await expect( + driver.executeTool('chat-test', 'browser_list_tabs', {}, 'tool-before-authorization') + ).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + }) + + it('settles native automation activity immediately when an active tool is cancelled', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + try { + const waiting = driver.executeTool( + 'chat-test', + 'browser_wait_for', + { timeoutMs: 120_000 }, + 'tool-waiting' + ) + await vi.advanceTimersByTimeAsync(0) + expect(session.getTabsState().automationActive).toBe(true) + + expect(driver.cancelActiveTool('chat-test')).toBe(true) + await vi.advanceTimersByTimeAsync(0) + expect(session.getTabsState().automationActive).toBe(false) + await expect(waiting).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + } finally { + vi.useRealTimers() + } + }) + + it('cancels queued pre-boundary tools while allowing later browser work', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + try { + const waiting = driver.executeTool( + 'chat-test', + 'browser_wait_for', + { timeoutMs: 120_000 }, + 'tool-active' + ) + await vi.advanceTimersByTimeAsync(0) + const queuedOpen = driver.executeTool('chat-test', 'browser_open_tab', {}, 'tool-queued') + + expect(driver.cancelActiveTool('chat-test')).toBe(true) + + await expect(waiting).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled'), + }) + await expect(queuedOpen).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + expect(session.getTabsState().tabs).toHaveLength(1) + + await expect( + driver.executeTool('chat-test', 'browser_open_tab', {}, 'tool-after-boundary') + ).resolves.toMatchObject({ ok: true }) + expect(session.getTabsState().tabs).toHaveLength(2) + } finally { + vi.useRealTimers() + } + }) + + it('releases an abandoned takeover when a newer browser action arrives', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + try { + const takeover = driver.executeTool('chat-test', 'browser_request_takeover', { + reason: 'Please finish in the browser', + }) + await vi.advanceTimersByTimeAsync(0) + + const listTabs = driver.executeTool('chat-test', 'browser_list_tabs', {}) + await vi.advanceTimersByTimeAsync(1_500) + + await expect(takeover).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('superseded by a newer browser action'), + }) + await expect(listTabs).resolves.toMatchObject({ + ok: true, + result: { tabs: expect.any(Array) }, + }) + } finally { + vi.useRealTimers() + } + }) + + it('returns a free-text takeover instruction to the browser agent', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + vi.useFakeTimers() + try { + const takeover = driver.executeTool('chat-test', 'browser_request_takeover', { + reason: 'Please pick a match in the draw', + }) + await vi.advanceTimersByTimeAsync(0) + + await driver.handlePanelAction('chat-test', { + action: 'takeover-done', + takeoverResponse: 'Open the second match', + }) + await vi.advanceTimersByTimeAsync(1_500) + + await expect(takeover).resolves.toMatchObject({ + ok: true, + result: { + completed: true, + userInstruction: 'Open the second match', + }, + }) + } finally { + vi.useRealTimers() + } + }) + it('publishes a settled tab when the main frame finishes before subresources', async () => { const onPageState = vi.fn() const onTabsState = vi.fn() @@ -200,6 +428,74 @@ describe('executeTool', () => { expect(refreshAvailability).toHaveBeenCalledWith(true) }) + it('keeps target-blank initiation user-owned while automation is active', async () => { + driver = freshDriver() + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const source = session.requireTab().view.webContents + session.setAutomationActive(true) + const beforeMouse = (source.on as unknown as ReturnType).mock.calls.find( + ([eventName]) => eventName === 'before-mouse-event' + )?.[1] as (event: unknown, mouse: { type: string }) => void + beforeMouse({}, { type: 'mouseDown' }) + const openWindow = vi.mocked(source.setWindowOpenHandler).mock.calls[0]?.[0] as (details: { + url: string + }) => { action: string } + + expect(openWindow({ url: 'https://user-popup.example/' })).toEqual({ action: 'deny' }) + const popup = session.activeTab()?.view.webContents + if (!popup) throw new Error('Expected user popup tab') + expect(session.automationTab()?.view.webContents).toBe(source) + expect(popup.loadURL).toHaveBeenCalledWith('https://user-popup.example/') + }) + + it('keeps agent-opened target-blank tabs agent-owned after dispatch ends', async () => { + driver = freshDriver() + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const source = session.requireTab().view.webContents + session.setAutomationActive(true) + const openWindow = vi.mocked(source.setWindowOpenHandler).mock.calls[0]?.[0] as (details: { + url: string + }) => { action: string } + + expect(openWindow({ url: 'https://agent-popup.example/' })).toEqual({ action: 'deny' }) + const popup = session.requireAutomationTab().view.webContents + expect(session.activeTab()?.view.webContents).toBe(source) + session.setAutomationActive(false) + expect(session.automationTab()?.view.webContents).toBe(popup) + expect(popup.loadURL).toHaveBeenCalledWith('https://agent-popup.example/') + }) + + it('keeps context-menu new tabs user-owned while automation is active', async () => { + driver = freshDriver() + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const source = session.requireTab().view.webContents + session.setAutomationActive(true) + vi.mocked(Menu.buildFromTemplate).mockClear() + const contextMenu = (source.on as unknown as ReturnType).mock.calls.find( + ([eventName]) => eventName === 'context-menu' + )?.[1] as (event: unknown, params: unknown) => void + contextMenu( + {}, + { + selectionText: '', + linkURL: 'https://context-link.example/', + isEditable: false, + editFlags: { canPaste: false }, + } + ) + const template = vi.mocked(Menu.buildFromTemplate).mock.calls.at(-1)?.[0] as + | MenuItemConstructorOptions[] + | undefined + template + ?.find((item) => item.label === 'Open Link in New Tab') + ?.click?.({} as never, undefined as never, {} as never) + + const popup = session.activeTab()?.view.webContents + if (!popup) throw new Error('Expected context-menu tab') + expect(session.automationTab()?.view.webContents).toBe(source) + expect(popup.loadURL).toHaveBeenCalledWith('https://context-link.example/') + }) + it('builds the native toolbar menu and routes renderer-owned actions back to its chat', async () => { await driver.executeTool('chat-test', 'browser_open_tab', {}) const win = new BrowserWindow() diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 33b1584572a..8d3674cbd73 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -72,7 +72,6 @@ const NAVIGATION_SETTLE_MS = 400 const DEFAULT_WAIT_FOR_TIMEOUT_MS = 10_000 const MAX_WAIT_FOR_TIMEOUT_MS = 120_000 const TAKEOVER_POLL_MS = 1_500 -const TAKEOVER_MAX_MS = 12 * 60 * 60 * 1000 /** * Hard ceiling on any single tool execution (takeover excepted): whatever * goes wrong, the Sim side always gets a response. Sits above the longest @@ -114,6 +113,17 @@ interface DriverScopeState { pendingNotices: string[] takeoverActive: boolean takeoverDone: boolean + takeoverResponse: string | null + /** Invocation that currently owns the shared takeover response state. */ + takeoverInvocationEpoch: number | null + /** Monotonic browser-tool invocation id used to supersede an abandoned takeover. */ + toolInvocationEpoch: number + /** Exact client tool currently at the head of this scope's serialized queue. */ + activeToolCallId: string | null + /** Rejects the active queue entry while its underlying Chromium work winds down. */ + activeToolCancel: (() => void) | null + /** Invalidates every invocation already queued when a scope-level cancel establishes a boundary. */ + toolQueueCancellationEpoch: number lastTabsStateFingerprint: string | null toolQueue: Promise /** True while activation is the only operation that has touched this scope. */ @@ -130,11 +140,23 @@ interface DriverScopeState { toolExecutionEpoch: number } +/** Captures the native queue boundary before an async authorization round trip. */ +export interface BrowserToolQueueBoundary { + scopeId: string + cancellationEpoch: number +} + function createDriverScopeState(): DriverScopeState { return { pendingNotices: [], takeoverActive: false, takeoverDone: false, + takeoverResponse: null, + takeoverInvocationEpoch: null, + toolInvocationEpoch: 0, + activeToolCallId: null, + activeToolCancel: null, + toolQueueCancellationEpoch: 0, lastTabsStateFingerprint: null, toolQueue: Promise.resolve(), activationOnly: true, @@ -154,6 +176,22 @@ function invalidateSnapshot(state = driverScopeState()): void { const driverScopeStates = new Map() const driverScopeAliases = new Map() +const CANCELLED_TOOL_TTL_MS = 5 * 60_000 +const MAX_CANCELLED_TOOL_TOMBSTONES = 256 +const cancelledToolCallIds = new Map() + +function pruneCancelledToolCallIds(now = Date.now()): void { + for (const [toolCallId, expiresAt] of cancelledToolCallIds) { + if (expiresAt > now && cancelledToolCallIds.size <= MAX_CANCELLED_TOOL_TOMBSTONES) break + cancelledToolCallIds.delete(toolCallId) + } +} + +function isToolCallCancelled(toolCallId: string | undefined): boolean { + if (!toolCallId) return false + pruneCancelledToolCallIds() + return cancelledToolCallIds.has(toolCallId) +} function resolveDriverScopeId(scopeId: string): string { let resolved = scopeId @@ -175,6 +213,19 @@ function driverScopeState(scopeId = session.getBrowserScopeId()): DriverScopeSta return state } +export function captureBrowserToolQueueBoundary(scopeId: string): BrowserToolQueueBoundary { + const resolvedScopeId = resolveDriverScopeId(scopeId) + return { + scopeId: resolvedScopeId, + cancellationEpoch: driverScopeState(resolvedScopeId).toolQueueCancellationEpoch, + } +} + +function isBrowserToolQueueBoundaryCurrent(boundary: BrowserToolQueueBoundary): boolean { + const state = driverScopeStates.get(resolveDriverScopeId(boundary.scopeId)) + return state?.toolQueueCancellationEpoch === boundary.cancellationEpoch +} + function recordNotice(notice: string): void { const state = driverScopeState() state.activationOnly = false @@ -182,8 +233,8 @@ function recordNotice(notice: string): void { } /** - * True while browser_request_takeover waits on the user. The Done chip on the - * chat's takeover tool row completes it via the `takeover-done` panel action; + * True while browser_request_takeover waits on the user. The question card on + * the chat's takeover tool row completes it via the `takeover-done` panel action; * the state lives here (session-level, not in the page) so it survives * navigations and tab switches. */ @@ -261,7 +312,7 @@ function instrumentTab(contents: WebContents): void { contents.on( 'did-navigate', inScope(() => { - if (session.activeTab()?.view.webContents === contents) { + if (session.automationTab()?.view.webContents === contents) { invalidateSnapshot() } knownSessions?.noteTopLevelNavigation(contents.getURL()) @@ -281,7 +332,7 @@ function instrumentTab(contents: WebContents): void { inScope(() => { if ( event === 'did-navigate-in-page' && - session.activeTab()?.view.webContents === contents + session.automationTab()?.view.webContents === contents ) { invalidateSnapshot() } @@ -309,6 +360,7 @@ export function initDriver( // first tab push as a duplicate. driverScopeStates.clear() driverScopeAliases.clear() + cancelledToolCallIds.clear() // The serialization chain, too. A takeover from the previous session can sit // unresolved indefinitely, and its `takeoverDone` flag is reset above — so // leaving the old chain head in place would queue the new session's first @@ -667,7 +719,7 @@ async function execInPage( )) as Result } if ('frameTreeNodeId' in target && typeof target.frameTreeNodeId === 'number') { - const contents = session.activeTab()?.view.webContents + const contents = session.automationTab()?.view.webContents const frame = target as WebFrameMain if ( !contents || @@ -967,7 +1019,7 @@ async function activeElementState(target: PageExecutionTarget): Promise { const state = driverScopeState() - const tab = session.requireTab() + const tab = session.requireAutomationTab() if (tab.view.webContents !== contents) { throw new ToolError('The active tab changed before the snapshot started. Try again.') } @@ -1473,7 +1526,7 @@ async function captureSnapshot(contents: WebContents, notAfter?: number): Promis const targets = new Map() const targetLineIndexes = new Map() const stillCurrent = (): boolean => { - const active = session.activeTab() + const active = session.automationTab() return ( state.snapshotCaptureEpoch === captureEpoch && active?.id === capturedTabId && @@ -1627,41 +1680,58 @@ async function captureSnapshot(contents: WebContents, notAfter?: number): Promis /** * Hands control to the user: the page is already natively interactive in the - * panel, and the chat's takeover tool row shows the reason with a Done chip. - * The tool resolves when that chip sends the `takeover-done` panel action. + * panel, and the chat's takeover tool row shows the reason as a question. + * The tool resolves when that question sends the `takeover-done` panel action. * Nothing is injected into the page, so nothing covers page content and the * pending state survives navigations. */ -async function runTakeover(purpose: string | undefined): Promise { - const tab = session.ensureTab() +async function runTakeover(purpose: string | undefined, invocationEpoch: number): Promise { + const tab = session.ensureAutomationTab() const contents = tab.view.webContents const state = driverScopeState() state.takeoverActive = true state.takeoverDone = false + state.takeoverResponse = null + state.takeoverInvocationEpoch = invocationEpoch + session.setAutomationNeedsAttention(true) const startedAt = Date.now() try { - while (Date.now() - startedAt < TAKEOVER_MAX_MS) { + for (;;) { await sleep(TAKEOVER_POLL_MS) if (!session.hasSession() || contents.isDestroyed()) { throw new ToolError( 'The browser session was closed during takeover. Ask the user what happened, then reopen with browser_navigate.' ) } + if (state.toolInvocationEpoch !== invocationEpoch) { + throw new ToolError('The browser takeover was superseded by a newer browser action.') + } if (state.takeoverDone) { if (purpose === 'sign_in') { - const activeContents = session.activeTab()?.view.webContents + const activeContents = session.automationTab()?.view.webContents if (activeContents && !activeContents.isDestroyed()) { knownSessions?.noteSignInCompleted(activeContents.getURL()) } } - return { completed: true, elapsedMs: Date.now() - startedAt } + return { + completed: true, + elapsedMs: Date.now() - startedAt, + ...(state.takeoverResponse ? { userInstruction: state.takeoverResponse } : {}), + } } } - throw new ToolError('Takeover timed out after 12 hours without the user finishing.') } finally { - state.takeoverActive = false - state.takeoverDone = false + // Cancellation releases the serialized tool queue before this polling + // loop observes its superseding epoch. Never let that delayed cleanup + // erase a newer takeover that has already claimed the same scope state. + if (state.takeoverInvocationEpoch === invocationEpoch) { + session.setAutomationNeedsAttention(false) + state.takeoverActive = false + state.takeoverDone = false + state.takeoverResponse = null + state.takeoverInvocationEpoch = null + } } } @@ -1669,7 +1739,8 @@ async function executeToolInner( tool: BrowserToolName, params: Record, assertCurrentExecution: () => void, - executionDeadline?: number + executionDeadline: number | undefined, + invocationEpoch: number ): Promise { switch (tool) { case 'browser_navigate': { @@ -1685,7 +1756,7 @@ async function executeToolInner( throw new ToolError(guard.error ?? 'That address was blocked.') } assertCurrentExecution() - const tab = session.ensureTab() + const tab = session.ensureAutomationTab() const contents = tab.view.webContents assertCurrentExecution() return await loadUrlAndGetResult(contents, url) @@ -1702,7 +1773,7 @@ async function executeToolInner( throw new ToolError(guard.error ?? 'That address was blocked.') } assertCurrentExecution() - const tab = session.ensureTab() + const tab = session.ensureAutomationTab() const contents = tab.view.webContents assertCurrentExecution() const nav = await loadUrlAndGetResult(contents, url) @@ -1718,7 +1789,7 @@ async function executeToolInner( case 'browser_go_back': case 'browser_go_forward': { invalidateSnapshot() - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const history = contents.navigationHistory assertCurrentExecution() let completion: Promise @@ -1746,7 +1817,7 @@ async function executeToolInner( } } assertCurrentExecution() - const tab = session.addTab() + const tab = session.addAutomationTab() const contents = tab.view.webContents if (url) { assertCurrentExecution() @@ -1758,7 +1829,7 @@ async function executeToolInner( case 'browser_switch_tab': { invalidateSnapshot() - const tab = session.switchTab(requireStr(params, 'tabId')) + const tab = session.switchAutomationTab(requireStr(params, 'tabId')) const contents = tab.view.webContents return { tabId: tab.id, url: contents.getURL(), title: contents.getTitle() } } @@ -1766,13 +1837,13 @@ async function executeToolInner( case 'browser_close_tab': { invalidateSnapshot() const tabId = requireStr(params, 'tabId') - session.closeTab(tabId) + session.closeAutomationTab(tabId) return { closed: tabId } } case 'browser_list_tabs': { session.restoreBrowserSession() - return session.getTabsState() + return session.getAutomationTabsState() } case 'browser_list_sessions': { @@ -1790,10 +1861,10 @@ async function executeToolInner( await sleep(timeoutMs) return { waitedMs: timeoutMs } } - const waitedTab = session.requireTab() + const waitedTab = session.requireAutomationTab() const contents = waitedTab.view.webContents while (Date.now() - startedAt < timeoutMs) { - const active = session.activeTab() + const active = session.automationTab() if (active?.id !== waitedTab.id || active.view.webContents !== contents) { throw new ToolError( 'The active tab changed while waiting. Start browser_wait_for again on the tab you want to inspect.' @@ -1832,13 +1903,13 @@ async function executeToolInner( } case 'browser_snapshot': { - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents assertCurrentExecution() return await captureSnapshot(contents, executionDeadline) } case 'browser_read_text': { - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const elementId = num(params, 'elementId') if (elementId === undefined) return await readWholePageText(contents, executionDeadline) const target = pageTargetForElement(contents, elementId) @@ -1848,7 +1919,7 @@ async function executeToolInner( } case 'browser_screenshot': { - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const dataUrl = await cdp.captureScreenshot(contents).catch(() => null) if (dataUrl === null) { throw new ToolError( @@ -1866,13 +1937,13 @@ async function executeToolInner( case 'browser_extract': { const instruction = requireStr(params, 'instruction') - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const page = await readWholePageText(contents, executionDeadline) return { instruction, page } } case 'browser_click': { - const clickedTab = session.requireTab() + const clickedTab = session.requireAutomationTab() const contents = clickedTab.view.webContents const elementId = requireNum(params, 'elementId') const target = pageTargetForElement(contents, elementId) @@ -2136,7 +2207,7 @@ async function executeToolInner( const topObservation = observeTopPage ? pageEffect(beforeTopPage, afterTopPage, beforeTopElement, afterTopElement) : observation - const activeTab = session.activeTab() + const activeTab = session.automationTab() const tabChanged = activeTab?.id !== clickedTab.id const effect = { ...observation.effect, @@ -2207,7 +2278,7 @@ async function executeToolInner( const elementId = requireNum(params, 'elementId') const text = requireStr(params, 'text') const submit = params.submit === true - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const target = pageTargetForElement(contents, elementId) const targetFrame = frameExecutionTarget(target, contents) @@ -2519,7 +2590,7 @@ async function executeToolInner( case 'browser_press_key': { const requestedKey = requireStr(params, 'key') const combo = parseKeyCombo(requestedKey) - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const pressedPageUrl = contents.getURL() let target: PageExecutionTarget = focusedPageTarget(contents) let pressedFrameUrl = target === contents ? undefined : (target as WebFrameMain).url @@ -2692,7 +2763,7 @@ async function executeToolInner( } case 'browser_scroll': { - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const elementId = num(params, 'elementId') const target = elementId !== undefined @@ -2728,7 +2799,7 @@ async function executeToolInner( } case 'browser_select_option': { - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const elementId = requireNum(params, 'elementId') const target = pageTargetForElement(contents, elementId) const targetFrame = frameExecutionTarget(target, contents) @@ -2793,7 +2864,7 @@ async function executeToolInner( } case 'browser_hover': { - const contents = session.requireTab().view.webContents + const contents = session.requireAutomationTab().view.webContents const elementId = requireNum(params, 'elementId') const target = pageTargetForElement(contents, elementId) const targetFrame = frameExecutionTarget(target, contents) @@ -2942,7 +3013,7 @@ async function executeToolInner( // The reason renders in the chat's tool row, not here — but require it // so the model always tells the user why control was handed over. requireStr(params, 'reason') - return await runTakeover(str(params, 'purpose')) + return await runTakeover(str(params, 'purpose'), invocationEpoch) } default: { @@ -2971,7 +3042,9 @@ function withNotices(result: unknown): unknown { export async function executeTool( scopeId: string, tool: BrowserToolName, - params: Record + params: Record, + toolCallId?: string, + authorizationBoundary?: BrowserToolQueueBoundary ): Promise<{ ok: boolean; result?: unknown; error?: string }> { const resolvedScopeId = resolveDriverScopeId(scopeId) if (session.isBrowserScopeSuspended(resolvedScopeId)) { @@ -2982,7 +3055,22 @@ export async function executeTool( } const state = driverScopeState(resolvedScopeId) state.activationOnly = false + const invocationEpoch = ++state.toolInvocationEpoch + const queueCancellationEpoch = state.toolQueueCancellationEpoch const run = async () => { + if ( + (authorizationBoundary && !isBrowserToolQueueBoundaryCurrent(authorizationBoundary)) || + queueCancellationEpoch !== state.toolQueueCancellationEpoch || + isToolCallCancelled(toolCallId) + ) { + throw new ToolError('This browser action was cancelled before it started.') + } + state.activeToolCallId = toolCallId ?? null + let cancelActiveExecution: () => void = () => {} + const cancellation = new Promise((_resolve, reject) => { + cancelActiveExecution = () => reject(new ToolError('This browser action was cancelled.')) + }) + state.activeToolCancel = cancelActiveExecution return await session.withBrowserScope(resolvedScopeId, async () => { logger.info('Executing browser tool', { tool, scopeId: resolvedScopeId }) const keepHiddenPageActive = tool !== 'browser_request_takeover' @@ -2998,21 +3086,31 @@ export async function executeTool( throw new ToolError('This browser action expired before it could dispatch input.') } } - const execution = executeToolInner(tool, params, assertCurrentExecution, executionDeadline) - return withNotices( - await (watchdogMs === null + const execution = executeToolInner( + tool, + params, + assertCurrentExecution, + executionDeadline, + invocationEpoch + ) + const guardedExecution = + watchdogMs === null ? execution : raceAgainstWatchdog(execution, watchdogMs, () => { if (state.toolExecutionEpoch === executionEpoch) state.toolExecutionEpoch++ if (tool === 'browser_snapshot' || tool === 'browser_open_url') { invalidateSnapshot(state) } - })) - ) + }) + return withNotices(await Promise.race([guardedExecution, cancellation])) } finally { if (keepHiddenPageActive) { session.setAutomationActive(false) } + if (state.activeToolCancel === cancelActiveExecution) { + state.activeToolCallId = null + state.activeToolCancel = null + } } }) } @@ -3034,6 +3132,40 @@ export async function executeTool( } } +/** + * Cancels one exact browser tool, including a cancellation that arrives while + * its authorization IPC is still in flight. The bounded tombstone lets that + * later invocation observe the stop without retaining call ids indefinitely. + */ +export function cancelTool(scopeId: string, toolCallId: string): boolean { + pruneCancelledToolCallIds() + cancelledToolCallIds.set(toolCallId, Date.now() + CANCELLED_TOOL_TTL_MS) + pruneCancelledToolCallIds() + + const resolvedScopeId = resolveDriverScopeId(scopeId) + const state = driverScopeStates.get(resolvedScopeId) + if (!state || state.activeToolCallId !== toolCallId) return true + + state.toolInvocationEpoch++ + state.toolExecutionEpoch++ + state.activeToolCancel?.() + void session.withBrowserScope(resolvedScopeId, () => { + session.setAutomationActive(false) + session.setAutomationNeedsAttention(false) + }) + return true +} + +/** Cancels the active tool and every older invocation already queued for this scope. */ +export function cancelActiveTool(scopeId: string): boolean { + const resolvedScopeId = resolveDriverScopeId(scopeId) + const state = driverScopeStates.get(resolvedScopeId) + if (!state) return false + state.toolQueueCancellationEpoch++ + const toolCallId = state.activeToolCallId + return toolCallId ? cancelTool(resolvedScopeId, toolCallId) : false +} + /** Browser-chrome commands from the panel header; fire-and-forget. */ export async function handlePanelAction( scopeId: string, @@ -3042,11 +3174,18 @@ export async function handlePanelAction( const resolvedScopeId = resolveDriverScopeId(scopeId) if (session.isBrowserScopeSuspended(resolvedScopeId)) return return await session.withBrowserScope(resolvedScopeId, async () => { - // The Done chip on the chat's takeover tool row: hands control back to the + // The question card on the chat's takeover tool row hands control back to the // agent. Meaningful only while a takeover is actually waiting. if (action.action === 'takeover-done') { const state = driverScopeState() - if (state.takeoverActive) state.takeoverDone = true + if (state.takeoverActive) { + session.returnAutomationTabToAgent() + state.takeoverResponse = + typeof action.takeoverResponse === 'string' && action.takeoverResponse.trim() + ? action.takeoverResponse.trim() + : null + state.takeoverDone = true + } return } // Navigate bootstraps the session: the user can open the panel manually @@ -3054,6 +3193,7 @@ export async function handlePanelAction( // bar. The other chrome actions need an existing page. if (action.action === 'navigate') { if (typeof action.url === 'string' && /^https?:\/\//i.test(action.url)) { + session.claimActiveTabForUser() const contents = session.ensureTab().view.webContents void contents.loadURL(action.url).catch(() => {}) } @@ -3081,6 +3221,7 @@ export async function handlePanelAction( } return } + session.claimActiveTabForUser() const tab = session.activeTab() if (!tab) return const contents = tab.view.webContents diff --git a/apps/desktop/src/main/browser-agent/panel.test.ts b/apps/desktop/src/main/browser-agent/panel.test.ts index 2ddb6720e85..4af126c5269 100644 --- a/apps/desktop/src/main/browser-agent/panel.test.ts +++ b/apps/desktop/src/main/browser-agent/panel.test.ts @@ -74,6 +74,7 @@ describe('panel chat scope', () => { const scopeId = panel.getActivePanelScopeId() vi.mocked(view.setVisible).mockClear() vi.mocked(view.setBounds).mockClear() + vi.mocked(view.webContents.invalidate).mockClear() await expect(panel.capturePanelSnapshot(win, scopeId)).resolves.toEqual({ dataUrl: 'data:image/png;base64,c2lt', @@ -97,9 +98,27 @@ describe('panel chat scope', () => { expect(panel.setPanelOccluded(false, win, scopeId)).toBe(true) expect(view.setVisible).toHaveBeenLastCalledWith(true) + expect(view.webContents.invalidate).toHaveBeenCalledOnce() expect(view.setBounds).not.toHaveBeenCalled() }) + it('requests a fresh compositor frame when a browser view is attached or revealed', () => { + const { win, view } = showPanel(panel) + + expect(view.webContents.invalidate).toHaveBeenCalledOnce() + + panel.setPanelBounds(null, win) + expect(view.setVisible).toHaveBeenLastCalledWith(false) + expect(view.webContents.invalidate).toHaveBeenCalledOnce() + + panel.setPanelBounds(PANEL_RECT, win) + expect(view.setVisible).toHaveBeenLastCalledWith(true) + expect(view.webContents.invalidate).toHaveBeenCalledTimes(2) + + panel.setPanelBounds(PANEL_RECT, win) + expect(view.webContents.invalidate).toHaveBeenCalledTimes(2) + }) + it('reports the exact applied native rectangle in renderer viewport coordinates', async () => { const { win, view } = showPanel(panel) const scopeId = panel.getActivePanelScopeId() diff --git a/apps/desktop/src/main/browser-agent/panel.ts b/apps/desktop/src/main/browser-agent/panel.ts index 905c13554c3..992f2f2b669 100644 --- a/apps/desktop/src/main/browser-agent/panel.ts +++ b/apps/desktop/src/main/browser-agent/panel.ts @@ -411,6 +411,9 @@ export function layout(): void { if (lastAppliedVisibility !== visible) { lastAppliedVisibility = visible active.view.setVisible(visible) + if (visible && !active.view.webContents.isDestroyed()) { + active.view.webContents.invalidate() + } } } diff --git a/apps/desktop/src/main/browser-agent/session.test.ts b/apps/desktop/src/main/browser-agent/session.test.ts index 4147f0cdc29..11d06defa59 100644 --- a/apps/desktop/src/main/browser-agent/session.test.ts +++ b/apps/desktop/src/main/browser-agent/session.test.ts @@ -28,6 +28,7 @@ interface MockView { getTitle: ReturnType close: ReturnType focus: ReturnType + invalidate: ReturnType isFocused: ReturnType isDestroyed: ReturnType isLoading: ReturnType @@ -929,11 +930,12 @@ describe('browser-agent session', () => { expect(session.listTabs()[0].tabId).not.toBe(blankTabId) session.setPanelFocused(false) + panel.setPanelBounds(null) expect(session.handleFocusedShortcut('close-tab')).toBe(false) expect(session.listTabs()).toHaveLength(1) }) - it('treats renderer browser chrome as browser focus', () => { + it('keeps close-tab routed to a visible browser through a transient focus loss', () => { panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) const first = session.requireTab() const second = session.addTab() @@ -945,17 +947,38 @@ describe('browser-agent session', () => { expect(session.listTabs()[0].tabId).not.toBe(second.id) session.setPanelFocused(false) + expect(session.handleFocusedShortcut('close-tab')).toBe(true) + expect(session.listTabs()).toHaveLength(1) + + panel.setPanelBounds(null) expect(session.handleFocusedShortcut('close-tab')).toBe(false) }) - it('opens a tab from the shared resource shortcut while browser chrome owns focus', () => { + it('keeps browser tab shortcuts routed while the visible panel has no DOM focus', () => { panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) session.requireTab() - session.setPanelFocused(true) + session.setPanelFocused(false) const before = session.listTabs().length expect(session.handleFocusedShortcut('new-tab')).toBe(true) expect(session.listTabs()).toHaveLength(before + 1) + expect(session.handleFocusedShortcut('close-tab')).toBe(true) + expect(session.listTabs()).toHaveLength(before) + expect(session.handleFocusedShortcut('reopen-closed-tab')).toBe(true) + expect(session.listTabs()).toHaveLength(before + 1) + + vi.mocked(win.webContents.send).mockClear() + expect(session.handleFocusedShortcut('focus-omnibox')).toBe(true) + expect(win.webContents.send).toHaveBeenCalledWith( + 'browser-agent:focus-omnibox', + 'select', + session.getBrowserScopeId() + ) + + panel.setPanelBounds(null) + expect(session.handleFocusedShortcut('new-tab')).toBe(false) + expect(session.handleFocusedShortcut('reopen-closed-tab')).toBe(false) + expect(session.handleFocusedShortcut('focus-omnibox')).toBe(false) }) it('reloads only the focused browser tab', () => { @@ -1037,7 +1060,7 @@ describe('browser-agent session', () => { expect(activeContents.setBackgroundThrottling).toHaveBeenLastCalledWith(true) }) - it('moves the automation exemption to whichever tab becomes active', () => { + it('moves the automation exemption with the agent cursor, not visible selection', () => { const first = session.ensureTab() const second = session.addTab() session.switchTab(first.id) @@ -1048,7 +1071,7 @@ describe('browser-agent session', () => { firstContents.setBackgroundThrottling.mockClear() secondContents.setBackgroundThrottling.mockClear() - session.switchTab(second.id) + session.switchAutomationTab(second.id) // The old active tab is re-throttled, the new one exempted — otherwise a // mid-tool switch would strand the wake on a tab the agent left behind. @@ -1056,6 +1079,98 @@ describe('browser-agent session', () => { expect(secondContents.setBackgroundThrottling).toHaveBeenLastCalledWith(false) }) + it('keeps the user-visible tab selected while the agent opens and switches background tabs', () => { + const first = session.ensureTab() + const visible = session.addTab() + + session.switchAutomationTab(first.id) + const background = session.addAutomationTab() + + expect(session.getTabsState().activeTabId).toBe(visible.id) + expect(session.getTabsState().automationTabId).toBe(background.id) + expect(session.getTabsState().tabs.find((tab) => tab.active)?.tabId).toBe(visible.id) + }) + + it('refuses to let automation close a visible tab claimed by the user', () => { + const visible = session.ensureTab() + session.switchTab(visible.id) + + expect(() => session.closeAutomationTab(visible.id)).toThrow('currently being used by the user') + expect(session.getTabsState().activeTabId).toBe(visible.id) + }) + + it('moves the next agent action to a background copy after the user claims its tab', () => { + const visible = session.ensureTab() + session.switchTab(visible.id) + + const agent = session.ensureAutomationTab() + + expect(agent.id).not.toBe(visible.id) + expect(session.getTabsState().activeTabId).toBe(visible.id) + expect(session.getTabsState().automationTabId).toBe(agent.id) + }) + + it('does not create a background tab until the agent resumes after a toolbar action', () => { + const visible = session.ensureTab() + + expect(session.claimActiveTabForUser()?.id).toBe(visible.id) + expect(session.listTabs()).toHaveLength(1) + + const agent = session.ensureAutomationTab() + expect(agent.id).not.toBe(visible.id) + expect(session.listTabs()).toHaveLength(2) + }) + + it('does not treat a passive native focus event as user takeover', () => { + const visible = session.ensureTab() + const contents = (visible.view as unknown as MockView).webContents + const focusListener = contents.on.mock.calls.find( + ([eventName]) => eventName === 'focus' + )?.[1] as (() => void) | undefined + + focusListener?.() + + expect(session.ensureAutomationTab().id).toBe(visible.id) + expect(session.listTabs()).toHaveLength(1) + }) + + it('moves automation to a background copy after real page interaction', () => { + const visible = session.ensureTab() + const contents = (visible.view as unknown as MockView).webContents + const beforeMouse = contents.on.mock.calls.find( + ([eventName]) => eventName === 'before-mouse-event' + )?.[1] as ((event: unknown, input: { type: string }) => void) | undefined + + beforeMouse?.({}, { type: 'mouseDown' }) + + expect(session.ensureAutomationTab().id).not.toBe(visible.id) + expect(session.getTabsState().activeTabId).toBe(visible.id) + expect(session.listTabs()).toHaveLength(2) + }) + + it('clears automation indicators instead of moving them when their tab closes', () => { + const target = session.ensureAutomationTab() + session.setAutomationActive(true) + session.setAutomationNeedsAttention(true) + + session.closeTab(target.id) + + expect(session.getTabsState()).toMatchObject({ + automationActive: false, + automationNeedsAttention: false, + }) + }) + + it('resumes takeover in the same tab after the user explicitly hands it back', () => { + const visible = session.ensureTab() + session.switchTab(visible.id) + + session.returnAutomationTabToAgent() + + expect(session.ensureAutomationTab().id).toBe(visible.id) + expect(session.listTabs()).toHaveLength(1) + }) + it('updates the native backdrop when Sim changes browser theme', () => { const tab = session.ensureTab() const view = tab.view as unknown as MockView @@ -1149,6 +1264,7 @@ describe('browser-agent session', () => { expect(contents?.focus).toHaveBeenCalled() session.setPanelFocused(false) + panel.setPanelBounds(null) expect(session.handleFocusedShortcut('reopen-closed-tab')).toBe(false) }) @@ -1305,14 +1421,17 @@ describe('browser-agent session', () => { win as unknown as { contentView: { removeChildView: ReturnType } } ).contentView.removeChildView view.setVisible.mockClear() + view.webContents.invalidate.mockClear() panel.setPanelBounds(null) expect(view.setVisible).toHaveBeenCalledWith(false) + expect(view.webContents.invalidate).not.toHaveBeenCalled() expect(removeChildView).not.toHaveBeenCalled() // Showing it again reuses the attached view rather than re-adding it. content.addChildView.mockClear() panel.setPanelBounds({ x: 100, y: 50, width: 800, height: 600 }) expect(view.setVisible).toHaveBeenLastCalledWith(true) + expect(view.webContents.invalidate).toHaveBeenCalledOnce() expect(content.addChildView).not.toHaveBeenCalled() }) @@ -1336,6 +1455,7 @@ describe('browser-agent session', () => { // two native views stacked in the window would composite over each other. expect(content.removeChildView).toHaveBeenCalledWith(first.view) expect(content.addChildView).toHaveBeenCalledWith(second.view) + expect(second.view.webContents.invalidate).toHaveBeenCalledOnce() }) // The measured report is the sole writer of bounds. A main-process @@ -1581,6 +1701,48 @@ describe('browser-agent session', () => { expect(contents.loadURL).not.toHaveBeenCalled() }) + it('keeps agent popups in the background and context-menu links user-owned', () => { + const onTabCreated = vi.fn() + session = freshSession(win, { onTabCreated }) + const sourceTab = session.ensureTab() + const source = (sourceTab.view as unknown as MockView).webContents + onTabCreated.mockClear() + session.setAutomationActive(true) + + const openWindow = source.setWindowOpenHandler.mock.calls[0]?.[0] as (details: { + url: string + }) => { action: string } + openWindow({ url: 'https://agent-popup.example/' }) + const agentPopup = session.automationTab() + expect(agentPopup).not.toBeNull() + expect(session.activeTab()).toBe(sourceTab) + expect(onTabCreated).toHaveBeenLastCalledWith(agentPopup?.view.webContents) + + const contextMenu = source.on.mock.calls.find( + ([eventName]) => eventName === 'context-menu' + )?.[1] as (event: unknown, params: unknown) => void + contextMenu( + {}, + { + selectionText: '', + linkURL: 'https://context-link.example/', + isEditable: false, + editFlags: { canPaste: false }, + } + ) + const template = vi.mocked(Menu.buildFromTemplate).mock.calls.at(-1)?.[0] as + | MenuItemConstructorOptions[] + | undefined + template + ?.find((item) => item.label === 'Open Link in New Tab') + ?.click?.({} as never, undefined as never, {} as never) + + const userTab = session.activeTab() + expect(userTab).not.toBe(sourceTab) + expect(session.automationTab()).toBe(agentPopup) + expect(onTabCreated).toHaveBeenLastCalledWith(userTab?.view.webContents) + }) + it('blocks controlled pages from moving or resizing the desktop window', () => { const tab = session.ensureTab() const contents = (tab.view as unknown as MockView).webContents diff --git a/apps/desktop/src/main/browser-agent/session.ts b/apps/desktop/src/main/browser-agent/session.ts index b8fe5fef5af..83f260440e0 100644 --- a/apps/desktop/src/main/browser-agent/session.ts +++ b/apps/desktop/src/main/browser-agent/session.ts @@ -37,6 +37,7 @@ import { shell, WebContentsView, } from 'electron' +import { isDispatchingAgentInput } from '@/main/browser-agent/cdp' import { attachAgentContextMenu, BASE_ZOOM_FACTOR, @@ -63,7 +64,12 @@ import { } from '@/main/browser-agent/url-guard' import type { BrowserSessionSnapshot } from '@/main/desktop-chat-session-store' import { suggestedFilename, uniqueDownloadPath } from '@/main/downloads' -import { type FocusedResourceShortcut, zoomActionForShortcut } from '@/main/resource-shortcuts' +import { + type FocusedResourceShortcut, + isResourceTabSelectionShortcut, + resourceTabTargetIndex, + zoomActionForShortcut, +} from '@/main/resource-shortcuts' const logger = createLogger('BrowserAgentSession') @@ -170,6 +176,8 @@ interface BrowserScopeState { tabs: AgentTab[] recentlyClosedTabUrls: string[] activeTabId: string | null + automationTabId: string | null + visibleTabUserSelected: boolean nextTabId: number /** True until anything beyond scope activation inspects or materializes this state. */ activationOnly: boolean @@ -179,6 +187,7 @@ interface BrowserScopeState { focusedBrowserTabId: string | null focusedBrowserClearTimer: ReturnType | null automationActive: boolean + automationNeedsAttention: boolean /** * Tab a find is currently running on. Tracked because the find outlives the * call that started it — Chromium keeps the highlights until it is told to @@ -195,6 +204,8 @@ function createBrowserScopeState(): BrowserScopeState { tabs: [], recentlyClosedTabUrls: [], activeTabId: null, + automationTabId: null, + visibleTabUserSelected: false, nextTabId: 1, activationOnly: true, restored: false, @@ -203,6 +214,7 @@ function createBrowserScopeState(): BrowserScopeState { focusedBrowserTabId: null, focusedBrowserClearTimer: null, automationActive: false, + automationNeedsAttention: false, findingTabId: null, findingRequestId: null, } @@ -542,6 +554,7 @@ function isActivationOnlyBrowserScope(scopeId: string): boolean { state.tabs.length === 0 && state.recentlyClosedTabUrls.length === 0 && state.activeTabId === null && + state.automationTabId === null && state.nextTabId === 1 && !state.restored && !state.restoring @@ -1048,10 +1061,10 @@ export function stopFindInActiveTab(focusPage: boolean): void { * inside the browser resource rather than spawn a native window, and both are * reached from an untrusted page, so the scheme is checked here once. */ -function openTabWithUrl(url: string): void { +function openTabWithUrl(url: string, agentOwned: boolean): void { if (!/^https?:\/\//i.test(url)) return try { - const tab = addTab() + const tab = agentOwned ? addAutomationTab() : addTab() void tab.view.webContents.loadURL(url).catch(() => {}) } catch (error) { logger.warn('Could not open a link in a new browser tab', { @@ -1090,7 +1103,7 @@ function createTabView(): WebContentsView { configureAgentPartition(contents.session) attachAgentContextMenu(contents, { addToChat: (text) => withBrowserScope(scopeId, () => addPageSelectionToChat(contents, text)), - openTab: (url) => withBrowserScope(scopeId, () => openTabWithUrl(url)), + openTab: (url) => withBrowserScope(scopeId, () => openTabWithUrl(url, false)), defaultZoomFactor: getBrowserDefaultZoomFactor, }) @@ -1105,6 +1118,19 @@ function createTabView(): WebContentsView { currentScope.focusedBrowserTabId = tab?.id ?? currentScope.activeTabId }) ) + contents.on( + 'before-mouse-event', + bindToBrowserScope(scopeId, (_event, mouse) => { + if ( + isDispatchingAgentInput(contents) || + !['mouseDown', 'contextMenu', 'mouseWheel'].includes(mouse.type) + ) { + return + } + const tab = tabs.find((entry) => entry.view.webContents === contents) + if (tab?.id === currentScope.activeTabId) currentScope.visibleTabUserSelected = true + }) + ) contents.on( 'blur', bindToBrowserScope(scopeId, () => { @@ -1131,7 +1157,7 @@ function createTabView(): WebContentsView { // Keep popups inside the browser resource: http(s) window.open and // target=_blank requests become a new internal tab, never a native window. contents.setWindowOpenHandler((details) => { - withBrowserScope(scopeId, () => openTabWithUrl(details.url)) + withBrowserScope(scopeId, () => openTabWithUrl(details.url, agentOwnsPopupFrom(contents))) return { action: 'deny' } }) @@ -1163,6 +1189,10 @@ function createTabView(): WebContentsView { contents.on( 'before-input-event', bindToBrowserScope(scopeId, (event, input) => { + const tab = tabs.find((entry) => entry.view === view) + if (!isDispatchingAgentInput(contents) && tab?.id === currentScope.activeTabId) { + currentScope.visibleTabUserSelected = true + } const shortcut = browserShortcutForInput(input) if (!shortcut) return @@ -1181,7 +1211,6 @@ function createTabView(): WebContentsView { return } - const tab = tabs.find((entry) => entry.view === view) if (tab) closeTabFromUser(tab.id) }) ) @@ -1284,8 +1313,16 @@ export function hasSession(): boolean { * touches it, and network loading is not throttled anyway. */ export function setAutomationActive(active: boolean): void { + if (currentScope.automationActive === active) return currentScope.automationActive = active applyActiveTabThrottling() + events?.onTabsChanged() +} + +export function setAutomationNeedsAttention(needsAttention: boolean): void { + if (currentScope.automationNeedsAttention === needsAttention) return + currentScope.automationNeedsAttention = needsAttention + events?.onTabsChanged() } /** @@ -1296,11 +1333,18 @@ export function setAutomationActive(active: boolean): void { function applyActiveTabThrottling(): void { for (const tab of tabs) { if (tab.view.webContents.isDestroyed()) continue - const exempt = currentScope.automationActive && tab.id === currentScope.activeTabId + const exempt = currentScope.automationActive && tab.id === currentScope.automationTabId tab.view.webContents.setBackgroundThrottling(!exempt) } } +/** A closed target must not transfer its activity marker to a replacement tab. */ +function clearAutomationIndicatorsForTab(tabId: string): void { + if (currentScope.automationTabId !== tabId) return + currentScope.automationActive = false + currentScope.automationNeedsAttention = false +} + function browserBackgroundColor(): string { const dark = browserTheme === 'dark' || (browserTheme === 'system' && nativeTheme.shouldUseDarkColors) @@ -1430,6 +1474,7 @@ function addTabInternal({ pinned, } insertPinnedAware(tab) + if (currentScope.automationTabId === null) currentScope.automationTabId = tab.id if (activate || currentScope.activeTabId === null) { currentScope.activeTabId = tab.id applyActiveTabThrottling() @@ -1444,6 +1489,35 @@ function addTabInternal({ return tab } +/** Forks the agent cursor before the user takes over the same live page. */ +function yieldAutomationTabToUser(tab: AgentTab): void { + if (currentScope.automationTabId !== tab.id) return + const replacement = addTabInternal({ activate: false, notify: false }) + currentScope.automationTabId = replacement.id + const url = sanitizeRestorableUrl(tabUrl(tab)) + if (url && url !== 'about:blank') { + void replacement.view.webContents.loadURL(url).catch(() => {}) + } + applyActiveTabThrottling() + persistBrowserSession() + events?.onTabsChanged() +} + +/** Marks the visible page as user-owned; automation forks only if it acts again. */ +export function claimActiveTabForUser(): AgentTab | null { + const tab = activeTab() + if (!tab) return null + currentScope.visibleTabUserSelected = true + return tab +} + +/** Explicit hand-back after takeover lets automation resume in the same page. */ +export function returnAutomationTabToAgent(): void { + if (currentScope.activeTabId === currentScope.automationTabId) { + currentScope.visibleTabUserSelected = false + } +} + export function restoreBrowserSession(): void { if (isBrowserScopeSuspended(getBrowserScopeId())) { throw new SessionError('This task browser is suspended until the task is reopened.') @@ -1481,6 +1555,7 @@ export function restoreBrowserSession(): void { } } currentScope.activeTabId = restoredTabs[snapshot.activeIndex]?.id ?? restoredTabs[0]?.id ?? null + currentScope.automationTabId = currentScope.activeTabId currentScope.lastPersistedSnapshot = JSON.stringify(snapshot) } @@ -1496,15 +1571,61 @@ export function restoreBrowserSession(): void { export function addTab(): AgentTab { restoreBrowserSession() + currentScope.visibleTabUserSelected = true return addTabInternal() } +/** Opens a tab for agent work without replacing the page the user is viewing. */ +export function addAutomationTab(): AgentTab { + restoreBrowserSession() + const tab = addTabInternal({ activate: false, notify: false }) + currentScope.automationTabId = tab.id + applyActiveTabThrottling() + persistBrowserSession() + events?.onTabsChanged() + return tab +} + +/** Agent target, creating or adopting a page without changing visible selection. */ +export function ensureAutomationTab(): AgentTab { + restoreBrowserSession() + let tab = automationTab() + if (tab?.id === currentScope.activeTabId && currentScope.visibleTabUserSelected) { + yieldAutomationTabToUser(tab) + tab = automationTab() + } + if (tab) return tab + tab = activeTab() + if (tab) { + currentScope.automationTabId = tab.id + applyActiveTabThrottling() + events?.onTabsChanged() + return tab + } + return addAutomationTab() +} + +/** Current agent target without creating one. */ +export function requireAutomationTab(): AgentTab { + restoreBrowserSession() + let tab = automationTab() + if (tab?.id === currentScope.activeTabId && currentScope.visibleTabUserSelected) { + yieldAutomationTabToUser(tab) + tab = automationTab() + } + if (!tab) { + throw new SessionError('No page is open yet — call browser_navigate or browser_open_tab first.') + } + return tab +} + /** Restores the most recently closed regular tab for the current app session. */ export function reopenClosedTab(): AgentTab | null { restoreBrowserSession() const url = recentlyClosedTabUrls.shift() if (!url) return null + currentScope.visibleTabUserSelected = true const tab = addTabInternal() if (url !== 'about:blank') { // No checkAgentUrl here, unlike the tool-driven navigations: the stored @@ -1528,6 +1649,7 @@ export function duplicateTab(tabId: string): AgentTab | null { if (!source) return null const url = sanitizeRestorableUrl(source.view.webContents.getURL()) + currentScope.visibleTabUserSelected = true const tab = addTabInternal() if (url && url !== 'about:blank') { // Sanitized to http(s) without embedded credentials above, and the @@ -1550,8 +1672,9 @@ export function switchTab(tabId: string): AgentTab { currentScope.focusedBrowserTabId !== null || tabs.some((entry) => entry.view.webContents.isFocused()) currentScope.activeTabId = tab.id - // The automation exemption follows the active tab, so a mid-tool switch - // unthrottles the new one and re-throttles the old. + currentScope.visibleTabUserSelected = true + // Visible selection does not move the automation exemption; the user may + // inspect another page while a tool continues in its background tab. applyActiveTabThrottling() layout() if (transferBrowserFocus) currentScope.focusedBrowserTabId = tab.id @@ -1561,6 +1684,17 @@ export function switchTab(tabId: string): AgentTab { return tab } +/** Moves the agent cursor without moving or focusing the user's visible tab. */ +export function switchAutomationTab(tabId: string): AgentTab { + restoreBrowserSession() + const tab = tabs.find((entry) => entry.id === tabId) + if (!tab) throw new SessionError(`No tab with id ${tabId} — call browser_list_tabs.`) + currentScope.automationTabId = tab.id + applyActiveTabThrottling() + events?.onTabsChanged() + return tab +} + /** * Moves a tab to a final list index while preserving the pinned/regular * boundary. Dragging across that boundary moves to its nearest valid edge. @@ -1601,6 +1735,7 @@ function forgetTab(tab: AgentTab): void { // on a tab that is going away keeps `findingTabId` naming a dead tab and // leaves the bar open counting matches on a page nobody can see. dismissFind(tab.id) + clearAutomationIndicatorsForTab(tab.id) tabs.splice(index, 1) const transferBrowserFocus = currentScope.focusedBrowserTabId === tab.id clearFocusedBrowserTab(tab.id) @@ -1613,6 +1748,10 @@ function forgetTab(tab: AgentTab): void { events?.onActiveTabChanged(active.view.webContents) } } + if (currentScope.automationTabId === tab.id) { + currentScope.automationTabId = (tabs[index] ?? tabs[index - 1])?.id ?? null + applyActiveTabThrottling() + } if (!hasSession() && getBrowserScopeId() === getActiveBrowserScopeId() && isPanelVisible()) { addTab() if (transferBrowserFocus) currentScope.focusedBrowserTabId = currentScope.activeTabId @@ -1635,6 +1774,7 @@ export function closeTab(tabId: string): void { } // Before the splice, while the tab is still resolvable — see forgetTab. dismissFind(tabId) + clearAutomationIndicatorsForTab(tabId) const [tab] = tabs.splice(index, 1) recentlyClosedTabUrls.unshift(sanitizeRestorableUrl(tabUrl(tab)) ?? 'about:blank') if (recentlyClosedTabUrls.length > MAX_RECENTLY_CLOSED_TABS) { @@ -1653,6 +1793,10 @@ export function closeTab(tabId: string): void { events?.onActiveTabChanged(active.view.webContents) } } + if (currentScope.automationTabId === tab.id) { + currentScope.automationTabId = (tabs[index] ?? tabs[index - 1])?.id ?? null + applyActiveTabThrottling() + } // Closing the last tab must not leave a visible browser resource with an // empty strip. Replace it with a fresh New tab, matching normal browser UI. if (!hasSession() && getBrowserScopeId() === getActiveBrowserScopeId() && isPanelVisible()) { @@ -1668,6 +1812,16 @@ export function closeTab(tabId: string): void { } } +/** Closes agent-owned work while protecting a visible tab the user claimed. */ +export function closeAutomationTab(tabId: string): void { + if (tabId === currentScope.activeTabId && currentScope.visibleTabUserSelected) { + throw new SessionError( + 'That tab is currently being used by the user. Switch to another agent tab instead of closing it.' + ) + } + closeTab(tabId) +} + /** * Pins or unpins a live tab. Pinned tabs form a stable group at the far left, * and their latest URLs are persisted locally for the next browser opening. @@ -1712,7 +1866,9 @@ export function showTabContextMenu(tabId: string): void { /** The live page whose browser surface owns a menu accelerator. */ function focusedTabForShortcut(ownerWindow?: BrowserWindow | null): AgentTab | null { - if (!panelUpdateAllowed(ownerWindow ?? undefined, getBrowserScopeId())) return null + if (!isPanelVisible() || !panelUpdateAllowed(ownerWindow ?? undefined, getBrowserScopeId())) { + return null + } return ( tabs.find( (tab) => @@ -1722,6 +1878,14 @@ function focusedTabForShortcut(ownerWindow?: BrowserWindow | null): AgentTab | n ) } +/** The active tab while this window owns an on-screen browser panel. */ +function visibleActiveTab(ownerWindow?: BrowserWindow | null): AgentTab | null { + if (!isPanelVisible() || !panelUpdateAllowed(ownerWindow ?? undefined, getBrowserScopeId())) { + return null + } + return activeTab() +} + /** * Claims a global resource shortcut only while this browser owns interaction. * @@ -1733,8 +1897,32 @@ export function handleFocusedShortcut( shortcut: FocusedResourceShortcut, ownerWindow?: BrowserWindow | null ): boolean { - const focusedTab = focusedTabForShortcut(ownerWindow) - if (!focusedTab) return false + // Tab-management accelerators belong to the visible Browser even after + // focus moves into Sim chrome. Otherwise Cmd-T/L/Shift-T silently stop + // behaving like browser shortcuts, and Cmd-W becomes especially unsafe. + const visibleBrowserShortcut = + shortcut === 'close-tab' || + shortcut === 'new-tab' || + shortcut === 'reopen-closed-tab' || + shortcut === 'focus-omnibox' + const shortcutTab = + focusedTabForShortcut(ownerWindow) ?? + (visibleBrowserShortcut ? visibleActiveTab(ownerWindow) : null) + if (!shortcutTab) return false + + if (isResourceTabSelectionShortcut(shortcut)) { + const targetIndex = resourceTabTargetIndex( + shortcut, + tabs.length, + tabs.findIndex((tab) => tab.id === shortcutTab.id) + ) + const target = targetIndex === null ? null : tabs[targetIndex] + if (target) { + switchTab(target.id) + target.view.webContents.focus() + } + return true + } switch (shortcut) { case 'new-tab': @@ -1747,15 +1935,18 @@ export function handleFocusedShortcut( return true } case 'close-tab': - closeTabFromUser(focusedTab.id) + closeTabFromUser(shortcutTab.id) + return true + case 'focus-omnibox': + focusRendererOmnibox('select') return true case 'reload-or-clear': - focusedTab.view.webContents.reload() + shortcutTab.view.webContents.reload() return true } const zoomAction = zoomActionForShortcut(shortcut) - const contents = focusedTab.view.webContents + const contents = shortcutTab.view.webContents const factor = zoomAction === 'reset' ? getBrowserDefaultZoomFactor() @@ -1781,6 +1972,7 @@ export function setPanelFocused( currentScope.focusedBrowserClearTimer = null } currentScope.focusedBrowserTabId = activeTab()?.id ?? null + currentScope.visibleTabUserSelected = true }) } @@ -1794,7 +1986,10 @@ function clearFocusedBrowserTab(tabId?: string): void { } function closeTabFromUser(tabId: string): void { - if (tabs.find((tab) => tab.id === tabId)?.pinned) return + if (tabs.find((tab) => tab.id === tabId)?.pinned) { + shell.beep() + return + } const closingLastTab = listTabs().length === 1 closeTab(tabId) const active = activeTab() @@ -1816,6 +2011,10 @@ function closeLiveTabs(): void { } recentlyClosedTabUrls.length = 0 currentScope.activeTabId = null + currentScope.automationTabId = null + currentScope.automationActive = false + currentScope.automationNeedsAttention = false + currentScope.visibleTabUserSelected = false clearFocusedBrowserTab() } @@ -1952,6 +2151,22 @@ export function getTabsState(): BrowserTabsState { scopeId: getBrowserScopeId(), tabs: listTabs(), activeTabId: activeTab()?.id ?? null, + automationTabId: automationTab()?.id ?? null, + automationActive: currentScope.automationActive, + automationNeedsAttention: currentScope.automationNeedsAttention, + } +} + +/** Tool-facing tab list whose active marker follows the agent cursor. */ +export function getAutomationTabsState(): BrowserTabsState { + const automationTabId = automationTab()?.id ?? null + return { + scopeId: getBrowserScopeId(), + tabs: listTabs().map((tab) => ({ ...tab, active: tab.tabId === automationTabId })), + activeTabId: automationTabId, + automationTabId, + automationActive: currentScope.automationActive, + automationNeedsAttention: currentScope.automationNeedsAttention, } } @@ -1965,3 +2180,25 @@ export function activeTab(): AgentTab | null { if (!tab || tab.view.webContents.isDestroyed()) return null return tab } + +export function automationTab(): AgentTab | null { + const tab = tabs.find((entry) => entry.id === currentScope.automationTabId) ?? null + if (!tab || tab.view.webContents.isDestroyed()) return null + return tab +} + +export function automationTabClaimedByUser(): boolean { + return ( + currentScope.visibleTabUserSelected && + currentScope.activeTabId !== null && + currentScope.activeTabId === currentScope.automationTabId + ) +} + +/** Keeps page-created tabs with the input owner that opened them. */ +function agentOwnsPopupFrom(contents: WebContents): boolean { + if (isDispatchingAgentInput(contents)) return true + if (automationTab()?.view.webContents !== contents) return false + if (automationTabClaimedByUser()) return false + return currentScope.automationActive +} diff --git a/apps/desktop/src/main/desktop-settings.test.ts b/apps/desktop/src/main/desktop-settings.test.ts index 25c137aa8ca..b7849a57558 100644 --- a/apps/desktop/src/main/desktop-settings.test.ts +++ b/apps/desktop/src/main/desktop-settings.test.ts @@ -1,7 +1,7 @@ import { mkdtempSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { TERMINAL_DARK_THEME } from '@sim/desktop-bridge' +import { TERMINAL_DARK_THEME, TERMINAL_LIGHT_THEME } from '@sim/desktop-bridge' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('electron', () => import('@/test/electron-mock')) @@ -15,6 +15,14 @@ const IMPORTED_PALETTE = { ...TERMINAL_DARK_THEME, background: '#101010', } +const IMPORTED_LIGHT_PALETTE = { + ...TERMINAL_LIGHT_THEME, + background: '#fafafa', +} +const IMPORTED_DARK_PALETTE = { + ...TERMINAL_DARK_THEME, + background: '#202020', +} function makeService() { const config = createConfigStore( @@ -191,7 +199,7 @@ describe('desktop settings service', () => { expect(preferences?.browserDownloadDirectory).toBe('/tmp/custom-downloads') }) - it('caches and selects a Terminal or iTerm2 profile', () => { + it('persists a Terminal or iTerm2 profile with appearance-specific palettes', () => { const { config, service } = makeService() const preferences = service.selectTerminalProfile({ @@ -199,6 +207,8 @@ describe('desktop settings service', () => { name: 'Ocean', source: 'iterm2', palette: IMPORTED_PALETTE, + lightPalette: IMPORTED_LIGHT_PALETTE, + darkPalette: IMPORTED_DARK_PALETTE, }) expect(config.get('terminalTheme')).toEqual({ @@ -206,6 +216,8 @@ describe('desktop settings service', () => { name: 'Ocean', source: 'iterm2', palette: IMPORTED_PALETTE, + lightPalette: IMPORTED_LIGHT_PALETTE, + darkPalette: IMPORTED_DARK_PALETTE, }) expect(preferences).toMatchObject({ terminalTheme: { id: 'iterm2:ocean', name: 'Ocean' }, diff --git a/apps/desktop/src/main/desktop-settings.ts b/apps/desktop/src/main/desktop-settings.ts index aeaef5a0852..f84b364b40b 100644 --- a/apps/desktop/src/main/desktop-settings.ts +++ b/apps/desktop/src/main/desktop-settings.ts @@ -1,5 +1,6 @@ import { isAbsolute } from 'node:path' import { + cloneTerminalSelectedProfile, type DesktopAppearanceTheme, type DesktopNotificationPayload, type DesktopPreferenceKey, @@ -177,12 +178,7 @@ export function createDesktopSettingsService( return read() }, selectTerminalProfile(profile) { - deps.config.set('terminalTheme', { - id: profile.id, - name: profile.name, - source: profile.source, - palette: { ...profile.palette }, - }) + deps.config.set('terminalTheme', cloneTerminalSelectedProfile(profile)) deps.config.flush() return read() }, diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index ad30bdb8b06..48762e5c9c4 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -664,8 +664,8 @@ function main(): void { newWindow: () => void createAndLoadAppWindow(), newChat: () => void openMainWindowAt(newChatRoute(config.get('lastRoute'))), handleFocusedResourceShortcut: (win, shortcut) => - terminal.handleFocusedShortcut(win, shortcut) || - handleFocusedBrowserShortcut(shortcut, win), + handleFocusedBrowserShortcut(shortcut, win) || + terminal.handleFocusedShortcut(win, shortcut), toggleSidebar: () => getMainWindow()?.webContents.send('desktop:command', 'toggle-sidebar'), signOut: signOutFromMenu, checkForUpdates: () => diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index 0376da1a065..a23f83b4745 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -669,6 +669,64 @@ describe('registerIpcHandlers', () => { ) }) + it('routes exact browser-tool cancellation without waiting for authorization', async () => { + const { invoke } = collectHandlers() + const cancel = vi.spyOn(browserDriver, 'cancelTool').mockReturnValue(true) + const cancelActive = vi.spyOn(browserDriver, 'cancelActiveTool').mockReturnValue(true) + const handler = invoke.get('browser-agent:cancel-tool') + const activeHandler = invoke.get('browser-agent:cancel-active-tool') + + await expect(handler?.(appEvent, 'tool-1', 'chat-1')).resolves.toBe(true) + expect(cancel).toHaveBeenCalledWith('chat-1', 'tool-1') + await expect(activeHandler?.(appEvent, 'chat-reloaded')).resolves.toBe(true) + expect(cancelActive).toHaveBeenCalledWith('chat-reloaded') + await expect(handler?.(evilEvent, 'tool-2', 'chat-1')).resolves.toBe(false) + await expect(activeHandler?.(evilEvent, 'chat-reloaded')).resolves.toBe(false) + expect(cancel).toHaveBeenCalledTimes(1) + expect(cancelActive).toHaveBeenCalledTimes(1) + cancel.mockRestore() + cancelActive.mockRestore() + }) + + it('rejects a browser tool authorized after its scope cancellation boundary', async () => { + const { invoke } = collectHandlers() + const executeHandler = invoke.get('browser-agent:execute-tool') + const cancelActiveHandler = invoke.get('browser-agent:cancel-active-tool') + let resolveAuthorization: (response: Response) => void = () => {} + const fetchAuthorization = vi.fn( + () => + new Promise((resolve) => { + resolveAuthorization = resolve + }) + ) + const delayedEvent = { + senderFrame: { url: `${APP}/workspace/ws1` }, + sender: { session: { fetch: fetchAuthorization } }, + } + + const execution = executeHandler?.( + delayedEvent, + 'tool-delayed-authorization', + 'browser_open_tab', + {}, + 'chat-delayed-authorization' + ) + await Promise.resolve() + await cancelActiveHandler?.(delayedEvent, 'chat-delayed-authorization') + resolveAuthorization( + Response.json({ + chatId: 'chat-delayed-authorization', + toolName: 'browser_open_tab', + args: {}, + }) + ) + + await expect(execution).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('cancelled before it started'), + }) + }) + it('routes terminal tools by the server-authorized chat, not renderer scope', async () => { const { invoke } = collectHandlers() const executeTool = vi.spyOn(deps.terminal, 'executeTool').mockResolvedValue({ ok: true }) @@ -919,6 +977,42 @@ describe('registerIpcHandlers', () => { restore.mockRestore() }) + it('creates browser tabs through an acknowledged active-scope operation', async () => { + const tabsState = { + scopeId: 'chat-b', + tabs: [ + { + tabId: '2', + url: '', + title: '', + loading: false, + active: true, + pinned: false, + }, + ], + activeTabId: '2', + } + const add = vi.spyOn(browserSession, 'addTab').mockReturnValue({} as never) + const peek = vi.spyOn(browserSession, 'peekTabsState').mockReturnValue(tabsState) + const { invoke } = collectHandlers() + + await invoke.get('browser-agent:activate-scope')?.(appEvent, 'chat-b') + peek.mockClear() + await expect(invoke.get('browser-agent:open-tab')?.(appEvent, 'chat-b')).resolves.toEqual( + tabsState + ) + await expect(invoke.get('browser-agent:open-tab')?.(evilEvent, 'chat-b')).resolves.toEqual({ + scopeId: '', + tabs: [], + activeTabId: null, + }) + + expect(add).toHaveBeenCalledOnce() + expect(peek).toHaveBeenCalledOnce() + add.mockRestore() + peek.mockRestore() + }) + it('routes browser scope events after activation and a valid provisional migration', async () => { const migrate = vi.spyOn(browserDriver, 'migrateBrowserScope').mockReturnValue(true) const { invoke } = collectHandlers() diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 93f95b8d6b3..4b4b48bb7d8 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -26,6 +26,10 @@ import { isRecordLike } from '@sim/utils/object' import type { BrowserWindow, IpcMainEvent, IpcMainInvokeEvent, WebContents } from 'electron' import { clipboard, ipcMain } from 'electron' import { + type BrowserToolQueueBoundary, + cancelActiveTool, + cancelTool, + captureBrowserToolQueueBoundary, clearBrowsingData, disposeBrowserScope, executeTool, @@ -38,6 +42,7 @@ import { } from '@/main/browser-agent/driver' import { isAgentWebContents } from '@/main/browser-agent/registry' import { + addTab, findInActiveTab, getBrowserDownloadsState, peekTabsState, @@ -745,12 +750,53 @@ export function registerIpcHandlers(deps: IpcDeps): void { gate: 'app-origin', requires: 'browser', denied: { ok: false, error: 'Browser automation is not allowed from this page.' }, - handler: (scope, tool, params) => { - if (typeof scope !== 'string' || typeof tool !== 'string' || !isBrowserToolName(tool)) { + handler: (scope, toolCallId, tool, params, authorizationBoundary) => { + if ( + typeof scope !== 'string' || + typeof toolCallId !== 'string' || + typeof tool !== 'string' || + !isBrowserToolName(tool) + ) { return { ok: false, error: `Unknown browser tool: ${String(tool)}` } } const toolParams = isRecordLike(params) ? params : {} - return executeTool(scope, tool, toolParams) + return executeTool( + scope, + tool, + toolParams, + toolCallId, + authorizationBoundary as BrowserToolQueueBoundary | undefined + ) + }, + }, + 'browser-agent:cancel-tool': { + kind: 'invoke', + gate: 'app-origin', + requires: 'browser', + passSender: true, + denied: false, + handler: (sender, toolCallId, rawScope) => { + const scope = rendererScope(browserScopeBySender, sender as WebContents, rawScope) + if ( + !scope || + typeof toolCallId !== 'string' || + toolCallId.length < 1 || + toolCallId.length > 256 + ) { + return false + } + return cancelTool(scope, toolCallId) + }, + }, + 'browser-agent:cancel-active-tool': { + kind: 'invoke', + gate: 'app-origin', + requires: 'browser', + passSender: true, + denied: false, + handler: (sender, rawScope) => { + const scope = rendererScope(browserScopeBySender, sender as WebContents, rawScope) + return scope ? cancelActiveTool(scope) : false }, }, 'browser-agent:get-tabs-state': { @@ -767,6 +813,22 @@ export function registerIpcHandlers(deps: IpcDeps): void { : { tabs: [], activeTabId: null } }, }, + 'browser-agent:open-tab': { + kind: 'invoke', + gate: 'app-origin', + requires: 'browser', + passSender: true, + denied: { scopeId: '', tabs: [], activeTabId: null }, + handler: (sender, rawScope) => { + const contents = sender as WebContents + const scope = activeRendererScope(browserScopeBySender, contents, rawScope) + if (!scope) return { scopeId: '', tabs: [], activeTabId: null } + return withBrowserScope(scope, () => { + addTab() + return peekTabsState() + }) + }, + }, 'browser-agent:activate-scope': { kind: 'invoke', gate: 'app-origin', @@ -1391,6 +1453,16 @@ export function registerIpcHandlers(deps: IpcDeps): void { if (scope) deps.terminal.setPanelFocused(scope, focused === true, sender as WebContents) }, }, + 'terminal:visible': { + kind: 'send', + gate: 'app-origin', + requires: 'terminal', + passSender: true, + handler: (sender, visible, rawScope) => { + const scope = rendererScope(terminalScopeBySender, sender as WebContents, rawScope) + if (scope) deps.terminal.setPanelVisible(scope, visible === true, sender as WebContents) + }, + }, 'terminal:paste': { kind: 'invoke', gate: 'app-origin', @@ -1554,6 +1626,24 @@ export function registerIpcHandlers(deps: IpcDeps): void { return { ...tabs, scopeId: scope } }, }, + 'terminal:reorder': { + kind: 'invoke', + gate: 'app-origin', + requires: 'terminal', + passSender: true, + denied: { tabs: [], activeTerminalId: null }, + handler: (sender, terminalId, targetIndex, rawScope) => { + const scope = rendererScope(terminalScopeBySender, sender as WebContents, rawScope) + if (!scope) return { tabs: [], activeTerminalId: null } + const tabs = + typeof terminalId === 'string' && + typeof targetIndex === 'number' && + Number.isFinite(targetIndex) + ? deps.terminal.reorderTerminal(scope, terminalId, targetIndex) + : deps.terminal.getTabs(scope) + return { ...tabs, scopeId: scope } + }, + }, 'terminal:close': { kind: 'invoke', gate: 'app-origin', @@ -1646,6 +1736,10 @@ export function registerIpcHandlers(deps: IpcDeps): void { let handlerArgs = args if (channel === 'browser-agent:execute-tool') { const requestedTool = args[1] + const requestedScope = parseDesktopScope(args[3]) + const authorizationBoundary = requestedScope + ? captureBrowserToolQueueBoundary(requestedScope) + : undefined const authorization = await fetchDesktopToolAuthorization(event, deps, args[0]) if ( !authorization || @@ -1658,7 +1752,13 @@ export function registerIpcHandlers(deps: IpcDeps): void { error: 'This browser action is not an authorized pending Copilot tool call.', } } - handlerArgs = [authorization.chatId, authorization.toolName, authorization.args] + handlerArgs = [ + authorization.chatId, + args[0], + authorization.toolName, + authorization.args, + authorizationBoundary, + ] } if (channel === 'terminal:execute-tool') { const requestedTool = args[1] diff --git a/apps/desktop/src/main/menu.test.ts b/apps/desktop/src/main/menu.test.ts index 12bb128655d..3f982bc35d5 100644 --- a/apps/desktop/src/main/menu.test.ts +++ b/apps/desktop/src/main/menu.test.ts @@ -67,6 +67,13 @@ describe('buildMenuTemplate', () => { 'New Chat', 'separator', 'Reopen Closed Tab', + 'Focus Address Bar', + 'separator', + 'Next Tab', + 'Previous Tab', + 'Select Tab', + 'separator', + 'Close Tab', 'Close Window', ]) expect(submenu(template, 'View').map((item) => item.label ?? item.role ?? item.type)).toEqual([ @@ -93,7 +100,7 @@ describe('buildMenuTemplate', () => { expect(view.some((item) => item.role === 'toggleDevTools')).toBe(false) }) - it('routes the close accelerator through the focused resource before closing a window', () => { + it('reserves the close-tab accelerator for resources and never closes the window', () => { const handleFocusedResourceShortcut = vi.fn(() => true) const deps = Object.assign(makeDeps(), { handleFocusedResourceShortcut }) const closeItem = submenu(buildMenuTemplate(deps), 'File').find( @@ -101,7 +108,7 @@ describe('buildMenuTemplate', () => { ) const focusedWindow = new BrowserWindow() - expect(closeItem).toMatchObject({ label: 'Close Window', accelerator: 'CmdOrCtrl+W' }) + expect(closeItem).toMatchObject({ label: 'Close Tab', accelerator: 'CmdOrCtrl+W' }) expect(closeItem?.role).toBeUndefined() const click = closeItem?.click as unknown as ( @@ -115,10 +122,52 @@ describe('buildMenuTemplate', () => { handleFocusedResourceShortcut.mockReturnValue(false) click({}, focusedWindow) + expect(focusedWindow.close).not.toHaveBeenCalled() + }) + + it('always offers a separate close-window accelerator', () => { + const deps = makeDeps() + const closeWindow = submenu(buildMenuTemplate(deps), 'File').find( + (item) => item.accelerator === 'CmdOrCtrl+Shift+W' + ) + const focusedWindow = new BrowserWindow() + + ;(closeWindow?.click as unknown as (menuItem: unknown, browserWindow: BrowserWindow) => void)( + {}, + focusedWindow + ) + + expect(closeWindow?.label).toBe('Close Window') expect(focusedWindow.close).toHaveBeenCalledOnce() + expect(deps.handleFocusedResourceShortcut).not.toHaveBeenCalled() + }) + + it('routes next, previous, and numbered tab accelerators to the focused resource', () => { + const handleFocusedResourceShortcut = vi.fn(() => true) + const template = buildMenuTemplate( + Object.assign(makeDeps(), { + handleFocusedResourceShortcut, + }) + ) + const file = submenu(template, 'File') + const selectTabs = submenu(file, 'Select Tab') + const focusedWindow = new BrowserWindow() + const invoke = (item: MenuItemConstructorOptions | undefined) => + (item?.click as unknown as (menuItem: unknown, browserWindow: BrowserWindow) => void)( + {}, + focusedWindow + ) + + invoke(file.find((item) => item.accelerator === 'Ctrl+Tab')) + invoke(file.find((item) => item.accelerator === 'Ctrl+Shift+Tab')) + invoke(selectTabs.find((item) => item.accelerator === 'CmdOrCtrl+9')) + + expect(handleFocusedResourceShortcut).toHaveBeenNthCalledWith(1, focusedWindow, 'next-tab') + expect(handleFocusedResourceShortcut).toHaveBeenNthCalledWith(2, focusedWindow, 'previous-tab') + expect(handleFocusedResourceShortcut).toHaveBeenNthCalledWith(3, focusedWindow, 'select-tab-9') }) - it('routes new and reopen accelerators through the focused resource', () => { + it('routes new, reopen, and address-bar accelerators through the active resource', () => { const handleFocusedResourceShortcut = vi.fn(() => true) const template = buildMenuTemplate( Object.assign(makeDeps(), { @@ -129,6 +178,7 @@ describe('buildMenuTemplate', () => { const reopenItem = submenu(template, 'File').find( (item) => item.accelerator === 'CmdOrCtrl+Shift+T' ) + const addressItem = submenu(template, 'File').find((item) => item.accelerator === 'CmdOrCtrl+L') expect(reopenItem).toMatchObject({ label: 'Reopen Closed Tab', @@ -143,12 +193,17 @@ describe('buildMenuTemplate', () => { {}, focusedWindow ) + ;(addressItem?.click as unknown as (menuItem: unknown, browserWindow: BrowserWindow) => void)( + {}, + focusedWindow + ) expect(handleFocusedResourceShortcut).toHaveBeenNthCalledWith(1, focusedWindow, 'new-tab') expect(handleFocusedResourceShortcut).toHaveBeenNthCalledWith( 2, focusedWindow, 'reopen-closed-tab' ) + expect(handleFocusedResourceShortcut).toHaveBeenNthCalledWith(3, focusedWindow, 'focus-omnibox') }) it('routes reload through the focused resource before falling back to the Sim renderer', () => { diff --git a/apps/desktop/src/main/menu.ts b/apps/desktop/src/main/menu.ts index 5261f241048..615fa1354db 100644 --- a/apps/desktop/src/main/menu.ts +++ b/apps/desktop/src/main/menu.ts @@ -2,7 +2,10 @@ import type { MenuItemConstructorOptions } from 'electron' import { app, BrowserWindow, Menu } from 'electron' import type { ConfigStore } from '@/main/config' import { openExternalSafe } from '@/main/navigation' -import type { FocusedResourceShortcut } from '@/main/resource-shortcuts' +import type { + FocusedResourceShortcut, + ResourceTabSelectionShortcut, +} from '@/main/resource-shortcuts' const DOCS_URL = 'https://docs.sim.ai' const STATUS_URL = 'https://status.sim.ai' @@ -46,6 +49,24 @@ export function buildMenuTemplate(deps: MenuDeps): MenuItemConstructorOptions[] const focusedOrMain = (focusedWindow: unknown): BrowserWindow | null => focusedWindow instanceof BrowserWindow ? focusedWindow : deps.getMainWindow() + const resourceShortcut = ( + shortcut: FocusedResourceShortcut + ): NonNullable => { + return (_item, focusedWindow) => { + deps.handleFocusedResourceShortcut(focusedOrMain(focusedWindow), shortcut) + } + } + + const numberedTabItems: MenuItemConstructorOptions[] = Array.from({ length: 9 }, (_, index) => { + const number = index + 1 + const shortcut = `select-tab-${number}` as ResourceTabSelectionShortcut + return { + label: number === 9 ? 'Last Tab' : `Tab ${number}`, + accelerator: `CmdOrCtrl+${number}`, + click: resourceShortcut(shortcut), + } + }) + const setZoom = ( action: 'in' | 'out' | 'reset' ): NonNullable => { @@ -144,11 +165,36 @@ export function buildMenuTemplate(deps: MenuDeps): MenuItemConstructorOptions[] }, }, { - label: 'Close Window', + label: 'Focus Address Bar', + accelerator: 'CmdOrCtrl+L', + click: resourceShortcut('focus-omnibox'), + }, + { type: 'separator' }, + { + label: 'Next Tab', + accelerator: 'Ctrl+Tab', + click: resourceShortcut('next-tab'), + }, + { + label: 'Previous Tab', + accelerator: 'Ctrl+Shift+Tab', + click: resourceShortcut('previous-tab'), + }, + { label: 'Select Tab', submenu: numberedTabItems }, + { type: 'separator' }, + { + label: 'Close Tab', accelerator: 'CmdOrCtrl+W', click: (_item, focusedWindow) => { const win = focusedOrMain(focusedWindow) - if (deps.handleFocusedResourceShortcut(win, 'close-tab')) return + deps.handleFocusedResourceShortcut(win, 'close-tab') + }, + }, + { + label: 'Close Window', + accelerator: 'CmdOrCtrl+Shift+W', + click: (_item, focusedWindow) => { + const win = focusedOrMain(focusedWindow) if (win && !win.isDestroyed()) win.close() }, }, diff --git a/apps/desktop/src/main/resource-shortcuts.test.ts b/apps/desktop/src/main/resource-shortcuts.test.ts new file mode 100644 index 00000000000..b59abaf36c5 --- /dev/null +++ b/apps/desktop/src/main/resource-shortcuts.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { resourceTabTargetIndex } from '@/main/resource-shortcuts' + +describe('resourceTabTargetIndex', () => { + it('cycles through tabs in both directions', () => { + expect(resourceTabTargetIndex('next-tab', 3, 2)).toBe(0) + expect(resourceTabTargetIndex('previous-tab', 3, 0)).toBe(2) + expect(resourceTabTargetIndex('next-tab', 3, -1)).toBe(0) + }) + + it('selects numbered tabs and reserves nine for the last tab', () => { + expect(resourceTabTargetIndex('select-tab-1', 5, 2)).toBe(0) + expect(resourceTabTargetIndex('select-tab-4', 5, 2)).toBe(3) + expect(resourceTabTargetIndex('select-tab-9', 5, 2)).toBe(4) + expect(resourceTabTargetIndex('select-tab-6', 5, 2)).toBeNull() + }) +}) diff --git a/apps/desktop/src/main/resource-shortcuts.ts b/apps/desktop/src/main/resource-shortcuts.ts index 3305fc89c72..2e5d8dab62f 100644 --- a/apps/desktop/src/main/resource-shortcuts.ts +++ b/apps/desktop/src/main/resource-shortcuts.ts @@ -9,9 +9,40 @@ export type FocusedResourceShortcut = | 'new-tab' | 'reopen-closed-tab' | 'close-tab' + | 'focus-omnibox' + | ResourceTabSelectionShortcut | 'reload-or-clear' | `zoom-${DesktopZoomAction}` +export type ResourceTabSelectionShortcut = + | 'next-tab' + | 'previous-tab' + | `select-tab-${1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9}` + +export function isResourceTabSelectionShortcut( + shortcut: FocusedResourceShortcut +): shortcut is ResourceTabSelectionShortcut { + return ( + shortcut === 'next-tab' || shortcut === 'previous-tab' || shortcut.startsWith('select-tab-') + ) +} + +/** Resolves browser-style next/previous and numbered tab shortcuts. */ +export function resourceTabTargetIndex( + shortcut: ResourceTabSelectionShortcut, + tabCount: number, + activeIndex: number +): number | null { + if (tabCount <= 0) return null + if (shortcut === 'next-tab') return activeIndex < 0 ? 0 : (activeIndex + 1) % tabCount + if (shortcut === 'previous-tab') { + return activeIndex < 0 ? tabCount - 1 : (activeIndex - 1 + tabCount) % tabCount + } + const requested = Number(shortcut.slice('select-tab-'.length)) + const target = requested === 9 ? tabCount - 1 : requested - 1 + return target >= 0 && target < tabCount ? target : null +} + export function zoomActionForShortcut(shortcut: `zoom-${DesktopZoomAction}`): DesktopZoomAction { switch (shortcut) { case 'zoom-in': diff --git a/apps/desktop/src/main/terminal-themes.test.ts b/apps/desktop/src/main/terminal-themes.test.ts index 55538455b0c..4887ea43342 100644 --- a/apps/desktop/src/main/terminal-themes.test.ts +++ b/apps/desktop/src/main/terminal-themes.test.ts @@ -1,4 +1,4 @@ -import { TERMINAL_DARK_THEME } from '@sim/desktop-bridge' +import { TERMINAL_DARK_THEME, TERMINAL_LIGHT_THEME } from '@sim/desktop-bridge' import { describe, expect, it } from 'vitest' import { parseTerminalThemeProfiles } from '@/main/terminal-themes' @@ -6,6 +6,10 @@ const PALETTE = { ...TERMINAL_DARK_THEME, background: '#101010', } +const LIGHT_PALETTE = { + ...TERMINAL_LIGHT_THEME, + background: '#fafafa', +} function profile(id: string, overrides: Record = {}) { return { @@ -22,10 +26,22 @@ describe('parseTerminalThemeProfiles', () => { expect(parseTerminalThemeProfiles([profile('iterm2:ocean')])).toEqual([profile('iterm2:ocean')]) }) + it('preserves separate iTerm2 light and dark palettes', () => { + const separateProfile = profile('iterm2:ocean', { + lightPalette: LIGHT_PALETTE, + darkPalette: PALETTE, + }) + + expect(parseTerminalThemeProfiles([separateProfile])).toEqual([separateProfile]) + }) + it('drops malformed colors and unsupported applications', () => { expect( parseTerminalThemeProfiles([ profile('bad-color', { palette: { ...PALETTE, background: 'rgb(0, 0, 0)' } }), + profile('bad-mode-color', { + lightPalette: { ...LIGHT_PALETTE, foreground: 'white' }, + }), profile('bad-source', { source: 'warp' }), ]) ).toEqual([]) diff --git a/apps/desktop/src/main/terminal-themes.ts b/apps/desktop/src/main/terminal-themes.ts index eecbe2e631c..dfd26cb60f5 100644 --- a/apps/desktop/src/main/terminal-themes.ts +++ b/apps/desktop/src/main/terminal-themes.ts @@ -1,6 +1,7 @@ import { execFile } from 'node:child_process' import { promisify } from 'node:util' import { + cloneTerminalSelectedProfile, isTerminalSelectedProfile, TERMINAL_DARK_THEME, TERMINAL_LIGHT_THEME, @@ -86,21 +87,25 @@ function terminalPalette(profile) { return palette } -function itermPalette(profile) { - const background = dictionaryColor(profile['Background Color'], LIGHT_THEME.background) +function itermColor(profile, key, suffix, fallback) { + return dictionaryColor(profile[key + suffix], dictionaryColor(profile[key], fallback)) +} + +function itermPalette(profile, suffix) { + const background = itermColor(profile, 'Background Color', suffix, LIGHT_THEME.background) const dark = isDark(background) const fallback = dark ? DARK_THEME : LIGHT_THEME const palette = { background: background, - foreground: dictionaryColor(profile['Foreground Color'], fallback.foreground), - cursor: dictionaryColor(profile['Cursor Color'], fallback.cursor), - cursorAccent: dictionaryColor(profile['Cursor Text Color'], background), - selectionBackground: dictionaryColor(profile['Selection Color'], fallback.selectionBackground), - selectionForeground: dictionaryColor(profile['Selected Text Color'], fallback.foreground) + foreground: itermColor(profile, 'Foreground Color', suffix, fallback.foreground), + cursor: itermColor(profile, 'Cursor Color', suffix, fallback.cursor), + cursorAccent: itermColor(profile, 'Cursor Text Color', suffix, background), + selectionBackground: itermColor(profile, 'Selection Color', suffix, fallback.selectionBackground), + selectionForeground: itermColor(profile, 'Selected Text Color', suffix, fallback.foreground) } for (let index = 0; index < PALETTE_KEYS.length; index += 1) { const key = PALETTE_KEYS[index] - palette[key] = dictionaryColor(profile['Ansi ' + index + ' Color'], fallback[key]) + palette[key] = itermColor(profile, 'Ansi ' + index + ' Color', suffix, fallback[key]) } return palette } @@ -131,12 +136,18 @@ try { const guid = String(profile.Guid || '') const name = String(profile.Name || '') if (!guid || !name) continue - profiles.push({ + const result = { id: 'iterm2:' + encodeURIComponent(guid), name: name, source: 'iterm2', - palette: itermPalette(profile) - }) + palette: itermPalette(profile, '') + } + const separateColors = profile['Use Separate Colors for Light and Dark Mode'] + if (separateColors === true || separateColors === 1) { + result.lightPalette = itermPalette(profile, ' (Light)') + result.darkPalette = itermPalette(profile, ' (Dark)') + } + profiles.push(result) } } catch (_) {} @@ -151,12 +162,7 @@ export function parseTerminalThemeProfiles(value: unknown): TerminalThemeProfile for (const candidate of value) { if (!isTerminalSelectedProfile(candidate) || seen.has(candidate.id)) continue seen.add(candidate.id) - profiles.push({ - id: candidate.id, - name: candidate.name, - source: candidate.source, - palette: { ...candidate.palette }, - }) + profiles.push(cloneTerminalSelectedProfile(candidate)) } return profiles.sort( (left, right) => left.source.localeCompare(right.source) || left.name.localeCompare(right.name) @@ -180,9 +186,8 @@ async function readTerminalThemeProfiles(): Promise { let cachedProfiles: TerminalThemeProfile[] | null = null let profileLoad: Promise | null = null -/** Reads Terminal.app and iTerm2 profiles once per desktop process. */ +/** Reads current Terminal.app and iTerm2 profiles, coalescing concurrent requests. */ export async function listTerminalThemeProfiles(): Promise { - if (cachedProfiles) return cachedProfiles profileLoad ??= readTerminalThemeProfiles() .then((profiles) => { cachedProfiles = profiles diff --git a/apps/desktop/src/main/terminal/index.ts b/apps/desktop/src/main/terminal/index.ts index 074cdc0776c..028282ba312 100644 --- a/apps/desktop/src/main/terminal/index.ts +++ b/apps/desktop/src/main/terminal/index.ts @@ -34,7 +34,11 @@ import { import { sleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import type { BrowserWindow, WebContents } from 'electron' -import type { FocusedResourceShortcut } from '@/main/resource-shortcuts' +import { + type FocusedResourceShortcut, + isResourceTabSelectionShortcut, + resourceTabTargetIndex, +} from '@/main/resource-shortcuts' import { elide, TerminalSession } from '@/main/terminal/session' import { activePane, @@ -158,6 +162,8 @@ export class TerminalService { /** Insertion-ordered, which is also the tab order the user sees. */ private readonly sessions = new Map() private activeId: string | null = null + private agentActiveId: string | null = null + private activeTerminalUserSelected = false /** True while tearing every shell down, so an exit does not respawn one. */ private disposing = false private nextId = 1 @@ -175,6 +181,9 @@ export class TerminalService { */ private focusOwner: WebContents | null = null private releaseFocusListeners: (() => void) | null = null + /** Renderer currently displaying this terminal panel, independent of DOM focus. */ + private visibleOwner: WebContents | null = null + private releaseVisibleListeners: (() => void) | null = null /** Directories of recently closed terminals, newest first, for reopening. */ private readonly recentlyClosedCwds: string[] = [] /** Terminals handed to the user; the value is whether they have handed back. */ @@ -232,13 +241,30 @@ export class TerminalService { session.tabState(session.terminalId === this.activeId) ), activeTerminalId: this.activeId, + agentActiveTerminalId: this.agentActiveId, + } + } + + /** Tool-facing tab list whose active marker follows the agent cursor. */ + private getAgentTabs(): TerminalTabsState { + const state = this.getTabs() + return { + ...state, + tabs: state.tabs.map((tab) => ({ + ...tab, + active: tab.terminalId === this.agentActiveId, + })), + activeTerminalId: this.agentActiveId, } } /** Opens the first terminal, or adopts what is already running. */ start(options: TerminalStartOptions): TerminalTabsState { if (this.sessions.size === 0) { - this.spawn(this.startingCwd(), options.cols, options.rows) + this.spawn(this.startingCwd(), options.cols, options.rows, { + activateVisible: true, + activateAgent: true, + }) } return this.getTabs() } @@ -273,20 +299,90 @@ export class TerminalService { const size = active ? { cols: active.cols, rows: active.rows } : { cols: 80, rows: 24 } // A new terminal opens where the current one is: the user is almost always // continuing the same piece of work in a second shell. - this.spawn(cwd ?? active?.currentCwd ?? this.startingCwd(), size.cols, size.rows) + this.activeTerminalUserSelected = true + this.spawn(cwd ?? active?.currentCwd ?? this.startingCwd(), size.cols, size.rows, { + activateVisible: true, + activateAgent: this.agentActiveId === null, + }) + return this.getTabs() + } + + /** Recreates a persisted tab without treating restoration as user interaction. */ + restoreTerminal(cwd?: string): TerminalTabsState { + const active = this.activeId ? this.sessions.get(this.activeId) : null + const size = active ? { cols: active.cols, rows: active.rows } : { cols: 80, rows: 24 } + this.spawn(cwd ?? active?.currentCwd ?? this.startingCwd(), size.cols, size.rows, { + activateVisible: true, + activateAgent: true, + }) + this.activeTerminalUserSelected = false + return this.getTabs() + } + + /** Restores the persisted visible/agent cursor without claiming user ownership. */ + restoreActiveTerminal(terminalId: string): TerminalTabsState { + if (!this.sessions.has(terminalId)) { + throw new TerminalError('NO_SUCH_TERMINAL', unknownTerminal(terminalId)) + } + this.activeId = terminalId + this.agentActiveId = terminalId + this.activeTerminalUserSelected = false + this.emitTabs() return this.getTabs() } + /** Opens a shell for agent work without changing the terminal the user sees. */ + private openAgentTerminal(cwd?: string): TerminalTabsState { + const agent = this.agentActiveId ? this.sessions.get(this.agentActiveId) : null + const visible = this.activeId ? this.sessions.get(this.activeId) : null + const source = agent ?? visible + const size = source ? { cols: source.cols, rows: source.rows } : { cols: 80, rows: 24 } + this.spawn(cwd ?? source?.currentCwd ?? this.startingCwd(), size.cols, size.rows, { + activateVisible: this.activeId === null, + activateAgent: true, + }) + return this.getAgentTabs() + } + switchTerminal(terminalId: string): TerminalTabsState { if (!this.sessions.has(terminalId)) { throw new TerminalError('NO_SUCH_TERMINAL', unknownTerminal(terminalId)) } this.activeId = terminalId + this.activeTerminalUserSelected = true this.emitTabs() void this.sessions.get(terminalId)?.refreshCwd() return this.getTabs() } + /** Moves the agent cursor without changing the visible terminal. */ + private switchAgentTerminal(terminalId: string): TerminalTabsState { + if (!this.sessions.has(terminalId)) { + throw new TerminalError('NO_SUCH_TERMINAL', unknownTerminal(terminalId)) + } + this.agentActiveId = terminalId + this.emitTabs() + return this.getAgentTabs() + } + + /** Moves one terminal to a final list index without changing the active shell. */ + reorderTerminal(terminalId: string, targetIndex: number): TerminalTabsState { + if (!this.sessions.has(terminalId)) { + throw new TerminalError('NO_SUCH_TERMINAL', unknownTerminal(terminalId)) + } + if (!Number.isFinite(targetIndex)) return this.getTabs() + const entries = [...this.sessions.entries()] + const currentIndex = entries.findIndex(([id]) => id === terminalId) + const nextIndex = Math.max(0, Math.min(entries.length - 1, Math.trunc(targetIndex))) + if (currentIndex === nextIndex) return this.getTabs() + const [entry] = entries.splice(currentIndex, 1) + entries.splice(nextIndex, 0, entry) + this.sessions.clear() + for (const [id, session] of entries) this.sessions.set(id, session) + this.emitTabs() + return this.getTabs() + } + /** * Closes a terminal, or resets it when it is the only one left. * @@ -309,6 +405,17 @@ export class TerminalService { return this.retire(terminalId) } + /** Closes agent-owned work without destroying the shell the user claimed. */ + private closeAgentTerminal(terminalId: string): TerminalTabsState { + if (terminalId === this.activeId && this.activeTerminalUserSelected) { + throw new TerminalError( + 'INVALID_REQUEST', + 'That terminal is currently being used by the user. Switch to another agent terminal instead of closing it.' + ) + } + return this.closeTerminal(terminalId) + } + /** * Removes the temp directories of tracked runs that have since finished. * @@ -359,7 +466,10 @@ export class TerminalService { this.releasePendingRuns(terminalId) if (this.sessions.size === 0) { - this.spawn(this.resolveCwd(closedCwd), cols, rows) + this.spawn(this.resolveCwd(closedCwd), cols, rows, { + activateVisible: true, + activateAgent: true, + }) return this.getTabs() } @@ -367,6 +477,9 @@ export class TerminalService { if (this.activeId === terminalId) { this.activeId = order[index + 1] ?? order[index - 1] ?? null } + if (this.agentActiveId === terminalId) { + this.agentActiveId = order[index + 1] ?? order[index - 1] ?? null + } this.emitTabs() return this.getTabs() } @@ -383,9 +496,27 @@ export class TerminalService { handleFocusedShortcut( shortcut: FocusedResourceShortcut, ownerWindow: BrowserWindow | null, - emitRendererCommand: (command: TerminalShortcutCommand, terminalId: string) => void + emitRendererCommand: (command: TerminalShortcutCommand, terminalId: string) => void, + confirmCloseRunning?: (running: string) => boolean ): boolean { - if (!this.ownsInteraction(ownerWindow)) return false + if (shortcut === 'focus-omnibox') return false + const visibleTabShortcut = + shortcut === 'new-tab' || shortcut === 'reopen-closed-tab' || shortcut === 'close-tab' + const ownsVisibleTabs = + visibleTabShortcut && this.sessions.size > 0 && this.ownsVisiblePanel(ownerWindow) + if (!this.ownsInteraction(ownerWindow) && !ownsVisibleTabs) return false + + if (isResourceTabSelectionShortcut(shortcut)) { + const ids = [...this.sessions.keys()] + const targetIndex = resourceTabTargetIndex( + shortcut, + ids.length, + this.activeId ? ids.indexOf(this.activeId) : -1 + ) + const targetId = targetIndex === null ? null : ids[targetIndex] + if (targetId) this.switchTerminal(targetId) + return true + } switch (shortcut) { case 'new-tab': @@ -397,7 +528,12 @@ export class TerminalService { return true } case 'close-tab': - if (this.activeId) this.closeTerminal(this.activeId) + if (this.activeId) { + const active = this.sessions.get(this.activeId) + const running = active?.isBusy ? (active.foreground ?? 'A process') : null + if (running && confirmCloseRunning && !confirmCloseRunning(running)) return true + this.closeTerminal(this.activeId) + } return true case 'reload-or-clear': if (this.activeId) { @@ -438,6 +574,7 @@ export class TerminalService { // renderer behind it there is nothing that could ever release it. if (!owner || owner.isDestroyed()) return this.focusOwner = owner + this.activeTerminalUserSelected = true const release = () => this.setPanelFocused(false, owner) const onNavigate = (details: { isMainFrame: boolean; isSameDocument: boolean }) => { // A same-document route change keeps the React tree that made the claim. @@ -452,6 +589,29 @@ export class TerminalService { } } + /** Records which app window is currently displaying this terminal resource. */ + setPanelVisible(visible: boolean, owner?: WebContents | null): void { + if (!visible) { + if (owner && this.visibleOwner && owner !== this.visibleOwner) return + this.releaseVisibleOwner() + return + } + this.releaseVisibleOwner() + if (!owner || owner.isDestroyed()) return + this.visibleOwner = owner + const release = () => this.setPanelVisible(false, owner) + const onNavigate = (details: { isMainFrame: boolean; isSameDocument: boolean }) => { + if (details.isMainFrame && !details.isSameDocument) release() + } + owner.once('destroyed', release) + owner.on('did-start-navigation', onNavigate) + this.releaseVisibleListeners = () => { + if (owner.isDestroyed()) return + owner.removeListener('destroyed', release) + owner.removeListener('did-start-navigation', onNavigate) + } + } + /** Drops the claim and unsubscribes from the owner's lifecycle. */ private releaseFocusOwner(): void { this.releaseFocusListeners?.() @@ -459,6 +619,12 @@ export class TerminalService { this.focusOwner = null } + private releaseVisibleOwner(): void { + this.releaseVisibleListeners?.() + this.releaseVisibleListeners = null + this.visibleOwner = null + } + /** * Whether a global accelerator fired in the window that actually holds the * focused terminal panel. Without the window check a claim made in one window @@ -477,6 +643,15 @@ export class TerminalService { return ownerWindow.webContents === this.focusOwner } + private ownsVisiblePanel(ownerWindow: BrowserWindow | null): boolean { + return Boolean( + ownerWindow && + this.visibleOwner && + !this.visibleOwner.isDestroyed() && + ownerWindow.webContents === this.visibleOwner + ) + } + private rememberClosed(cwd: string | null): void { this.recentlyClosedCwds.unshift(cwd ?? '') if (this.recentlyClosedCwds.length > MAX_RECENTLY_CLOSED_TERMINALS) { @@ -521,8 +696,11 @@ export class TerminalService { } this.pendingRuns.clear() this.activeId = null + this.agentActiveId = null + this.activeTerminalUserSelected = false // A stale claim here is what let Cmd-W close a shell that no longer exists. this.setPanelFocused(false) + this.setPanelVisible(false) this.disposing = false } @@ -552,16 +730,16 @@ export class TerminalService { ): Promise { switch (operation) { case 'list': - return this.getTabs() + return this.getAgentTabs() case 'new': - return this.openTerminal(typeof args.cwd === 'string' ? args.cwd : undefined) + return this.openAgentTerminal(typeof args.cwd === 'string' ? args.cwd : undefined) case 'switch': - return this.switchTerminal(this.requireId(args)) + return this.switchAgentTerminal(this.requireId(args)) case 'close': // A named pane is a tmux thing and needs the session resolved below; // without one, close means the Sim terminal. if (typeof args.pane !== 'string' || !args.pane.trim()) { - return this.closeTerminal(this.requireId(args)) + return this.closeAgentTerminal(this.requireId(args)) } break default: @@ -718,7 +896,11 @@ export class TerminalService { /** The user pressing the hand-back button on a waiting handoff. */ finishHandoff(terminalId: string): void { - if (this.handoffs.has(terminalId)) this.handoffs.set(terminalId, true) + if (!this.handoffs.has(terminalId)) return + this.handoffs.set(terminalId, true) + if (terminalId === this.activeId && terminalId === this.agentActiveId) { + this.activeTerminalUserSelected = false + } } /** @@ -904,7 +1086,12 @@ export class TerminalService { return session.runCommand(command, toolCallId, resolveWaitMs(args.waitSeconds)) } - private spawn(cwd: string, cols: number, rows: number): TerminalSession { + private spawn( + cwd: string, + cols: number, + rows: number, + options: { activateVisible: boolean; activateAgent: boolean } + ): TerminalSession { const terminalId = String(this.nextId++) try { const session = TerminalSession.create({ @@ -927,7 +1114,8 @@ export class TerminalService { }, }) this.sessions.set(terminalId, session) - this.activeId = terminalId + if (options.activateVisible || this.activeId === null) this.activeId = terminalId + if (options.activateAgent || this.agentActiveId === null) this.agentActiveId = terminalId this.emitTabs() return session } catch (error) { @@ -945,6 +1133,19 @@ export class TerminalService { private requireSession(args: TerminalToolArgs): TerminalSession { const requested = typeof args.terminalId === 'string' ? args.terminalId : null if (requested) { + if (requested === this.activeId && this.activeTerminalUserSelected) { + const claimed = this.sessions.get(requested) + if (claimed?.alive && requested === this.agentActiveId) { + return this.spawn(claimed.currentCwd ?? this.startingCwd(), claimed.cols, claimed.rows, { + activateVisible: false, + activateAgent: true, + }) + } + throw new TerminalError( + 'INVALID_REQUEST', + 'That terminal is currently being used by the user. Open or switch to another agent terminal first.' + ) + } const session = this.sessions.get(requested) if (!session?.alive) { throw new TerminalError('NO_SUCH_TERMINAL', unknownTerminal(requested)) @@ -952,10 +1153,19 @@ export class TerminalService { return session } - const active = this.activeId ? this.sessions.get(this.activeId) : null + let active = this.agentActiveId ? this.sessions.get(this.agentActiveId) : null + if (active?.terminalId === this.activeId && this.activeTerminalUserSelected) { + active = this.spawn(active.currentCwd ?? this.startingCwd(), active.cols, active.rows, { + activateVisible: false, + activateAgent: true, + }) + } if (active?.alive) return active - const spawned = this.spawn(this.startingCwd(), 80, 24) + const spawned = this.spawn(this.startingCwd(), 80, 24, { + activateVisible: this.activeId === null, + activateAgent: true, + }) if (!spawned.alive) { throw new TerminalError('SPAWN_FAILED', 'Could not open a terminal on this machine.') } diff --git a/apps/desktop/src/main/terminal/registry.ts b/apps/desktop/src/main/terminal/registry.ts index c9f6a9dcd65..39951cd436c 100644 --- a/apps/desktop/src/main/terminal/registry.ts +++ b/apps/desktop/src/main/terminal/registry.ts @@ -7,7 +7,7 @@ import type { TerminalToolArgs, TerminalToolResponse, } from '@sim/terminal-protocol' -import type { BrowserWindow, WebContents } from 'electron' +import { type BrowserWindow, dialog, type WebContents } from 'electron' import type { TerminalSessionSnapshot } from '@/main/desktop-chat-session-store' import type { FocusedResourceShortcut } from '@/main/resource-shortcuts' import { TerminalService, type TerminalServiceOptions, type TerminalSink } from '@/main/terminal' @@ -123,6 +123,10 @@ export class TerminalRegistry { return this.serviceFor(scope).switchTerminal(terminalId) } + reorderTerminal(scope: string, terminalId: string, targetIndex: number): TerminalTabsState { + return this.serviceFor(scope).reorderTerminal(terminalId, targetIndex) + } + closeTerminal(scope: string, terminalId: string): TerminalTabsState { return this.serviceFor(scope).closeTerminal(terminalId) } @@ -198,6 +202,18 @@ export class TerminalRegistry { this.serviceFor(scope).setPanelFocused(focused, owner) } + /** Records the visible terminal scope without treating visibility as keyboard focus. */ + setPanelVisible(scope: string, visible: boolean, owner?: WebContents | null): void { + if (this.suspendedScopes.has(scope)) return + if (!visible && !this.entries.has(scope)) return + if (visible && owner) { + for (const entry of this.entries.values()) { + if (entry.scope !== scope) entry.service.setPanelVisible(false, owner) + } + } + this.serviceFor(scope).setPanelVisible(visible, owner) + } + /** Handles a menu accelerator, which has a window but no renderer chat scope. */ handleFocusedShortcut( ownerWindow: BrowserWindow | null, @@ -205,15 +221,34 @@ export class TerminalRegistry { ): boolean { for (const entry of this.entries.values()) { if ( - entry.service.handleFocusedShortcut(shortcut, ownerWindow, (command, terminalId) => { - if (!ownerWindow || ownerWindow.webContents.isDestroyed()) return - ownerWindow.webContents.send( - 'terminal:shortcut-command', - command, - entry.scope, - terminalId - ) - }) + entry.service.handleFocusedShortcut( + shortcut, + ownerWindow, + (command, terminalId) => { + if (!ownerWindow || ownerWindow.webContents.isDestroyed()) return + ownerWindow.webContents.send( + 'terminal:shortcut-command', + command, + entry.scope, + terminalId + ) + }, + (running) => { + if (!ownerWindow || ownerWindow.isDestroyed()) return false + return ( + dialog.showMessageBoxSync(ownerWindow, { + type: 'warning', + title: 'Close Running Terminal?', + message: `${running} is still running.`, + detail: 'Closing this terminal will stop the process.', + buttons: ['Close Terminal', 'Cancel'], + defaultId: 1, + cancelId: 1, + noLink: true, + }) === 0 + ) + } + ) ) { return true } @@ -320,11 +355,11 @@ export class TerminalRegistry { const persisted = entry.persisted if (persisted) { for (const tab of persisted.tabs.slice(1)) { - entry.service.openTerminal(restorableCwd(tab.cwd)) + entry.service.restoreTerminal(restorableCwd(tab.cwd)) } const restored = entry.service.getTabs() const active = restored.tabs[persisted.activeIndex] - if (active) entry.service.switchTerminal(active.terminalId) + if (active) entry.service.restoreActiveTerminal(active.terminalId) } tabs = entry.service.getTabs() } finally { diff --git a/apps/desktop/src/main/terminal/service.test.ts b/apps/desktop/src/main/terminal/service.test.ts index 662ba9f27cf..019ee518890 100644 --- a/apps/desktop/src/main/terminal/service.test.ts +++ b/apps/desktop/src/main/terminal/service.test.ts @@ -129,6 +129,32 @@ describe('closing terminals', () => { }) }) +describe('reordering terminals', () => { + it('moves a terminal without changing the active shell', () => { + const terminal = service() + const first = terminal.start({ cols: 80, rows: 24 }).activeTerminalId as string + const second = terminal.openTerminal().activeTerminalId as string + const third = terminal.openTerminal().activeTerminalId as string + + const after = terminal.reorderTerminal(first, 2) + + expect(after.tabs.map((tab) => tab.terminalId)).toEqual([second, third, first]) + expect(after.activeTerminalId).toBe(third) + }) + + it('clamps the destination and rejects an unknown terminal', () => { + const terminal = service() + const first = terminal.start({ cols: 80, rows: 24 }).activeTerminalId as string + const second = terminal.openTerminal().activeTerminalId as string + + expect(terminal.reorderTerminal(second, -100).tabs.map((tab) => tab.terminalId)).toEqual([ + second, + first, + ]) + expect(() => terminal.reorderTerminal('no-such-terminal', 0)).toThrow() + }) +}) + type OwnerWindow = Parameters[1] function runShortcut( @@ -188,6 +214,57 @@ describe('focus-gated shortcuts', () => { expect(terminal.getTabs().tabs).toHaveLength(1) }) + it('keeps tab shortcuts routed while the terminal panel is visible without focus', () => { + const terminal = service() + terminal.start({ cols: 80, rows: 24 }) + const renderer = rendererStub() + terminal.setPanelVisible(true, renderer.contents) + + expect(runShortcut(terminal, 'new-tab', renderer.window)).toBe(true) + expect(terminal.getTabs().tabs).toHaveLength(2) + expect(runShortcut(terminal, 'close-tab', renderer.window)).toBe(true) + expect(terminal.getTabs().tabs).toHaveLength(1) + expect(runShortcut(terminal, 'reopen-closed-tab', renderer.window)).toBe(true) + expect(terminal.getTabs().tabs).toHaveLength(2) + expect(runShortcut(terminal, 'focus-omnibox', renderer.window)).toBe(false) + + terminal.setPanelVisible(false, renderer.contents) + expect(runShortcut(terminal, 'new-tab', renderer.window)).toBe(false) + }) + + it('moves agent work to a background shell after the user claims the visible terminal', async () => { + const terminal = service() + const started = terminal.start({ cols: 80, rows: 24 }) + const visibleId = started.activeTerminalId as string + const renderer = rendererStub() + terminal.setPanelFocused(true, renderer.contents) + + const response = await terminal.executeTool('call-cwd', 'cwd', { terminalId: visibleId }) + const result = response.result as { terminalId: string } | undefined + + expect(response.ok).toBe(true) + expect(result?.terminalId).not.toBe(visibleId) + expect(terminal.getTabs().activeTerminalId).toBe(visibleId) + expect(terminal.getTabs().agentActiveTerminalId).toBe(result?.terminalId) + }) + + it('keeps a running terminal open when close confirmation is declined', () => { + const terminal = service() + const started = terminal.start({ cols: 80, rows: 24 }) + const activeId = started.activeTerminalId as string + stubSessions.get(activeId)?.setBusy(true) + const renderer = rendererStub() + const confirmClose = vi.fn(() => false) + terminal.setPanelFocused(true, renderer.contents) + + expect( + terminal.handleFocusedShortcut('close-tab', renderer.window, vi.fn(), confirmClose) + ).toBe(true) + + expect(confirmClose).toHaveBeenCalledWith('sleep 1') + expect(terminal.getTabs().activeTerminalId).toBe(activeId) + }) + it('opens tabs in main and sends canvas commands to the focused renderer', () => { const terminal = service() terminal.start({ cols: 80, rows: 24 }) @@ -212,6 +289,24 @@ describe('focus-gated shortcuts', () => { expect(emit).toHaveBeenLastCalledWith('zoom-reset', terminal.getTabs().activeTerminalId) }) + it('switches tabs with cycle and numbered shortcuts', () => { + const terminal = service() + const first = terminal.start({ cols: 80, rows: 24 }).activeTerminalId as string + const second = terminal.openTerminal().activeTerminalId as string + const third = terminal.openTerminal().activeTerminalId as string + const renderer = rendererStub() + terminal.setPanelFocused(true, renderer.contents) + + expect(runShortcut(terminal, 'next-tab', renderer.window)).toBe(true) + expect(terminal.getTabs().activeTerminalId).toBe(first) + expect(runShortcut(terminal, 'previous-tab', renderer.window)).toBe(true) + expect(terminal.getTabs().activeTerminalId).toBe(third) + expect(runShortcut(terminal, 'select-tab-2', renderer.window)).toBe(true) + expect(terminal.getTabs().activeTerminalId).toBe(second) + expect(runShortcut(terminal, 'select-tab-9', renderer.window)).toBe(true) + expect(terminal.getTabs().activeTerminalId).toBe(third) + }) + it('claims reopen even with no history, then reopens the latest closed terminal', () => { const terminal = service() terminal.start({ cols: 80, rows: 24 }) @@ -379,6 +474,29 @@ describe('handing the terminal to the user', () => { expect(() => terminal.finishHandoff(started.activeTerminalId as string)).not.toThrow() }) + it('resumes in the same terminal after the user explicitly hands it back', async () => { + const terminal = service() + const started = terminal.start({ cols: 80, rows: 24 }) + const id = started.activeTerminalId as string + stubSessions.get(id)?.setBusy(true) + const renderer = rendererStub() + + const handoff = terminal.executeTool('call-handoff', 'handoff', { + terminalId: id, + reason: 'Sign in', + }) + terminal.setPanelFocused(true, renderer.contents) + setTimeout(() => { + terminal.finishHandoff(id) + stubSessions.get(id)?.setBusy(false) + }, 20) + await handoff + + const resumed = await terminal.executeTool('call-cwd', 'cwd', { terminalId: id }) + expect((resumed.result as { terminalId: string }).terminalId).toBe(id) + expect(terminal.getTabs().tabs).toHaveLength(1) + }) + it('fails the handoff if the terminal is closed while it waits', async () => { const terminal = service() const started = terminal.start({ cols: 80, rows: 24 }) @@ -395,7 +513,7 @@ describe('handing the terminal to the user', () => { }) describe('closing', () => { - it('closes the Sim terminal when no pane is named', async () => { + it('does not let the agent close the visible terminal the user selected', async () => { const terminal = service() terminal.start({ cols: 80, rows: 24 }) const second = terminal.openTerminal() @@ -404,8 +522,31 @@ describe('closing', () => { terminalId: second.activeTerminalId as string, }) - expect(response.ok).toBe(true) - expect(terminal.getTabs().tabs).toHaveLength(1) + expect(response.ok).toBe(false) + expect(response.code).toBe('INVALID_REQUEST') + expect(terminal.getTabs().tabs).toHaveLength(2) + }) + + it('opens and closes an agent terminal without changing visible selection', async () => { + const terminal = service() + const started = terminal.start({ cols: 80, rows: 24 }) + const visible = terminal.openTerminal().activeTerminalId as string + + const opened = await terminal.executeTool('call-new', 'new', {}) + const agentTerminalId = (opened.result as { activeTerminalId: string | null } | undefined) + ?.activeTerminalId + + expect(opened.ok).toBe(true) + expect(agentTerminalId).toBeTruthy() + expect(terminal.getTabs().activeTerminalId).toBe(visible) + expect(terminal.getTabs().agentActiveTerminalId).toBe(agentTerminalId) + expect(terminal.getTabs().activeTerminalId).not.toBe(started.activeTerminalId) + + const closed = await terminal.executeTool('call-close', 'close', { + terminalId: agentTerminalId as string, + }) + expect(closed.ok).toBe(true) + expect(terminal.getTabs().activeTerminalId).toBe(visible) }) it('refuses to close a pane in a terminal that has no tmux', async () => { diff --git a/apps/desktop/src/main/terminal/session.test.ts b/apps/desktop/src/main/terminal/session.test.ts index d11886d3071..0224a2f1e0f 100644 --- a/apps/desktop/src/main/terminal/session.test.ts +++ b/apps/desktop/src/main/terminal/session.test.ts @@ -1,5 +1,46 @@ -import { describe, expect, it } from 'vitest' -import { stripAnsi, stripTerminalQueries, toInputChunks } from '@/main/terminal/session' +import type { TerminalCommandEvent } from '@sim/terminal-protocol' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + stripAnsi, + stripTerminalQueries, + TerminalSession, + toInputChunks, +} from '@/main/terminal/session' + +const ptyStub = vi.hoisted(() => ({ + dataHandler: null as ((data: string) => void) | null, + exitHandler: null as (() => void) | null, + writes: [] as string[], +})) + +vi.mock('@lydell/node-pty', () => ({ + spawn: () => ({ + pid: 1234, + onData: (handler: (data: string) => void) => { + ptyStub.dataHandler = handler + }, + onExit: (handler: () => void) => { + ptyStub.exitHandler = handler + }, + write: (data: string) => ptyStub.writes.push(data), + resize: vi.fn(), + kill: vi.fn(), + pause: vi.fn(), + resume: vi.fn(), + }), +})) + +vi.mock('@/main/terminal/shell-integration', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, createNonce: () => 'test-nonce' } +}) + +afterEach(() => { + vi.useRealTimers() + ptyStub.dataHandler = null + ptyStub.exitHandler = null + ptyStub.writes.length = 0 +}) describe('toInputChunks', () => { it('separates Enter from the text so the text is actually submitted', () => { @@ -93,3 +134,91 @@ describe('stripTerminalQueries', () => { expect(stripTerminalQueries('plain output\n')).toBe('plain output\n') }) }) + +describe('TerminalSession command lifecycle', () => { + it('keeps normal long-command activity until the foreground process exits', async () => { + vi.useFakeTimers() + const originalShell = process.env.SHELL + process.env.SHELL = '/bin/zsh' + const commandEvents: TerminalCommandEvent[] = [] + const session = TerminalSession.create({ + terminalId: 'terminal-1', + cwd: '/tmp', + cols: 80, + rows: 24, + callbacks: { + onData: () => {}, + onState: () => {}, + onCommand: (event) => commandEvents.push(event), + onExit: () => {}, + }, + }) + + try { + const resultPromise = session.runCommand('sleep 10', 'tool-call-long', 100) + ptyStub.dataHandler?.('\u001b]633;C;test-nonce\u0007working\n') + await vi.advanceTimersByTimeAsync(100) + expect(await resultPromise).toMatchObject({ status: 'running' }) + expect(commandEvents.filter((event) => event.phase === 'end')).toEqual([]) + + ptyStub.dataHandler?.('\u001b]633;D;0;test-nonce\u0007') + expect(commandEvents.at(-1)).toMatchObject({ + terminalId: 'terminal-1', + phase: 'end', + command: 'sleep 10', + toolCallId: 'tool-call-long', + exitCode: 0, + }) + } finally { + session.dispose() + if (originalShell === undefined) process.env.SHELL = undefined + else process.env.SHELL = originalShell + } + }) + + it('ends tool activity when an interactive command detaches, not when its process exits', async () => { + vi.useFakeTimers() + const originalShell = process.env.SHELL + process.env.SHELL = '/bin/zsh' + const commandEvents: TerminalCommandEvent[] = [] + const session = TerminalSession.create({ + terminalId: 'terminal-1', + cwd: '/tmp', + cols: 80, + rows: 24, + callbacks: { + onData: () => {}, + onState: () => {}, + onCommand: (event) => commandEvents.push(event), + onExit: () => {}, + }, + }) + + try { + const resultPromise = session.runCommand('vim', 'tool-call-1', 10_000) + ptyStub.dataHandler?.('\u001b]633;C;test-nonce\u0007\u001b[?1049h') + expect(await resultPromise).toMatchObject({ status: 'interactive' }) + + expect(commandEvents.at(-1)).toMatchObject({ + terminalId: 'terminal-1', + phase: 'end', + command: 'vim', + toolCallId: 'tool-call-1', + }) + + ptyStub.dataHandler?.('\u001b]633;D;0;test-nonce\u0007') + + expect(commandEvents.at(-1)).toMatchObject({ + terminalId: 'terminal-1', + phase: 'end', + command: 'vim', + exitCode: 0, + }) + expect(commandEvents.at(-1)?.toolCallId).toBeUndefined() + } finally { + session.dispose() + if (originalShell === undefined) process.env.SHELL = undefined + else process.env.SHELL = originalShell + } + }) +}) diff --git a/apps/desktop/src/main/terminal/session.ts b/apps/desktop/src/main/terminal/session.ts index 9a5a3810b24..350c1aa070d 100644 --- a/apps/desktop/src/main/terminal/session.ts +++ b/apps/desktop/src/main/terminal/session.ts @@ -315,6 +315,7 @@ export class TerminalSession { private shellIntegration = false private altScreen = false private foregroundCommand: string | null = null + private foregroundToolCallId: string | null = null private pendingCommand: PendingCommand | null = null /** Command line reported by the shell but not yet bracketed by output-start. */ private announcedCommand: string | null = null @@ -595,6 +596,7 @@ export class TerminalSession { resolve, } this.foregroundCommand = command + this.foregroundToolCallId = toolCallId this.emitState() this.callbacks.onCommand({ terminalId: this.terminalId, phase: 'start', command, toolCallId }) @@ -845,25 +847,22 @@ export class TerminalSession { * returned, since they are frames rather than output. */ private resolveInteractiveCommand(): void { - this.detachStillRunning((pending) => ({ - command: pending.command, - output: - 'This opened a full-screen interactive program, which now holds the terminal until it exits. terminal_read renders its current screen, so you can watch it: if it is doing work the user is waiting on, keep polling with wait + terminal_read until it finishes, exactly as you would a long command. Type into it with terminal_input and stop it with terminal_kill. The user can also drive it in the panel. terminal_run reports BUSY until it exits.', - status: 'interactive', - exitCode: null, - durationMs: Date.now() - pending.startedAt, - cwd: this.cwd, - terminalId: this.terminalId, - truncated: false, - })) + this.detachStillRunning( + (pending) => ({ + command: pending.command, + output: + 'This opened a full-screen interactive program, which now holds the terminal until it exits. terminal_read renders its current screen, so you can watch it: if it is doing work the user is waiting on, keep polling with wait + terminal_read until it finishes, exactly as you would a long command. Type into it with terminal_input and stop it with terminal_kill. The user can also drive it in the panel. terminal_run reports BUSY until it exits.', + status: 'interactive', + exitCode: null, + durationMs: Date.now() - pending.startedAt, + cwd: this.cwd, + terminalId: this.terminalId, + truncated: false, + }), + true + ) } - /** - * Hands back a command that is still going when the wait window elapses, - * with whatever it has printed so far. Not a failure: the agent polls from - * here with wait + terminal_read, which keeps the user seeing progress and - * lets the agent notice a prompt or an error as it appears. - */ /** * Hands a command back early when it has stopped mid-line and gone quiet — * the shape of something sitting on a prompt. Output that ends with a @@ -896,7 +895,7 @@ export class TerminalSession { truncated, ...(awaitingInput ? { awaitingInput: true } : {}), } - }) + }, false) } /** @@ -905,13 +904,27 @@ export class TerminalSession { * slot would let the next terminal_run interleave with it instead of * correctly reporting BUSY. */ - private detachStillRunning(build: (pending: PendingCommand) => TerminalRunResult): void { + private detachStillRunning( + build: (pending: PendingCommand) => TerminalRunResult, + endActivity: boolean + ): void { const pending = this.pendingCommand if (!pending) return clearTimeout(pending.timer) clearInterval(pending.promptWatchdog) this.pendingCommand = null - pending.resolve(build(pending)) + const result = build(pending) + if (endActivity) { + this.callbacks.onCommand({ + terminalId: this.terminalId, + phase: 'end', + command: pending.command, + toolCallId: pending.toolCallId, + durationMs: result.durationMs, + }) + this.foregroundToolCallId = null + } + pending.resolve(result) } private capturedText(pending: PendingCommand): string { @@ -956,11 +969,13 @@ export class TerminalSession { terminalId: this.terminalId, phase: 'end', command, + ...(this.foregroundToolCallId ? { toolCallId: this.foregroundToolCallId } : {}), ...(exitCode === null ? {} : { exitCode }), }) } this.foregroundCommand = null + this.foregroundToolCallId = null this.announcedCommand = null this.altScreen = false this.emitState() diff --git a/apps/desktop/src/main/updater.test.ts b/apps/desktop/src/main/updater.test.ts index 0ba00f9fdca..b2ad5db2670 100644 --- a/apps/desktop/src/main/updater.test.ts +++ b/apps/desktop/src/main/updater.test.ts @@ -25,6 +25,7 @@ import { isNewerVersion, parseSemver, resolveUpdateChannel, + updateCheckIntervalMs, } from '@/main/updater' describe('resolveUpdateChannel', () => { @@ -39,6 +40,17 @@ describe('resolveUpdateChannel', () => { }) }) +describe('updateCheckIntervalMs', () => { + it('checks dev and staging builds every five minutes', () => { + expect(updateCheckIntervalMs('1.2.3-alpha.2')).toBe(5 * 60 * 1000) + expect(updateCheckIntervalMs('1.2.3-beta.1')).toBe(5 * 60 * 1000) + }) + + it('checks production builds every thirty minutes', () => { + expect(updateCheckIntervalMs('1.2.3')).toBe(30 * 60 * 1000) + }) +}) + describe('parseSemver', () => { it('parses plain and v-prefixed versions', () => { expect(parseSemver('1.2.3')).toEqual({ major: 1, minor: 2, patch: 3, prerelease: '' }) @@ -262,6 +274,22 @@ describe('initUpdater state machine', () => { vi.mocked(app.getVersion).mockReturnValue('1.0.0') } }) + + it.each([ + ['1.0.1-alpha.7', 5 * 60 * 1000], + ['1.0.1-beta.7', 5 * 60 * 1000], + ['1.0.1', 30 * 60 * 1000], + ])('schedules %s update polling every %i milliseconds', async (version, interval) => { + vi.mocked(app.getVersion).mockReturnValue(version) + const intervalSpy = vi.spyOn(globalThis, 'setInterval') + try { + await createUpdater({ feedAvailable: true }) + expect(intervalSpy).toHaveBeenCalledWith(expect.any(Function), interval) + } finally { + intervalSpy.mockRestore() + vi.mocked(app.getVersion).mockReturnValue('1.0.0') + } + }) }) function manifest(version: string): string { diff --git a/apps/desktop/src/main/updater.ts b/apps/desktop/src/main/updater.ts index 711875cfdf6..c35d5e9eaec 100644 --- a/apps/desktop/src/main/updater.ts +++ b/apps/desktop/src/main/updater.ts @@ -10,7 +10,8 @@ import type { EventRecorder } from '@/main/observability' const logger = createLogger('DesktopUpdater') const INITIAL_CHECK_DELAY_MS = 10_000 -const CHECK_INTERVAL_MS = 4 * 60 * 60 * 1000 +const PRERELEASE_CHECK_INTERVAL_MS = 5 * 60 * 1000 +const STABLE_CHECK_INTERVAL_MS = 30 * 60 * 1000 export type UpdateChannel = 'latest' | 'beta' | 'alpha' @@ -72,6 +73,13 @@ export function resolveUpdateChannel(version: string): UpdateChannel { return 'latest' } +/** Dev/staging shells poll rapidly; production shells use a quieter cadence. */ +export function updateCheckIntervalMs(version: string): number { + return resolveUpdateChannel(version) === 'latest' + ? STABLE_CHECK_INTERVAL_MS + : PRERELEASE_CHECK_INTERVAL_MS +} + interface ParsedSemver { major: number minor: number @@ -263,7 +271,8 @@ interface UpdateEngine { /** * Keeps installed shells current against the per-environment update feed: - * checks on launch and every four hours, and mirrors pipeline state to the + * checks on launch, then every five minutes for dev/staging builds or every + * thirty minutes for production builds, and mirrors pipeline state to the * renderer for the settings update UI and the minimum-shell-version gate. * * Developer-ID-signed builds use electron-updater (background download, @@ -528,7 +537,7 @@ export function initUpdater(deps: UpdaterDeps): UpdaterHandle { } const check = () => engine?.check() setTimeout(check, INITIAL_CHECK_DELAY_MS) - setInterval(check, CHECK_INTERVAL_MS) + setInterval(check, updateCheckIntervalMs(currentVersion)) }) return { diff --git a/apps/desktop/src/main/window.test.ts b/apps/desktop/src/main/window.test.ts index 97096cb5c77..85644923119 100644 --- a/apps/desktop/src/main/window.test.ts +++ b/apps/desktop/src/main/window.test.ts @@ -1,16 +1,18 @@ -import { describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('electron', () => import('@/test/electron-mock')) -import { BrowserWindow } from 'electron' +import { BrowserWindow, systemPreferences } from 'electron' import type { ConfigStore } from '@/main/config' import type { EventRecorder } from '@/main/observability' import { backgroundColorFor, createMainWindow, createSecureWebPreferences, + ensureMicrophoneAccess, resolvePermission, sanitizeBounds, + setupPermissionHandlers, } from '@/main/window' const APP = 'https://sim.ai' @@ -28,22 +30,161 @@ describe('resolvePermission', () => { expect(resolvePermission('clipboard-read', '', APP)).toBe(false) }) - it('default-denies everything else, including media and unknown future permissions', () => { + it('allows audio-only media from the trusted origin, so voice input works', () => { + expect(resolvePermission('media', APP, APP, ['audio'])).toBe(true) + expect(resolvePermission('media', 'https://evil.example', APP, ['audio'])).toBe(false) + expect(resolvePermission('media', '', APP, ['audio'])).toBe(false) + }) + + it('denies media that is not narrowed to audio', () => { + expect(resolvePermission('media', APP, APP, ['video'])).toBe(false) + expect(resolvePermission('media', APP, APP, ['audio', 'video'])).toBe(false) + expect(resolvePermission('media', APP, APP, ['unknown'])).toBe(false) + expect(resolvePermission('media', APP, APP, [])).toBe(false) + expect(resolvePermission('media', APP, APP)).toBe(false) + }) + + it('default-denies everything else, including unknown future permissions', () => { for (const permission of [ - 'media', 'geolocation', 'notifications', 'camera', + 'display-capture', 'midi', 'pointerLock', 'openExternal', 'some-future-permission', ]) { expect(resolvePermission(permission, APP, APP)).toBe(false) + expect(resolvePermission(permission, APP, APP, ['audio'])).toBe(false) } }) }) +describe('ensureMicrophoneAccess', () => { + const realPlatform = process.platform + + function setPlatform(platform: NodeJS.Platform) { + Object.defineProperty(process, 'platform', { value: platform, configurable: true }) + } + + beforeEach(() => { + vi.mocked(systemPreferences.getMediaAccessStatus).mockReturnValue('granted') + vi.mocked(systemPreferences.askForMediaAccess).mockResolvedValue(true) + }) + + afterEach(() => { + setPlatform(realPlatform) + vi.clearAllMocks() + }) + + it('skips the OS check off macOS, where there is no TCC gate', async () => { + setPlatform('win32') + await expect(ensureMicrophoneAccess()).resolves.toBe(true) + expect(systemPreferences.getMediaAccessStatus).not.toHaveBeenCalled() + }) + + it('raises the macOS prompt when access has never been decided', async () => { + setPlatform('darwin') + vi.mocked(systemPreferences.getMediaAccessStatus).mockReturnValue('not-determined') + await expect(ensureMicrophoneAccess()).resolves.toBe(true) + expect(systemPreferences.askForMediaAccess).toHaveBeenCalledWith('microphone') + }) + + it('does not re-prompt once macOS already granted access', async () => { + setPlatform('darwin') + await expect(ensureMicrophoneAccess()).resolves.toBe(true) + expect(systemPreferences.askForMediaAccess).not.toHaveBeenCalled() + }) + + it('reports a blocked microphone without prompting, since macOS would not show one', async () => { + setPlatform('darwin') + vi.mocked(systemPreferences.getMediaAccessStatus).mockReturnValue('denied') + await expect(ensureMicrophoneAccess()).resolves.toBe(false) + expect(systemPreferences.askForMediaAccess).not.toHaveBeenCalled() + }) + + it('denies when the OS request itself fails', async () => { + setPlatform('darwin') + vi.mocked(systemPreferences.getMediaAccessStatus).mockReturnValue('not-determined') + vi.mocked(systemPreferences.askForMediaAccess).mockRejectedValue(new Error('boom')) + await expect(ensureMicrophoneAccess()).resolves.toBe(false) + }) +}) + +describe('setupPermissionHandlers', () => { + const realPlatform = process.platform + + function createSession() { + const session = { + setPermissionRequestHandler: vi.fn(), + setPermissionCheckHandler: vi.fn(), + } + setupPermissionHandlers(session as never, () => APP) + return { + request: session.setPermissionRequestHandler.mock.calls[0][0] as ( + contents: unknown, + permission: string, + callback: (granted: boolean) => void, + details: Record + ) => void, + check: session.setPermissionCheckHandler.mock.calls[0][0] as ( + contents: unknown, + permission: string, + requestingOrigin: string, + details: Record + ) => boolean, + } + } + + afterEach(() => { + Object.defineProperty(process, 'platform', { value: realPlatform, configurable: true }) + vi.clearAllMocks() + }) + + it('grants a microphone request only after the OS agrees', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }) + vi.mocked(systemPreferences.getMediaAccessStatus).mockReturnValue('denied') + const { request } = createSession() + const callback = vi.fn() + + request(null, 'media', callback, { requestingUrl: `${APP}/workspace`, mediaTypes: ['audio'] }) + await vi.waitFor(() => expect(callback).toHaveBeenCalledWith(false)) + + vi.mocked(systemPreferences.getMediaAccessStatus).mockReturnValue('granted') + request(null, 'media', callback, { requestingUrl: `${APP}/workspace`, mediaTypes: ['audio'] }) + await vi.waitFor(() => expect(callback).toHaveBeenLastCalledWith(true)) + }) + + it('rejects a camera request without touching the OS', () => { + const { request } = createSession() + const callback = vi.fn() + + request(null, 'media', callback, { requestingUrl: `${APP}/workspace`, mediaTypes: ['video'] }) + + expect(callback).toHaveBeenCalledWith(false) + expect(systemPreferences.getMediaAccessStatus).not.toHaveBeenCalled() + }) + + it('answers a clipboard request synchronously', () => { + const { request } = createSession() + const callback = vi.fn() + + request(null, 'clipboard-read', callback, { requestingUrl: `${APP}/workspace` }) + + expect(callback).toHaveBeenCalledWith(true) + }) + + it('reports microphone as permitted on the check path', () => { + const { check } = createSession() + + expect(check(null, 'media', APP, { mediaType: 'audio' })).toBe(true) + expect(check(null, 'media', APP, { mediaType: 'video' })).toBe(false) + expect(check(null, 'media', APP, {})).toBe(false) + expect(check(null, 'media', 'https://evil.example', { mediaType: 'audio' })).toBe(false) + }) +}) + describe('backgroundColorFor', () => { it('matches the persisted web-app theme', () => { expect(backgroundColorFor('dark', false)).toBe('#0c0c0c') diff --git a/apps/desktop/src/main/window.ts b/apps/desktop/src/main/window.ts index e5bdfb979b7..5ec4c02b7f8 100644 --- a/apps/desktop/src/main/window.ts +++ b/apps/desktop/src/main/window.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import type { Session, WebPreferences } from 'electron' -import { app, BrowserWindow, dialog, nativeTheme } from 'electron' +import { app, BrowserWindow, dialog, nativeTheme, systemPreferences } from 'electron' import { type ConfigStore, isSafeInternalPath, type WindowBounds } from '@/main/config' import { isAppOrigin, isAuthSurfacePath } from '@/main/navigation' import type { EventRecorder } from '@/main/observability' @@ -51,27 +52,73 @@ export function createSecureWebPreferences( } /** - * The permission matrix: clipboard access for the trusted app origin, - * default-deny for everything else including unknown future permissions - * (media/camera/microphone stay denied). + * The permission matrix: clipboard and microphone access for the trusted app + * origin, default-deny for everything else including unknown future + * permissions (camera and screen capture stay denied). * * Clipboard reads are what the terminal's Paste action runs on — xterm has no * native paste target to fall back to, so a denied read is a Paste that fails. - * The grant is scoped to the app's own origin, which already reaches far more - * sensitive surfaces through the preload bridge, so it widens nothing that a - * compromise of that origin would not already own. + * `media` is what the composer's voice input runs on, and is narrowed to + * audio-only requests so a `getUserMedia({ video: true })` still gets nothing. + * Both grants are scoped to the app's own origin, which already reaches far + * more sensitive surfaces through the preload bridge, so they widen nothing + * that a compromise of that origin would not already own. + * + * `mediaTypes` is the capture kind Chromium asked for. It is absent on + * non-media permissions and, on the check path, may arrive as `unknown` — an + * un-narrowable request is treated as a camera request and denied. */ export function resolvePermission( permission: string, requestingOrigin: string, - appOrigin: string + appOrigin: string, + mediaTypes?: readonly string[] ): boolean { if (!requestingOrigin || requestingOrigin !== appOrigin) { return false } + if (permission === 'media') { + return ( + mediaTypes !== undefined && + mediaTypes.length > 0 && + mediaTypes.every((type) => type === 'audio') + ) + } return permission === 'clipboard-sanitized-write' || permission === 'clipboard-read' } +/** + * macOS gates microphone capture behind TCC on top of Chromium's own + * permission, and Chromium does not raise that system prompt for an Electron + * app — an un-granted app just gets a hard `NotAllowedError`. So the shell + * asks for OS access itself and only then answers the page's request. + * + * A `denied`/`restricted` status is not re-askable: macOS shows no second + * prompt, so this resolves false and the renderer surfaces the "blocked" + * message rather than the click doing nothing at all. + */ +export async function ensureMicrophoneAccess(): Promise { + if (process.platform !== 'darwin') { + return true + } + const status = systemPreferences.getMediaAccessStatus('microphone') + if (status === 'granted') { + return true + } + if (status === 'denied' || status === 'restricted') { + logger.warn('Microphone access is blocked by macOS privacy settings', { status }) + return false + } + try { + const granted = await systemPreferences.askForMediaAccess('microphone') + logger.info('Requested macOS microphone access', { granted }) + return granted + } catch (error) { + logger.error('Could not request macOS microphone access', { error: getErrorMessage(error) }) + return false + } +} + function originOf(raw: string): string { try { return new URL(raw).origin @@ -87,11 +134,21 @@ function originOf(raw: string): string { export function setupPermissionHandlers(session: Session, getAppOrigin: () => string): void { session.setPermissionRequestHandler((webContents, permission, callback, details) => { const requestingUrl = details.requestingUrl || webContents?.getURL() || '' - callback(resolvePermission(permission, originOf(requestingUrl), getAppOrigin())) + const mediaTypes = 'mediaTypes' in details ? details.mediaTypes : undefined + if (!resolvePermission(permission, originOf(requestingUrl), getAppOrigin(), mediaTypes)) { + callback(false) + return + } + if (permission === 'media') { + void ensureMicrophoneAccess().then(callback) + return + } + callback(true) }) - session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) => { - return resolvePermission(permission, originOf(requestingOrigin), getAppOrigin()) + session.setPermissionCheckHandler((_webContents, permission, requestingOrigin, details) => { + const mediaTypes = details.mediaType ? [details.mediaType] : undefined + return resolvePermission(permission, originOf(requestingOrigin), getAppOrigin(), mediaTypes) }) } diff --git a/apps/desktop/src/preload/index.test.ts b/apps/desktop/src/preload/index.test.ts index 698bac6e381..dea8ac89579 100644 --- a/apps/desktop/src/preload/index.test.ts +++ b/apps/desktop/src/preload/index.test.ts @@ -27,11 +27,15 @@ describe('desktop preload bridge', () => { if (!exposed) throw new Error('Expected the desktop preload API to be exposed') expect(exposed.browserAgent.supportsAtomicPanelOcclusion).toBe(true) + await exposed.browserAgent.cancelTool?.('tool-1', 'chat-default') + await exposed.browserAgent.cancelActiveTool?.('chat-reloaded') await exposed.browserAgent.setPanelOccluded(true, 'chat-default') await exposed.browserAgent.setPanelOccluded(false, 'chat-explicit-false', false) await exposed.browserAgent.setPanelOccluded(true, 'chat-force', true) expect(invoke.mock.calls).toEqual([ + ['browser-agent:cancel-tool', 'tool-1', 'chat-default'], + ['browser-agent:cancel-active-tool', 'chat-reloaded'], ['browser-agent:set-panel-occluded', true, 'chat-default', false], ['browser-agent:set-panel-occluded', false, 'chat-explicit-false', false], ['browser-agent:set-panel-occluded', true, 'chat-force', true], diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index c9a5f404cfb..ffb5314df73 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -185,9 +185,15 @@ const api: SimDesktopApi = { scopeId: string ): Promise => ipcRenderer.invoke('browser-agent:execute-tool', toolCallId, tool, params, scopeId), + cancelTool: (toolCallId: string, scopeId: string): Promise => + ipcRenderer.invoke('browser-agent:cancel-tool', toolCallId, scopeId), + cancelActiveTool: (scopeId: string): Promise => + ipcRenderer.invoke('browser-agent:cancel-active-tool', scopeId), panelAction: (action: BrowserPanelAction, scopeId: string): void => { ipcRenderer.send('browser-agent:panel-action', action, scopeId) }, + openTab: (scopeId: string): Promise => + ipcRenderer.invoke('browser-agent:open-tab', scopeId), activateScope: (scopeId: string): Promise => ipcRenderer.invoke('browser-agent:activate-scope', scopeId), restoreScope: (scopeId: string): Promise => @@ -438,6 +444,12 @@ const api: SimDesktopApi = { ipcRenderer.invoke('terminal:open', cwd, scopeId), switchTerminal: (terminalId: string, scopeId: string): Promise => ipcRenderer.invoke('terminal:switch', terminalId, scopeId), + reorderTerminal: ( + terminalId: string, + targetIndex: number, + scopeId: string + ): Promise => + ipcRenderer.invoke('terminal:reorder', terminalId, targetIndex, scopeId), closeTerminal: (terminalId: string, scopeId: string): Promise => ipcRenderer.invoke('terminal:close', terminalId, scopeId), getTabs: (scopeId: string): Promise => @@ -470,6 +482,9 @@ const api: SimDesktopApi = { setFocused: (focused: boolean, scopeId: string): void => { ipcRenderer.send('terminal:focused', focused, scopeId) }, + setVisible: (visible: boolean, scopeId: string): void => { + ipcRenderer.send('terminal:visible', visible, scopeId) + }, finishHandoff: (terminalId: string, scopeId: string): void => { ipcRenderer.send('terminal:handoff-done', terminalId, scopeId) }, diff --git a/apps/desktop/src/test/electron-mock.ts b/apps/desktop/src/test/electron-mock.ts index 8d3db2a1b30..fafa34ee4ac 100644 --- a/apps/desktop/src/test/electron-mock.ts +++ b/apps/desktop/src/test/electron-mock.ts @@ -35,6 +35,7 @@ export const crashReporter = { } export const shell = { + beep: vi.fn(), openExternal: vi.fn(() => Promise.resolve()), openPath: vi.fn(() => Promise.resolve('')), showItemInFolder: vi.fn(), @@ -57,6 +58,13 @@ export const clipboard = { readText: vi.fn(() => ''), } +export const systemPreferences = { + getMediaAccessStatus: vi.fn(() => 'granted'), + askForMediaAccess: vi.fn(() => Promise.resolve(true)), + canPromptTouchID: vi.fn(() => false), + promptTouchID: vi.fn(() => Promise.resolve()), +} + export const nativeTheme = { shouldUseDarkColors: false, on: vi.fn(), @@ -135,6 +143,7 @@ function createWebContentsMock() { reload: vi.fn(), print: vi.fn(), focus: vi.fn(), + invalidate: vi.fn(), isFocused: vi.fn(() => false), close: vi.fn(), isDestroyed: vi.fn(() => false), diff --git a/apps/sim/app/_styles/globals.css b/apps/sim/app/_styles/globals.css index b467e4bfc93..14508122541 100644 --- a/apps/sim/app/_styles/globals.css +++ b/apps/sim/app/_styles/globals.css @@ -376,6 +376,12 @@ html.sidebar-booting .sidebar-shell-inner { --text-icon: #5a5a5a; --text-icon-muted: #5c5c5c; + /* Shared ThinkingLoader ink treatment. */ + --thinking-ink-current: #2c2c2c; + --thinking-ink-inner: #2c2c2c; + --thinking-ink-outer: #5f5f5f; + --thinking-ink-glow: rgba(255, 255, 255, 0.6); + --text-inverse: #ffffff; --text-muted-inverse: #a0a0a0; --text-error: #ef4444; @@ -531,6 +537,12 @@ html.sidebar-booting .sidebar-shell-inner { --text-icon: #969696; --text-icon-muted: #949494; + /* Shared ThinkingLoader ink treatment. */ + --thinking-ink-current: #d6d6d6; + --thinking-ink-inner: #a7a7a7; + --thinking-ink-outer: #d6d6d6; + --thinking-ink-glow: rgba(255, 255, 255, 0.9); + --text-inverse: #1b1b1b; --text-muted-inverse: #b3b3b3; --text-error: #ef4444; diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group.test.ts index 842034b9d75..4808893b4eb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group.test.ts @@ -1,10 +1,22 @@ /** - * @vitest-environment node + * @vitest-environment jsdom */ -import { describe, expect, it } from 'vitest' +import { act, createElement } from 'react' +import { createRoot } from 'react-dom/client' +import { describe, expect, it, vi } from 'vitest' import type { ToolCallData, ToolCallStatus } from '../../../../types' import type { AgentGroupItem } from './agent-group' -import { isAgentGroupResolved } from './agent-group' +import { AgentGroup, isAgentGroupResolved } from './agent-group' + +vi.mock('@/lib/browser-agent/transport', () => ({ + isBrowserAgentAvailable: () => true, +})) + +vi.mock('../special-tags', () => ({ + CredentialDisplay: ({ data }: { data: Array<{ name?: string }> }) => data[0]?.name ?? '', + BrowserTakeoverQuestion: ({ reason, answer }: { reason?: string; answer?: string }) => + createElement('div', { 'data-takeover-answer': 'true' }, `${reason}: ${answer}`), +})) let toolSeq = 0 @@ -37,6 +49,20 @@ function group(items: AgentGroupItem[], isDelegating = false): AgentGroupItem { } } +function browserTakeover(reason: string): Extract { + toolSeq += 1 + return { + type: 'tool', + data: { + id: `takeover-${toolSeq}`, + toolName: 'browser_request_takeover', + displayTitle: `Waiting for you: ${reason}`, + status: 'executing', + params: { reason }, + }, + } +} + describe('isAgentGroupResolved', () => { it('is unresolved when there is no work yet', () => { expect(isAgentGroupResolved([])).toBe(false) @@ -69,3 +95,130 @@ describe('isAgentGroupResolved', () => { expect(isAgentGroupResolved([group([group([tool('executing')])])])).toBe(false) }) }) + +describe('AgentGroup browser takeover', () => { + it('collapses the browser log and renders the question outside its viewport', () => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + const container = document.createElement('div') + const root = createRoot(container) + const reason = 'Please pick a match in the draw.' + + act(() => { + root.render( + createElement(AgentGroup, { + agentName: 'browser', + agentLabel: 'Browser Agent', + items: [tool('success'), browserTakeover(reason)], + isStreaming: true, + isCurrentSection: true, + isLaneOpen: true, + }) + ) + }) + + const collapsedLog = container.querySelector('[data-state="closed"]') + const liftedQuestion = Array.from(container.querySelectorAll('.animate-stream-fade-in')).find( + (element) => element.textContent === reason + ) + expect(collapsedLog).not.toBeNull() + expect(liftedQuestion).toBeDefined() + expect(collapsedLog?.contains(liftedQuestion ?? null)).toBe(false) + + const header = Array.from(container.querySelectorAll('button')).find((button) => + button.textContent?.includes('Browser Agent') + ) + act(() => header?.click()) + expect(container.querySelector('[data-state="open"]')).not.toBeNull() + expect(liftedQuestion?.textContent).toBe(reason) + + act(() => root.unmount()) + }) + + it('clears a stale question when the lane closes or a newer tool starts', () => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + const container = document.createElement('div') + const root = createRoot(container) + const reason = 'Please finish in the browser.' + const takeover = browserTakeover(reason) + + act(() => { + root.render( + createElement(AgentGroup, { + agentName: 'browser', + agentLabel: 'Browser Agent', + items: [takeover, tool('executing')], + isStreaming: true, + isLaneOpen: true, + }) + ) + }) + expect(container.querySelector('.animate-stream-fade-in')).toBeNull() + + act(() => { + root.render( + createElement(AgentGroup, { + agentName: 'browser', + agentLabel: 'Browser Agent', + items: [takeover], + isStreaming: false, + isLaneOpen: false, + }) + ) + }) + expect(container.querySelector('.animate-stream-fade-in')).toBeNull() + + act(() => root.unmount()) + }) + + it('moves the answered question back inside the resumed browser agent', () => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + const container = document.createElement('div') + const root = createRoot(container) + const reason = 'Pick a match from the draw.' + const takeover = browserTakeover(reason) + + act(() => { + root.render( + createElement(AgentGroup, { + agentName: 'browser', + agentLabel: 'Browser Agent', + items: [takeover], + isStreaming: true, + isCurrentSection: true, + isLaneOpen: true, + }) + ) + }) + expect(container.querySelector('.animate-stream-fade-in')).not.toBeNull() + + const completedTakeover: AgentGroupItem = { + type: 'tool', + data: { + ...takeover.data, + status: 'success', + result: { success: true, output: { userInstruction: 'Open the second match' } }, + }, + } + act(() => { + root.render( + createElement(AgentGroup, { + agentName: 'browser', + agentLabel: 'Browser Agent', + items: [completedTakeover], + isStreaming: true, + isCurrentSection: true, + isLaneOpen: true, + }) + ) + }) + + expect(container.querySelector('.animate-stream-fade-in')).toBeNull() + const resumedLog = container.querySelector('[data-state="open"]') + const answeredQuestion = container.querySelector('[data-takeover-answer="true"]') + expect(answeredQuestion?.textContent).toContain(reason) + expect(answeredQuestion?.textContent).toContain('Open the second match') + expect(resumedLog?.contains(answeredQuestion)).toBe(true) + + act(() => root.unmount()) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group.tsx index 012158e64f5..103e6ba6e4f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/agent-group/agent-group.tsx @@ -3,9 +3,12 @@ import { useEffect, useLayoutEffect, useRef, useState } from 'react' import { ChevronDown, cn, Expandable, ExpandableContent } from '@sim/emcn' import { ShimmerText } from '@/components/ui' +import { isBrowserAgentAvailable } from '@/lib/browser-agent/transport' +import { BrowserRequestTakeover } from '@/lib/copilot/generated/tool-catalog-v1' import { useSmoothText } from '@/hooks/use-smooth-text' import { type ToolCallData, ToolCallStatus } from '../../../../types' import { getAgentIcon, isToolDone } from '../../utils' +import { CredentialDisplay } from '../special-tags' import { renderInlineMarkdown } from './inline-markdown' import { ToolCallItem } from './tool-call-item' @@ -49,6 +52,44 @@ function hasAwaitingApproval(items: AgentGroupItem[]): boolean { }) } +interface ActiveBrowserTakeover { + id: string + reason: string +} + +/** Returns this group's own active browser hand-back, if any. */ +function getActiveBrowserTakeover(items: AgentGroupItem[]): ActiveBrowserTakeover | null { + for (let index = items.length - 1; index >= 0; index--) { + const item = items[index] + if (item.type !== 'tool') continue + if ( + item.data.toolName === BrowserRequestTakeover.id && + item.data.status === ToolCallStatus.executing + ) { + const reason = item.data.params?.reason + return { + id: item.data.id, + reason: typeof reason === 'string' ? reason.trim() : '', + } + } + // Browser-agent tools are serialized. Once a newer tool exists, an older + // executing takeover is stale and must not keep a question on screen. + return null + } + return null +} + +/** True when a nested group owns a browser hand-back question. */ +function hasNestedBrowserTakeover(items: AgentGroupItem[]): boolean { + return items.some( + (item) => + item.type === 'agent_group' && + item.group.isOpen && + (getActiveBrowserTakeover(item.group.items) !== null || + hasNestedBrowserTakeover(item.group.items)) + ) +} + export function isAgentGroupResolved(items: AgentGroupItem[]): boolean { let hasWork = false for (const item of items) { @@ -75,7 +116,12 @@ export function AgentGroup({ const AgentIcon = getAgentIcon(agentName) const hasItems = items.length > 0 const resolved = isAgentGroupResolved(items) - const isWorking = (isDelegating && !resolved) || (isStreaming && isLaneOpen) + const browserAgentAvailable = isBrowserAgentAvailable() + const activeBrowserTakeover = + browserAgentAvailable && isLaneOpen ? getActiveBrowserTakeover(items) : null + const nestedBrowserTakeover = browserAgentAvailable && hasNestedBrowserTakeover(items) + const isWorking = + !activeBrowserTakeover && ((isDelegating && !resolved) || (isStreaming && isLaneOpen)) // Expand while the turn is live and any of: the lane is open (the subagent is // actively running), this is the current/latest section, or there is unresolved @@ -89,17 +135,31 @@ export function AgentGroup({ // (isStreaming false) collapses everything; a manual toggle pins the choice. const autoExpanded = isStreaming && (isCurrentSection || isLaneOpen || !resolved) const [manualExpanded, setManualExpanded] = useState(null) + const [expandedTakeoverId, setExpandedTakeoverId] = useState(null) // An outstanding permission prompt overrides a manual collapse: the turn // cannot proceed until it is answered, so hiding it would deadlock the chat // with nothing on screen to explain why. - const expanded = hasAwaitingApproval(items) || (manualExpanded ?? autoExpanded) + const expanded = + hasAwaitingApproval(items) || + nestedBrowserTakeover || + (activeBrowserTakeover + ? expandedTakeoverId === activeBrowserTakeover.id + : (manualExpanded ?? autoExpanded)) + + const toggleExpanded = () => { + if (activeBrowserTakeover) { + setExpandedTakeoverId(expanded ? null : activeBrowserTakeover.id) + return + } + setManualExpanded(!expanded) + } return (
{hasItems ? (
)} - {!disabled && ( + {!disabled && dismissible && ( + /> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.test.ts index fb88e6780a5..8226865967a 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.test.ts @@ -581,7 +581,7 @@ describe('completed tool titles', () => { ) const presentTitle = getToolDisplayTitle(toolName, args) - const expectedTitle = getToolStatusDisplayTitle(presentTitle, 'success') + const expectedTitle = getToolStatusDisplayTitle(presentTitle, 'success', toolName) const actualTitle = firstToolTitle(modelToContentBlocks(model)) if (actualTitle !== expectedTitle) { failures.push(`${toolName}: expected ${expectedTitle}, received ${actualTitle}`) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx index a2462865053..cc65ba262d6 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/message-content.tsx @@ -189,7 +189,7 @@ function toToolData(tc: NonNullable): ToolCallData { const overrideDisplayTitle = getOverrideDisplayTitle(tc) const resolvedTitle = overrideDisplayTitle || tc.displayTitle || getToolDisplayTitle(tc.name, tc.params) - const displayTitle = getToolStatusDisplayTitle(resolvedTitle, tc.status) + const displayTitle = getToolStatusDisplayTitle(resolvedTitle, tc.status, tc.name) return { id: tc.id, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.module.css b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.module.css new file mode 100644 index 00000000000..1938a59f8d4 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.module.css @@ -0,0 +1,84 @@ +.track { + position: absolute; + right: 0; + bottom: -1px; + left: 0; + z-index: 30; + height: 2px; + overflow: hidden; + pointer-events: none; +} + +.indicator { + position: absolute; + top: 0; + bottom: 0; + left: 0; + width: 100%; + transform: scaleX(0.08); + transform-origin: left; + /* The ThinkingLoader and browser progress share one theme-owned ink palette. */ + background: linear-gradient(90deg, var(--thinking-ink-inner), var(--thinking-ink-outer)); + box-shadow: 0 0 3px color-mix(in srgb, var(--thinking-ink-outer) 55%, transparent); + animation: browser-page-loading 12s ease-out forwards; +} + +@keyframes browser-page-loading { + 0% { + transform: scaleX(0.08); + } + + 10% { + transform: scaleX(0.3); + } + + 35% { + transform: scaleX(0.52); + } + + 70% { + transform: scaleX(0.72); + } + + 100% { + transform: scaleX(0.86); + } +} + +.completing { + animation: browser-page-loading-fade 200ms linear forwards; +} + +.completing .indicator { + transform: scaleX(1); + animation: none; + transition: transform 100ms ease-out; +} + +@keyframes browser-page-loading-fade { + 0%, + 50% { + opacity: 1; + } + + 100% { + opacity: 0; + } +} + +@media (prefers-reduced-motion: reduce) { + .indicator { + transform: scaleX(0.7); + animation: none; + } + + .completing { + opacity: 1; + animation: none; + } + + .completing .indicator { + transform: scaleX(1); + transition: none; + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.test.tsx new file mode 100644 index 00000000000..212f6966dff --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.test.tsx @@ -0,0 +1,59 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { BrowserLoadingBar } from './browser-loading-bar' + +let container: HTMLDivElement +let root: Root + +function render(loading: boolean): void { + act(() => root.render()) +} + +beforeEach(() => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + vi.useFakeTimers() + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) +}) + +afterEach(() => { + act(() => root.unmount()) + container.remove() + vi.useRealTimers() +}) + +describe('BrowserLoadingBar', () => { + it('reaches 100% before it disappears after loading settles', () => { + render(true) + expect(container.querySelector('[role="progressbar"]')?.getAttribute('data-phase')).toBe( + 'loading' + ) + + render(false) + const completing = container.querySelector('[role="progressbar"]') + expect(completing?.getAttribute('data-phase')).toBe('completing') + expect(completing?.getAttribute('aria-valuenow')).toBe('100') + + act(() => vi.advanceTimersByTime(199)) + expect(container.querySelector('[role="progressbar"]')).not.toBeNull() + + act(() => vi.advanceTimersByTime(1)) + expect(container.querySelector('[role="progressbar"]')).toBeNull() + }) + + it('cancels completion when another navigation starts', () => { + render(true) + render(false) + render(true) + + act(() => vi.advanceTimersByTime(500)) + expect(container.querySelector('[role="progressbar"]')?.getAttribute('data-phase')).toBe( + 'loading' + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.tsx new file mode 100644 index 00000000000..3b698531803 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar.tsx @@ -0,0 +1,49 @@ +'use client' + +import { useEffect, useState } from 'react' +import { cn } from '@sim/emcn' +import styles from './browser-loading-bar.module.css' + +const COMPLETION_DURATION_MS = 200 + +type LoadingBarPhase = 'hidden' | 'loading' | 'completing' + +/** + * Browser-style progress without fabricated percentages: it trickles below + * completion while navigation is active, then visibly reaches the right edge + * and fades only after Electron reports the page settled. + */ +export function BrowserLoadingBar({ loading }: { loading: boolean }) { + const [phase, setPhase] = useState(loading ? 'loading' : 'hidden') + + useEffect(() => { + if (loading) { + setPhase('loading') + return + } + setPhase((current) => (current === 'hidden' ? current : 'completing')) + }, [loading]) + + useEffect(() => { + if (phase !== 'completing') return + const timeout = setTimeout(() => setPhase('hidden'), COMPLETION_DURATION_MS) + return () => clearTimeout(timeout) + }, [phase]) + + if (phase === 'hidden') return null + + return ( +
+ +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-panel-occlusion.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-panel-occlusion.test.ts index 1c0a5a20d2f..63c974fcbcf 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-panel-occlusion.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-panel-occlusion.test.ts @@ -536,6 +536,34 @@ describe('useBrowserPanelOcclusion modal lifecycle', () => { hook.unmount() }) + it('replaces one popover with another without revealing or recapturing the native view', async () => { + const firstFallback = vi.fn() + const secondFallback = vi.fn() + const hook = renderOcclusionHook() + let firstRequest!: Promise + let secondRequest!: Promise + + act(() => { + firstRequest = hook.result().requestOverlay('resources', firstFallback) + }) + await flushOcclusionLifecycle() + expect(await firstRequest).toBe(true) + expect(hook.result().activeOverlay).toBe('resources') + + act(() => { + secondRequest = hook.result().requestOverlay('tab', secondFallback) + }) + await flushOcclusionLifecycle() + + expect(await secondRequest).toBe(true) + expect(hook.result().activeOverlay).toBe('tab') + expect(captureBrowserPanelSnapshot).toHaveBeenCalledOnce() + expect(setBrowserPanelOccluded).toHaveBeenCalledTimes(1) + expect(firstFallback).not.toHaveBeenCalled() + expect(secondFallback).not.toHaveBeenCalled() + hook.unmount() + }) + it('retires an active popover and replacement when the Browser becomes ineligible', async () => { const hook = renderOcclusionHook() act(() => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-panel-occlusion.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-panel-occlusion.ts index c90f9a66523..b4b76eb9367 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-panel-occlusion.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-panel-occlusion.ts @@ -36,7 +36,8 @@ export type BrowserPanelOverlay = | 'toolbar' export interface BrowserPanelOverlayController { - requestOverlay: (overlay: BrowserPanelOverlay, fallback: () => void) => Promise + /** True when the renderer overlay owns the painted frame; false when fallback handled it. */ + requestOverlay: (overlay: BrowserPanelOverlay, fallback: () => void) => Promise closeOverlay: (overlay: BrowserPanelOverlay) => Promise } @@ -446,30 +447,36 @@ export function useBrowserPanelOcclusion( ) const requestOverlay = useCallback( - async (overlay: BrowserPanelOverlay, fallback: () => void) => { - if ( - !panelVisibleRef.current || - screenOcclusionPresentRef.current || - activeOverlayRef.current || - pendingOverlayRef.current - ) { - return + async (overlay: BrowserPanelOverlay, fallback: () => void): Promise => { + if (!panelVisibleRef.current || screenOcclusionPresentRef.current) return false + if (activeOverlayRef.current === overlay) return true + if (pendingOverlayRef.current === overlay) { + await reconcileChainRef.current + return activeOverlayRef.current === overlay } + // A different renderer popover can replace the current one without + // revealing the native page between them. Keeping a pending popover owns + // the existing captured frame while the old controlled menu closes. pendingOverlayRef.current = overlay + if (activeOverlayRef.current) { + activeOverlayRef.current = null + setActiveOverlay(null) + } const ready = await scheduleReconcile() - if (!mountedRef.current || pendingOverlayRef.current !== overlay) return + if (!mountedRef.current || pendingOverlayRef.current !== overlay) return false if (ready && nativeHiddenRef.current && desiredLayer() === 'popover') { pendingOverlayRef.current = null activeOverlayRef.current = overlay setActiveOverlay(overlay) - return + return true } pendingOverlayRef.current = null await scheduleReconcile() fallback() + return false }, [desiredLayer, scheduleReconcile] ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.test.ts index 81bd64563e2..0edd0e9f3f0 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.test.ts @@ -6,8 +6,11 @@ import { browserPanelSnapshotStyle, browserSelectionContext, clearOmniboxSelection, + hasConfirmedBrowserTabCreation, + initialUrlSuggestionIndex, resolveUrlBarInput, selectFocusedOmniboxOnNextFrame, + shouldOpenUrlSuggestions, shouldRemoveBrowserResource, shouldReportBrowserBounds, } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session' @@ -153,6 +156,54 @@ describe('clearOmniboxSelection', () => { }) }) +describe('shouldOpenUrlSuggestions', () => { + it('opens only once the renderer owns the painted frame', () => { + expect(shouldOpenUrlSuggestions('suggestions', 3)).toBe(true) + }) + + it('stays closed while the native page is still on top', () => { + // The rows are ranked and ready, but a click would land on the + // WebContentsView rather than the list — the "clicking a suggestion does + // nothing" bug. Not opening is the honest outcome; there is no native menu + // to fall back to. + expect(shouldOpenUrlSuggestions(null, 3)).toBe(false) + }) + + it('stays closed when another overlay holds the lease', () => { + expect(shouldOpenUrlSuggestions('credentials', 3)).toBe(false) + expect(shouldOpenUrlSuggestions('tab', 3)).toBe(false) + }) + + it('stays closed with nothing to suggest, however the frame is owned', () => { + expect(shouldOpenUrlSuggestions('suggestions', 0)).toBe(false) + expect(shouldOpenUrlSuggestions(null, 0)).toBe(false) + }) +}) + +describe('initialUrlSuggestionIndex', () => { + it('selects the first suggestion on a new tab', () => { + expect(initialUrlSuggestionIndex('', 3)).toBe(0) + expect(initialUrlSuggestionIndex('about:blank', 3)).toBe(0) + }) + + it('leaves existing pages unselected so Enter submits the current URL', () => { + expect(initialUrlSuggestionIndex('https://sim.ai', 3)).toBeNull() + }) + + it('selects nothing when there are no suggestions', () => { + expect(initialUrlSuggestionIndex('', 0)).toBeNull() + }) +}) + +describe('hasConfirmedBrowserTabCreation', () => { + it('requires both a larger strip and a distinct active tab', () => { + expect(hasConfirmedBrowserTabCreation('tab-1', 1, 'tab-2', 2)).toBe(true) + expect(hasConfirmedBrowserTabCreation('tab-1', 1, 'tab-1', 2)).toBe(false) + expect(hasConfirmedBrowserTabCreation('tab-1', 1, 'tab-2', 1)).toBe(false) + expect(hasConfirmedBrowserTabCreation('tab-1', 1, null, 2)).toBe(false) + }) +}) + describe('suspended browser resource lifecycle', () => { it('does not remove a resource when administrative suspension clears its tabs', () => { expect(shouldRemoveBrowserResource(false, true, true)).toBe(false) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx index a1df8816bd2..3d241ca738b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx @@ -2,6 +2,7 @@ import { type CSSProperties, useCallback, useEffect, useMemo, useRef, useState } from 'react' import type { + BrowserOmniboxFocusMode, BrowserPanelAnchor, BrowserPanelBounds, BrowserPanelSnapshot, @@ -29,10 +30,12 @@ import { PopoverAnchor, PopoverContent, PopoverItem, + toast, } from '@sim/emcn' import { ArrowLeft, ArrowRight, Globe, Key, Link, RefreshCw, Search } from '@sim/emcn/icons' import { useTheme } from 'next-themes' import { createPortal } from 'react-dom' +import { onFocusVisibleBrowserOmnibox } from '@/lib/browser-agent/renderer-shortcuts' import { fillBrowserCredential, loadBrowserFillOptions, @@ -44,6 +47,7 @@ import { onBrowserFindOpen, onBrowserOmniboxFocus, onBrowserToolbarCommand, + openBrowserTab, reorderBrowserTab, reportBrowserPanelBounds, reportBrowserPanelFocused, @@ -57,6 +61,7 @@ import { supportsAtomicBrowserPanelOcclusion, } from '@/lib/browser-agent/transport' import { BROWSER_SESSION_RESOURCE_ID } from '@/lib/copilot/resources/types' +import { faviconUrl } from '@/lib/core/utils/favicon' import { loadDesktopBrowserAppearanceTheme, resolveDesktopAppearanceTheme, @@ -66,7 +71,9 @@ import { addMothershipContext } from '@/lib/mothership/events' import { useMothershipResources } from '@/app/workspace/[workspaceId]/home/components/mothership-resources-context' import { BrowserDownloads } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-downloads' import { BrowserFindBar } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-find-bar' +import { BrowserLoadingBar } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar' import { + type BrowserPanelOverlay, type BrowserPanelOverlayController, type BrowserPanelSnapshotLayer, createBrowserPanelGeometryOcclusionLease, @@ -90,10 +97,29 @@ import type { ChatContext } from '@/stores/panel' /** Ties the omnibox to its listbox for assistive tech. */ const SUGGESTIONS_LIST_ID = 'browser-url-suggestions' +const NEW_TAB_CONFIRM_TIMEOUT_MS = 10_000 const EMPTY_BROWSER_TABS: BrowserTabState[] = [] const suggestionRowId = (index: number) => `${SUGGESTIONS_LIST_ID}-${index}` +function BrowserSuggestionIcon({ suggestion }: { suggestion: UrlSuggestion }) { + const [failed, setFailed] = useState(false) + const source = suggestion.icon || faviconUrl(suggestion.hostname, 32) + + if (failed) { + return + } + + return ( + setFailed(true)} + /> + ) +} + /** Converts the native page selection into the browser-tab mention shown in chat. */ export function browserSelectionContext({ text, @@ -247,6 +273,61 @@ export function shouldReportBrowserBounds(visible: boolean, suspended: boolean): return visible && !suspended } +/** + * Whether the omnibox suggestion list may be shown, which is not the same + * question as whether it has anything to show. + * + * Ranking a list is synchronous; making it clickable is not. The rows hang over + * the native page, so a click only reaches them once the shell has swapped that + * page for its captured frame — and that handshake can fail (a capture that + * never lands, a decode or paint that times out, a hide the shell refuses). + * Gating on the row count alone painted the list *underneath* the + * WebContentsView on those failures: visible, arrow-key navigable, and silently + * swallowing every click below the toolbar, while rows still inside the chrome + * kept working. + * + * The other browser popovers survive the same failure because each hands + * `requestOverlay` a native Electron menu to fall back to. An omnibox dropdown + * has no native equivalent, so its fallback is a no-op and failure has to mean + * "do not open" rather than "open something unusable". + * + * Keyed on the live overlay rather than on the request's result: the lease is + * also lost when a modal seizes the frame or another popover takes it, neither + * of which re-runs the request. + */ +export function shouldOpenUrlSuggestions( + activeOverlay: BrowserPanelOverlay | null, + suggestionCount: number +): boolean { + return activeOverlay === 'suggestions' && suggestionCount > 0 +} + +/** New tabs submit the best suggestion; existing pages submit their current URL. */ +export function initialUrlSuggestionIndex( + pageUrl: string | undefined, + suggestionCount: number +): number | null { + if (suggestionCount === 0) return null + return !pageUrl || pageUrl === 'about:blank' ? 0 : null +} + +/** A new-tab request is complete only after the authoritative strip grows and activates a new id. */ +export function hasConfirmedBrowserTabCreation( + previousActiveTabId: string | null, + previousTabCount: number, + activeTabId: string | null, + tabCount: number +): boolean { + return tabCount > previousTabCount && activeTabId !== null && activeTabId !== previousActiveTabId +} + +interface PendingNewTabFocus { + scopeId: string + previousActiveTabId: string | null + previousTabCount: number + timeoutId: number +} + export function BrowserSession({ visible, scopeId, @@ -270,6 +351,18 @@ export function BrowserSession({ const activeTabId = useBrowserSessionStore( (state) => state.sessions[scopeId]?.activeTabId ?? null ) + const automationTabId = useBrowserSessionStore( + (state) => state.sessions[scopeId]?.automationTabId ?? null + ) + const automationActive = useBrowserSessionStore( + (state) => state.sessions[scopeId]?.automationActive ?? false + ) + const automationNeedsAttention = useBrowserSessionStore( + (state) => state.sessions[scopeId]?.automationNeedsAttention ?? false + ) + const browserAgentActive = useBrowserSessionStore( + (state) => (state.sessions[scopeId]?.agentRunIds.length ?? 0) > 0 + ) const sessionAlive = useBrowserSessionStore( (state) => state.sessions[scopeId]?.sessionAlive ?? true ) @@ -280,6 +373,10 @@ export function BrowserSession({ const findInputRef = useRef(null) const fillButtonRef = useRef(null) const toolbarMenuButtonRef = useRef(null) + const omniboxFocusRafRef = useRef(null) + const pendingNewTabFocusRef = useRef(null) + const visibleRef = useRef(visible) + visibleRef.current = visible const { removeResource } = useMothershipResources() const { navigateToSettings } = useSettingsNavigation() @@ -322,12 +419,16 @@ export function BrowserSession({ const [fillAvailable, setFillAvailable] = useState(false) /** Accounts the active page can accept, loaded only when its key menu opens. */ const [fillOptions, setFillOptions] = useState([]) - /** Hosts worth suggesting: signed into, holding a saved password, or imported. */ + /** Visited hosts worth suggesting, optionally decorated by imported credentials. */ const [suggestionCorpus, setSuggestionCorpus] = useState([]) - /** Null until the user arrows into the list, so Enter still means "go to what I typed". */ + /** Highlighted row, or null when Enter should submit the omnibox text. */ const [activeSuggestion, setActiveSuggestion] = useState(null) + /** Page URL captured when the current omnibox edit began. */ + const [suggestionOriginUrl, setSuggestionOriginUrl] = useState('') /** Whether the user has asked to see suggestions for the current omnibox edit. */ const [suggestionsVisible, setSuggestionsVisible] = useState(false) + /** Empty on initial focus; follows the typed text once the user edits it. */ + const [suggestionQuery, setSuggestionQuery] = useState(null) /** Whether the find bar is docked above the page. */ const [findOpen, setFindOpen] = useState(false) const { @@ -412,45 +513,86 @@ export function BrowserSession({ } }, [appearanceTheme, theme]) - // Claiming focus is tied to being on screen, not to being mounted: a hidden - // panel that announced itself as focused would take keystrokes meant for - // whichever resource is actually showing. + // Renderer-owned browser chrome claims shortcuts only after real user + // interaction. The desktop shell observes focus in the native page itself. useEffect(() => { const panel = panelRef.current if (!panel || !panelVisible) return - // Claimed up front, which the terminal deliberately does NOT do. The - // difference is that the terminal has a real signal to wait for — xterm - // focuses its textarea and that `focusin` bubbles — while this panel's - // content is a native view that emits no DOM events at all. Nothing else - // seeds the claim either: attaching a view does not focus it, so - // `webContents.isFocused()` is false until the user clicks the page. Drop - // this and Cmd-W closes the whole window instead of the browser tab. - reportBrowserPanelFocused(true, scopeId) return trackPanelFocus(panel, (focused) => reportBrowserPanelFocused(focused, scopeId)) }, [panelVisible, scopeId]) + const focusOmnibox = useCallback((mode: BrowserOmniboxFocusMode) => { + // Selecting an existing URL means the user is choosing where to go next, + // exactly like clicking the omnibox. A freshly opened blank tab stays + // quiet until the user clicks or types. + setSuggestionsVisible(mode === 'select') + setSuggestionQuery(mode === 'select' ? '' : null) + setActiveSuggestion(null) + setSuggestionOriginUrl(mode === 'clear' ? '' : pageUrlRef.current) + setUrlDraft(mode === 'clear' ? '' : pageUrlRef.current) + if (omniboxFocusRafRef.current !== null) { + cancelAnimationFrame(omniboxFocusRafRef.current) + } + omniboxFocusRafRef.current = requestAnimationFrame(() => { + omniboxFocusRafRef.current = null + if (!visibleRef.current) return + urlInputRef.current?.focus() + urlInputRef.current?.select() + }) + }, []) + + const clearPendingNewTabFocus = useCallback((pending?: PendingNewTabFocus): boolean => { + const current = pendingNewTabFocusRef.current + if (!current || (pending && current !== pending)) return false + window.clearTimeout(current.timeoutId) + pendingNewTabFocusRef.current = null + return true + }, []) + + // New shells acknowledge tab creation directly; older installed shells only + // publish the resulting strip. Both paths land here so neither clears the + // current page's omnibox before a distinct tab actually exists. + useEffect(() => { + const pending = pendingNewTabFocusRef.current + if ( + !pending || + pending.scopeId !== scopeId || + !hasConfirmedBrowserTabCreation( + pending.previousActiveTabId, + pending.previousTabCount, + activeTabId, + tabs.length + ) + ) { + return + } + if (!clearPendingNewTabFocus(pending)) return + if (visible) focusOmnibox('clear') + }, [activeTabId, clearPendingNewTabFocus, focusOmnibox, scopeId, tabs.length, visible]) + + useEffect(() => { + return () => { + const pending = pendingNewTabFocusRef.current + if (pending?.scopeId === scopeId) clearPendingNewTabFocus(pending) + } + }, [clearPendingNewTabFocus, scopeId]) + + useEffect(() => onBrowserOmniboxFocus(focusOmnibox, scopeId), [focusOmnibox, scopeId]) + + // Sim owns keyboard events while its renderer has focus. Claim Cmd+L here + // before the workspace's global "Go to Logs" command can navigate away. + useEffect(() => { + if (!panelVisible || !activeTabId) return + return onFocusVisibleBrowserOmnibox(() => focusOmnibox('select')) + }, [activeTabId, focusOmnibox, panelVisible]) + useEffect(() => { - let focusRaf: number | null = null - const unsubscribe = onBrowserOmniboxFocus((mode) => { - setSuggestionsVisible(false) - setActiveSuggestion(null) - setUrlDraft(mode === 'clear' ? '' : pageUrlRef.current) - if (focusRaf !== null) { - cancelAnimationFrame(focusRaf) - } - focusRaf = requestAnimationFrame(() => { - focusRaf = null - urlInputRef.current?.focus() - urlInputRef.current?.select() - }) - }, scopeId) return () => { - unsubscribe() - if (focusRaf !== null) { - cancelAnimationFrame(focusRaf) + if (omniboxFocusRafRef.current !== null) { + cancelAnimationFrame(omniboxFocusRafRef.current) } } - }, [scopeId]) + }, []) // The page is a separate WebContentsView, so clicking it blurs this renderer // without reliably blurring its active DOM input. Collapse the selection as @@ -693,10 +835,16 @@ export function BrowserSession({ */ const suggestions = useMemo( () => - suggestionsVisible && urlDraft !== null ? rankSuggestions(suggestionCorpus, urlDraft) : [], - [suggestionCorpus, suggestionsVisible, urlDraft] + suggestionsVisible && suggestionQuery !== null + ? rankSuggestions(suggestionCorpus, suggestionQuery) + : [], + [suggestionCorpus, suggestionQuery, suggestionsVisible] ) + useEffect(() => { + setActiveSuggestion(initialUrlSuggestionIndex(suggestionOriginUrl, suggestions.length)) + }, [suggestionOriginUrl, suggestions]) + // The suggestion list is renderer UI that extends over the native page. // Keep the page's exact captured frame underneath it while it is open so // pointer events reach the Sim popover instead of the WebContentsView. @@ -708,18 +856,24 @@ export function BrowserSession({ void closeOverlay('suggestions') }, [closeOverlay, requestOverlay, suggestions.length]) + const suggestionsOpen = shouldOpenUrlSuggestions(activeOverlay, suggestions.length) + const navigateTo = useCallback( (url: string) => { sendBrowserPanelAction('navigate', { url }, scopeId) setSuggestionsVisible(false) + setSuggestionQuery(null) setActiveSuggestion(null) + setSuggestionOriginUrl('') urlInputRef.current?.blur() }, [scopeId] ) const submitUrl = useCallback(() => { - const highlighted = activeSuggestion === null ? undefined : suggestions[activeSuggestion] + // Enter can only take a highlight from a list the user can actually see. + const highlighted = + suggestionsOpen && activeSuggestion !== null ? suggestions[activeSuggestion] : undefined if (highlighted) { navigateTo(highlighted.url) return @@ -730,16 +884,47 @@ export function BrowserSession({ return } urlInputRef.current?.blur() - }, [activeSuggestion, navigateTo, suggestions, urlDraft]) + }, [activeSuggestion, navigateTo, suggestions, suggestionsOpen, urlDraft]) const handleNewTab = useCallback(() => { setSuggestionsVisible(false) + setSuggestionQuery(null) setActiveSuggestion(null) - setUrlDraft('') - sendBrowserPanelAction('new-tab', {}, scopeId) - urlInputRef.current?.focus() - urlInputRef.current?.select() - }, [scopeId]) + setSuggestionOriginUrl('') + clearPendingNewTabFocus() + const pending: PendingNewTabFocus = { + scopeId, + previousActiveTabId: activeTabId, + previousTabCount: tabs.length, + timeoutId: 0, + } + pending.timeoutId = window.setTimeout(() => { + if (clearPendingNewTabFocus(pending)) { + toast.error('Could not open a new browser tab. Please try again.') + } + }, NEW_TAB_CONFIRM_TIMEOUT_MS) + pendingNewTabFocusRef.current = pending + void openBrowserTab(scopeId) + .then((state) => { + // Older shells resolve null and confirm through the tab-state effect. + if (!state) return + if ( + !hasConfirmedBrowserTabCreation( + pending.previousActiveTabId, + pending.previousTabCount, + state.activeTabId, + state.tabs.length + ) + ) { + throw new Error('The desktop browser did not create a distinct tab.') + } + }) + .catch(() => { + if (clearPendingNewTabFocus(pending)) { + toast.error('Could not open a new browser tab. Please try again.') + } + }) + }, [activeTabId, clearPendingNewTabFocus, scopeId, tabs.length]) /** * Opens the shell's native account chooser under the key icon. Called @@ -772,6 +957,7 @@ export function BrowserSession({ const handleSwitchTab = useCallback( (tabId: string) => { setSuggestionsVisible(false) + setSuggestionQuery(null) setUrlDraft(null) urlInputRef.current?.blur() sendBrowserPanelAction('switch-tab', { tabId }, scopeId) @@ -782,6 +968,7 @@ export function BrowserSession({ const handleCloseTab = useCallback( (tabId: string) => { setSuggestionsVisible(false) + setSuggestionQuery(null) setUrlDraft(null) urlInputRef.current?.blur() sendBrowserPanelAction('close-tab', { tabId }, scopeId) @@ -812,17 +999,20 @@ export function BrowserSession({ return (
-
+
- void requestOverlay('tab', () => showBrowserTabContextMenu(tabId, scopeId)) + requestOverlay('tab', () => showBrowserTabContextMenu(tabId, scopeId)) } onCloseTabMenu={() => void closeOverlay('tab')} onReorderTab={handleReorderTab} @@ -863,10 +1053,11 @@ export function BrowserSession({ {/* URL bar: Enter navigates the agent browser. */} 0} + open={suggestionsOpen} onOpenChange={(open) => { if (open) return setSuggestionsVisible(false) + setSuggestionQuery(null) setActiveSuggestion(null) urlInputRef.current?.blur() }} @@ -884,14 +1075,23 @@ export function BrowserSession({ placeholder='Search Google or enter a URL' autoComplete='off' role='combobox' - aria-expanded={suggestions.length > 0} + aria-expanded={suggestionsOpen} aria-controls={SUGGESTIONS_LIST_ID} aria-activedescendant={ - activeSuggestion === null ? undefined : suggestionRowId(activeSuggestion) + suggestionsOpen && activeSuggestion !== null + ? suggestionRowId(activeSuggestion) + : undefined } - onPointerDown={() => setSuggestionsVisible(true)} + onPointerDown={(event) => { + setSuggestionsVisible(true) + if (document.activeElement !== event.currentTarget) { + setSuggestionOriginUrl(pageState?.url ?? '') + setSuggestionQuery('') + } + }} onChange={(event) => { setSuggestionsVisible(true) + setSuggestionQuery(event.target.value) setUrlDraft(event.target.value) // The old highlight pointed at a row that may no longer be // in the list, let alone in the same position. @@ -899,18 +1099,23 @@ export function BrowserSession({ }} onFocus={(event) => { setUrlDraft((current) => current ?? pageState?.url ?? '') + setSuggestionOriginUrl(pageState?.url ?? '') + setSuggestionQuery('') selectFocusedOmniboxOnNextFrame(event.currentTarget) }} onBlur={(event) => { clearOmniboxSelection(event.currentTarget) setSuggestionsVisible(false) + setSuggestionQuery(null) setUrlDraft(null) setActiveSuggestion(null) + setSuggestionOriginUrl('') }} onKeyDown={(event) => { event.stopPropagation() if (event.key === 'ArrowDown' || event.key === 'ArrowUp') { - if (suggestions.length === 0) return + // Never move a highlight through a list that is not on screen. + if (!suggestionsOpen) return // Otherwise the caret jumps to either end of the text. event.preventDefault() setActiveSuggestion((current) => @@ -955,19 +1160,15 @@ export function BrowserSession({ onMouseDown={(event) => event.preventDefault()} onClick={() => navigateTo(suggestion.url)} > -
- {suggestion.icon ? ( - - ) : ( - - )} +
+ {suggestion.name ? ( - <> - {suggestion.name} - +
+ {suggestion.name} + — {suggestion.hostname} - +
) : ( {suggestion.hostname} )} @@ -1084,6 +1285,7 @@ export function BrowserSession({ {themeNoticeVersion > 0 && ( )} +
{/* Host area: the real page is overlaid exactly on this rect. */}
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-label.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-label.ts new file mode 100644 index 00000000000..17395528660 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-label.ts @@ -0,0 +1,27 @@ +import type { BrowserTabState } from '@sim/browser-protocol' + +export function browserTabHostname(url: string): string | null { + if (!/^https?:\/\//i.test(url)) return null + try { + return new URL(url).hostname + } catch { + return null + } +} + +/** Page loading needs a spinner only until the tab has a usable favicon. */ +export function shouldShowBrowserTabSpinner( + loading: boolean, + hostname: string | null, + loadedHostname: string | null +): boolean { + return loading && (!hostname || loadedHostname !== hostname) +} + +/** A settled blank-title page is identified by its host, never as still loading. */ +export function browserTabTitle(tab: BrowserTabState): string { + const title = tab.title.trim() + if (title) return title + if (tab.loading) return 'Loading…' + return browserTabHostname(tab.url) ?? 'New tab' +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.test.ts index cf840296435..4afac4a57e9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { browserTabHostname } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip' +import { + browserTabHostname, + browserTabTitle, + shouldShowBrowserTabSpinner, +} from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-label' // Drop-index and title-truncation behaviour moved to the shared TabStrip in // @sim/emcn along with the component; see its own tests. Only the browser's @@ -16,3 +20,45 @@ describe('browserTabHostname', () => { expect(browserTabHostname('not a url')).toBeNull() }) }) + +describe('browserTabTitle', () => { + const tab = { + tabId: '1', + title: '', + url: 'https://docs.sim.ai/guides', + loading: false, + active: false, + pinned: false, + } + + it('never labels a settled blank-title page as loading', () => { + expect(browserTabTitle(tab)).toBe('docs.sim.ai') + }) + + it('uses the loading label only while the tab is actually loading', () => { + expect(browserTabTitle({ ...tab, loading: true })).toBe('Loading…') + }) + + it('keeps a resolved title when one is available', () => { + expect(browserTabTitle({ ...tab, title: ' Sim Docs ' })).toBe('Sim Docs') + }) +}) + +describe('shouldShowBrowserTabSpinner', () => { + it('uses a spinner while a blank tab has no favicon', () => { + expect(shouldShowBrowserTabSpinner(true, null, null)).toBe(true) + }) + + it('keeps a known favicon visible while its page loads', () => { + expect(shouldShowBrowserTabSpinner(true, 'sim.ai', 'sim.ai')).toBe(false) + }) + + it('keeps spinning until a new hostname favicon loads', () => { + expect(shouldShowBrowserTabSpinner(true, 'sim.ai', null)).toBe(true) + expect(shouldShowBrowserTabSpinner(true, 'sim.ai', 'docs.sim.ai')).toBe(true) + }) + + it('never spins after loading finishes', () => { + expect(shouldShowBrowserTabSpinner(false, null, null)).toBe(false) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.tsx index c3276e7f27f..839455c141e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-strip.tsx @@ -10,59 +10,71 @@ import { useState, } from 'react' import type { BrowserTabState } from '@sim/browser-protocol' -import { TabStrip, type TabStripItem } from '@sim/emcn' -import { Link, Loader } from '@sim/emcn/icons' +import { cn, TabStrip, type TabStripItem, toast } from '@sim/emcn' +import { Globe, Loader } from '@sim/emcn/icons' +import { ThinkingLoader } from '@/components/ui' import { SIM_RESOURCE_DRAG_TYPE } from '@/lib/copilot/resource-types' import { faviconUrl } from '@/lib/core/utils/favicon' +import { getDesktopBridge } from '@/lib/desktop' +import { + browserTabHostname, + browserTabTitle, + shouldShowBrowserTabSpinner, +} from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-label' import { ContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/context-menu/context-menu' interface BrowserTabStripProps { tabs: BrowserTabState[] activeTabId: string | null + automationTabId: string | null + automationActive: boolean + automationNeedsAttention: boolean onNewTab: () => void onSwitchTab: (tabId: string) => void onCloseTab: (tabId: string) => void onDuplicateTab: (tabId: string) => void onSetTabPinned: (tabId: string, pinned: boolean) => void - onOpenTabMenu: (tabId: string) => void + /** Resolves true only when the renderer context menu may safely open. */ + onOpenTabMenu: (tabId: string) => Promise onCloseTabMenu: () => void onReorderTab: (tabId: string, targetIndex: number) => void contextMenuOpen: boolean } -function tabTitle(tab: BrowserTabState): string { - return tab.title.trim() || (tab.url ? 'Loading…' : 'New tab') -} - -export function browserTabHostname(url: string): string | null { - if (!/^https?:\/\//i.test(url)) return null - try { - return new URL(url).hostname - } catch { - return null - } -} - function BrowserTabIcon({ tab }: { tab: BrowserTabState }) { - if (tab.loading) { - return - } - const hostname = browserTabHostname(tab.url) - if (!hostname) { - return - } + const [loadedHostname, setLoadedHostname] = useState(null) + const [failedHostname, setFailedHostname] = useState(null) + const faviconLoaded = Boolean(hostname && loadedHostname === hostname) + const faviconFailed = Boolean(hostname && failedHostname === hostname) + const showSpinner = shouldShowBrowserTabSpinner(tab.loading, hostname, loadedHostname) return ( - { - event.currentTarget.style.display = 'none' - }} - /> + + {hostname && !faviconFailed && ( + setLoadedHostname(hostname)} + onError={() => setFailedHostname(hostname)} + /> + )} + {showSpinner ? ( + + ) : !faviconLoaded || faviconFailed ? ( + + ) : null} + ) } @@ -78,6 +90,9 @@ function BrowserTabIcon({ tab }: { tab: BrowserTabState }) { export function BrowserTabStrip({ tabs, activeTabId, + automationTabId, + automationActive, + automationNeedsAttention, onNewTab, onSwitchTab, onCloseTab, @@ -91,18 +106,38 @@ export function BrowserTabStrip({ const [contextTabId, setContextTabId] = useState(null) const [contextMenuPosition, setContextMenuPosition] = useState({ x: 0, y: 0 }) const contextMenuRef = useRef(null) + const contextRequestRef = useRef(0) const contextTab = tabs.find((tab) => tab.tabId === contextTabId) const items = useMemo( () => - tabs.map((tab) => ({ - id: tab.tabId, - title: tabTitle(tab), - icon: , - active: tab.tabId === activeTabId, - ...(tab.pinned ? { pinned: true } : {}), - })), - [tabs, activeTabId] + tabs.map((tab) => { + const isAutomationRunning = automationActive && tab.tabId === automationTabId + return { + id: tab.tabId, + title: browserTabTitle(tab), + icon: ( + + + + + {isAutomationRunning && ( + + + + )} + + ), + active: tab.tabId === activeTabId, + attention: + !automationActive && + automationNeedsAttention && + tab.tabId === automationTabId && + tab.tabId !== activeTabId, + ...(tab.pinned ? { pinned: true } : {}), + } + }), + [tabs, activeTabId, automationTabId, automationActive, automationNeedsAttention] ) // Dragging a tab into the chat attaches it as context. `copyMove` because @@ -115,7 +150,7 @@ export function BrowserTabStrip({ event.dataTransfer.effectAllowed = 'copyMove' event.dataTransfer.setData( SIM_RESOURCE_DRAG_TYPE, - JSON.stringify({ type: 'browser', id: tab.tabId, title: tabTitle(tab) }) + JSON.stringify({ type: 'browser', id: tab.tabId, title: browserTabTitle(tab) }) ) }, [tabs] @@ -126,14 +161,19 @@ export function BrowserTabStrip({ event.preventDefault() event.stopPropagation() window.getSelection()?.removeAllRanges() - setContextTabId(tabId) - setContextMenuPosition({ x: event.clientX, y: event.clientY }) - onOpenTabMenu(tabId) + const request = ++contextRequestRef.current + const position = { x: event.clientX, y: event.clientY } + void onOpenTabMenu(tabId).then((opened) => { + if (!opened || request !== contextRequestRef.current) return + setContextTabId(tabId) + setContextMenuPosition(position) + }) }, [onOpenTabMenu] ) const closeTabContextMenu = useCallback(() => { + contextRequestRef.current++ setContextTabId(null) onCloseTabMenu() }, [onCloseTabMenu]) @@ -145,6 +185,19 @@ export function BrowserTabStrip({ if (contextMenuOpen && contextTabId && !contextTab) closeTabContextMenu() }, [closeTabContextMenu, contextMenuOpen, contextTab, contextTabId]) + const openTabInExternalBrowser = useCallback(() => { + const url = contextTab?.url + const bridge = getDesktopBridge() + if (!url || url === 'about:blank' || !bridge) return + + void bridge.openExternal(url).then( + (opened) => { + if (!opened) toast.error('Could not open this page in your browser.') + }, + () => toast.error('Could not open this page in your browser.') + ) + }, [contextTab?.url]) + return ( onSetTabPinned(contextTab.tabId, !contextTab.pinned) : undefined } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/url-suggestions.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/url-suggestions.test.ts index e433ec37874..f4407ab3f6a 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/url-suggestions.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/url-suggestions.test.ts @@ -70,7 +70,11 @@ describe('mergeSuggestionSources', () => { [session('github.com', '2026-01-01T00:00:00.000Z', 'sign-in-completed')], [] ) - const [saved] = mergeSuggestionSources([], [credential('https://gitlab.com')]) + const [saved] = mergeSuggestionSources( + [], + [credential('https://gitlab.com')], + [site('gitlab.com', { visits: 1 })] + ) expect(signedIn.tier).toBe(SUGGESTION_TIER.ACCOUNT) expect(saved.tier).toBe(SUGGESTION_TIER.ACCOUNT) @@ -79,7 +83,8 @@ describe('mergeSuggestionSources', () => { it('holds a host known only by its cookie below one with an account', () => { const merged = mergeSuggestionSources( [session('cookies.com')], - [credential('https://saved.com')] + [credential('https://saved.com')], + [site('saved.com', { visits: 1 })] ) expect(merged.find((entry) => entry.hostname === 'cookies.com')?.tier).toBe( @@ -88,10 +93,11 @@ describe('mergeSuggestionSources', () => { expect(hostnames(rankSuggestions(merged, ''))).toEqual(['saved.com', 'cookies.com']) }) - it('suggests hosts with a saved password, carrying the imported icon', () => { + it('decorates a visited host with its saved-password icon', () => { const merged = mergeSuggestionSources( [], - [credential('https://news.ycombinator.com', { icon: 'data:image/png;base64,AAA' })] + [credential('https://news.ycombinator.com', { icon: 'data:image/png;base64,AAA' })], + [site('news.ycombinator.com', { visits: 12 })] ) expect(merged[0]).toMatchObject({ @@ -101,6 +107,10 @@ describe('mergeSuggestionSources', () => { }) }) + it('does not suggest a host known only from a saved credential', () => { + expect(mergeSuggestionSources([], [credential('https://saved-only.com')])).toEqual([]) + }) + it('lists a host present in both sources once', () => { const merged = mergeSuggestionSources( [session('github.com')], @@ -169,7 +179,7 @@ describe('mergeSuggestionSources with an imported directory', () => { const merged = mergeSuggestionSources( [], [credential('https://github.com', { icon: 'data:from-vault' })], - [site('github.com', { name: 'GitHub', icon: 'data:from-directory' })] + [site('github.com', { name: 'GitHub', icon: 'data:from-directory', visits: 1 })] ) expect(merged).toHaveLength(1) @@ -195,6 +205,16 @@ describe('mergeSuggestionSources with an imported directory', () => { ]) }) + it('does not offer an imported host without positive visit evidence', () => { + expect( + mergeSuggestionSources( + [], + [], + [site('cookie-only.com'), site('zero-visits.com', { visits: 0 })] + ) + ).toEqual([]) + }) + it('lists a host that was both imported and saved once, at the tier its account earned', () => { const merged = mergeSuggestionSources( [], @@ -244,13 +264,21 @@ describe('mergeSuggestionSources with an imported directory', () => { }) it('reaches an imported host by the name the source browser gave it', () => { - const merged = mergeSuggestionSources([], [], [site('mail.google.com', { name: 'Gmail' })]) + const merged = mergeSuggestionSources( + [], + [], + [site('mail.google.com', { name: 'Gmail', visits: 1 })] + ) expect(hostnames(rankSuggestions(merged, 'gmail'))).toEqual(['mail.google.com']) }) it('skips a site record with no hostname rather than offering a bare https://', () => { - const merged = mergeSuggestionSources([], [], [site(''), site('github.com')]) + const merged = mergeSuggestionSources( + [], + [], + [site('', { visits: 1 }), site('github.com', { visits: 1 })] + ) expect(hostnames(merged)).toEqual(['github.com']) }) @@ -260,8 +288,8 @@ describe('mergeSuggestionSources with an imported directory', () => { [], [], [ - site('github.com', { importedAt: 'whenever' }), - site('gitlab.com', { importedAt: undefined }), + site('github.com', { importedAt: 'whenever', visits: 1 }), + site('gitlab.com', { importedAt: undefined, visits: 1 }), ] ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/url-suggestions.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/url-suggestions.ts index 090c571d4ca..c5ae7399c7f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/url-suggestions.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/url-suggestions.ts @@ -71,18 +71,16 @@ function hostnameOf(origin: string): string | null { /** * Builds the omnibox's corpus from what the browser already knows. * - * Three sources, each of which exists for a reason of its own: hosts with a - * saved password, hosts this browser has been to that still hold a cookie, and - * hosts brought over from the browser the user imported from. None of them is - * a log of where this browser has been — it keeps no history, and an agent - * drives it, so a visit log would blend the agent's browsing into the user's - * suggestions and the user's into the agent's reach. + * Two sources admit a host: a top-level visit in this browser that still has + * session evidence, or positive aggregate visit evidence imported from the + * user's other browser. Saved credentials can promote and decorate an admitted + * host, but cannot create a suggestion by themselves: owning a password is not + * evidence that someone chose to visit the site. * - * The imported source is the reason the list is not almost empty on a fresh - * install, and it is admitted at the weakest tier: hosts nobody has signed into - * here can be offered without displacing the ones somebody has. A host found in - * more than one source appears once, at its strongest tier, keeping whichever - * favicon and whichever timestamp is the more useful of the two. + * This remains an aggregate directory rather than a history log: no page URL, + * visit timestamp, or sequence is retained. A host found in more than one + * source appears once, at its strongest tier, keeping whichever favicon and + * whichever timestamp is the more useful of the two. */ export function mergeSuggestionSources( sessions: readonly BrowserKnownSession[], @@ -91,6 +89,12 @@ export function mergeSuggestionSources( ): UrlSuggestion[] { const byHostname = new Map() const known = new Map(sites.map((site) => [site.hostname, site])) + const visitedHosts = new Set(sessions.map((session) => session.hostname)) + for (const site of sites) { + if (typeof site.visits === 'number' && Number.isFinite(site.visits) && site.visits > 0) { + visitedHosts.add(site.hostname) + } + } const record = (hostname: string, tier: SuggestionTier, seenAt: number, icon?: string) => { const existing = byHostname.get(hostname) @@ -114,7 +118,7 @@ export function mergeSuggestionSources( for (const credential of credentials) { const hostname = hostnameOf(credential.origin) - if (!hostname) continue + if (!hostname || !visitedHosts.has(hostname)) continue record( hostname, SUGGESTION_TIER.ACCOUNT, @@ -135,7 +139,9 @@ export function mergeSuggestionSources( // Last, so a host with real evidence keeps the tier and timestamp that // evidence earned it rather than being flattened to the moment of the import. for (const site of sites) { - if (!site.hostname || byHostname.has(site.hostname)) continue + if (!site.hostname || !visitedHosts.has(site.hostname) || byHostname.has(site.hostname)) { + continue + } byHostname.set(site.hostname, { hostname: site.hostname, url: `https://${site.hostname}`, @@ -242,10 +248,8 @@ function bestMatch(suggestion: UrlSuggestion, query: string): { matched: boolean /** * Where the arrow keys land next. * - * Nothing is highlighted until the user actually arrows into the list, so - * Enter keeps meaning "go to what I typed" rather than silently redirecting to - * a suggestion. Both ends wrap, and Up from that neutral state enters at the - * bottom. + * Both ends wrap. A null selection enters at the first row with Down and the + * final row with Up, which also keeps this helper safe while results change. */ export function moveActiveIndex( current: number | null, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/generic-resource-content/generic-resource-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/generic-resource-content/generic-resource-content.tsx index a05459786b1..af92c312255 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/generic-resource-content/generic-resource-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/generic-resource-content/generic-resource-content.tsx @@ -41,7 +41,7 @@ export function GenericResourceContent({ data }: GenericResourceContentProps) { /> )} - {getToolStatusDisplayTitle(entry.displayTitle, entry.status)} + {getToolStatusDisplayTitle(entry.displayTitle, entry.status, entry.toolName)} {entry.status === 'error' && ( Error diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-session.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-session.tsx index 3dde6afcb0f..5a32639a574 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-session.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-session.tsx @@ -11,15 +11,11 @@ import { useState, } from 'react' import { - type DesktopAppearanceTheme, type DesktopZoomAction, type DesktopZoomPercent, resolveDesktopZoom, - TERMINAL_DARK_THEME, - TERMINAL_LIGHT_THEME, type TerminalAppearanceTheme, type TerminalShortcutCommand, - type TerminalThemePalette, type TerminalThemeProfile, } from '@sim/desktop-bridge' import { @@ -27,6 +23,7 @@ import { NATIVE_SURFACE_OCCLUSION_PREPARE_EVENT, TabStrip, type TabStripItem, + type TabStripSelectionSource, toast, } from '@sim/emcn' import { TerminalWindow } from '@sim/emcn/icons' @@ -45,7 +42,8 @@ import { getDesktopBridge } from '@/lib/desktop' import { loadDesktopTerminalAppearance, loadDesktopTerminalThemeProfiles, - resolveDesktopAppearanceTheme, + refreshSelectedTerminalProfile, + resolveTerminalThemePalette, withSelectedProfile, } from '@/lib/desktop/appearance' import { trackPanelFocus } from '@/lib/desktop/panel-focus' @@ -59,7 +57,9 @@ import { onTerminalShortcutCommand, openTerminal, pasteIntoTerminal, + reorderTerminal, reportTerminalFocused, + reportTerminalVisible, resizeTerminal, startTerminalSession, switchTerminal, @@ -67,6 +67,7 @@ import { } from '@/lib/terminal/transport' import { useMothershipResources } from '@/app/workspace/[workspaceId]/home/components/mothership-resources-context' import { TerminalContextMenu } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-context-menu' +import { TerminalTabIcon } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-tab-icon' import { ContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/context-menu/context-menu' import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks' import { useDesktopPreferenceMutation } from '@/hooks/use-desktop-preference-mutation' @@ -75,9 +76,16 @@ import type { ChatContext, TerminalTextSelection } from '@/stores/panel' const logger = createLogger('TerminalSession') const EMPTY_TERMINAL_TABS: TerminalTabsState = { tabs: [], activeTerminalId: null } +const EMPTY_AGENT_COMMAND_TERMINAL_IDS: Record = {} const TERMINAL_BASE_FONT_SIZE = 12 const TERMINAL_ZOOM_BOUNDS = { min: 50, max: 300 } as const +/** Fits xterm to its current host while preserving the addon's method binding. */ +function fitTerminal(addon: FitAddon): void { + const fitToHost = addon.fit.bind(addon) + fitToHost() +} + /** * Radix keeps closed menus mounted for their exit animation. A full-screen * effect starts in a layout effect, so hide the active menu hierarchy in the @@ -294,6 +302,7 @@ function zoomActionForTerminalCommand(command: TerminalShortcutCommand): Desktop const TerminalView = memo(function TerminalView({ terminalId, + running, active, visible, scopeId, @@ -302,8 +311,10 @@ const TerminalView = memo(function TerminalView({ onAppearanceThemeChange, appearanceThemePending, defaultZoom, + focusRequest, }: { terminalId: string + running: string | null active: boolean visible: boolean scopeId: string @@ -312,17 +323,10 @@ const TerminalView = memo(function TerminalView({ onAppearanceThemeChange?: (theme: TerminalAppearanceTheme) => void appearanceThemePending?: boolean defaultZoom: DesktopZoomPercent + focusRequest: number }) { const { resolvedTheme } = useTheme() - const profileTheme = typeof appearanceTheme === 'string' ? undefined : appearanceTheme - const builtInTheme: DesktopAppearanceTheme = - typeof appearanceTheme === 'string' ? appearanceTheme : 'app' - const colorScheme = resolveDesktopAppearanceTheme(builtInTheme, resolvedTheme) - const terminalTheme: TerminalThemePalette = profileTheme - ? profileTheme.palette - : colorScheme === 'dark' - ? TERMINAL_DARK_THEME - : TERMINAL_LIGHT_THEME + const terminalTheme = resolveTerminalThemePalette(appearanceTheme, resolvedTheme) const hostRef = useRef(null) const terminalRef = useRef(null) const fitRef = useRef(null) @@ -381,8 +385,8 @@ const TerminalView = memo(function TerminalView({ scrollback: 10_000, theme: terminalTheme, }) - const fit = new FitAddon() - terminal.loadAddon(fit) + const fitAddon = new FitAddon() + terminal.loadAddon(fitAddon) terminal.loadAddon(new WebLinksAddon()) const unicode = new Unicode11Addon() terminal.loadAddon(unicode) @@ -391,7 +395,7 @@ const TerminalView = memo(function TerminalView({ terminal.open(host) terminalRef.current = terminal - fitRef.current = fit + fitRef.current = fitAddon terminal.attachCustomKeyEventHandler((event) => handleTerminalLocalShortcut(event, clearScreen)) const disposeData = terminal.onData((data) => writeToTerminal(terminalId, data, scopeId)) @@ -511,8 +515,7 @@ const TerminalView = memo(function TerminalView({ // not that it shrank. Fitting to that would resize the pty to nonsense. if (!onscreenRef.current || host.clientWidth <= 0 || host.clientHeight <= 0) return try { - // biome-ignore lint/suspicious/noFocusedTests: xterm FitAddon.fit(), not a focused test - fit.fit() + fitTerminal(fitAddon) } catch { // Zero-sized while animating; the next observation refits. } @@ -563,9 +566,8 @@ const TerminalView = memo(function TerminalView({ const frame = requestAnimationFrame(() => { if (host.clientWidth <= 0 || host.clientHeight <= 0) return try { - // biome-ignore lint/suspicious/noFocusedTests: xterm FitAddon.fit(), not a focused test - fitRef.current?.fit() - terminal.focus() + const fitAddon = fitRef.current + if (fitAddon) fitTerminal(fitAddon) } catch { // Panel still animating; the ResizeObserver refits. } @@ -573,6 +575,12 @@ const TerminalView = memo(function TerminalView({ return () => cancelAnimationFrame(frame) }, [currentZoom, onscreen]) + useEffect(() => { + if (!onscreen || focusRequest === 0) return + const frame = requestAnimationFrame(() => terminalRef.current?.focus()) + return () => cancelAnimationFrame(frame) + }, [focusRequest, onscreen]) + useEffect(() => { if (!onscreen) return return onTerminalShortcutCommand( @@ -665,7 +673,9 @@ const TerminalView = memo(function TerminalView({ }, [terminalId, scopeId]) const newTab = useCallback(() => { - void openTerminal(undefined, scopeId) + void openTerminal(undefined, scopeId).catch(() => { + toast.error('Could not open a new terminal. Please try again.') + }) }, [scopeId]) // Scoped to the terminal that was right-clicked, not the active one. @@ -674,8 +684,16 @@ const TerminalView = memo(function TerminalView({ // for the action to do — and hiding it here while the tab strip's own close // stays available would just be the two menus disagreeing. const closeThisTerminal = useCallback(() => { - void closeTerminal(terminalId, scopeId) - }, [terminalId, scopeId]) + if ( + running && + !window.confirm(`${running} is still running. Close this terminal and stop it?`) + ) { + return + } + void closeTerminal(terminalId, scopeId).catch(() => { + toast.error('Could not close that terminal. Please try again.') + }) + }, [running, terminalId, scopeId]) // An inactive tab is `display: none`, not merely invisible. xterm watches its // element with an IntersectionObserver and pauses rendering once it stops @@ -687,6 +705,7 @@ const TerminalView = memo(function TerminalView({ <>
terminalRef.current?.focus()} onContextMenu={openMenu} className={cn('absolute inset-0 pt-[7px] pr-2 pb-1 pl-1.5', !active && 'hidden')} style={{ backgroundColor: terminalTheme.background }} @@ -746,36 +765,41 @@ export function TerminalSession({ visible, scopeId }: TerminalSessionProps) { (state) => state.sessions[scopeId]?.tabs ?? EMPTY_TERMINAL_TABS ) const suspended = useCopilotTerminalStore((state) => state.sessions[scopeId]?.suspended ?? false) + const agentCommandTerminalIds = useCopilotTerminalStore( + (state) => state.sessions[scopeId]?.agentCommandTerminalIds ?? EMPTY_AGENT_COMMAND_TERMINAL_IDS + ) + const activityResetEpoch = useCopilotTerminalStore( + (state) => state.sessions[scopeId]?.activityResetEpoch ?? 0 + ) const { tabs, activeTerminalId } = tabsState + const agentCommandTargets = useMemo( + () => new Set(Object.values(agentCommandTerminalIds)), + [agentCommandTerminalIds] + ) const settledCommands = useSettledCommands(tabs) const { removeResource } = useMothershipResources() const [startError, setStartError] = useState(null) + const [focusRequest, setFocusRequest] = useState({ terminalId: '', nonce: 0 }) const availableProfiles = useMemo( () => withSelectedProfile(profiles, appearanceTheme), [appearanceTheme, profiles] ) useEffect(() => { + if (!visible) return let active = true - void loadDesktopTerminalAppearance().then((next) => { - if (!active) return - setAppearanceTheme(next.theme) - setDefaultZoom(next.defaultZoom) - }) - return () => { - active = false - } - }, []) - - useEffect(() => { - let active = true - void loadDesktopTerminalThemeProfiles().then((next) => { - if (active) setProfiles(next) - }) + void Promise.all([loadDesktopTerminalAppearance(), loadDesktopTerminalThemeProfiles()]).then( + ([nextAppearance, nextProfiles]) => { + if (!active) return + setProfiles(nextProfiles) + setAppearanceTheme(refreshSelectedTerminalProfile(nextProfiles, nextAppearance.theme)) + setDefaultZoom(nextAppearance.defaultZoom) + } + ) return () => { active = false } - }, []) + }, [visible]) useEffect(() => { let active = true @@ -805,21 +829,18 @@ export function TerminalSession({ visible, scopeId }: TerminalSessionProps) { [setTerminalAppearanceTheme] ) - // Interaction ownership is reported once for the whole panel, never per tab. - // The shell holds a single focus flag, so a per-tab reporter would let one - // tab's unmount erase a sibling's live claim — and Cmd-W would then fall - // through to closing the window out from under a running shell. - // - // No claim on appear: xterm focuses its textarea once the panel is measurably - // on screen, and that focusin is the claim. Appearing is not enough, because - // the agent opens this panel on the user's behalf — claiming then would let a - // Cmd-W meant for the chat close a shell the user never touched. useEffect(() => { + if (!visible || suspended) return const panel = panelRef.current - if (!panel || !visible || suspended) return + if (!panel) return return trackPanelFocus(panel, (focused) => reportTerminalFocused(focused, scopeId)) }, [visible, suspended, scopeId]) + useEffect(() => { + reportTerminalVisible(visible && !suspended, scopeId) + return () => reportTerminalVisible(false, scopeId) + }, [scopeId, suspended, visible]) + useEffect(() => { if (suspended) { setStartError(null) @@ -856,29 +877,38 @@ export function TerminalSession({ visible, scopeId }: TerminalSessionProps) { } }, [tabs.length, suspended, removeResource]) - // Every tab carries the same glyph. A spinner would have to mean "transient - // work", and nothing here can tell that from a coding agent sitting open for - // an hour: the alternate screen is the only signal available, and the tools - // people leave running — Claude Code, Codex — draw inline without it. A - // spinner that is wrong for the longest-lived tabs is worse than none, and - // the tab already says what it is running. - const items = useMemo( - () => - tabs.map((tab) => { - const naming = namesItsCommand(tab, settledCommands) ? tab.running : null - return { - id: tab.terminalId, - title: naming ?? tab.title, - // The label is a basename, and the tab may be running something it - // is not naming yet, so hovering gives the whole picture: where the - // shell is, and what it is doing there. - tooltip: terminalTooltip(tab), - icon: , - active: tab.terminalId === activeTerminalId, - } - }), - [tabs, activeTerminalId, settledCommands] - ) + // Shell process state is not activity state: a coding tool can run for hours. + // The agent-command lifecycle is precise, though, so the targeted terminal + // replaces its regular glyph while Mothership is actively driving it. + const items = useMemo(() => { + const labels = tabs.map((tab) => + namesItsCommand(tab, settledCommands) ? (tab.running ?? tab.title) : tab.title + ) + const counts = new Map() + for (const label of labels) counts.set(label, (counts.get(label) ?? 0) + 1) + const occurrences = new Map() + return tabs.map((tab, index) => { + const label = labels[index] + const occurrence = (occurrences.get(label) ?? 0) + 1 + occurrences.set(label, occurrence) + const isAgentCommandRunning = agentCommandTargets.has(tab.terminalId) + return { + id: tab.terminalId, + title: counts.get(label) === 1 ? label : `${label} ${occurrence}`, + // The label is a basename, and the tab may be running something it + // is not naming yet, so hovering gives the whole picture: where the + // shell is, and what it is doing there. + tooltip: terminalTooltip(tab), + icon: ( + + ), + active: tab.terminalId === activeTerminalId, + } + }) + }, [tabs, activeTerminalId, agentCommandTargets, activityResetEpoch, settledCommands]) const [contextTerminalId, setContextTerminalId] = useState(null) const { @@ -899,13 +929,45 @@ export function TerminalSession({ visible, scopeId }: TerminalSessionProps) { return () => window.removeEventListener(NATIVE_SURFACE_OCCLUSION_PREPARE_EVENT, handlePrepare) }, [closeContextMenu, isContextMenuOpen, visible]) const contextTab = tabs.find((tab) => tab.terminalId === contextTerminalId) + const canReorderTabs = Boolean(getDesktopBridge()?.terminal.reorderTerminal) + + useEffect(() => { + if (isContextMenuOpen && contextTerminalId && !contextTab) { + setContextTerminalId(null) + closeContextMenu() + } + }, [closeContextMenu, contextTab, contextTerminalId, isContextMenuOpen]) const handleNew = useCallback(() => { void openTerminal(undefined, scopeId) + .then((state) => { + if (state.activeTerminalId) { + setFocusRequest((current) => ({ + terminalId: state.activeTerminalId ?? '', + nonce: current.nonce + 1, + })) + } + }) + .catch(() => { + toast.error('Could not open a new terminal. Please try again.') + }) }, [scopeId]) const handleSwitch = useCallback( - (terminalId: string) => { - void switchTerminal(terminalId, scopeId) + (terminalId: string, source?: TabStripSelectionSource) => { + if (source !== 'keyboard') { + setFocusRequest((current) => ({ terminalId, nonce: current.nonce + 1 })) + } + void switchTerminal(terminalId, scopeId).catch(() => { + toast.error('Could not switch terminals. Please try again.') + }) + }, + [scopeId] + ) + const handleReorder = useCallback( + (terminalId: string, targetIndex: number) => { + void reorderTerminal(terminalId, targetIndex, scopeId).catch(() => { + toast.error('Could not reorder that terminal. Please try again.') + }) }, [scopeId] ) @@ -913,9 +975,18 @@ export function TerminalSession({ visible, scopeId }: TerminalSessionProps) { // desktop app decides that, so the button means the same thing at any count. const handleClose = useCallback( (terminalId: string) => { - void closeTerminal(terminalId, scopeId) + const tab = tabs.find((entry) => entry.terminalId === terminalId) + if ( + tab?.running && + !window.confirm(`${tab.running} is still running. Close this terminal and stop it?`) + ) { + return + } + void closeTerminal(terminalId, scopeId).catch(() => { + toast.error('Could not close that terminal. Please try again.') + }) }, - [scopeId] + [scopeId, tabs] ) // A duplicate is a new shell in the same directory, not a copy of the @@ -923,18 +994,66 @@ export function TerminalSession({ visible, scopeId }: TerminalSessionProps) { const handleDuplicate = useCallback( (cwd: string | null) => { void openTerminal(cwd ?? undefined, scopeId) + .then((state) => { + if (state.activeTerminalId) { + setFocusRequest((current) => ({ + terminalId: state.activeTerminalId ?? '', + nonce: current.nonce + 1, + })) + } + }) + .catch(() => { + toast.error('Could not duplicate that terminal. Please try again.') + }) }, [scopeId] ) - // Dragging a terminal tab into the chat attaches it as context. This strip - // has no reordering, so supplying this is also what makes its tabs - // draggable at all. + const handleCloseMany = useCallback( + (terminalIds: string[]) => { + const runningCount = tabs.filter( + (tab) => terminalIds.includes(tab.terminalId) && Boolean(tab.running) + ).length + if ( + runningCount > 0 && + !window.confirm( + `${runningCount} selected ${runningCount === 1 ? 'terminal has' : 'terminals have'} a running process. Close ${runningCount === 1 ? 'it' : 'them'} anyway?` + ) + ) { + return + } + for (const terminalId of terminalIds) { + void closeTerminal(terminalId, scopeId).catch(() => { + toast.error('Could not close one of those terminals. Please try again.') + }) + } + }, + [scopeId, tabs] + ) + + const closeOtherTabs = useCallback(() => { + if (!contextTab) return + handleCloseMany( + tabs.filter((tab) => tab.terminalId !== contextTab.terminalId).map((tab) => tab.terminalId) + ) + }, [contextTab, handleCloseMany, tabs]) + + const closeTabsToRight = useCallback(() => { + if (!contextTab) return + const contextIndex = tabs.findIndex((tab) => tab.terminalId === contextTab.terminalId) + handleCloseMany(tabs.slice(contextIndex + 1).map((tab) => tab.terminalId)) + }, [contextTab, handleCloseMany, tabs]) + + const contextIndex = contextTab + ? tabs.findIndex((tab) => tab.terminalId === contextTab.terminalId) + : -1 + + // A terminal can move inside the strip or be copied into chat as context. const startTabDrag = useCallback( (event: ReactDragEvent, terminalId: string) => { const tab = tabs.find((entry) => entry.terminalId === terminalId) if (!tab) return - event.dataTransfer.effectAllowed = 'copy' + event.dataTransfer.effectAllowed = 'copyMove' event.dataTransfer.setData( SIM_RESOURCE_DRAG_TYPE, JSON.stringify({ type: 'terminal', id: tab.terminalId, title: tab.title }) @@ -954,38 +1073,42 @@ export function TerminalSession({ visible, scopeId }: TerminalSessionProps) { return (
- {tabs.length > 0 && ( - - handleDuplicate(contextTab.cwd) : undefined} - {...(contextTab - ? { onCloseTab: () => handleClose(contextTab.terminalId), showCloseTab: true } - : {})} - onDelete={() => {}} - showRename={false} - showDuplicate={Boolean(contextTab)} - showDelete={false} - /> - - )} + + handleDuplicate(contextTab.cwd) : undefined} + onCloseOtherTabs={contextTab ? closeOtherTabs : undefined} + onCloseTabsToRight={contextTab ? closeTabsToRight : undefined} + disableCloseOtherTabs={tabs.length <= 1} + disableCloseTabsToRight={contextIndex < 0 || contextIndex === tabs.length - 1} + {...(contextTab + ? { onCloseTab: () => handleClose(contextTab.terminalId), showCloseTab: true } + : {})} + onDelete={() => {}} + showRename={false} + showDuplicate={Boolean(contextTab)} + showDelete={false} + /> +
{tabs.map((tab) => ( ))} {startError && ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-tab-icon.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-tab-icon.test.tsx new file mode 100644 index 00000000000..0c7d4f49ee6 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-tab-icon.test.tsx @@ -0,0 +1,79 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { TerminalTabIcon } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-tab-icon' + +vi.mock('@/components/ui', async () => { + const { createElement } = await import('react') + return { + ThinkingLoader: () => createElement('span', { 'aria-label': 'Thinking' }), + } +}) + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function render(active: boolean, resetEpoch = 0): void { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + if (!container) { + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + } + act(() => root?.render()) +} + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null + vi.useRealTimers() +}) + +describe('TerminalTabIcon', () => { + it('shows a fast command for at least one second', () => { + vi.useFakeTimers() + + render(true) + expect(container?.querySelector('[aria-label="Thinking"]')).not.toBeNull() + + render(false) + act(() => vi.advanceTimersByTime(999)) + expect(container?.querySelector('[aria-label="Thinking"]')).not.toBeNull() + + act(() => vi.advanceTimersByTime(1)) + expect(container?.querySelector('[aria-label="Thinking"]')).toBeNull() + }) + + it('bypasses the visibility floor when the activity epoch resets', () => { + vi.useFakeTimers() + + render(true) + act(() => vi.advanceTimersByTime(0)) + act(() => vi.advanceTimersByTime(100)) + render(false) + expect(container?.querySelector('[aria-label="Thinking"]')).not.toBeNull() + + render(false, 1) + expect(container?.querySelector('[aria-label="Thinking"]')).toBeNull() + }) + + it('keeps the visibility floor when stale cleanup preserves the activity epoch', () => { + vi.useFakeTimers() + + render(true) + act(() => vi.advanceTimersByTime(100)) + render(false) + act(() => vi.advanceTimersByTime(100)) + + render(false) + expect(container?.querySelector('[aria-label="Thinking"]')).not.toBeNull() + + act(() => vi.advanceTimersByTime(800)) + expect(container?.querySelector('[aria-label="Thinking"]')).toBeNull() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-tab-icon.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-tab-icon.tsx new file mode 100644 index 00000000000..5b88cc2ba1d --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/terminal-session/terminal-tab-icon.tsx @@ -0,0 +1,23 @@ +'use client' + +import { TerminalWindow } from '@sim/emcn/icons' +import { ThinkingLoader } from '@/components/ui' +import { useStableFlag } from '@/hooks/use-stable-flag' + +const TERMINAL_ACTIVITY_MIN_VISIBLE_MS = 1_000 + +interface TerminalTabIconProps { + active: boolean +} + +/** Keeps brief terminal activity visible long enough to register. */ +export function TerminalTabIcon({ active }: TerminalTabIconProps) { + const visible = useStableFlag(active, { minVisibleMs: TERMINAL_ACTIVITY_MIN_VISIBLE_MS }) + return visible ? ( + + + + ) : ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx index 83fc70dc7b3..4378ba9a3cf 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx @@ -1,6 +1,6 @@ 'use client' -import { lazy, memo, Suspense, useEffect, useMemo, useRef, useState } from 'react' +import { lazy, memo, Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Button, PlayOutline, Skeleton, Tooltip, toast } from '@sim/emcn' import { Download, @@ -24,6 +24,7 @@ import { reportManualRunToolStop, } from '@/lib/copilot/tools/client/run-tool-execution' import { canonicalWorkspaceFilePath } from '@/lib/copilot/vfs/path-utils' +import { prefersInPlaceNavigation } from '@/lib/desktop' import { triggerFileDownload } from '@/lib/uploads/client/download' import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' import { @@ -73,6 +74,25 @@ const LOADING_SKELETON = (
) +/** + * Opens an internal app link the way the host expects: a new browser tab on the + * web, and the current view in the desktop app, whose shell would otherwise turn + * the same-origin `window.open` into a second Sim window. + */ +function useOpenInternalLink() { + const router = useRouter() + return useCallback( + (href: string) => { + if (prefersInPlaceNavigation()) { + router.push(href) + return + } + window.open(href, '_blank') + }, + [router] + ) +} + interface ResourceContentProps { workspaceId: string desktopScopeId: string @@ -350,6 +370,7 @@ interface EmbeddedWorkflowActionsProps { } export function EmbeddedWorkflowActions({ workspaceId, workflowId }: EmbeddedWorkflowActionsProps) { + const openInternalLink = useOpenInternalLink() const { navigateToSettings } = useSettingsNavigation() const { data: session } = useSession() const hostContext = useWorkspaceHostContext() @@ -404,7 +425,7 @@ export function EmbeddedWorkflowActions({ workspaceId, workflowId }: EmbeddedWor } const handleOpenWorkflow = () => { - window.open(`/workspace/${workspaceId}/w/${workflowId}`, '_blank') + openInternalLink(`/workspace/${workspaceId}/w/${workflowId}`) } return ( @@ -727,6 +748,7 @@ interface EmbeddedFolderProps { } function EmbeddedFolder({ workspaceId, folderId }: EmbeddedFolderProps) { + const openInternalLink = useOpenInternalLink() const { data: folderList, isPending: isFoldersPending } = useFolders(workspaceId) const { data: workflowList = [] } = useWorkflows(workspaceId) @@ -760,7 +782,7 @@ function EmbeddedFolder({ workspaceId, folderId }: EmbeddedFolderProps) { - - -

Collapse

- - - )}
- {addResourceDropdown} -
- ) : ( - addResourceDropdown - )} +
+ {addResourceDropdown} +
{(actions || (previewMode && onCyclePreviewMode)) && (
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx index 3ff07cf21ac..d82d73db634 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx @@ -64,14 +64,16 @@ interface MothershipViewProps { desktopScopeId: string resources: MothershipResource[] activeResourceId: string | null + activityResourceIds?: ReadonlySet isCollapsed: boolean - useFixedResourceToggle: boolean className?: string previewSession?: FilePreviewSession | null isAgentResponding?: boolean genericResourceData?: GenericResourceData /** Resolved server-side by the home page; forwarded to the embedded table. */ tableViewsEnabled?: boolean + /** Claims the current resource selection after direct panel interaction. */ + onUserInteraction?: () => void } export const MothershipView = memo( @@ -82,17 +84,18 @@ export const MothershipView = memo( desktopScopeId, resources, activeResourceId, + activityResourceIds, isCollapsed, - useFixedResourceToggle, className, previewSession, isAgentResponding, genericResourceData, tableViewsEnabled, + onUserInteraction, }: MothershipViewProps, ref ) { - const active = resources.find((r) => r.id === activeResourceId) ?? resources[0] ?? null + const active = resources.find((r) => r.id === activeResourceId) ?? null const { canEdit } = useUserPermissionsContext() const { removeResource } = useMothershipResources() const browserOverlayControllerRef = useRef(null) @@ -169,6 +172,8 @@ export const MothershipView = memo( // Read by the browser panel to declare its resize anchor: an inline px // width means a divider drag pinned it, otherwise `w-1/2` governs. data-mothership-panel='' + onPointerDownCapture={onUserInteraction} + onKeyDownCapture={onUserInteraction} className={cn( 'relative z-10 flex h-full flex-col overflow-hidden border-[var(--border)] bg-[var(--bg)] transition-[width,min-width,border-width] duration-200 [transition-timing-function:cubic-bezier(0.25,0.1,0.25,1)]', isCollapsed ? 'w-0 min-w-0 border-l-0' : 'w-1/2 border-l', @@ -186,7 +191,7 @@ export const MothershipView = memo( chatId={chatId} resources={resources} activeId={active?.id ?? null} - useFixedResourceToggle={useFixedResourceToggle} + activityIds={activityResourceIds} actions={ active ? : null } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/constants.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/constants.ts index 75c0b4da123..f24b1890ee7 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/constants.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/constants.ts @@ -109,10 +109,9 @@ export const SPEECH_RECOGNITION_LANG = 'en-US' * so adding a new resource type fails compilation here until a conversion is * supplied — preventing silent drift between the two taxonomies. */ -// A dragged `browser`/`terminal` resource is one TAB, and its `id` is that -// tab's id — the panel itself is a singleton with nothing to point at. Both -// become pointers the agent resolves with its own tools rather than content -// captured here, so what it reads is the tab as it stands when it looks. +// Browser/terminal resources may name either the singleton panel or one live +// inner tab. The singleton ids ask the agent to inspect the whole resource; +// every other id is a precise live-tab pointer. const RESOURCE_TO_CONTEXT: Record< MothershipResourceType, (resource: MothershipResource) => ChatContext diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx index 39f9c4b0ae2..1855d99eced 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx @@ -15,10 +15,16 @@ import { } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown' import { getResourceConfig } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry' import type { PlusMenuHandle } from '@/app/workspace/[workspaceId]/home/components/user-input/components/constants' +import { + resourceMentionMatches, + withDesktopTabMentions, +} from '@/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items' import type { MothershipResource, MothershipResourceType, } from '@/app/workspace/[workspaceId]/home/types' +import { useBrowserSessionStore } from '@/stores/browser-session/store' +import { useCopilotTerminalStore } from '@/stores/copilot-terminal/store' /** * Resource types that are only offered via `@`-mention autocomplete and hidden @@ -32,6 +38,8 @@ import type { */ const MENTION_ONLY_RESOURCE_TYPES = new Set(['integration']) const NON_ATTACHABLE_RESOURCE_TYPES = new Set(['browser']) +const EMPTY_BROWSER_TABS = [] as const +const EMPTY_TERMINAL_TABS = [] as const interface PlusMenuDropdownProps { workspaceId: string @@ -63,6 +71,16 @@ export const PlusMenuDropdown = React.memo( const [activeIndex, setActiveIndex] = useState(0) const searchRef = useRef(null) const contentRef = useRef(null) + const browserTabs = useBrowserSessionStore((state) => { + const scopeId = state.activeScopeId + return scopeId ? (state.sessions[scopeId]?.tabs ?? EMPTY_BROWSER_TABS) : EMPTY_BROWSER_TABS + }) + const terminalTabs = useCopilotTerminalStore((state) => { + const scopeId = state.activeScopeId + return scopeId + ? (state.sessions[scopeId]?.tabs.tabs ?? EMPTY_TERMINAL_TABS) + : EMPTY_TERMINAL_TABS + }) // Gated so an idle chat surface never fetches the workspace lists. const { @@ -88,16 +106,18 @@ export const PlusMenuDropdown = React.memo( setOpen(false) }, []) - // The `+` browse menu hides mention-only resource types; `@`-mention mode - // exposes the full catalog so integrations remain searchable inline. + // The `+` browse menu hides non-attachable and mention-only resource types. + // `@` mode exposes the full catalog and adds each live Browser/Terminal tab + // after its always-present whole-resource row. const visibleResources = useMemo(() => { + if (isMention) { + return withDesktopTabMentions(availableResources, browserTabs, terminalTabs) + } const attachable = availableResources.filter( ({ type }) => !NON_ATTACHABLE_RESOURCE_TYPES.has(type) ) - return isMention - ? attachable - : attachable.filter(({ type }) => !MENTION_ONLY_RESOURCE_TYPES.has(type)) - }, [isMention, availableResources]) + return attachable.filter(({ type }) => !MENTION_ONLY_RESOURCE_TYPES.has(type)) + }, [availableResources, browserTabs, isMention, terminalTabs]) const treeSections = useResourceTreeSections({ groups: visibleResources, @@ -113,7 +133,7 @@ export const PlusMenuDropdown = React.memo( return visibleResources.flatMap(({ type, items }) => items.map((item) => ({ type, item }))) } return visibleResources.flatMap(({ type, items }) => - items.filter((item) => item.name.toLowerCase().includes(q)).map((item) => ({ type, item })) + items.filter((item) => resourceMentionMatches(item, q)).map((item) => ({ type, item })) ) }, [isMention, mentionQuery, search, visibleResources]) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.test.ts new file mode 100644 index 00000000000..bed14025882 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest' +import { + BROWSER_SESSION_RESOURCE_ID, + TERMINAL_SESSION_RESOURCE_ID, +} from '@/lib/copilot/resources/types' +import { + resourceMentionMatches, + withDesktopTabMentions, +} from '@/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items' + +const groups = [ + { type: 'workflow' as const, items: [{ id: 'wf-1', name: 'Deploy' }] }, + { + type: 'browser' as const, + items: [{ id: BROWSER_SESSION_RESOURCE_ID, name: 'Browser' }], + }, + { + type: 'terminal' as const, + items: [{ id: TERMINAL_SESSION_RESOURCE_ID, name: 'Terminal' }], + }, +] + +describe('withDesktopTabMentions', () => { + it('keeps Browser and Terminal as flat resource mentions with no live tabs', () => { + const result = withDesktopTabMentions(groups, [], []) + + expect(result.find((group) => group.type === 'browser')?.items).toEqual([ + expect.objectContaining({ + id: BROWSER_SESSION_RESOURCE_ID, + name: 'Browser', + mentionLevel: 'resource', + }), + ]) + expect(result.find((group) => group.type === 'terminal')?.items).toEqual([ + expect.objectContaining({ + id: TERMINAL_SESSION_RESOURCE_ID, + name: 'Terminal', + mentionLevel: 'resource', + }), + ]) + }) + + it('offers the whole resources first and every live tab after them', () => { + const result = withDesktopTabMentions( + groups, + [ + { + tabId: 'browser-1', + title: 'Sim Docs', + url: 'https://docs.sim.ai', + loading: false, + active: true, + pinned: false, + }, + { + tabId: 'browser-2', + title: '', + url: 'https://github.com/simstudioai/sim', + loading: false, + active: false, + pinned: false, + }, + ], + [ + { + terminalId: 'terminal-1', + title: 'sim', + cwd: '/code/sim', + running: null, + interactive: false, + active: true, + }, + { + terminalId: 'terminal-2', + title: 'sim', + cwd: '/tmp/sim', + running: null, + interactive: false, + active: false, + }, + ] + ) + + expect(result.find((group) => group.type === 'browser')?.items).toMatchObject([ + { id: BROWSER_SESSION_RESOURCE_ID, name: 'Browser', mentionLevel: 'resource' }, + { id: 'browser-1', name: 'Sim Docs', mentionLevel: 'tab' }, + { id: 'browser-2', name: 'github.com', mentionLevel: 'tab' }, + ]) + expect(result.find((group) => group.type === 'terminal')?.items).toMatchObject([ + { id: TERMINAL_SESSION_RESOURCE_ID, name: 'Terminal', mentionLevel: 'resource' }, + { id: 'terminal-1', name: 'sim 1', mentionLevel: 'tab' }, + { id: 'terminal-2', name: 'sim 2', mentionLevel: 'tab' }, + ]) + }) + + it('keeps specific tabs discoverable by either their title or resource family', () => { + const tab = { + id: 'browser-1', + name: 'Sim Docs', + mentionFamily: 'Browser', + mentionLevel: 'tab', + } + + expect(resourceMentionMatches(tab, 'docs')).toBe(true) + expect(resourceMentionMatches(tab, 'browser')).toBe(true) + expect(resourceMentionMatches(tab, 'terminal')).toBe(false) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.ts new file mode 100644 index 00000000000..bbbe84ad29e --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.ts @@ -0,0 +1,94 @@ +import type { BrowserTabState } from '@sim/browser-protocol' +import type { TerminalTabState } from '@sim/terminal-protocol' +import { + BROWSER_SESSION_RESOURCE_ID, + TERMINAL_SESSION_RESOURCE_ID, +} from '@/lib/copilot/resources/types' +import type { AvailableItem } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree' +import { browserTabTitle } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-tab-label' +import type { MothershipResourceType } from '@/app/workspace/[workspaceId]/home/types' + +export interface ResourceMentionGroup { + type: MothershipResourceType + items: AvailableItem[] +} + +export type ResourceMentionLevel = 'resource' | 'tab' + +/** A family query such as "browser" keeps that resource's live tabs visible. */ +export function resourceMentionMatches(item: AvailableItem, query: string): boolean { + const normalized = query.toLowerCase().trim() + if (!normalized) return true + return ( + item.name.toLowerCase().includes(normalized) || + (typeof item.mentionFamily === 'string' && + item.mentionFamily.toLowerCase().includes(normalized)) + ) +} + +function uniqueTabNames(tabs: readonly T[], nameOf: (tab: T) => string): string[] { + const names = tabs.map(nameOf) + const counts = new Map() + for (const name of names) counts.set(name, (counts.get(name) ?? 0) + 1) + const occurrences = new Map() + return names.map((name) => { + if (counts.get(name) === 1) return name + const occurrence = (occurrences.get(name) ?? 0) + 1 + occurrences.set(name, occurrence) + return `${name} ${occurrence}` + }) +} + +function resourceItem(id: string, name: string, existing?: AvailableItem): AvailableItem { + return { + ...existing, + id, + name, + mentionFamily: name, + mentionLevel: 'resource' satisfies ResourceMentionLevel, + } +} + +/** Adds live inner tabs after each always-present desktop resource mention. */ +export function withDesktopTabMentions( + groups: readonly ResourceMentionGroup[], + browserTabs: readonly BrowserTabState[], + terminalTabs: readonly TerminalTabState[] +): ResourceMentionGroup[] { + const browserNames = uniqueTabNames(browserTabs, browserTabTitle) + const terminalNames = uniqueTabNames(terminalTabs, (tab) => tab.title.trim() || 'Terminal') + + return groups.map((group) => { + if (group.type === 'browser') { + const existing = group.items.find((item) => item.id === BROWSER_SESSION_RESOURCE_ID) + return { + ...group, + items: [ + resourceItem(BROWSER_SESSION_RESOURCE_ID, 'Browser', existing), + ...browserTabs.map((tab, index) => ({ + id: tab.tabId, + name: browserNames[index], + mentionFamily: 'Browser', + mentionLevel: 'tab' satisfies ResourceMentionLevel, + })), + ], + } + } + if (group.type === 'terminal') { + const existing = group.items.find((item) => item.id === TERMINAL_SESSION_RESOURCE_ID) + return { + ...group, + items: [ + resourceItem(TERMINAL_SESSION_RESOURCE_ID, 'Terminal', existing), + ...terminalTabs.map((tab, index) => ({ + id: tab.terminalId, + name: terminalNames[index], + mentionFamily: 'Terminal', + mentionLevel: 'tab' satisfies ResourceMentionLevel, + })), + ], + } + } + return group + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/resource-context.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/resource-context.test.ts index 9b45baa5c3f..47e0216319f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/resource-context.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/resource-context.test.ts @@ -10,6 +10,17 @@ function resource(partial: Partial & Pick { + it('turns the singleton panels into whole-resource pointers', () => { + expect( + mapResourceToContext(resource({ type: 'browser', id: 'browser-session', title: 'Browser' })) + ).toEqual({ kind: 'browser_tab', tabId: 'browser-session', label: 'Browser' }) + expect( + mapResourceToContext( + resource({ type: 'terminal', id: 'terminal-session', title: 'Terminal' }) + ) + ).toEqual({ kind: 'terminal_tab', terminalId: 'terminal-session', label: 'Terminal' }) + }) + it('turns a dragged browser tab into a pointer at that tab', () => { // The id is the TAB's, not the panel's: the panel is a singleton and // pointing at it would not say which page the user meant. diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 1f78a7199af..d8bf4142639 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -15,6 +15,7 @@ import { createLogger } from '@sim/logger' import { useParams } from 'next/navigation' import { getMothershipAttachmentPreviewUrl } from '@/lib/copilot/chat/attachment-preview' import { SIM_RESOURCE_DRAG_TYPE, SIM_RESOURCES_DRAG_TYPE } from '@/lib/copilot/resource-types' +import { isDesktopApp } from '@/lib/desktop' import { MOTHERSHIP_ADD_CONTEXT_EVENT } from '@/lib/mothership/events' import { MOTHERSHIP_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { useChatSurface } from '@/app/workspace/[workspaceId]/home/components/chat-surface-context' @@ -37,7 +38,7 @@ import { useFileAttachments } from '@/app/workspace/[workspaceId]/w/[workflowId] import type { AttachedFile } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments' import { mentionifyIntegrations } from '@/blocks/integration-matcher' import { useSettingsNavigation } from '@/hooks/use-settings-navigation' -import { useSpeechToText } from '@/hooks/use-speech-to-text' +import { type SpeechToTextError, useSpeechToText } from '@/hooks/use-speech-to-text' import { useMothershipDraftsStore } from '@/stores/mothership-drafts/store' import type { ChatContext } from '@/stores/panel' @@ -288,6 +289,22 @@ const UserInputImpl = forwardRef(function UserI ) } + function handleSpeechError(error: SpeechToTextError) { + if (error === 'microphone-blocked') { + toast.error( + isDesktopApp() + ? 'Microphone access is blocked. Allow Sim to use the microphone in your system privacy settings.' + : 'Microphone access is blocked. Allow it for this site and try again.' + ) + return + } + if (error === 'microphone-unavailable') { + toast.error('No microphone found. Connect one and try again.') + return + } + toast.error('Could not start voice input. Try again.') + } + const { isListening, isSupported: isSttSupported, @@ -296,6 +313,7 @@ const UserInputImpl = forwardRef(function UserI } = useSpeechToText({ onTranscript: handleTranscript, onUsageLimitExceeded: handleUsageLimitExceeded, + onError: handleSpeechError, workspaceId, }) @@ -313,6 +331,7 @@ const UserInputImpl = forwardRef(function UserI const isSendingRef = useRef(isSending) isSendingRef.current = isSending const wasSendingRef = useRef(false) + const composerOwnsFocusRef = useRef(false) useImperativeHandle( ref, @@ -421,13 +440,13 @@ const UserInputImpl = forwardRef(function UserI }, []) useEffect(() => { - if (wasSendingRef.current && !isSending) { - const active = document.activeElement - const isEditingElsewhere = - active instanceof HTMLTextAreaElement || active instanceof HTMLInputElement - if (!isEditingElsewhere) { - textareaRef.current?.focus() - } + if ( + wasSendingRef.current && + !isSending && + composerOwnsFocusRef.current && + document.hasFocus() + ) { + textareaRef.current?.focus() } wasSendingRef.current = isSending }, [isSending, textareaRef]) @@ -435,9 +454,9 @@ const UserInputImpl = forwardRef(function UserI useEffect(() => { const raf = window.requestAnimationFrame(() => { const active = document.activeElement - const isEditingElsewhere = - active instanceof HTMLTextAreaElement || active instanceof HTMLInputElement - if (!isEditingElsewhere) { + const pageHasFocus = document.hasFocus() + const hasNeutralFocus = active === document.body || active === document.documentElement + if (pageHasFocus && hasNeutralFocus) { textareaRef.current?.focus() } }) @@ -529,6 +548,14 @@ const UserInputImpl = forwardRef(function UserI return (
{ + composerOwnsFocusRef.current = true + }} + onBlurCapture={(event) => { + if (!event.currentTarget.contains(event.relatedTarget as Node | null)) { + composerOwnsFocusRef.current = false + } + }} className={cn( 'relative z-10 mx-auto w-full max-w-chat cursor-text rounded-2xl border border-[var(--border-1)] bg-[var(--white)] px-2.5 py-2 dark:bg-[var(--surface-4)]', isInitialView && 'shadow-ambient' diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 03008c597ae..b274c12b864 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -3,6 +3,7 @@ import { type Dispatch, lazy, + type PointerEvent, type SetStateAction, Suspense, useCallback, @@ -10,7 +11,6 @@ import { useMemo, useRef, useState, - useSyncExternalStore, } from 'react' import { Button, cn, toast } from '@sim/emcn' import { PanelLeft } from '@sim/emcn/icons' @@ -27,7 +27,6 @@ import { LandingWorkflowSeedStorage, MothershipHandoffStorage, } from '@/lib/core/utils/browser-storage' -import { isDesktopApp } from '@/lib/desktop' import { addMothershipContexts, MOTHERSHIP_SEND_MESSAGE_EVENT, @@ -65,8 +64,6 @@ import type { } from './types' const logger = createLogger('Home') -const subscribeToDesktopApp = () => () => {} -const getServerDesktopAppSnapshot = () => false /** * The resource preview panel pulls in the file-viewer stack (rich-markdown @@ -89,11 +86,6 @@ interface HomeProps { export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) { useOAuthReturnRouter() - const isDesktop = useSyncExternalStore( - subscribeToDesktopApp, - isDesktopApp, - getServerDesktopAppSnapshot - ) const { workspaceId } = useParams<{ workspaceId: string }>() const router = useRouter() const queryClient = useQueryClient() @@ -212,13 +204,32 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) const [isResourceCollapsed, setIsResourceCollapsed] = useState(true) const [skipResourceTransition, setSkipResourceTransition] = useState(false) + const [resourceActivityIds, setResourceActivityIds] = useState>(new Set()) const isResourceCollapsedRef = useRef(isResourceCollapsed) isResourceCollapsedRef.current = isResourceCollapsed - - function handleResourceEvent() { - if (isResourceCollapsedRef.current) { - setIsResourceCollapsed(false) + const userOwnsResourceViewRef = useRef(false) + const activeResourceParamRef = useRef(activeResourceParam) + activeResourceParamRef.current = activeResourceParam + + function handleResourceEvent(resourceId: string) { + // Agent work should always make the resource surface available. Expanding + // the panel is independent from selecting a resource: once the user has + // chosen another resource, the agent may work in the background without + // taking that selection away. + if (isResourceCollapsedRef.current) setIsResourceCollapsed(false) + + const activeResourceId = activeResourceParamRef.current + if (userOwnsResourceViewRef.current && activeResourceId && activeResourceId !== resourceId) { + setResourceActivityIds((current) => new Set(current).add(resourceId)) + return } + setResourceActivityIds((current) => { + if (!current.has(resourceId)) return current + const next = new Set(current) + next.delete(resourceId) + return next + }) + if (activeResourceId !== resourceId) setActiveResourceUrl(resourceId) } const { @@ -263,13 +274,58 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) ) const { mothershipRef, handleResizePointerDown, clearWidth } = useMothershipResize(desktopScopeId) + const effectiveActiveResourceIdRef = useRef(activeResourceId) + effectiveActiveResourceIdRef.current = activeResourceId + const resourceAttentionChatIdRef = useRef(resolvedChatId) const collapseResource = useCallback(() => { + userOwnsResourceViewRef.current = true clearWidth() setIsResourceCollapsed(true) }, [clearWidth]) + const selectResourceFromUser = useCallback( + (resourceId: string) => { + userOwnsResourceViewRef.current = true + setResourceActivityIds((current) => { + if (!current.has(resourceId)) return current + const next = new Set(current) + next.delete(resourceId) + return next + }) + if (effectiveActiveResourceIdRef.current === resourceId) return + effectiveActiveResourceIdRef.current = resourceId + activeResourceParamRef.current = resourceId + setActiveResourceId(resourceId) + }, + [setActiveResourceId] + ) + + const addResourceFromUser = useCallback( + (resource: MothershipResource) => { + userOwnsResourceViewRef.current = true + addResource(resource) + selectResourceFromUser(resource.id) + setIsResourceCollapsed(false) + }, + [addResource, selectResourceFromUser] + ) + + const handleResourceResizePointerDown = useCallback( + (event: PointerEvent) => { + userOwnsResourceViewRef.current = true + handleResizePointerDown(event) + }, + [handleResizePointerDown] + ) + + const handleResourceInteraction = useCallback(() => { + userOwnsResourceViewRef.current = true + }, []) + useEffect(() => { + const previousChatId = resourceAttentionChatIdRef.current + resourceAttentionChatIdRef.current = resolvedChatId wasSendingRef.current = false if (resolvedChatId) { markRead(resolvedChatId) @@ -277,6 +333,10 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) clearWidth() setIsResourceCollapsed(true) } + if (!resolvedChatId || (previousChatId && previousChatId !== resolvedChatId)) { + userOwnsResourceViewRef.current = false + setResourceActivityIds(new Set()) + } }, [resolvedChatId, markRead, clearWidth]) useEffect(() => { @@ -287,7 +347,12 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) }, [isSending, resolvedChatId, markRead]) useEffect(() => { - if (!(resources.length > 0 && isResourceCollapsedRef.current)) return + if ( + !(resources.length > 0 && isResourceCollapsedRef.current) || + userOwnsResourceViewRef.current + ) { + return + } setIsResourceCollapsed(false) setSkipResourceTransition(true) const id = requestAnimationFrame(() => setSkipResourceTransition(false)) @@ -296,9 +361,18 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) useEffect(() => { if (resources.length === 0 && !isResourceCollapsedRef.current) { - collapseResource() + clearWidth() + setIsResourceCollapsed(true) } - }, [resources, collapseResource]) + }, [resources, clearWidth]) + + useEffect(() => { + const resourceIds = new Set(resources.map((resource) => resource.id)) + setResourceActivityIds((current) => { + const next = new Set([...current].filter((id) => resourceIds.has(id))) + return next.size === current.size ? current : next + }) + }, [resources]) const handleStopGeneration = useCallback(() => { captureEvent(posthogRef.current, 'task_generation_aborted', { @@ -325,6 +399,8 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) setIsInputEntering(true) } + userOwnsResourceViewRef.current = false + setResourceActivityIds(new Set()) sendMessage(trimmed || 'Analyze the attached file(s).', fileAttachments, contexts) }, [workspaceId, chatId, sendMessage] @@ -425,8 +501,7 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) function handleContextAdd(context: ChatContext) { const resolved = resolveResourceFromContext(context) if (resolved) { - addResource({ ...resolved, title: resourceTitleForContext(context) }) - handleResourceEvent() + addResourceFromUser({ ...resolved, title: resourceTitleForContext(context) }) } } @@ -446,11 +521,7 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) } function openWorkspaceResource(resource: MothershipResource) { - const wasAdded = addResource(resource) - if (!wasAdded) { - setActiveResourceId(resource.id) - } - handleResourceEvent() + addResourceFromUser(resource) } /** @@ -513,9 +584,7 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) className={cn( 'absolute z-10', RESOURCE_HEADER_CLASSES.contentTop, - isDesktop || isResourceCollapsed - ? RESOURCE_HEADER_CLASSES.adjacentEndPosition - : RESOURCE_HEADER_CLASSES.endPosition + RESOURCE_HEADER_CLASSES.adjacentEndPosition )} > @@ -589,14 +658,14 @@ export function Home({ chatId, userName, userId, tableViewsEnabled }: HomeProps) role='separator' aria-orientation='vertical' aria-label='Resize resource panel' - onPointerDown={handleResizePointerDown} + onPointerDown={handleResourceResizePointerDown} />
)} - {isDesktop ? ( -
+ -
- ) : ( - isResourceCollapsed && ( -
0 && ( + )} - > - -
- ) - )} + + +
) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-controller.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-controller.ts index d413d080bfb..02ca681bf37 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-controller.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-controller.ts @@ -28,6 +28,7 @@ interface FilePreviewControllerDeps { setResources: Dispatch> setActiveResourceId: Dispatch> activeResourceIdRef: MutableRefObject + onResourceEventRef: MutableRefObject<((resourceId: string) => void) | undefined> } function asPayloadRecord(value: unknown): Record | undefined { @@ -49,6 +50,7 @@ export function useFilePreviewController({ setResources, setActiveResourceId, activeResourceIdRef, + onResourceEventRef, }: FilePreviewControllerDeps) { const queryClient = useQueryClient() @@ -72,6 +74,7 @@ export function useFilePreviewController({ if (!session.fileId) { return false } + if (onResourceEventRef.current) return true const currentActiveResourceId = activeResourceIdRef.current const activationOwnerId = previewActivationOwnerRef.current.get(session.id) return ( @@ -81,7 +84,16 @@ export function useFilePreviewController({ currentActiveResourceId === activationOwnerId ) }, - [activeResourceIdRef] + [activeResourceIdRef, onResourceEventRef] + ) + + const requestResourceAttention = useCallback( + (resourceId: string) => { + const onResourceEvent = onResourceEventRef.current + if (onResourceEvent) onResourceEvent(resourceId) + else setActiveResourceId(resourceId) + }, + [onResourceEventRef, setActiveResourceId] ) const seedCompletedPreviewContentCache = useCallback( @@ -239,18 +251,18 @@ export function useFilePreviewController({ { type: 'file', id: 'streaming-file', title: session.fileName || 'Writing file...' }, ] }) - setActiveResourceId('streaming-file') + requestResourceAttention('streaming-file') return } if (session.fileId && hasRenderableFilePreviewContent(session)) { promoteFileResource(session.fileId, session.fileName || 'File') if (options?.activate !== false) { - setActiveResourceId(session.fileId) + requestResourceAttention(session.fileId) } } }, - [promoteFileResource, setActiveResourceId, setResources] + [promoteFileResource, requestResourceAttention, setResources] ) const seedPreviewSessions = useCallback( @@ -354,7 +366,7 @@ export function useFilePreviewController({ (!wasRenderableBeforeComplete && hasRenderableFilePreviewContent(nextSession))) && shouldAutoActivatePreviewSession(nextSession) if (shouldActivateOnComplete) { - setActiveResourceId(fileId) + requestResourceAttention(fileId) } completedPreviewResourceHandoffRef.current.set(fileId, { sessionId: nextSession.id, @@ -385,7 +397,7 @@ export function useFilePreviewController({ queryClient, rememberPreviewActivationOwner, seedCompletedPreviewContentCache, - setActiveResourceId, + requestResourceAttention, shouldAutoActivatePreviewSession, syncPreviewResourceChrome, workspaceId, diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/resource-activity.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/resource-activity.test.ts new file mode 100644 index 00000000000..21b4d4fd096 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/resource-activity.test.ts @@ -0,0 +1,119 @@ +import { beforeEach, describe, expect, it } from 'vitest' +import { + clearTrackedResourceActivity, + createResourceActivityTracker, + excludeActivityOwnedBy, + setTrackedBrowserRun, +} from '@/app/workspace/[workspaceId]/home/hooks/resource-activity' +import { getBrowserSession, useBrowserSessionStore } from '@/stores/browser-session/store' +import { getCopilotTerminalSession, useCopilotTerminalStore } from '@/stores/copilot-terminal/store' + +describe('resource activity tracker', () => { + beforeEach(() => { + useBrowserSessionStore.setState({ activeScopeId: null, sessions: {} }) + useCopilotTerminalStore.setState({ + activeScopeId: null, + sessions: {}, + settledAgentCommandIds: [], + }) + }) + + it('clears exact old-stream activity after migration without touching a newer stream', () => { + const browserStore = useBrowserSessionStore.getState() + const terminalStore = useCopilotTerminalStore.getState() + browserStore.activateScope('pending:chat') + terminalStore.activateScope('pending:chat') + browserStore.setAgentRunActive('pending:chat', 'browser-old', true) + terminalStore.applyCommandEvent({ + scopeId: 'pending:chat', + terminalId: 'terminal-1', + phase: 'start', + command: 'old', + toolCallId: 'terminal-old', + }) + const tracker = createResourceActivityTracker(1, ['pending:chat']) + + browserStore.migrateScope('pending:chat', 'chat-1') + terminalStore.migrateScope('pending:chat', 'chat-1') + browserStore.setAgentRunActive('chat-1', 'browser-new', true) + terminalStore.applyCommandEvent({ + scopeId: 'chat-1', + terminalId: 'terminal-1', + phase: 'start', + command: 'new', + toolCallId: 'terminal-new', + }) + + clearTrackedResourceActivity(tracker, { hardResetActivity: false }) + + expect(getBrowserSession('chat-1').agentRunIds).toEqual(['browser-new']) + expect(getCopilotTerminalSession('chat-1')).toMatchObject({ + agentCommandTerminalIds: { 'terminal-new': 'terminal-1' }, + activityResetEpoch: 0, + }) + }) + + it('settles a browser span end after its pending scope migrates', () => { + const browserStore = useBrowserSessionStore.getState() + browserStore.activateScope('pending:chat') + const tracker = createResourceActivityTracker(1, ['pending:chat']) + setTrackedBrowserRun(tracker, 'pending:chat', 'browser-old', true) + browserStore.migrateScope('pending:chat', 'chat-1') + + setTrackedBrowserRun(tracker, 'pending:chat', 'browser-old', false) + + expect(getBrowserSession('chat-1').agentRunIds).toEqual([]) + }) + + it('does not absorb current activity when constructing a stale-generation tracker', () => { + const browserStore = useBrowserSessionStore.getState() + const terminalStore = useCopilotTerminalStore.getState() + browserStore.activateScope('chat-1') + terminalStore.activateScope('chat-1') + browserStore.setAgentRunActive('chat-1', 'browser-new', true) + terminalStore.applyCommandEvent({ + scopeId: 'chat-1', + terminalId: 'terminal-1', + phase: 'start', + command: 'new', + toolCallId: 'terminal-new', + }) + + const staleTracker = createResourceActivityTracker(1, ['chat-1'], { + captureExisting: false, + }) + clearTrackedResourceActivity(staleTracker, { hardResetActivity: false }) + + expect(getBrowserSession('chat-1').agentRunIds).toEqual(['browser-new']) + expect(getCopilotTerminalSession('chat-1').agentCommandTerminalIds).toEqual({ + 'terminal-new': 'terminal-1', + }) + expect(getCopilotTerminalSession('chat-1').activityResetEpoch).toBe(0) + }) + + it('preserves exact ids adopted by a newer reader when stale cleanup arrives late', () => { + const browserStore = useBrowserSessionStore.getState() + const terminalStore = useCopilotTerminalStore.getState() + browserStore.activateScope('chat-1') + terminalStore.activateScope('chat-1') + browserStore.setAgentRunActive('chat-1', 'shared-browser-run', true) + terminalStore.applyCommandEvent({ + scopeId: 'chat-1', + terminalId: 'terminal-1', + phase: 'start', + command: 'shared', + toolCallId: 'shared-terminal-tool', + }) + const staleTracker = createResourceActivityTracker(1, ['chat-1']) + const currentTracker = createResourceActivityTracker(2, ['chat-1']) + + excludeActivityOwnedBy(staleTracker, currentTracker) + clearTrackedResourceActivity(staleTracker, { hardResetActivity: false }) + + expect(getBrowserSession('chat-1').agentRunIds).toEqual(['shared-browser-run']) + expect(getCopilotTerminalSession('chat-1')).toMatchObject({ + agentCommandTerminalIds: { 'shared-terminal-tool': 'terminal-1' }, + activityResetEpoch: 0, + }) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/resource-activity.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/resource-activity.ts new file mode 100644 index 00000000000..57b4b852a2d --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/resource-activity.ts @@ -0,0 +1,143 @@ +import { getBrowserSession, useBrowserSessionStore } from '@/stores/browser-session/store' +import { getCopilotTerminalSession, useCopilotTerminalStore } from '@/stores/copilot-terminal/store' + +/** Activity owned by one chat-stream generation across reconnect legs and scope migration. */ +export interface ResourceActivityTracker { + generation: number + scopeIds: Set + currentScopeId: string + browserRunIds: Set + terminalToolCallIds: Set + cleared: boolean +} + +/** Captures activity already visible in a scope, such as when reconnecting mid-turn. */ +export function captureResourceActivityScope( + tracker: ResourceActivityTracker, + scopeId: string +): void { + if (!scopeId) return + tracker.scopeIds.add(scopeId) + tracker.currentScopeId = scopeId + for (const runId of getBrowserSession(scopeId).agentRunIds) { + tracker.browserRunIds.add(runId) + } + for (const toolCallId of Object.keys( + getCopilotTerminalSession(scopeId).agentCommandTerminalIds + )) { + tracker.terminalToolCallIds.add(toolCallId) + } +} + +/** Creates a tracker and seeds it with any activity restored before the stream reconnects. */ +export function createResourceActivityTracker( + generation: number, + scopeIds: Iterable, + options?: { captureExisting?: boolean } +): ResourceActivityTracker { + const tracker: ResourceActivityTracker = { + generation, + scopeIds: new Set(), + currentScopeId: '', + browserRunIds: new Set(), + terminalToolCallIds: new Set(), + cleared: false, + } + for (const scopeId of scopeIds) { + if (options?.captureExisting === false) { + tracker.scopeIds.add(scopeId) + tracker.currentScopeId = scopeId + } else captureResourceActivityScope(tracker, scopeId) + } + return tracker +} + +/** Records a browser span against this stream while preserving exact run identity. */ +export function setTrackedBrowserRun( + tracker: ResourceActivityTracker, + scopeId: string, + runId: string, + active: boolean +): void { + if (scopeId) { + tracker.scopeIds.add(scopeId) + tracker.currentScopeId = scopeId + } + tracker.cleared = false + const browserStore = useBrowserSessionStore.getState() + if (active) { + tracker.browserRunIds.add(runId) + browserStore.setAgentRunActive(scopeId, runId, true) + return + } + tracker.browserRunIds.delete(runId) + browserStore.clearAgentRunIds([runId]) +} + +/** Records an exact terminal tool before its native command-start event arrives. */ +export function trackTerminalToolCall( + tracker: ResourceActivityTracker, + scopeId: string, + toolCallId: string +): void { + if (scopeId) { + tracker.scopeIds.add(scopeId) + tracker.currentScopeId = scopeId + } + tracker.cleared = false + tracker.terminalToolCallIds.add(toolCallId) +} + +/** Prevents stale cleanup from settling exact ids adopted by a newer reader. */ +export function excludeActivityOwnedBy( + tracker: ResourceActivityTracker, + currentOwner: ResourceActivityTracker +): void { + for (const runId of currentOwner.browserRunIds) tracker.browserRunIds.delete(runId) + for (const toolCallId of currentOwner.terminalToolCallIds) { + tracker.terminalToolCallIds.delete(toolCallId) + } +} + +/** + * Immediately settles only this stream's resource activity. Exact ids keep a + * late reader from clearing work started by a newer generation in the same chat. + */ +export function clearTrackedResourceActivity( + tracker: ResourceActivityTracker, + options: { hardResetActivity: boolean } +): void { + if (tracker.cleared) return + const browserStore = useBrowserSessionStore.getState() + browserStore.clearAgentRunIds( + [...tracker.browserRunIds], + options.hardResetActivity ? { hardResetScopeIds: [...tracker.scopeIds] } : undefined + ) + + const terminalStore = useCopilotTerminalStore.getState() + const terminalToolCallIds = [...tracker.terminalToolCallIds] + const resetScopeId = tracker.currentScopeId || tracker.scopeIds.values().next().value + if (resetScopeId) { + terminalStore.clearAgentCommands(resetScopeId, terminalToolCallIds, options) + } + + tracker.browserRunIds.clear() + tracker.terminalToolCallIds.clear() + tracker.cleared = true +} + +/** Settles stale renderer activity when a detached chat hydrates as terminal. */ +export function clearResourceActivityScope(scopeId: string): void { + if (!scopeId) return + const browserSession = getBrowserSession(scopeId) + useBrowserSessionStore + .getState() + .clearAgentRunIds(browserSession.agentRunIds, { hardResetScopeIds: [scopeId] }) + + const terminalSession = getCopilotTerminalSession(scopeId) + useCopilotTerminalStore + .getState() + .clearAgentCommands(scopeId, Object.keys(terminalSession.agentCommandTerminalIds), { + hardResetActivity: true, + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-complete-event.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-complete-event.ts index c4eb9889811..6b4fa6f8c39 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-complete-event.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-complete-event.ts @@ -9,6 +9,8 @@ type CompleteEvent = Extract * async pause). This handler only records the terminal flag and flushes. */ export function handleCompleteEvent(ctx: StreamLoopContext, _parsed: CompleteEvent): void { + ctx.deps.clearBrowserAgentRuns() + ctx.state.browserAgentRunIds.clear() ctx.state.sawCompleteEvent = true ctx.ops.flush() } diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts index 1283db400e5..5eb8df1154d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts @@ -85,7 +85,10 @@ describe('handleResourceEvent removal', () => { }) it('normalizes a page-shaped browser event into the singleton Browser panel', () => { - const deps = makeStreamLoopDeps() + const onResourceEvent = vi.fn() + const deps = makeStreamLoopDeps({ + onResourceEventRef: { current: onResourceEvent }, + }) const ctx = { deps } as StreamLoopContext handleResourceEvent( @@ -98,5 +101,7 @@ describe('handleResourceEvent removal', () => { id: 'browser-session', title: 'Browser', }) + expect(deps.setActiveResourceId).not.toHaveBeenCalled() + expect(onResourceEvent).toHaveBeenCalledWith('browser-session') }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts index 90f34f721ef..12c1a2d6f94 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts @@ -33,7 +33,6 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven addResource, removeResource, setResources, - setActiveResourceId, resourcesRef, activeResourceIdRef, previewSessionsRef, @@ -59,7 +58,6 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven removeWorkflowFromActiveCache(queryClient, workspaceId, resource.id) } invalidateResourceQueries(queryClient, workspaceId, resourceType, resource.id) - onResourceEvent?.() return } @@ -115,14 +113,7 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven } invalidateResourceQueries(queryClient, workspaceId, resource.type, resource.id) - if ( - !shouldSuppressFileResourceActivation && - !wasAdded && - activeResourceIdRef.current !== resource.id - ) { - setActiveResourceId(resource.id) - } - onResourceEvent?.() + if (!shouldSuppressFileResourceActivation) onResourceEvent?.(resource.id) if (resource.type === 'workflow') { const wasRegistered = ensureWorkflowInRegistry(resource.id, resource.title, workspaceId) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-span-event.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-span-event.test.ts new file mode 100644 index 00000000000..e74693687e5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-span-event.test.ts @@ -0,0 +1,87 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + MothershipStreamV1SpanLifecycleEvent, + MothershipStreamV1SpanPayloadKind, +} from '@/lib/copilot/generated/mothership-stream-v1' +import type { PersistedStreamEventEnvelope } from '@/lib/copilot/request/session/contract' +import { handleCompleteEvent } from './handle-complete-event' +import { handleSpanEvent } from './handle-span-event' +import { createStreamLoopContext } from './stream-context' +import { makeStreamLoopDeps } from './stream-test-helpers' + +function spanEvent( + event: MothershipStreamV1SpanLifecycleEvent, + agent = 'browser' +): Extract { + return { + v: 1, + seq: event === MothershipStreamV1SpanLifecycleEvent.start ? 1 : 2, + ts: '2026-01-01T00:00:00Z', + stream: { streamId: 'stream-1' }, + type: 'span', + payload: { + kind: MothershipStreamV1SpanPayloadKind.subagent, + event, + agent, + data: { tool_call_id: 'browser-call-1' }, + }, + } as Extract +} + +describe('browser subagent span activity', () => { + it('stays active for the whole browser span, independently of tool calls', () => { + const deps = makeStreamLoopDeps() + const ctx = createStreamLoopContext(deps) + const scope = { + scopedParentToolCallId: 'browser-call-1', + scopedAgentId: 'browser', + scopedSpanId: 'browser-span-1', + } + + handleSpanEvent(ctx, spanEvent(MothershipStreamV1SpanLifecycleEvent.start), scope) + + expect(ctx.state.browserAgentRunIds).toEqual(new Set(['browser-span-1'])) + expect(deps.startBrowserAgentRun).toHaveBeenCalledWith('browser-span-1') + expect(deps.endBrowserAgentRun).not.toHaveBeenCalled() + + handleSpanEvent(ctx, spanEvent(MothershipStreamV1SpanLifecycleEvent.end), scope) + + expect(ctx.state.browserAgentRunIds.size).toBe(0) + expect(deps.endBrowserAgentRun).toHaveBeenCalledWith('browser-span-1') + }) + + it('clears a browser span when the stream completes without an explicit end', () => { + const deps = makeStreamLoopDeps() + const ctx = createStreamLoopContext(deps) + ctx.state.browserAgentRunIds.add('browser-span-1') + + handleCompleteEvent(ctx, { + v: 1, + seq: 2, + ts: '2026-01-01T00:00:01Z', + stream: { streamId: 'stream-1' }, + type: 'complete', + payload: { status: 'complete' }, + } as Extract) + + expect(ctx.state.browserAgentRunIds.size).toBe(0) + expect(deps.clearBrowserAgentRuns).toHaveBeenCalledOnce() + }) + + it('does not mark other subagents as browser activity', () => { + const deps = makeStreamLoopDeps() + const ctx = createStreamLoopContext(deps) + + handleSpanEvent(ctx, spanEvent(MothershipStreamV1SpanLifecycleEvent.start, 'workflow'), { + scopedParentToolCallId: 'workflow-call-1', + scopedAgentId: 'workflow', + scopedSpanId: 'workflow-span-1', + }) + + expect(ctx.state.browserAgentRunIds.size).toBe(0) + expect(deps.startBrowserAgentRun).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-span-event.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-span-event.ts index 7c0902470da..17ebd4c228f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-span-event.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-span-event.ts @@ -14,6 +14,8 @@ import { type SpanEvent = Extract +const BROWSER_SUBAGENT_ID = 'browser' + /** * Side effects for subagent span lifecycle. The model owns the subagent * group/nesting/close (via `reduceEvent`); this handler only seeds the file @@ -41,6 +43,17 @@ export function handleSpanEvent( const parentToolCallId = scopedParentToolCallId ?? parentToolCallIdFromData const isPendingPause = spanData?.pending === true const name = typeof payload.agent === 'string' ? payload.agent : scopedAgentId + const runId = scopedSpanId ?? parentToolCallId + + if (name === BROWSER_SUBAGENT_ID && runId) { + if (payload.event === MothershipStreamV1SpanLifecycleEvent.start) { + state.browserAgentRunIds.add(runId) + deps.startBrowserAgentRun(runId) + } else { + state.browserAgentRunIds.delete(runId) + deps.endBrowserAgentRun(runId) + } + } if (payload.event === MothershipStreamV1SpanLifecycleEvent.start && name === FILE_SUBAGENT_ID) { // Seed the pending preview session only on a freshly-opened lane (the agent @@ -78,7 +91,7 @@ export function handleSpanEvent( ) deps.setResources((rs) => rs.filter((r) => r.id !== 'streaming-file')) if (lastFileResource) { - deps.setActiveResourceId(lastFileResource.id) + deps.onResourceEventRef.current?.(lastFileResource.id) } } ops.flush() diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.test.ts index 70c45e3b59e..450f4d5b6b8 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.test.ts @@ -195,7 +195,7 @@ describe('tool events (dispatch → model + side effects)', () => { // user, and pop the collapsed panel open. Create/edit and the explicit // open_resource tool are the only things that should open the panel. const addResource = vi.fn(() => true) - const onResourceEventRef = ref<(() => void) | undefined>(vi.fn()) + const onResourceEventRef = ref<((resourceId: string) => void) | undefined>(vi.fn()) const ctx = createStreamLoopContext(makeStreamLoopDeps({ addResource, onResourceEventRef })) dispatchStreamEvent( diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.ts index d375705c2aa..e00e851626d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.ts @@ -92,13 +92,7 @@ function runToolResultSideEffects(ctx: StreamLoopContext, node: ToolNode): void deps.previewSessionRef.current?.fileName ?? 'File' deps.promoteFileResource(editedFileId, editedFileName) - if ( - deps.activeResourceIdRef.current === null || - deps.activeResourceIdRef.current === 'streaming-file' || - deps.activeResourceIdRef.current === editedFileId - ) { - deps.setActiveResourceId(editedFileId) - } + deps.onResourceEventRef.current?.(editedFileId) invalidateResourceQueries(deps.queryClient, deps.workspaceId, 'file', editedFileId) } } @@ -122,7 +116,7 @@ function runToolResultSideEffects(ctx: StreamLoopContext, node: ToolNode): void const fileResource = extractedResources.find((r) => r.type === 'file') if (fileResource) { deps.promoteFileResource(fileResource.id, fileResource.title) - deps.setActiveResourceId(fileResource.id) + deps.onResourceEventRef.current?.(fileResource.id) invalidateResourceQueries(deps.queryClient, deps.workspaceId, 'file', fileResource.id) } else if (calledBy !== FILE_SUBAGENT_ID) { deps.setResources((rs) => rs.filter((r) => r.id !== 'streaming-file')) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts index b7052b2ebfd..5d700ffae64 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts @@ -68,6 +68,7 @@ export interface StreamLoopState { streamRequestId: string | undefined sawStreamError: boolean sawCompleteEvent: boolean + browserAgentRunIds: Set scheduledTextFlushFrame: number | null /** Trailing timer for the min-interval text-flush gate (see flushText). */ scheduledTextFlushTimer: ReturnType | null @@ -110,6 +111,9 @@ export interface StreamLoopDeps { args: Record, ts?: string ) => void + startBrowserAgentRun: (runId: string) => void + endBrowserAgentRun: (runId: string) => void + clearBrowserAgentRuns: () => void upsertMothershipChatHistory: ( chatId: string, updater: (current: MothershipChatHistory) => MothershipChatHistory @@ -159,7 +163,7 @@ export interface StreamLoopDeps { onToolResultRef: MutableRefObject< ((toolName: string, success: boolean, result: unknown) => void) | undefined > - onResourceEventRef: MutableRefObject<(() => void) | undefined> + onResourceEventRef: MutableRefObject<((resourceId: string) => void) | undefined> previewSessionRef: MutableRefObject previewSessionsRef: MutableRefObject> latestPreviewTargetToolCallIdRef: MutableRefObject @@ -205,6 +209,7 @@ export function createStreamLoopContext(deps: StreamLoopDeps): StreamLoopContext streamRequestId: undefined, sawStreamError: false, sawCompleteEvent: false, + browserAgentRunIds: new Set(), scheduledTextFlushFrame: null, scheduledTextFlushTimer: null, } diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-test-helpers.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-test-helpers.ts index 75d69828835..f7b1755763b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-test-helpers.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-test-helpers.ts @@ -43,6 +43,9 @@ export function makeStreamLoopDeps(overrides: Partial = {}): Str startClientLocalFilesystemTool: vi.fn(), startClientBrowserTool: vi.fn(), startClientTerminalTool: vi.fn(), + startBrowserAgentRun: vi.fn(), + endBrowserAgentRun: vi.fn(), + clearBrowserAgentRuns: vi.fn(), upsertMothershipChatHistory: vi.fn(), ensureWorkflowInRegistry: vi.fn(() => false), onPreviewPhase: vi.fn(), @@ -78,7 +81,7 @@ export function makeStreamLoopDeps(overrides: Partial = {}): Str onToolResultRef: ref< ((toolName: string, success: boolean, result: unknown) => void) | undefined >(undefined), - onResourceEventRef: ref<(() => void) | undefined>(undefined), + onResourceEventRef: ref<((resourceId: string) => void) | undefined>(undefined), previewSessionRef: ref(null), previewSessionsRef: ref>({}), latestPreviewTargetToolCallIdRef: ref(null), diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index fe4d1d582a0..acbea0bd61f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -27,7 +27,11 @@ import { import { cancelWorkflowExecutionContract } from '@/lib/api/contracts/workflows' import { buildResourceAttachments } from '@/lib/browser-agent/attachments' import { onOpenInBrowserPanel } from '@/lib/browser-agent/open-in-panel' -import { initBrowserAgentTransport, sendBrowserPanelAction } from '@/lib/browser-agent/transport' +import { + cancelActiveBrowserTools, + initBrowserAgentTransport, + sendBrowserPanelAction, +} from '@/lib/browser-agent/transport' import { getMothershipAttachmentPreviewUrl } from '@/lib/copilot/chat/attachment-preview' import { toDisplayMessage } from '@/lib/copilot/chat/display-message' import { getLiveAssistantMessageId } from '@/lib/copilot/chat/effective-transcript' @@ -96,6 +100,16 @@ import { sendMothershipMessage } from '@/lib/mothership/events' import { initTerminalTransport } from '@/lib/terminal/transport' import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { useFilePreviewController } from '@/app/workspace/[workspaceId]/home/hooks/preview' +import { + captureResourceActivityScope, + clearResourceActivityScope, + clearTrackedResourceActivity, + createResourceActivityTracker, + excludeActivityOwnedBy, + type ResourceActivityTracker, + setTrackedBrowserRun, + trackTerminalToolCall, +} from '@/app/workspace/[workspaceId]/home/hooks/resource-activity' import { applyTurnTerminal, createStreamLoopContext, @@ -1175,7 +1189,7 @@ function ensureWorkflowInRegistry(resourceId: string, title: string, workspaceId } export interface UseChatOptions { - onResourceEvent?: () => void + onResourceEvent?: (resourceId: string) => void apiPath?: string stopPath?: string workflowId?: string @@ -1313,8 +1327,8 @@ export function useChat( const inFlightResourceAddsRef = useRef>>(new Map()) const reorderNeededAfterFlushRef = useRef(false) - // Derive the effective active resource ID — auto-selects the last resource when the stored ID is - // absent or no longer in the list, avoiding a separate Effect-based state correction loop. + // Derive the effective active resource ID for rendering without writing a + // passive fallback back into the user's URL selection. const effectiveActiveResourceId = useMemo(() => { if (resources.length === 0) return null if (activeResourceId && resources.some((r) => r.id === activeResourceId)) @@ -1345,6 +1359,7 @@ export function useChat( setResources, setActiveResourceId, activeResourceIdRef, + onResourceEventRef, }) const upsertChatHistory = useCallback( @@ -1457,6 +1472,7 @@ export function useChat( const activeStreamReturnRecoveryRef = useRef(null) const sendingRef = useRef(false) const streamGenRef = useRef(0) + const resourceActivityTrackerRef = useRef(null) const streamingContentRef = useRef('') const streamingBlocksRef = useRef([]) const handledClientWorkflowToolIdsRef = useRef>(new Set()) @@ -1646,11 +1662,29 @@ export function useChat( : pendingChatKeyRef.current chatIdRef.current = chatId const resolvedDesktopScopeId = desktopChatScopeId(workspaceId, chatId) + const activeActivityTracker = resourceActivityTrackerRef.current + if (activeActivityTracker?.generation === streamGenRef.current) { + if (wasPending) { + // Do not switch writes to the durable bucket until its async native + // + renderer migration has completed; pre-populating it makes the + // scoped-store migration treat the destination as conflicting. + activeActivityTracker.scopeIds.add(resolvedDesktopScopeId) + } else { + captureResourceActivityScope(activeActivityTracker, resolvedDesktopScopeId) + } + } const migrateDesktopResources = wasPending ? migrateDesktopChatScopes(pendingDesktopScopeId, resolvedDesktopScopeId) : Promise.resolve() void migrateDesktopResources .then(() => { + if ( + wasPending && + activeActivityTracker?.generation === streamGenRef.current && + resourceActivityTrackerRef.current === activeActivityTracker + ) { + captureResourceActivityScope(activeActivityTracker, resolvedDesktopScopeId) + } // Migration crosses IPC. The user can select another chat while it // is in flight, so re-read the live selection before activating; // the value captured above is only valid for the synchronous state @@ -1713,8 +1747,6 @@ export function useChat( if (exists) return prev return [...prev, resource] }) - setActiveResourceId(resource.id) - // Synthetic result/preview panels are in-memory only. The browser tab // metadata is persisted even though its live page remains desktop-owned. if (isEphemeralResource(resource)) { @@ -1842,15 +1874,12 @@ export function useChat( } const meta = getWorkflowById(workspaceId, targetWorkflowId) - const wasAdded = addResource({ + addResource({ type: 'workflow', id: targetWorkflowId, title: meta?.name ?? 'Workflow', }) - if (!wasAdded && activeResourceIdRef.current !== targetWorkflowId) { - setActiveResourceId(targetWorkflowId) - } - onResourceEventRef.current?.() + onResourceEventRef.current?.(targetWorkflowId) return targetWorkflowId }, @@ -1895,66 +1924,120 @@ export function useChat( ) const openBrowserResource = useCallback(() => { - const wasAdded = addResource({ + addResource({ type: 'browser', id: BROWSER_SESSION_RESOURCE_ID, title: 'Browser', }) - if (!wasAdded && activeResourceIdRef.current !== BROWSER_SESSION_RESOURCE_ID) { - setActiveResourceId(BROWSER_SESSION_RESOURCE_ID) - } - // Browser actions should always surface the panel, including when its - // persisted tab already exists but the viewer is collapsed. - onResourceEventRef.current?.() - }, [addResource, setActiveResourceId]) + onResourceEventRef.current?.(BROWSER_SESSION_RESOURCE_ID) + }, [addResource]) + + const getResourceActivityTracker = useCallback( + (generation: number, targetChatId?: string) => { + let tracker = resourceActivityTrackerRef.current + if (!tracker || tracker.generation !== generation) { + const isCurrentGeneration = generation === streamGenRef.current + tracker = createResourceActivityTracker( + generation, + [activeTurnRef.current?.desktopScopeId ?? desktopScopeIdRef.current], + { + captureExisting: isCurrentGeneration, + } + ) + if (isCurrentGeneration) { + resourceActivityTrackerRef.current = tracker + } + } + if (targetChatId) { + const targetScopeId = desktopChatScopeId(workspaceId, targetChatId) + if ( + tracker.generation === streamGenRef.current && + resourceActivityTrackerRef.current === tracker + ) { + captureResourceActivityScope(tracker, targetScopeId) + } else { + tracker.scopeIds.add(targetScopeId) + tracker.currentScopeId = targetScopeId + } + } + return tracker + }, + [workspaceId] + ) + + const clearResourceActivity = useCallback( + (tracker: ResourceActivityTracker, captureCurrentScope: boolean) => { + const isCurrentBoundary = + captureCurrentScope && + tracker.generation === streamGenRef.current && + resourceActivityTrackerRef.current === tracker + if (isCurrentBoundary) { + captureResourceActivityScope(tracker, desktopScopeIdRef.current) + if (chatIdRef.current) { + captureResourceActivityScope(tracker, desktopChatScopeId(workspaceId, chatIdRef.current)) + } + } + const currentTracker = resourceActivityTrackerRef.current + if (!isCurrentBoundary && currentTracker && currentTracker !== tracker) { + excludeActivityOwnedBy(tracker, currentTracker) + } + if (isCurrentBoundary) { + // The native tool may outlive an SSE reader or its AbortController. + // Fire cancellation without delaying the stream boundary below. + void cancelActiveBrowserTools(new Set(tracker.scopeIds)) + } + clearTrackedResourceActivity(tracker, { hardResetActivity: isCurrentBoundary }) + if (resourceActivityTrackerRef.current === tracker) { + resourceActivityTrackerRef.current = null + } + }, + [workspaceId] + ) const startClientBrowserTool = useCallback( - (toolCallId: string, toolName: string, toolArgs: Record, eventTs?: string) => { + ( + toolCallId: string, + toolName: string, + toolArgs: Record, + scopeId: string, + eventTs?: string, + signal?: AbortSignal + ) => { if (!isBrowserToolName(toolName)) { return } openBrowserResource() // Replay/exactly-once guarding lives in executeBrowserToolOnClient // (sessionStorage-backed, so reloads cannot re-run an action). - executeBrowserToolOnClient( - toolCallId, - toolName, - toolArgs, - chatIdRef.current ?? selectedChatIdRef.current ?? desktopScopeIdRef.current, - eventTs - ) + executeBrowserToolOnClient(toolCallId, toolName, toolArgs, scopeId, eventTs, signal) }, [openBrowserResource] ) const openTerminalResource = useCallback(() => { - const wasAdded = addResource({ + addResource({ type: 'terminal', id: TERMINAL_SESSION_RESOURCE_ID, title: 'Terminal', }) - if (!wasAdded && activeResourceIdRef.current !== TERMINAL_SESSION_RESOURCE_ID) { - setActiveResourceId(TERMINAL_SESSION_RESOURCE_ID) - } - // The panel must be visible before a command runs: it is where the user - // sees what is about to execute and approves or declines it. - onResourceEventRef.current?.() - }, [addResource, setActiveResourceId]) + onResourceEventRef.current?.(TERMINAL_SESSION_RESOURCE_ID) + }, [addResource]) const startClientTerminalTool = useCallback( - (toolCallId: string, toolName: string, toolArgs: Record, eventTs?: string) => { + ( + toolCallId: string, + toolName: string, + toolArgs: Record, + scopeId: string, + eventTs?: string + ) => { if (!isTerminalToolName(toolName)) { return } openTerminalResource() // Replay/exactly-once guarding lives in executeTerminalToolOnClient // (sessionStorage-backed, so reloads cannot re-run a command). - executeTerminalToolOnClient( - toolCallId, - toolArgs, - chatIdRef.current ?? selectedChatIdRef.current ?? desktopScopeIdRef.current, - eventTs - ) + executeTerminalToolOnClient(toolCallId, toolArgs, scopeId, eventTs) }, [openTerminalResource] ) @@ -2190,6 +2273,15 @@ export function useChat( activeStreamId !== locallyTerminalStreamIdRef.current && !isTerminalStreamStatus(chatHistory.streamSnapshot?.status) + if ( + !sendingRef.current && + (!activeStreamId || isTerminalStreamStatus(chatHistory.streamSnapshot?.status)) + ) { + const hydratedScopeId = desktopChatScopeId(workspaceId, chatHistory.id) + clearResourceActivityScope(hydratedScopeId) + void cancelActiveBrowserTools([hydratedScopeId]) + } + if (!activeStreamId && locallyTerminalStreamIdRef.current) { locallyTerminalStreamIdRef.current = undefined } @@ -2370,6 +2462,41 @@ export function useChat( shouldContinue?: () => boolean } ) => { + const streamAbortSignal = abortControllerRef.current?.signal + const activityTracker = getResourceActivityTracker( + expectedGen ?? streamGenRef.current, + options?.targetChatId + ) + const activityScopeId = () => activityTracker.currentScopeId + const startBrowserAgentRunForStream = (runId: string) => { + openBrowserResource() + const scopeId = activityScopeId() + setTrackedBrowserRun(activityTracker, scopeId, runId, true) + } + const endBrowserAgentRunForStream = (runId: string) => { + const scopeId = activityScopeId() + setTrackedBrowserRun(activityTracker, scopeId, runId, false) + } + const startClientBrowserToolForStream = ( + toolCallId: string, + toolName: string, + toolArgs: Record, + eventTs?: string + ) => { + const scopeId = activityScopeId() + startClientBrowserTool(toolCallId, toolName, toolArgs, scopeId, eventTs, streamAbortSignal) + } + const startClientTerminalToolForStream = ( + toolCallId: string, + toolName: string, + toolArgs: Record, + eventTs?: string + ) => { + const scopeId = activityScopeId() + trackTerminalToolCall(activityTracker, scopeId, toolCallId) + startClientTerminalTool(toolCallId, toolName, toolArgs, scopeId, eventTs) + } + const clearStreamResourceActivity = () => clearResourceActivity(activityTracker, true) const ctx = createStreamLoopContext({ workspaceId, queryClient, @@ -2386,8 +2513,11 @@ export function useChat( removeResource, startClientWorkflowTool, startClientLocalFilesystemTool, - startClientBrowserTool, - startClientTerminalTool, + startClientBrowserTool: startClientBrowserToolForStream, + startClientTerminalTool: startClientTerminalToolForStream, + startBrowserAgentRun: startBrowserAgentRunForStream, + endBrowserAgentRun: endBrowserAgentRunForStream, + clearBrowserAgentRuns: clearStreamResourceActivity, upsertMothershipChatHistory: upsertChatHistory, ensureWorkflowInRegistry, onPreviewPhase, @@ -2468,6 +2598,13 @@ export function useChat( }, }) } finally { + // A transport read failure may reconnect this same generation. Keep + // its exact resource activity alive until a terminal stream event or + // finalize/Stop establishes the real boundary. + if (state.sawStreamError) { + clearStreamResourceActivity() + state.browserAgentRunIds.clear() + } if (state.sawStreamError && !state.sawCompleteEvent) { applyTurnTerminal(state.model, 'error') ops.flush() @@ -2503,6 +2640,9 @@ export function useChat( startClientLocalFilesystemTool, startClientBrowserTool, startClientTerminalTool, + getResourceActivityTracker, + clearResourceActivity, + openBrowserResource, adoptResolvedChatId, upsertChatHistory, onPreviewPhase, @@ -3468,6 +3608,10 @@ export function useChat( }) } reconcileTerminalPreviewSessions() + const completedActivityTracker = resourceActivityTrackerRef.current + if (completedActivityTracker?.generation === streamGenRef.current) { + clearResourceActivity(completedActivityTracker, true) + } locallyTerminalStreamIdRef.current = streamIdRef.current ?? activeTurnRef.current?.userMessageId ?? undefined clearActiveTurn() @@ -3480,6 +3624,7 @@ export function useChat( notifyTurnEnded({ error: isError }) }, [ + clearResourceActivity, clearActiveTurn, invalidateChatQueries, notifyTurnEnded, @@ -4350,6 +4495,22 @@ export function useChat( const stopTraceparentSnapshot = streamTraceparentRef.current ?? initialStopTraceparentSnapshot locallyTerminalStreamIdRef.current = sid + const stopActivityTracker = + resourceActivityTrackerRef.current?.generation === streamGenRef.current + ? resourceActivityTrackerRef.current + : getResourceActivityTracker(streamGenRef.current, activeChatId) + captureResourceActivityScope(stopActivityTracker, desktopScopeIdRef.current) + if (chatIdRef.current) { + captureResourceActivityScope( + stopActivityTracker, + desktopChatScopeId(workspaceId, chatIdRef.current) + ) + } + clearResourceActivity(stopActivityTracker, true) + + // Establish the stream boundary immediately after synchronous activity + // settlement. Native cancellation above is deliberately fire-and-forget, + // so a slow shell cannot delay the server-side abort below. streamGenRef.current++ clearActiveTurn() streamReaderRef.current?.cancel().catch(() => {}) @@ -4545,6 +4706,7 @@ export function useChat( }, [ cancelActiveWorkflowExecutions, + cancelActiveBrowserTools, invalidateChatQueries, notifyTurnEnded, persistPartialResponse, @@ -4553,7 +4715,9 @@ export function useChat( resetEphemeralPreviewState, upsertChatHistory, adoptResolvedChatId, + clearResourceActivity, clearActiveTurn, + getResourceActivityTracker, setTransportIdle, workspaceId, ] diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/context-menu/context-menu.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/context-menu/context-menu.tsx index 98bb5442769..f381806cea1 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/context-menu/context-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/context-menu/context-menu.tsx @@ -33,6 +33,10 @@ interface ContextMenuProps { menuRef: React.RefObject onClose: () => void onOpenInNewTab?: () => void + openInNewTabLabel?: string + openInNewTabPosition?: 'first' | 'last' + separateNavigationAction?: boolean + groupNonDestructiveActions?: boolean onMarkAsRead?: () => void onMarkAsUnread?: () => void onTogglePin?: () => void @@ -60,6 +64,8 @@ interface ContextMenuProps { * it cannot be confused with `onClose`, which dismisses this menu. */ onCloseTab?: () => void + onCloseOtherTabs?: () => void + onCloseTabsToRight?: () => void showOpenInNewTab?: boolean showMarkAsRead?: boolean showMarkAsUnread?: boolean @@ -87,6 +93,8 @@ interface ContextMenuProps { isLocked?: boolean showDelete?: boolean showCloseTab?: boolean + disableCloseOtherTabs?: boolean + disableCloseTabsToRight?: boolean onUploadLogo?: () => void showUploadLogo?: boolean disableUploadLogo?: boolean @@ -102,6 +110,10 @@ export function ContextMenu({ menuRef, onClose, onOpenInNewTab, + openInNewTabLabel = 'Open in new tab', + openInNewTabPosition = 'first', + separateNavigationAction = false, + groupNonDestructiveActions = false, onMarkAsRead, onMarkAsUnread, onTogglePin, @@ -113,6 +125,8 @@ export function ContextMenu({ onExport, onDelete, onCloseTab, + onCloseOtherTabs, + onCloseTabsToRight, showOpenInNewTab = false, showMarkAsRead = false, showMarkAsUnread = false, @@ -140,6 +154,8 @@ export function ContextMenu({ isLocked = false, showDelete = true, showCloseTab = false, + disableCloseOtherTabs = false, + disableCloseTabsToRight = false, onUploadLogo, showUploadLogo = false, disableUploadLogo = false, @@ -195,7 +211,7 @@ export function ContextMenu({ } }} > - {showOpenInNewTab && onOpenInNewTab && ( + {openInNewTabPosition === 'first' && showOpenInNewTab && onOpenInNewTab && ( { onOpenInNewTab() @@ -203,12 +219,13 @@ export function ContextMenu({ }} > - Open in new tab + {openInNewTabLabel} )} - {hasNavigationSection && (hasStatusSection || hasEditSection || hasCopySection) && ( - - )} + {openInNewTabPosition === 'first' && + (!groupNonDestructiveActions || separateNavigationAction) && + hasNavigationSection && + (hasStatusSection || hasEditSection || hasCopySection) && } {showMarkAsRead && onMarkAsRead && ( )} - {hasStatusSection && (hasEditSection || hasCopySection) && } + {!groupNonDestructiveActions && hasStatusSection && (hasEditSection || hasCopySection) && ( + + )} {showRename && onRename && ( )} - {hasEditSection && hasCopySection && } + {!groupNonDestructiveActions && hasEditSection && hasCopySection && ( + + )} {showDuplicate && onDuplicate && ( )} + {openInNewTabPosition === 'last' && + (!groupNonDestructiveActions || separateNavigationAction) && + hasNavigationSection && + (hasStatusSection || hasEditSection || hasCopySection) && } + {openInNewTabPosition === 'last' && showOpenInNewTab && onOpenInNewTab && ( + { + onOpenInNewTab() + onClose() + }} + > + + {openInNewTabLabel} + + )} {(hasNavigationSection || hasStatusSection || hasEditSection || hasCopySection) && - (showLeave || showDelete || (showCloseTab && onCloseTab)) && } + (showLeave || + showDelete || + (showCloseTab && onCloseTab) || + onCloseOtherTabs || + onCloseTabsToRight) && } {showLeave && onLeave && ( )} + {onCloseOtherTabs && ( + { + onCloseOtherTabs() + onClose() + }} + > + + Close Others + + )} + {onCloseTabsToRight && ( + { + onCloseTabsToRight() + onClose() + }} + > + + Close Tabs to the Right + + )} ) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx index b5aa4e82855..7cc21bf9233 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx @@ -37,6 +37,7 @@ import Link from 'next/link' import { useParams, usePathname, useRouter } from 'next/navigation' import { usePostHog } from 'posthog-js/react' import { useSession } from '@/lib/auth/auth-client' +import { focusVisibleBrowserOmnibox } from '@/lib/browser-agent/renderer-shortcuts' import { SIM_RESOURCES_DRAG_TYPE } from '@/lib/copilot/resource-types' import { isChatEnabled } from '@/lib/core/config/env-flags' import { isMacPlatform } from '@/lib/core/utils/platform' @@ -1355,6 +1356,7 @@ export const Sidebar = memo(function Sidebar({ { id: 'goto-logs', handler: () => { + if (focusVisibleBrowserOmnibox()) return try { const pathWorkspaceId = resolveWorkspaceIdFromPath() if (pathWorkspaceId) { diff --git a/apps/sim/components/ui/thinking-loader.module.css b/apps/sim/components/ui/thinking-loader.module.css index fb8c72e4c0e..52703157b5e 100644 --- a/apps/sim/components/ui/thinking-loader.module.css +++ b/apps/sim/components/ui/thinking-loader.module.css @@ -19,16 +19,16 @@ /* currentColor paints the squeeze ring's stroked arcs (the only non-filled geometry). It tracks the loader polarity — dark on light, light on dark — matching the gradient stops below. */ - color: #2c2c2c; + color: var(--thinking-ink-current); flex: none; /* Loader fill is a radial gradient (center → edge) plus a soft white inner glow, per the Figma loader spec. Stops and glow opacity are theme vars, inherited by the gradient stops and feFlood inside the SVG, so one set of defs serves both themes. Light mode = the DARK-grey loader (reads on white): #2C2C2C → #5F5F5F, glow white 0.6. */ - --tl-grad-inner: #2c2c2c; - --tl-grad-outer: #5f5f5f; - --tl-glow: rgba(255, 255, 255, 0.6); + --tl-grad-inner: var(--thinking-ink-inner); + --tl-grad-outer: var(--thinking-ink-outer); + --tl-glow: var(--thinking-ink-glow); } /* Gradient stops + glow flood read the theme vars via CSS (a raw `var()` in an @@ -58,26 +58,6 @@ animation-delay: var(--tl-sync, 0s); } -:global(.dark) .frame { - color: #d6d6d6; - /* Dark mode = the LIGHT-grey loader (reads on dark): #A7A7A7 → #D6D6D6, - glow white 0.9. */ - --tl-grad-inner: #a7a7a7; - --tl-grad-outer: #d6d6d6; - --tl-glow: rgba(255, 255, 255, 0.9); -} - -/* An explicit `.light` wrapper (e.g. the always-light landing) re-asserts the - dark-grey loader even when an outer `.dark` theme is on the document — the - nearer theme wins. Ordered after the `.dark` rule so on equal specificity - (both ancestors present) this takes precedence. */ -:global(.light) .frame { - color: #2c2c2c; - --tl-grad-inner: #2c2c2c; - --tl-grad-outer: #5f5f5f; - --tl-glow: rgba(255, 255, 255, 0.6); -} - .frame.inheritInk { color: inherit; --tl-grad-inner: currentColor; diff --git a/apps/sim/hooks/use-speech-to-text.ts b/apps/sim/hooks/use-speech-to-text.ts index 6a1bcb2f726..bfef16a6bd8 100644 --- a/apps/sim/hooks/use-speech-to-text.ts +++ b/apps/sim/hooks/use-speech-to-text.ts @@ -16,6 +16,29 @@ import { useVoiceSettings } from '@/hooks/queries/voice' const logger = createLogger('useSpeechToText') +/** + * Why a session could not start. `microphone-blocked` is the recoverable one — + * the user has to grant access outside the app (OS privacy settings on the + * desktop shell, the site permission prompt in a browser) before retrying. + */ +export type SpeechToTextError = 'microphone-blocked' | 'microphone-unavailable' | 'start-failed' + +/** + * Maps a `getUserMedia` rejection onto {@link SpeechToTextError}. Anything that + * is not a recognized capture failure — a token fetch, the WebSocket handshake — + * falls through to the generic case. + */ +function classifyStartError(error: unknown): SpeechToTextError { + const name = (error as { name?: string } | null)?.name + if (name === 'NotAllowedError' || name === 'SecurityError' || name === 'PermissionDeniedError') { + return 'microphone-blocked' + } + if (name === 'NotFoundError' || name === 'DevicesNotFoundError') { + return 'microphone-unavailable' + } + return 'start-failed' +} + interface UseSpeechToTextProps { onTranscript: (text: string) => void /** @@ -23,6 +46,8 @@ interface UseSpeechToTextProps { * whether it was a per-member cap (which only an org admin can raise). */ onUsageLimitExceeded?: (message?: string, isMemberLimit?: boolean) => void + /** Called when a session fails to start, so the click is never a silent no-op. */ + onError?: (error: SpeechToTextError) => void /** Attributes the voice-input cost to this workspace for per-member usage. */ workspaceId?: string } @@ -37,6 +62,7 @@ interface UseSpeechToTextReturn { export function useSpeechToText({ onTranscript, onUsageLimitExceeded, + onError, workspaceId, }: UseSpeechToTextProps): UseSpeechToTextReturn { const [isListening, setIsListening] = useState(false) @@ -55,6 +81,7 @@ export function useSpeechToText({ const onTranscriptRef = useRef(onTranscript) const onUsageLimitExceededRef = useRef(onUsageLimitExceeded) + const onErrorRef = useRef(onError) const workspaceIdRef = useRef(workspaceId) const mountedRef = useRef(true) const startingRef = useRef(false) @@ -73,6 +100,7 @@ export function useSpeechToText({ onTranscriptRef.current = onTranscript onUsageLimitExceededRef.current = onUsageLimitExceeded + onErrorRef.current = onError workspaceIdRef.current = workspaceId const flushAudioBuffer = useCallback(() => { @@ -283,6 +311,9 @@ export function useSpeechToText({ } catch (error) { logger.error('Failed to start speech streaming', error) cleanup() + if (mountedRef.current) { + onErrorRef.current?.(classifyStartError(error)) + } return false } finally { startingRef.current = false diff --git a/apps/sim/hooks/use-stable-flag.test.ts b/apps/sim/hooks/use-stable-flag.test.ts index f4d90503e0f..e88ff6f9fae 100644 --- a/apps/sim/hooks/use-stable-flag.test.ts +++ b/apps/sim/hooks/use-stable-flag.test.ts @@ -149,11 +149,10 @@ describe('createStableFlagController', () => { expect(probe.states).toEqual([true]) // hide never fired }) - it('with zero options, mirrors the value on the next tick', () => { + it('with zero options, mirrors the value immediately', () => { const probe = setup({ delayMs: 0, minVisibleMs: 0 }) probe.controller.setValue(true) - vi.advanceTimersByTime(0) expect(probe.active).toBe(true) probe.controller.setValue(false) diff --git a/apps/sim/hooks/use-stable-flag.ts b/apps/sim/hooks/use-stable-flag.ts index 2fcd704f8b1..700a9a17ab1 100644 --- a/apps/sim/hooks/use-stable-flag.ts +++ b/apps/sim/hooks/use-stable-flag.ts @@ -64,6 +64,10 @@ export function createStableFlagController( if (active || showTimer !== null) { return } + if (delayMs <= 0) { + show() + return + } showTimer = setTimeout(show, delayMs) return } @@ -95,7 +99,7 @@ export function createStableFlagController( * - Rising edge — `value` must hold true for `delayMs` before the flag turns on. * - Falling edge — once on, the flag stays on for at least `minVisibleMs`. * - * With both options at `0` it returns `value` unchanged (after a tick). Useful for + * With both options at `0` it returns `value` unchanged. Useful for * connection/loading indicators that would otherwise flicker on sub-second changes. */ export function useStableFlag(value: boolean, options: StableFlagOptions = {}): boolean { diff --git a/apps/sim/lib/browser-agent/attachments.test.ts b/apps/sim/lib/browser-agent/attachments.test.ts index a5b834a036b..4d23aaa27a7 100644 --- a/apps/sim/lib/browser-agent/attachments.test.ts +++ b/apps/sim/lib/browser-agent/attachments.test.ts @@ -15,6 +15,10 @@ describe('buildResourceAttachments', () => { pageState: null, tabs: [], activeTabId: null, + automationTabId: null, + automationActive: false, + automationNeedsAttention: false, + agentRunIds: [], sessionAlive: true, suspended: false, } diff --git a/apps/sim/lib/browser-agent/renderer-shortcuts.test.ts b/apps/sim/lib/browser-agent/renderer-shortcuts.test.ts new file mode 100644 index 00000000000..b50a230e37a --- /dev/null +++ b/apps/sim/lib/browser-agent/renderer-shortcuts.test.ts @@ -0,0 +1,35 @@ +/** + * @vitest-environment jsdom + */ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + focusVisibleBrowserOmnibox, + onFocusVisibleBrowserOmnibox, +} from '@/lib/browser-agent/renderer-shortcuts' + +const cleanups: Array<() => void> = [] + +afterEach(() => { + for (const cleanup of cleanups.splice(0)) cleanup() +}) + +describe('visible browser omnibox shortcut', () => { + it('is unclaimed when no browser panel is visible', () => { + expect(focusVisibleBrowserOmnibox()).toBe(false) + }) + + it('is claimed synchronously by the visible browser panel', () => { + const focus = vi.fn() + cleanups.push(onFocusVisibleBrowserOmnibox(focus)) + + expect(focusVisibleBrowserOmnibox()).toBe(true) + expect(focus).toHaveBeenCalledOnce() + }) + + it('stops claiming the shortcut when the panel unregisters', () => { + const cleanup = onFocusVisibleBrowserOmnibox(vi.fn()) + cleanup() + + expect(focusVisibleBrowserOmnibox()).toBe(false) + }) +}) diff --git a/apps/sim/lib/browser-agent/renderer-shortcuts.ts b/apps/sim/lib/browser-agent/renderer-shortcuts.ts new file mode 100644 index 00000000000..9c02897350d --- /dev/null +++ b/apps/sim/lib/browser-agent/renderer-shortcuts.ts @@ -0,0 +1,23 @@ +const FOCUS_VISIBLE_BROWSER_OMNIBOX_EVENT = 'sim:focus-visible-browser-omnibox' + +/** + * Gives the visible browser panel first refusal on Sim renderer shortcuts. + * Returns true only when a live panel claimed and handled the request. + */ +export function focusVisibleBrowserOmnibox(): boolean { + if (typeof window === 'undefined') return false + const event = new Event(FOCUS_VISIBLE_BROWSER_OMNIBOX_EVENT, { cancelable: true }) + window.dispatchEvent(event) + return event.defaultPrevented +} + +/** Registers the currently visible browser panel as the renderer-side Cmd+L owner. */ +export function onFocusVisibleBrowserOmnibox(callback: () => void): () => void { + if (typeof window === 'undefined') return () => {} + const listener = (event: Event) => { + event.preventDefault() + callback() + } + window.addEventListener(FOCUS_VISIBLE_BROWSER_OMNIBOX_EVENT, listener) + return () => window.removeEventListener(FOCUS_VISIBLE_BROWSER_OMNIBOX_EVENT, listener) +} diff --git a/apps/sim/lib/browser-agent/transport.test.ts b/apps/sim/lib/browser-agent/transport.test.ts index f5031306353..75701bedfa7 100644 --- a/apps/sim/lib/browser-agent/transport.test.ts +++ b/apps/sim/lib/browser-agent/transport.test.ts @@ -3,6 +3,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { activateScope, capturePanelSnapshot, + cancelActiveTool, + cancelTool, discardScope, disposeScope, fillCredential, @@ -10,6 +12,7 @@ const { markScopeSuspended, migrateStoreScope, nativeMigrateScope, + executeTool, onPageState, onSessionStatus, onTabsState, @@ -21,7 +24,10 @@ const { onOpenFind, onScopeSuspended, onToolbarCommand, + openTab, + panelAction, reorderTab, + reorderStoreTab, restoreScope, nativeSuspendScope, setPageState, @@ -38,6 +44,8 @@ const { } = vi.hoisted(() => ({ activateScope: vi.fn(async (scopeId: string) => ({ scopeId, tabs: [], activeTabId: null })), capturePanelSnapshot: vi.fn(), + cancelActiveTool: vi.fn(), + cancelTool: vi.fn(), discardScope: vi.fn(), disposeScope: vi.fn(async () => true), fillCredential: vi.fn(async () => true), @@ -45,6 +53,7 @@ const { markScopeSuspended: vi.fn(), migrateStoreScope: vi.fn(), nativeMigrateScope: vi.fn(), + executeTool: vi.fn(), onPageState: vi.fn(), onSessionStatus: vi.fn(), onTabsState: vi.fn(), @@ -56,7 +65,10 @@ const { onOpenFind: vi.fn(), onScopeSuspended: vi.fn(), onToolbarCommand: vi.fn(), + openTab: vi.fn(), + panelAction: vi.fn(), reorderTab: vi.fn(), + reorderStoreTab: vi.fn(), restoreScope: vi.fn(), nativeSuspendScope: vi.fn(async () => true), setPageState: vi.fn(), @@ -78,7 +90,9 @@ vi.mock('@/lib/desktop', () => ({ browserAgent: { supportsAtomicPanelOcclusion: true, activateScope, - executeTool: vi.fn(), + cancelActiveTool, + cancelTool, + executeTool, capturePanelSnapshot, disposeScope, migrateScope: nativeMigrateScope, @@ -92,7 +106,8 @@ vi.mock('@/lib/desktop', () => ({ onToolbarCommand, onSessionStatus, onTabsState, - panelAction: vi.fn(), + openTab, + panelAction, reorderTab, restoreScope, suspendScope: nativeSuspendScope, @@ -120,6 +135,7 @@ vi.mock('@/stores/browser-session/store', () => ({ activateScope, discardScope, migrateScope: migrateStoreScope, + reorderTab: reorderStoreTab, suspendScope: markScopeSuspended, setPageState, setSessionAlive, @@ -130,8 +146,10 @@ vi.mock('@/stores/browser-session/store', () => ({ import { activateBrowserScope, + cancelActiveBrowserTools, captureBrowserPanelSnapshot, discardBrowserScope, + executeBrowserTool, fillBrowserCredential, initBrowserAgentTransport, loadBrowserFillOptions, @@ -143,6 +161,7 @@ import { onBrowserFindResult, onBrowserOmniboxFocus, onBrowserToolbarCommand, + openBrowserTab, reorderBrowserTab, reportBrowserPanelBounds, reportBrowserPanelFocused, @@ -168,12 +187,20 @@ describe('browser panel transport', () => { setSessionAlive.mockClear() setTabsState.mockClear() reorderTab.mockClear() + reorderStoreTab.mockClear() restoreScope.mockReset() nativeSuspendScope.mockReset() nativeSuspendScope.mockResolvedValue(true) markScopeSuspended.mockClear() migrateStoreScope.mockClear() nativeMigrateScope.mockReset() + cancelActiveTool.mockReset() + cancelActiveTool.mockResolvedValue(true) + cancelTool.mockReset() + cancelTool.mockResolvedValue(true) + executeTool.mockReset() + panelAction.mockClear() + openTab.mockReset() setTabPinned.mockClear() showTabContextMenu.mockClear() showToolbarMenu.mockClear() @@ -185,6 +212,37 @@ describe('browser panel transport', () => { disposeScope.mockClear() }) + it('opens a browser tab through the acknowledged bridge and applies its state', async () => { + const state = { + scopeId: 'chat-test', + activeTabId: '2', + tabs: [ + { + tabId: '1', + title: 'Existing', + url: 'https://example.com', + loading: false, + active: false, + pinned: false, + }, + { + tabId: '2', + title: '', + url: '', + loading: false, + active: true, + pinned: false, + }, + ], + } + openTab.mockResolvedValue(state) + + await expect(openBrowserTab('chat-test')).resolves.toEqual(state) + + expect(openTab).toHaveBeenCalledWith('chat-test') + expect(setTabsState).toHaveBeenCalledWith(state) + }) + it('forwards panel bounds directly to the native view', () => { const initialBounds = { x: 10, y: 20, width: 300, height: 200 } const updatedBounds = { x: 20, y: 30, width: 320, height: 220 } @@ -333,6 +391,7 @@ describe('browser panel transport', () => { it('forwards tab reordering to the native browser', () => { reorderBrowserTab('tab-3', 1) + expect(reorderStoreTab).toHaveBeenCalledWith('chat-test', 'tab-3', 1) expect(reorderTab).toHaveBeenCalledWith('tab-3', 1, 'chat-test') }) @@ -357,6 +416,141 @@ describe('browser panel transport', () => { expect(disposeScope).not.toHaveBeenCalled() }) + it('cancels a detached native tool by its captured scope without an AbortController', async () => { + let settleNative: (response: { ok: boolean; error?: string }) => void = () => {} + executeTool.mockImplementation( + () => + new Promise((resolve) => { + settleNative = resolve + }) + ) + const onCancel = vi.fn() + const execution = executeBrowserTool( + 'tool-detached', + 'browser_request_takeover', + { reason: 'Please sign in' }, + null, + 'chat-detached', + onCancel + ) + await Promise.resolve() + + await cancelActiveBrowserTools(['chat-detached']) + + expect(onCancel).toHaveBeenCalledOnce() + expect(cancelTool).toHaveBeenCalledWith('tool-detached', 'chat-detached') + expect(cancelActiveTool).toHaveBeenCalledWith('chat-detached') + settleNative({ ok: false, error: 'cancelled' }) + await expect(execution).rejects.toThrow('cancelled') + }) + + it('starts the native scope boundary without waiting for exact cancellation', async () => { + let settleNative: (response: { ok: boolean; error?: string }) => void = () => {} + let settleExactCancellation: (cancelled: boolean) => void = () => {} + executeTool.mockImplementation( + () => + new Promise((resolve) => { + settleNative = resolve + }) + ) + cancelTool.mockImplementation( + () => + new Promise((resolve) => { + settleExactCancellation = resolve + }) + ) + const execution = executeBrowserTool( + 'tool-boundary', + 'browser_request_takeover', + { reason: 'Please sign in' }, + null, + 'chat-boundary' + ) + await Promise.resolve() + + const stopping = cancelActiveBrowserTools(['chat-boundary']) + await Promise.resolve() + + expect(cancelTool).toHaveBeenCalledWith('tool-boundary', 'chat-boundary') + expect(cancelActiveTool).toHaveBeenCalledWith('chat-boundary') + + settleExactCancellation(true) + await stopping + settleNative({ ok: false, error: 'cancelled' }) + await expect(execution).rejects.toThrow('cancelled') + }) + + it('moves active tool ownership when a pending browser scope migrates', async () => { + let settleNative: (response: { ok: boolean; error?: string }) => void = () => {} + executeTool.mockImplementation( + () => + new Promise((resolve) => { + settleNative = resolve + }) + ) + nativeMigrateScope.mockResolvedValue({ + scopeId: 'chat-real', + tabs: [], + activeTabId: null, + }) + const execution = executeBrowserTool( + 'tool-migrated', + 'browser_request_takeover', + { reason: 'Please sign in' }, + null, + 'pending:new' + ) + await Promise.resolve() + + await migrateBrowserScope('pending:new', 'chat-real') + await cancelActiveBrowserTools(['chat-real']) + + expect(cancelTool).toHaveBeenCalledWith('tool-migrated', 'chat-real') + settleNative({ ok: false, error: 'cancelled' }) + await expect(execution).rejects.toThrow('cancelled') + }) + + it('uses takeover hand-back when the installed shell cannot cancel exact tools', async () => { + let settleNative: (response: { ok: boolean; result?: unknown }) => void = () => {} + executeTool.mockImplementation( + () => + new Promise((resolve) => { + settleNative = resolve + }) + ) + cancelTool.mockResolvedValue(false) + const execution = executeBrowserTool( + 'tool-old-shell', + 'browser_request_takeover', + { reason: 'Please sign in' }, + null, + 'chat-old-shell' + ) + await Promise.resolve() + + await cancelActiveBrowserTools(['chat-old-shell']) + + expect(panelAction).toHaveBeenCalledWith({ action: 'takeover-done' }, 'chat-old-shell') + settleNative({ ok: true, result: { completed: true } }) + await expect(execution).resolves.toEqual({ completed: true }) + }) + + it('cancels the active native scope when renderer tool ownership was lost on reload', async () => { + await cancelActiveBrowserTools(['chat-reloaded']) + + expect(cancelTool).not.toHaveBeenCalled() + expect(cancelActiveTool).toHaveBeenCalledWith('chat-reloaded') + expect(panelAction).not.toHaveBeenCalled() + }) + + it('hands back a reloaded takeover when the installed shell lacks scope cancellation', async () => { + cancelActiveTool.mockResolvedValue(false) + + await cancelActiveBrowserTools(['chat-old-reloaded']) + + expect(panelAction).toHaveBeenCalledWith({ action: 'takeover-done' }, 'chat-old-reloaded') + }) + it('discards a provisional browser scope when the durable destination wins', async () => { nativeMigrateScope.mockResolvedValue({ tabs: [], activeTabId: null }) diff --git a/apps/sim/lib/browser-agent/transport.ts b/apps/sim/lib/browser-agent/transport.ts index cb751b22515..51815f9755f 100644 --- a/apps/sim/lib/browser-agent/transport.ts +++ b/apps/sim/lib/browser-agent/transport.ts @@ -44,6 +44,16 @@ let activeScopeId: string | null = null /** Last VISIBLE rect per scope; a hidden/unmounted panel has no entry. */ const latestPanelBoundsByScope = new Map() +interface ActiveBrowserTool { + toolCallId: string + tool: BrowserToolName + scopeId: string + onCancel?: () => void +} + +/** Native browser work outlives any one SSE reader or reconnect AbortController. */ +const activeBrowserTools = new Map() + function bridge(): SimDesktopBrowserAgentApi | null { return getDesktopBridge()?.browserAgent ?? null } @@ -116,6 +126,9 @@ export async function migrateBrowserScope(fromScopeId: string, toScopeId: string } useBrowserSessionStore.getState().migrateScope(fromScopeId, toScopeId) + for (const activeTool of activeBrowserTools.values()) { + if (activeTool.scopeId === fromScopeId) activeTool.scopeId = toScopeId + } if (activeScopeId === fromScopeId) activeScopeId = toScopeId const movedBounds = latestPanelBoundsByScope.get(fromScopeId) latestPanelBoundsByScope.delete(fromScopeId) @@ -164,29 +177,103 @@ export async function executeBrowserTool( tool: BrowserToolName, params: Record, timeoutMs: number | null, - scopeId = currentBrowserScopeId() + scopeId = currentBrowserScopeId(), + onCancel?: () => void ): Promise { const agent = bridge() if (!agent) { throw new Error('The Sim desktop browser agent is unavailable.') } - const invocation = agent.executeTool(toolCallId, tool, params, scopeId) - const response = - timeoutMs === null - ? await invocation - : await Promise.race([ - invocation, - new Promise((_, reject) => { - setTimeout( - () => reject(new Error(`The browser did not respond within ${timeoutMs}ms`)), - timeoutMs - ) - }), - ]) - if (!response.ok) { - throw new Error(response.error || 'The browser agent reported an error') + const activeTool = { toolCallId, tool, scopeId, onCancel } + activeBrowserTools.set(toolCallId, activeTool) + try { + const invocation = agent.executeTool(toolCallId, tool, params, scopeId) + const response = + timeoutMs === null + ? await invocation + : await Promise.race([ + invocation, + new Promise((_, reject) => { + setTimeout( + () => reject(new Error(`The browser did not respond within ${timeoutMs}ms`)), + timeoutMs + ) + }), + ]) + if (!response.ok) { + throw new Error(response.error || 'The browser agent reported an error') + } + return response.result + } finally { + if (activeBrowserTools.get(toolCallId) === activeTool) { + activeBrowserTools.delete(toolCallId) + } } - return response.result +} + +async function cancelRegisteredBrowserTool(activeTool: ActiveBrowserTool): Promise { + activeTool.onCancel?.() + const agent = bridge() + if (!agent) return false + + try { + if ((await agent.cancelTool?.(activeTool.toolCallId, activeTool.scopeId)) === true) return true + } catch { + // Older or transitioning shells fall through to the takeover hand-back. + } + + if (activeTool.tool !== 'browser_request_takeover') return false + try { + agent.panelAction({ action: 'takeover-done' }, activeTool.scopeId) + return true + } catch { + return false + } +} + +/** Requests cancellation of one exact native browser tool. */ +export async function cancelBrowserTool( + toolCallId: string, + scopeId: string, + tool: BrowserToolName +): Promise { + return await cancelRegisteredBrowserTool( + activeBrowserTools.get(toolCallId) ?? { toolCallId, scopeId, tool } + ) +} + +/** Cancels native browser work owned by the stopped stream's captured scopes. */ +export async function cancelActiveBrowserTools(scopeIds: Iterable): Promise { + const scopes = new Set(scopeIds) + const activeTools = [...activeBrowserTools.values()].filter((activeTool) => + scopes.has(activeTool.scopeId) + ) + const exactCancellations = activeTools.map(cancelRegisteredBrowserTool) + + // A renderer reload can lose an older native tool, then register newer work + // in the same scope after reconnect. Establish the scope boundary even when + // every currently known renderer tool was cancelled exactly, so that older + // native work and queued pre-boundary calls cannot survive Stop. + const agent = bridge() + const scopeCancellations = agent + ? [...scopes].map(async (scopeId) => { + try { + if ((await agent.cancelActiveTool?.(scopeId)) === true) return + } catch { + // Older or transitioning shells fall through to takeover hand-back. + } + try { + agent.panelAction({ action: 'takeover-done' }, scopeId) + } catch { + // Best-effort recovery for a renderer-owned registry that no longer exists. + } + }) + : [] + + // Both IPC paths are started before yielding. A new stream can begin while + // cancellation settles, but its tools must land after the native scope + // boundary rather than being swept up by the previous stream's Stop. + await Promise.all([...exactCancellations, ...scopeCancellations]) } /** Browser-chrome commands from the panel header; fire-and-forget. */ @@ -198,6 +285,28 @@ export function sendBrowserPanelAction( bridge()?.panelAction({ action, ...payload }, scopeId) } +/** + * Creates a tab through an acknowledged IPC path when supported. Older shells + * retain the fire-and-forget fallback, but callers must not assume completion + * until a tab-state push arrives in that case. + */ +export async function openBrowserTab( + scopeId = currentBrowserScopeId() +): Promise { + const agent = bridge() + if (!agent) throw new Error('The Sim desktop browser agent is unavailable.') + if (!agent.openTab) { + agent.panelAction({ action: 'new-tab' }, scopeId) + return null + } + const state = await agent.openTab(scopeId) + if (state.scopeId !== scopeId || !state.activeTabId) { + throw new Error('The desktop browser did not confirm the new tab.') + } + useBrowserSessionStore.getState().setTabsState(state) + return state +} + /** Pins or unpins a live browser tab. */ export function setBrowserTabPinned( tabId: string, @@ -218,7 +327,10 @@ export function reorderBrowserTab( targetIndex: number, scopeId = currentBrowserScopeId() ): void { - bridge()?.reorderTab(tabId, targetIndex, scopeId) + const agent = bridge() + if (!agent) return + useBrowserSessionStore.getState().reorderTab(scopeId, tabId, targetIndex) + agent.reorderTab(tabId, targetIndex, scopeId) } /** Mirrors Sim's raw light/dark/system preference into embedded pages. */ diff --git a/apps/sim/lib/copilot/chat/persisted-message.test.ts b/apps/sim/lib/copilot/chat/persisted-message.test.ts index d600f6ca5b1..304a9dcfce7 100644 --- a/apps/sim/lib/copilot/chat/persisted-message.test.ts +++ b/apps/sim/lib/copilot/chat/persisted-message.test.ts @@ -393,6 +393,41 @@ describe('stripToolResultOutput', () => { }) }) + it('keeps only the answered browser takeover instruction for its question recap', () => { + const message: PersistedMessage = { + id: 'msg-takeover', + role: 'assistant', + content: '', + timestamp: '2026-01-01T00:00:00.000Z', + contentBlocks: [ + { + type: 'tool', + phase: 'call', + toolCall: { + id: 'takeover-1', + name: 'browser_request_takeover', + state: 'success', + result: { + success: true, + output: { + completed: true, + elapsedMs: 5_000, + userInstruction: ' Open the second match ', + }, + }, + }, + }, + ], + } + + const stripped = stripToolResultOutput(message) + expect(stripped.contentBlocks?.[0].toolCall?.result).toEqual({ + success: true, + output: { userInstruction: 'Open the second match' }, + }) + expect(stripToolResultOutput(stripped)).toBe(stripped) + }) + it('returns the same reference when there is nothing to strip', () => { const noBlocks: PersistedMessage = { id: 'u', diff --git a/apps/sim/lib/copilot/chat/persisted-message.ts b/apps/sim/lib/copilot/chat/persisted-message.ts index a48841f135f..76372f25d9e 100644 --- a/apps/sim/lib/copilot/chat/persisted-message.ts +++ b/apps/sim/lib/copilot/chat/persisted-message.ts @@ -1,4 +1,5 @@ import { generateId } from '@sim/utils/id' +import { isPlainRecord } from '@sim/utils/object' import { mergeAndRedactPersistedBlocks, redactSensitiveContent, @@ -14,6 +15,7 @@ import { MothershipStreamV1ToolOutcome, MothershipStreamV1ToolPhase, } from '@/lib/copilot/generated/mothership-stream-v1' +import { BrowserRequestTakeover } from '@/lib/copilot/generated/tool-catalog-v1' import type { ContentBlock, LocalToolCallStatus, @@ -126,10 +128,11 @@ export interface PersistedMessage { } /** - * Drop the `output` of every persisted tool result, keeping `success` and - * `error`. Tool outputs are never rendered (the chat thread shows only the tool - * name/title/status) and never replayed to the model (the upstream copilot - * service owns conversation memory), so storing them only bloats + * Drop persisted tool outputs, keeping `success` and `error`. The one narrow + * UI-state exception is a browser takeover's user-authored instruction, which + * restores its answered question recap after reload. Other outputs are never + * rendered or replayed to the model (the upstream service owns conversation + * memory), so storing them only bloats * `copilot_messages.content` — a single `get_workflow_logs`/`run_workflow` * result can reach hundreds of MB and stall task loads. * @@ -145,8 +148,25 @@ export function stripToolResultOutput(message: PersistedMessage): PersistedMessa const toolCall = block.toolCall const result = toolCall?.result if (!toolCall || !result || typeof result !== 'object' || !('output' in result)) return block + const output = result.output + const userInstruction = + toolCall.name === BrowserRequestTakeover.id && isPlainRecord(output) + ? output.userInstruction + : undefined + const normalizedInstruction = typeof userInstruction === 'string' ? userInstruction.trim() : '' + if ( + normalizedInstruction && + isPlainRecord(output) && + Object.keys(output).length === 1 && + output.userInstruction === normalizedInstruction + ) { + return block + } changed = true - const strippedResult: { success: boolean; error?: string } = { success: result.success } + const strippedResult: { success: boolean; output?: unknown; error?: string } = { + success: result.success, + ...(normalizedInstruction ? { output: { userInstruction: normalizedInstruction } } : {}), + } if (result.error !== undefined) strippedResult.error = result.error return { ...block, toolCall: { ...toolCall, result: strippedResult } } }) diff --git a/apps/sim/lib/copilot/chat/process-contents.test.ts b/apps/sim/lib/copilot/chat/process-contents.test.ts index be2bcd989b8..6570628e3bb 100644 --- a/apps/sim/lib/copilot/chat/process-contents.test.ts +++ b/apps/sim/lib/copilot/chat/process-contents.test.ts @@ -317,6 +317,31 @@ describe('processContextsServer - MCP contexts', () => { }) describe('processContextsServer - browser and terminal selections', () => { + it('describes whole Browser and Terminal mentions without inventing tab ids', async () => { + const result = await processContextsServer( + [ + { kind: 'browser_tab', tabId: 'browser-session', label: 'Browser' }, + { kind: 'terminal_tab', terminalId: 'terminal-session', label: 'Terminal' }, + ], + 'user-1' + ) + + expect(result).toMatchObject([ + { + type: 'browser_tab', + tag: '@Browser', + content: expect.stringContaining('resource as a whole'), + }, + { + type: 'terminal_tab', + tag: '@Terminal', + content: expect.stringContaining('resource as a whole'), + }, + ]) + expect(result[0].content).toContain('browser_list_tabs') + expect(result[1].content).toContain('terminal list operation') + }) + it('keeps the live browser pointer and appends quoted untrusted page text', async () => { const result = await processContextsServer( [ diff --git a/apps/sim/lib/copilot/chat/process-contents.ts b/apps/sim/lib/copilot/chat/process-contents.ts index f586267eb56..fc6ef5ce837 100644 --- a/apps/sim/lib/copilot/chat/process-contents.ts +++ b/apps/sim/lib/copilot/chat/process-contents.ts @@ -14,6 +14,10 @@ import { truncateSelectionText, } from '@/lib/copilot/chat/selection-context' import { QueryLogs } from '@/lib/copilot/generated/tool-catalog-v1' +import { + BROWSER_SESSION_RESOURCE_ID, + TERMINAL_SESSION_RESOURCE_ID, +} from '@/lib/copilot/resources/types' import { buildVfsFolderPathMap, canonicalBlockVfsPath, @@ -196,6 +200,14 @@ export async function processContextsServer( // additionally carries the quoted snapshot they chose, while the pointer // lets the agent inspect or act on the current page/shell when needed. if (ctx.kind === 'browser_tab' && ctx.tabId) { + if (ctx.tabId === BROWSER_SESSION_RESOURCE_ID) { + return { + type: 'browser_tab', + tag: ctx.label ? `@${ctx.label}` : '@Browser', + content: + 'The user tagged the Browser resource as a whole, not a specific tab. Inspect the live tabs with browser_list_tabs and choose the relevant one from their request. If no browser tab is open yet, open or navigate one as needed.', + } + } const pointer = `The user pointed at an open browser tab: "${ctx.label}" (tabId ${ctx.tabId}). Act on THIS tab — switch to it with browser_switch_tab and read it with browser_snapshot rather than assuming which tab they meant.` return { type: 'browser_tab', @@ -206,6 +218,14 @@ export async function processContextsServer( } } if (ctx.kind === 'terminal_tab' && ctx.terminalId) { + if (ctx.terminalId === TERMINAL_SESSION_RESOURCE_ID) { + return { + type: 'terminal_tab', + tag: ctx.label ? `@${ctx.label}` : '@Terminal', + content: + 'The user tagged the Terminal resource as a whole, not a specific shell. Inspect the live terminals with the terminal list operation and choose the relevant one from their request. If no terminal is open yet, create one as needed.', + } + } const pointer = `The user pointed at an open terminal: "${ctx.label}" (terminalId ${ctx.terminalId}). Act on THIS terminal — pass that terminalId to the terminal tool, and read its screen before assuming what is in it.` return { type: 'terminal_tab', diff --git a/apps/sim/lib/copilot/persistence/tool-confirm/index.ts b/apps/sim/lib/copilot/persistence/tool-confirm/index.ts index dd1a41006b2..d6e4fe3f8ad 100644 --- a/apps/sim/lib/copilot/persistence/tool-confirm/index.ts +++ b/apps/sim/lib/copilot/persistence/tool-confirm/index.ts @@ -14,6 +14,7 @@ import { createPubSubChannel, type PubSubChannel } from '@/lib/events/pubsub' const logger = createLogger('CopilotOrchestratorPersistence') const TOOL_CONFIRMATION_TTL_SECONDS = 60 * 10 +const DURABLE_CONFIRMATION_POLL_MS = 5_000 const toolConfirmationKey = (toolCallId: string) => `copilot:tool-confirmation:${toolCallId}` type ToolConfirmGlobal = typeof globalThis & { @@ -111,7 +112,7 @@ export function publishToolConfirmation(event: AsyncCompletionEnvelope): void { */ export async function waitForToolConfirmation( toolCallId: string, - timeoutMs: number, + timeoutMs: number | null, abortSignal?: AbortSignal, options: { acceptStatus?: (status: AsyncConfirmationState['status']) => boolean @@ -121,10 +122,12 @@ export async function waitForToolConfirmation( return new Promise((resolve) => { let settled = false let timeoutId: ReturnType | null = null + let pollId: ReturnType | null = null let unsubscribe: (() => void) | null = null const cleanup = () => { if (timeoutId) clearTimeout(timeoutId) + if (pollId) clearTimeout(pollId) if (unsubscribe) unsubscribe() abortSignal?.removeEventListener('abort', onAbort) } @@ -138,6 +141,27 @@ export async function waitForToolConfirmation( const onAbort = () => settle(null) + const checkDurableConfirmation = async (source: 'subscribe' | 'pubsub' | 'poll') => { + const latest = await getToolConfirmation(toolCallId) + if (!latest || !acceptStatus(latest.status)) return false + logger.info('Resolved tool confirmation from durable state', { + toolCallId, + status: latest.status, + source, + }) + settle(latest) + return true + } + + const scheduleDurablePoll = () => { + if (settled || timeoutMs !== null) return + pollId = setTimeout(async () => { + pollId = null + await checkDurableConfirmation('poll') + scheduleDurablePoll() + }, DURABLE_CONFIRMATION_POLL_MS) + } + unsubscribe = toolConfirmationChannel.subscribe((event) => { if (event.toolCallId !== toolCallId) return if (isAsyncEphemeralConfirmationStatus(event.status) && acceptStatus(event.status)) { @@ -149,31 +173,16 @@ export async function waitForToolConfirmation( }) return } - void getToolConfirmation(toolCallId).then((latest) => { - if (!latest || !acceptStatus(latest.status)) return - logger.info('Resolved tool confirmation from pubsub', { - toolCallId, - status: latest.status, - }) - settle(latest) - }) + void checkDurableConfirmation('pubsub') }) - timeoutId = setTimeout(() => settle(null), timeoutMs) + if (timeoutMs !== null) timeoutId = setTimeout(() => settle(null), timeoutMs) if (abortSignal?.aborted) { settle(null) return } abortSignal?.addEventListener('abort', onAbort, { once: true }) - void getToolConfirmation(toolCallId).then((latest) => { - if (latest && acceptStatus(latest.status)) { - logger.info('Resolved tool confirmation after subscribe', { - toolCallId, - status: latest.status, - }) - settle(latest) - } - }) + void checkDurableConfirmation('subscribe').then(scheduleDurablePoll) }) } diff --git a/apps/sim/lib/copilot/persistence/tool-confirm/tool-confirm.test.ts b/apps/sim/lib/copilot/persistence/tool-confirm/tool-confirm.test.ts index 7b72bce8255..cc5f7b02338 100644 --- a/apps/sim/lib/copilot/persistence/tool-confirm/tool-confirm.test.ts +++ b/apps/sim/lib/copilot/persistence/tool-confirm/tool-confirm.test.ts @@ -184,4 +184,109 @@ describe('copilot orchestrator persistence', () => { timestamp: '2026-01-01T00:00:01.000Z', }) }) + + it('keeps a no-deadline human wait alive until confirmation arrives', async () => { + vi.useFakeTimers() + try { + row = { + status: 'pending', + error: null, + result: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + } + let settled = false + const waitPromise = waitForToolConfirmation('tool-1', null, undefined, { + acceptStatus: (status) => + status === 'success' || status === 'error' || status === 'cancelled', + }).then((result) => { + settled = true + return result + }) + + await vi.advanceTimersByTimeAsync(24 * 60 * 60 * 1000) + expect(settled).toBe(false) + + row = { + status: 'completed', + error: null, + result: { ok: true }, + updatedAt: new Date('2026-01-02T00:00:00.000Z'), + } + publishToolConfirmation({ + toolCallId: 'tool-1', + status: 'success', + timestamp: '2026-01-02T00:00:00.000Z', + }) + + await expect(waitPromise).resolves.toEqual({ + status: 'success', + message: undefined, + data: { ok: true }, + timestamp: '2026-01-02T00:00:00.000Z', + }) + } finally { + vi.useRealTimers() + } + }) + + it('catches up from durable state when a no-deadline waiter misses pubsub', async () => { + vi.useFakeTimers() + try { + row = { + status: 'pending', + error: null, + result: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + } + const waitPromise = waitForToolConfirmation('tool-1', null, undefined, { + acceptStatus: (status) => + status === 'success' || status === 'error' || status === 'cancelled', + }) + await vi.advanceTimersByTimeAsync(0) + + row = { + status: 'completed', + error: null, + result: { recovered: true }, + updatedAt: new Date('2026-01-01T00:00:01.000Z'), + } + await vi.advanceTimersByTimeAsync(5_000) + + await expect(waitPromise).resolves.toMatchObject({ + status: 'success', + data: { recovered: true }, + }) + const callsAfterSettle = getAsyncToolCalls.mock.calls.length + await vi.advanceTimersByTimeAsync(5_000) + expect(getAsyncToolCalls).toHaveBeenCalledTimes(callsAfterSettle) + } finally { + vi.useRealTimers() + } + }) + + it('stops durable catch-up polling when aborted', async () => { + vi.useFakeTimers() + try { + row = { + status: 'pending', + error: null, + result: null, + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + } + const controller = new AbortController() + const waitPromise = waitForToolConfirmation('tool-1', null, controller.signal, { + acceptStatus: (status) => + status === 'success' || status === 'error' || status === 'cancelled', + }) + await vi.advanceTimersByTimeAsync(0) + controller.abort() + await expect(waitPromise).resolves.toBeNull() + + const callsAfterAbort = getAsyncToolCalls.mock.calls.length + await vi.advanceTimersByTimeAsync(5_000) + expect(getAsyncToolCalls).toHaveBeenCalledTimes(callsAfterAbort) + } finally { + vi.useRealTimers() + } + }) }) diff --git a/apps/sim/lib/copilot/request/handlers/handlers.test.ts b/apps/sim/lib/copilot/request/handlers/handlers.test.ts index 65843584f16..9bcf12d9d7c 100644 --- a/apps/sim/lib/copilot/request/handlers/handlers.test.ts +++ b/apps/sim/lib/copilot/request/handlers/handlers.test.ts @@ -695,6 +695,43 @@ describe('sse-handlers tool lifecycle', () => { expect(executeTool).not.toHaveBeenCalled() }) + it('waits for a browser takeover without a client-tool deadline', async () => { + isSimExecuted.mockReturnValue(false) + waitForClientToolCompletion.mockResolvedValueOnce({ + status: 'success', + message: 'Browser hand-back completed', + data: { completed: true }, + }) + + await sseHandlers.tool( + { + type: MothershipStreamV1EventType.tool, + payload: { + toolCallId: 'tool-browser-takeover', + toolName: 'browser_request_takeover', + arguments: { reason: 'Please sign in' }, + executor: MothershipStreamV1ToolExecutor.client, + mode: MothershipStreamV1ToolMode.async, + phase: MothershipStreamV1ToolPhase.call, + }, + } satisfies StreamEvent, + context, + execContext, + { interactive: true, timeout: 1000 } + ) + + await Promise.allSettled(context.pendingToolPromises.values()) + + expect(waitForClientToolCompletion).toHaveBeenCalledWith({ + toolCallId: 'tool-browser-takeover', + runId: context.runId, + userId: 'user-1', + timeoutMs: null, + abortSignal: undefined, + registry: execContext.resolvedSecretTraceRegistry, + }) + }) + it('keeps an ordinary static VFS read on the Sim executor', async () => { await sseHandlers.tool( { diff --git a/apps/sim/lib/copilot/request/handlers/tool.ts b/apps/sim/lib/copilot/request/handlers/tool.ts index 14c0afb259e..415e7475eed 100644 --- a/apps/sim/lib/copilot/request/handlers/tool.ts +++ b/apps/sim/lib/copilot/request/handlers/tool.ts @@ -12,6 +12,7 @@ import { MothershipStreamV1ToolOutcome, type MothershipStreamV1ToolResultPayload, } from '@/lib/copilot/generated/mothership-stream-v1' +import { BrowserRequestTakeover } from '@/lib/copilot/generated/tool-catalog-v1' import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1' import { withCopilotSpan } from '@/lib/copilot/request/otel' @@ -742,12 +743,14 @@ async function dispatchToolExecution( */ function waitForClientExecution(): Promise { toolCall.status = 'executing' + const waitsForHuman = toolName === BrowserRequestTakeover.id + const timeoutMs = waitsForHuman ? null : options.timeout || STREAM_TIMEOUT_MS return withCopilotSpan( TraceSpan.CopilotToolWaitForClientResult, { [TraceAttr.ToolName]: toolName, [TraceAttr.ToolCallId]: toolCallId, - [TraceAttr.ToolTimeoutMs]: options.timeout || STREAM_TIMEOUT_MS, + ...(timeoutMs !== null ? { [TraceAttr.ToolTimeoutMs]: timeoutMs } : {}), ...(context.runId ? { [TraceAttr.RunId]: context.runId } : {}), }, async (span) => { @@ -755,7 +758,7 @@ async function dispatchToolExecution( ? await waitForWorkflowToolCompletion({ toolCallId, workflowId: resolveWorkflowToolTargetId(args, execContext.workflowId), - timeoutMs: options.timeout || STREAM_TIMEOUT_MS, + timeoutMs: timeoutMs ?? STREAM_TIMEOUT_MS, abortSignal: options.abortSignal, registry: execContext.resolvedSecretTraceRegistry, }) @@ -763,7 +766,7 @@ async function dispatchToolExecution( toolCallId, runId: context.runId, userId: execContext.userId, - timeoutMs: options.timeout || STREAM_TIMEOUT_MS, + timeoutMs, abortSignal: options.abortSignal, registry: execContext.resolvedSecretTraceRegistry, }) diff --git a/apps/sim/lib/copilot/request/lifecycle/run.test.ts b/apps/sim/lib/copilot/request/lifecycle/run.test.ts index 94f79e5c6c5..5f535532f55 100644 --- a/apps/sim/lib/copilot/request/lifecycle/run.test.ts +++ b/apps/sim/lib/copilot/request/lifecycle/run.test.ts @@ -30,7 +30,7 @@ const { mockPrepareCopilotEnvironmentContext: vi.fn(), mockPrepareExecutionContext: vi.fn(), mockRunStreamLoop: vi.fn(), - mockPendingToolWaitBudgetMs: vi.fn(() => 60_000), + mockPendingToolWaitBudgetMs: vi.fn((_toolCall?: { name?: string }) => 60_000 as number | null), mockGetAutoAllowedTools: vi.fn(async () => new Set()), mockFilterModelSafeWorkspaceFileAttachments: vi.fn(async (attachments: unknown[]) => attachments), mockUpdateRunStatus: vi.fn(), @@ -162,6 +162,7 @@ describe('runCopilotLifecycle', () => { isCopilotToolPermissionsEnabled: false, }) mockGetAutoAllowedTools.mockResolvedValue(new Set()) + mockPendingToolWaitBudgetMs.mockImplementation(() => 60_000) mockGetMothershipBaseURL.mockResolvedValue('http://mothership.test') mockGetMothershipSourceEnvHeaders.mockReturnValue({}) mockPrepareCopilotEnvironmentContext.mockResolvedValue({ @@ -1916,6 +1917,114 @@ describe('runCopilotLifecycle', () => { expect(result.errors).toEqual(['The provider is overloaded']) }) + it('keeps a human wait durable while force-failing a hung parallel tool', async () => { + vi.useFakeTimers() + try { + let releaseTakeover = () => {} + let lifecycleSettled = false + const fetchUrls: string[] = [] + const executionContext: ExecutionContext = { + userId: 'user-1', + workflowId: '', + workspaceId: 'ws-1', + chatId: 'chat-1', + } + + mockPendingToolWaitBudgetMs.mockImplementation((toolCall) => + toolCall?.name === 'browser_request_takeover' ? null : 60_000 + ) + mockForceFailHungToolCall.mockImplementation( + async (toolCallId: string, context: StreamingContext, message: string) => { + const tool = context.toolCalls.get(toolCallId) + if (!tool) return + tool.status = MothershipStreamV1ToolOutcome.error + tool.endTime = Date.now() + tool.result = { success: false } + tool.error = message + } + ) + + mockRunStreamLoop.mockImplementationOnce( + async ( + fetchUrl: string, + _fetchOptions: RequestInit, + context: StreamingContext + ): Promise => { + fetchUrls.push(fetchUrl) + const takeoverId = 'tool-takeover' + context.toolCalls.set(takeoverId, { + id: takeoverId, + name: 'browser_request_takeover', + status: 'executing', + }) + const takeover = new Promise<{ status: 'success' }>((resolve) => { + releaseTakeover = () => { + const tool = context.toolCalls.get(takeoverId) + if (tool) { + tool.status = MothershipStreamV1ToolOutcome.success + tool.endTime = Date.now() + tool.result = { success: true, output: { completed: true } } + } + context.pendingToolPromises.delete(takeoverId) + resolve({ status: 'success' }) + } + }) + context.pendingToolPromises.set(takeoverId, takeover) + + context.toolCalls.set('tool-hung', { + id: 'tool-hung', + name: 'read', + status: 'executing', + }) + context.pendingToolPromises.set('tool-hung', new Promise(() => {})) + context.awaitingAsyncContinuation = { + checkpointId: 'ckpt-1', + pendingToolCallIds: [takeoverId, 'tool-hung'], + } + } + ) + mockRunStreamLoop.mockImplementationOnce( + async (fetchUrl: string, _fetchOptions: RequestInit, context: StreamingContext) => { + fetchUrls.push(fetchUrl) + context.accumulatedContent = 'Continued after browser control resumed.' + } + ) + + const lifecycle = runCopilotLifecycle( + { message: 'hello', messageId: 'stream-1' }, + { + userId: 'user-1', + workspaceId: 'ws-1', + chatId: 'chat-1', + executionId: 'exec-1', + runId: 'run-1', + executionContext, + } + ).finally(() => { + lifecycleSettled = true + }) + + await vi.advanceTimersByTimeAsync(91_000) + expect(mockForceFailHungToolCall).toHaveBeenCalledTimes(1) + expect(mockForceFailHungToolCall).toHaveBeenCalledWith( + 'tool-hung', + expect.anything(), + expect.stringContaining('hung') + ) + expect(lifecycleSettled).toBe(false) + expect(fetchUrls).toEqual(['http://mothership.test/api/copilot']) + + releaseTakeover() + await vi.advanceTimersByTimeAsync(0) + const result = await lifecycle + + expect(fetchUrls[1]).toBe('http://mothership.test/api/tools/resume') + expect(result.success).toBe(true) + } finally { + vi.useRealTimers() + } + }) + it('force-fails a hung tool promise and resumes with an error result instead of wedging', async () => { vi.useFakeTimers() try { diff --git a/apps/sim/lib/copilot/request/lifecycle/run.ts b/apps/sim/lib/copilot/request/lifecycle/run.ts index dfa0ca2ea91..389f31564a2 100644 --- a/apps/sim/lib/copilot/request/lifecycle/run.ts +++ b/apps/sim/lib/copilot/request/lifecycle/run.ts @@ -950,36 +950,52 @@ async function runCheckpointLoop( } if (context.pendingToolPromises.size > 0) { - // Bounded by the slowest pending tool's watchdog plus grace. The - // per-tool watchdog already guarantees each promise settles; this gate - // is the structural backstop so that no tool failure mode — known or - // unknown — can park the checkpoint loop (and the chat's pending-stream - // lock) forever. - const waitBudgetMs = - Array.from(context.pendingToolPromises.keys()).reduce( - (max, toolCallId) => - Math.max(max, pendingToolWaitBudgetMs(context.toolCalls.get(toolCallId))), - 0 - ) + TOOL_WATCHDOG_RESUME_GRACE_MS + // Snapshot the gate by tool. Human waits remain durable, but they must + // not disable the structural watchdog for an unrelated parallel tool. + const pendingTools = Array.from(context.pendingToolPromises.entries()).map( + ([toolCallId, promise]) => ({ + toolCallId, + promise, + waitBudgetMs: pendingToolWaitBudgetMs(context.toolCalls.get(toolCallId)), + }) + ) + const durableTools = pendingTools.filter((tool) => tool.waitBudgetMs === null) + const boundedTools = pendingTools.flatMap((tool) => + tool.waitBudgetMs === null ? [] : [{ ...tool, waitBudgetMs: tool.waitBudgetMs }] + ) + const boundedWaitBudgetMs = + boundedTools.length > 0 + ? Math.max(...boundedTools.map((tool) => tool.waitBudgetMs)) + + TOOL_WATCHDOG_RESUME_GRACE_MS + : null const waitSpan = context.trace.startSpan('Wait for Tools', 'lifecycle.wait_tools', { checkpointId: continuation.checkpointId, pendingCount: context.pendingToolPromises.size, - waitBudgetMs, + durableCount: durableTools.length, + ...(boundedWaitBudgetMs !== null ? { waitBudgetMs: boundedWaitBudgetMs } : {}), }) logger.info('Waiting for in-flight tool executions before resume', { checkpointId: continuation.checkpointId, pendingCount: context.pendingToolPromises.size, - waitBudgetMs, + durableCount: durableTools.length, + waitBudgetMs: boundedWaitBudgetMs, }) - const settledInTime = await Promise.race([ - Promise.allSettled(context.pendingToolPromises.values()).then(() => true), - sleep(waitBudgetMs).then(() => false), - ]) - if (!settledInTime) { - const hungToolCallIds = Array.from(context.pendingToolPromises.keys()) + const boundedSettledInTime = + boundedWaitBudgetMs === null + ? true + : await Promise.race([ + Promise.allSettled(boundedTools.map((tool) => tool.promise)).then(() => true), + sleep(boundedWaitBudgetMs).then(() => false), + ]) + if (!boundedSettledInTime) { + const hungToolCallIds = boundedTools + .filter( + ({ toolCallId, promise }) => context.pendingToolPromises.get(toolCallId) === promise + ) + .map(({ toolCallId }) => toolCallId) logger.error('Pending tool executions exceeded the resume wait budget; force-failing', { checkpointId: continuation.checkpointId, - waitBudgetMs, + waitBudgetMs: boundedWaitBudgetMs, hungToolCallIds, }) for (const toolCallId of hungToolCallIds) { @@ -991,7 +1007,8 @@ async function runCheckpointLoop( context.pendingToolPromises.delete(toolCallId) } } - waitSpan.attributes = { ...waitSpan.attributes, settledInTime } + await Promise.allSettled(durableTools.map((tool) => tool.promise)) + waitSpan.attributes = { ...waitSpan.attributes, settledInTime: boundedSettledInTime } context.trace.endSpan(waitSpan) } diff --git a/apps/sim/lib/copilot/request/tools/client.ts b/apps/sim/lib/copilot/request/tools/client.ts index de792abd5fc..663482dc028 100644 --- a/apps/sim/lib/copilot/request/tools/client.ts +++ b/apps/sim/lib/copilot/request/tools/client.ts @@ -37,7 +37,7 @@ const logger = createLogger('CopilotClientToolWaiter') */ export async function waitForToolCompletion( toolCallId: string, - timeoutMs: number, + timeoutMs: number | null, abortSignal?: AbortSignal ): Promise { const decision = await waitForToolConfirmation(toolCallId, timeoutMs, abortSignal, { @@ -57,7 +57,8 @@ interface WaitForClientToolCompletionOptions { toolCallId: string runId?: string userId: string - timeoutMs: number + /** Null for a durable human-interaction wait that ends only on answer or abort. */ + timeoutMs: number | null abortSignal?: AbortSignal registry?: ResolvedSecretTraceRegistry } diff --git a/apps/sim/lib/copilot/request/tools/executor.test.ts b/apps/sim/lib/copilot/request/tools/executor.test.ts index 52bb6dcef6c..06c86e224fa 100644 --- a/apps/sim/lib/copilot/request/tools/executor.test.ts +++ b/apps/sim/lib/copilot/request/tools/executor.test.ts @@ -119,6 +119,12 @@ describe('toolWatchdogTimeoutMs', () => { }) describe('pendingToolWaitBudgetMs', () => { + it('does not put a deadline on an executing browser takeover', () => { + expect( + pendingToolWaitBudgetMs({ name: 'browser_request_takeover', status: 'executing' }) + ).toBeNull() + }) + it('waits on a person for as long as the whole turn allows', () => { // The 60s default would force-fail a permission prompt while the user was // still reading it, resuming Go before they ever answered. diff --git a/apps/sim/lib/copilot/request/tools/executor.ts b/apps/sim/lib/copilot/request/tools/executor.ts index 2db7570d004..693a8c325d2 100644 --- a/apps/sim/lib/copilot/request/tools/executor.ts +++ b/apps/sim/lib/copilot/request/tools/executor.ts @@ -20,6 +20,7 @@ import { MothershipStreamV1ToolPhase, } from '@/lib/copilot/generated/mothership-stream-v1' import { + BrowserRequestTakeover, CrawlWebsite, CreateFile, CreateWorkflow, @@ -246,7 +247,8 @@ export function toolWatchdogTimeoutMs(toolName: string | undefined): number { } /** - * How long the resume gate may wait on one pending tool call. + * How long the resume gate may wait on one pending tool call. Null means the + * tool is durably waiting on a person and has no deadline. * * A call sitting on a permission prompt is waiting on a person, not on the * executor, so the tool's own watchdog is the wrong bound — the 60s default @@ -255,7 +257,10 @@ export function toolWatchdogTimeoutMs(toolName: string | undefined): number { */ export function pendingToolWaitBudgetMs( toolCall: Pick | undefined -): number { +): number | null { + if (toolCall?.name === BrowserRequestTakeover.id && toolCall.status === 'executing') { + return null + } if (toolCall?.status === 'awaiting_approval') return TOOL_WATCHDOG_LONG_RUNNING_MS return toolWatchdogTimeoutMs(toolCall?.name) } diff --git a/apps/sim/lib/copilot/tools/client/browser-tool-execution.test.ts b/apps/sim/lib/copilot/tools/client/browser-tool-execution.test.ts index 0b95a3dd022..8731aafeedc 100644 --- a/apps/sim/lib/copilot/tools/client/browser-tool-execution.test.ts +++ b/apps/sim/lib/copilot/tools/client/browser-tool-execution.test.ts @@ -4,15 +4,20 @@ import { sleep } from '@sim/utils/helpers' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockExecuteBrowserTool, mockReportCompletion, mockRestoreBrowserScope } = vi.hoisted( - () => ({ - mockExecuteBrowserTool: vi.fn(), - mockReportCompletion: vi.fn(), - mockRestoreBrowserScope: vi.fn(), - }) -) +const { + mockCancelBrowserTool, + mockExecuteBrowserTool, + mockReportCompletion, + mockRestoreBrowserScope, +} = vi.hoisted(() => ({ + mockCancelBrowserTool: vi.fn(), + mockExecuteBrowserTool: vi.fn(), + mockReportCompletion: vi.fn(), + mockRestoreBrowserScope: vi.fn(), +})) vi.mock('@/lib/browser-agent/transport', () => ({ + cancelBrowserTool: mockCancelBrowserTool, executeBrowserTool: mockExecuteBrowserTool, restoreBrowserScope: mockRestoreBrowserScope, })) @@ -44,6 +49,10 @@ describe('executeBrowserToolOnClient', () => { pageState: null, tabs: [], activeTabId: null, + automationTabId: null, + automationActive: false, + automationNeedsAttention: false, + agentRunIds: [], sessionAlive: true, suspended: false, } @@ -54,6 +63,7 @@ describe('executeBrowserToolOnClient', () => { }) mockReportCompletion.mockResolvedValue(undefined) mockRestoreBrowserScope.mockResolvedValue(false) + mockCancelBrowserTool.mockResolvedValue(true) }) it('executes the tool and reports success when the session is alive', async () => { @@ -68,13 +78,166 @@ describe('executeBrowserToolOnClient', () => { 'browser_snapshot', {}, 30_000, - CHAT_SCOPE + CHAT_SCOPE, + expect.any(Function) ) expect(mockReportCompletion).toHaveBeenCalledWith(toolCallId, 'success', expect.any(String), { text: 'page content', }) }) + it('preserves a takeover instruction and waits without a renderer deadline', async () => { + mockExecuteBrowserTool.mockResolvedValue({ + completed: true, + userInstruction: 'Open the second match', + }) + const toolCallId = nextToolCallId() + + executeBrowserToolOnClient(toolCallId, 'browser_request_takeover', { + reason: 'Please pick a match', + }) + await flush() + + expect(mockExecuteBrowserTool).toHaveBeenCalledWith( + toolCallId, + 'browser_request_takeover', + { reason: 'Please pick a match' }, + null, + CHAT_SCOPE, + expect.any(Function) + ) + expect(mockReportCompletion).toHaveBeenCalledWith(toolCallId, 'success', expect.any(String), { + completed: true, + userInstruction: 'Open the second match', + }) + }) + + it('cancels the exact native tool and suppresses a stale completion after Chat Stop', async () => { + let resolveTool: (value: unknown) => void = () => {} + mockExecuteBrowserTool.mockImplementation( + () => + new Promise((resolve) => { + resolveTool = resolve + }) + ) + const controller = new AbortController() + const toolCallId = nextToolCallId() + + executeBrowserToolOnClient( + toolCallId, + 'browser_request_takeover', + { reason: 'Please sign in' }, + CHAT_SCOPE, + undefined, + controller.signal + ) + controller.abort() + await flush() + + expect(mockCancelBrowserTool).toHaveBeenCalledWith( + toolCallId, + CHAT_SCOPE, + 'browser_request_takeover' + ) + resolveTool({ completed: true }) + await flush() + expect(mockReportCompletion).not.toHaveBeenCalled() + }) + + it('suppresses completion when scope cancellation outlives the stream AbortController', async () => { + let resolveTool: (value: unknown) => void = () => {} + let markCancelled: (() => void) | undefined + mockExecuteBrowserTool.mockImplementation( + ( + _toolCallId: string, + _toolName: string, + _params: Record, + _timeoutMs: number | null, + _scopeId: string, + onCancel: () => void + ) => { + markCancelled = onCancel + return new Promise((resolve) => { + resolveTool = resolve + }) + } + ) + const toolCallId = nextToolCallId() + + executeBrowserToolOnClient( + toolCallId, + 'browser_request_takeover', + { reason: 'Please sign in' }, + CHAT_SCOPE + ) + markCancelled?.() + resolveTool({ completed: true }) + await flush() + + expect(mockReportCompletion).not.toHaveBeenCalled() + }) + + it('delegates older-shell takeover cancellation to the shared transport fallback', async () => { + mockExecuteBrowserTool.mockImplementation(() => new Promise(() => {})) + const controller = new AbortController() + const toolCallId = nextToolCallId() + + executeBrowserToolOnClient( + toolCallId, + 'browser_request_takeover', + { reason: 'Please sign in' }, + CHAT_SCOPE, + undefined, + controller.signal + ) + controller.abort() + await flush() + + expect(mockCancelBrowserTool).toHaveBeenCalledWith( + toolCallId, + CHAT_SCOPE, + 'browser_request_takeover' + ) + expect(mockReportCompletion).not.toHaveBeenCalled() + }) + + it('does not dispatch a takeover after Stop wins a session restore race', async () => { + useBrowserSessionStore.getState().setSessionAlive(false, CHAT_SCOPE) + let finishRestore: () => void = () => {} + mockRestoreBrowserScope.mockImplementation( + () => + new Promise((resolve) => { + finishRestore = () => { + useBrowserSessionStore.getState().setSessionAlive(true, CHAT_SCOPE) + resolve(true) + } + }) + ) + const controller = new AbortController() + const toolCallId = nextToolCallId() + + executeBrowserToolOnClient( + toolCallId, + 'browser_request_takeover', + { reason: 'Please sign in' }, + CHAT_SCOPE, + undefined, + controller.signal + ) + await flush() + controller.abort() + finishRestore() + await flush() + + expect(mockCancelBrowserTool).toHaveBeenCalledWith( + toolCallId, + CHAT_SCOPE, + 'browser_request_takeover' + ) + expect(mockExecuteBrowserTool).not.toHaveBeenCalled() + expect(mockReportCompletion).not.toHaveBeenCalled() + }) + // The copilot serializes a result carrying this `attachment` shape into a // real image content block, so the data URL has to be reshaped rather than // passed through — an inline data URL would be charged against the tool @@ -139,7 +302,8 @@ describe('executeBrowserToolOnClient', () => { 'browser_wait_for', params, expected, - CHAT_SCOPE + CHAT_SCOPE, + expect.any(Function) ) } ) @@ -185,7 +349,8 @@ describe('executeBrowserToolOnClient', () => { toolName, params, expect.any(Number), - scopeId + scopeId, + expect.any(Function) ) expect(mockRestoreBrowserScope.mock.invocationCallOrder[0]).toBeLessThan( mockExecuteBrowserTool.mock.invocationCallOrder[0] @@ -205,7 +370,8 @@ describe('executeBrowserToolOnClient', () => { 'browser_navigate', { url: 'https://example.com' }, 45_000, - CHAT_SCOPE + CHAT_SCOPE, + expect.any(Function) ) expect(mockRestoreBrowserScope).not.toHaveBeenCalled() expect(mockReportCompletion).toHaveBeenCalledWith(toolCallId, 'success', expect.any(String), { @@ -262,7 +428,8 @@ describe('executeBrowserToolOnClient', () => { 'browser_snapshot', {}, 30_000, - 'chat-b' + 'chat-b', + expect.any(Function) ) expect(mockReportCompletion).toHaveBeenCalledWith(toolCallId, 'success', expect.any(String), { text: 'B page', diff --git a/apps/sim/lib/copilot/tools/client/browser-tool-execution.ts b/apps/sim/lib/copilot/tools/client/browser-tool-execution.ts index 3bc40f663e2..e0a13d08125 100644 --- a/apps/sim/lib/copilot/tools/client/browser-tool-execution.ts +++ b/apps/sim/lib/copilot/tools/client/browser-tool-execution.ts @@ -11,7 +11,11 @@ import type { BrowserToolName } from '@sim/browser-protocol' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' -import { executeBrowserTool, restoreBrowserScope } from '@/lib/browser-agent/transport' +import { + cancelBrowserTool, + executeBrowserTool, + restoreBrowserScope, +} from '@/lib/browser-agent/transport' import { ASYNC_TOOL_CONFIRMATION_STATUS } from '@/lib/copilot/async-runs/lifecycle' import { COPILOT_CONFIRM_API_PATH } from '@/lib/copilot/constants' import { reportClientToolCompletion } from '@/lib/copilot/tools/client/completion' @@ -166,7 +170,8 @@ export function executeBrowserToolOnClient( toolName: BrowserToolName, params: Record, scopeId = useBrowserSessionStore.getState().activeScopeId, - eventTs?: string + eventTs?: string, + abortSignal?: AbortSignal ): void { if (!scopeId) { logger.error('Cannot execute browser tool without a chat scope', { toolCallId, toolName }) @@ -182,7 +187,7 @@ export function executeBrowserToolOnClient( return } markExecuted(toolCallId) - void doExecuteBrowserTool(toolCallId, toolName, params, scopeId).catch((err) => { + void doExecuteBrowserTool(toolCallId, toolName, params, scopeId, abortSignal).catch((err) => { logger.error('Unhandled error in client-side browser tool execution', { toolCallId, toolName, @@ -200,8 +205,31 @@ async function doExecuteBrowserTool( toolCallId: string, toolName: BrowserToolName, params: Record, - scopeId: string + scopeId: string, + abortSignal?: AbortSignal ): Promise { + let cancelled = abortSignal?.aborted === true + const cancelNativeTool = async () => { + cancelled = true + try { + await cancelBrowserTool(toolCallId, scopeId, toolName) + } catch (error) { + logger.warn('Could not cancel native browser tool', { + toolCallId, + toolName, + error: toError(error).message, + }) + } + } + const onAbort = () => { + void cancelNativeTool() + } + if (cancelled) { + void cancelNativeTool() + } else { + abortSignal?.addEventListener('abort', onAbort, { once: true }) + } + const needsLivePage = !SESSION_REVIVAL_TOOLS.has(toolName) if (needsLivePage && isSessionClosed(scopeId)) { try { @@ -219,6 +247,10 @@ async function doExecuteBrowserTool( toolCallId, toolName, }) + if (cancelled) { + abortSignal?.removeEventListener('abort', onAbort) + return + } await reportClientToolCompletion( toolCallId, ASYNC_TOOL_CONFIRMATION_STATUS.error, @@ -230,11 +262,20 @@ async function doExecuteBrowserTool( error: toError(reportErr).message, }) }) + abortSignal?.removeEventListener('abort', onAbort) + return + } + // A restore can outlive the stream that requested it. Do not dispatch the + // tool afterward—older shells have no cancellation tombstone to catch a + // takeover-done signal that arrived before the takeover itself existed. + if (cancelled) { + abortSignal?.removeEventListener('abort', onAbort) return } // If the user leaves the page mid-action the awaited result is lost; tell // the waiter so the turn fails fast instead of hanging until its timeout. const onPageHide = () => { + if (cancelled) return navigator.sendBeacon( COPILOT_CONFIRM_API_PATH, new Blob( @@ -262,8 +303,12 @@ async function doExecuteBrowserTool( toolName, params, timeoutForTool(toolName, params), - scopeId + scopeId, + () => { + cancelled = true + } ) + if (cancelled) return await reportClientToolCompletion( toolCallId, ASYNC_TOOL_CONFIRMATION_STATUS.success, @@ -271,6 +316,7 @@ async function doExecuteBrowserTool( sanitizeResultForModel(toolName, result) ) } catch (err) { + if (cancelled) return // The session dying mid-call (e.g. during a takeover) surfaces as a // generic timeout; tag it so the model learns the real, terminal cause // instead of retrying against a dead session. @@ -289,6 +335,7 @@ async function doExecuteBrowserTool( }) }) } finally { + abortSignal?.removeEventListener('abort', onAbort) if (typeof window !== 'undefined') { window.removeEventListener('pagehide', onPageHide) } diff --git a/apps/sim/lib/copilot/tools/tool-display.test.ts b/apps/sim/lib/copilot/tools/tool-display.test.ts index 9fd352aff15..027ce68d915 100644 --- a/apps/sim/lib/copilot/tools/tool-display.test.ts +++ b/apps/sim/lib/copilot/tools/tool-display.test.ts @@ -462,6 +462,18 @@ describe('getToolDisplayTitle for context management', () => { }) }) +describe('getToolStatusDisplayTitle for browser takeover', () => { + it('uses a neutral completed title after browser control resumes', () => { + expect( + getToolStatusDisplayTitle( + 'Waiting for you: Pick a match in the draw', + 'success', + 'browser_request_takeover' + ) + ).toBe('Resumed browser control') + }) +}) + describe('wait titles', () => { // The row is on screen for the whole pause, so a bare "Wait" reads as a // stall. The duration is the entire content of this tool. diff --git a/apps/sim/lib/copilot/tools/tool-display.ts b/apps/sim/lib/copilot/tools/tool-display.ts index e81ab9a54e2..9d3e30c37c6 100644 --- a/apps/sim/lib/copilot/tools/tool-display.ts +++ b/apps/sim/lib/copilot/tools/tool-display.ts @@ -1064,6 +1064,13 @@ export function getToolCompletedTitle(title: string): string | undefined { * running/error row remains truthful; every successful renderer calls this to * project the corresponding completed title from the canonical verb map. */ -export function getToolStatusDisplayTitle(title: string, status: string): string { +export function getToolStatusDisplayTitle( + title: string, + status: string, + toolName?: string +): string { + if (status === 'success' && toolName === 'browser_request_takeover') { + return 'Resumed browser control' + } return status === 'success' ? (getToolCompletedTitle(title) ?? title) : title } diff --git a/apps/sim/lib/desktop/appearance.test.ts b/apps/sim/lib/desktop/appearance.test.ts index f2d9284d585..76ded478bec 100644 --- a/apps/sim/lib/desktop/appearance.test.ts +++ b/apps/sim/lib/desktop/appearance.test.ts @@ -1,4 +1,4 @@ -import { TERMINAL_DARK_THEME } from '@sim/desktop-bridge' +import { TERMINAL_DARK_THEME, TERMINAL_LIGHT_THEME } from '@sim/desktop-bridge' import { afterEach, describe, expect, it, vi } from 'vitest' const { mockBridge } = vi.hoisted(() => ({ mockBridge: { current: undefined as unknown } })) @@ -10,7 +10,9 @@ vi.mock('@/lib/desktop', () => ({ import { loadDesktopTerminalAppearance, loadDesktopTerminalThemeProfiles, + refreshSelectedTerminalProfile, resolveDesktopAppearanceTheme, + resolveTerminalThemePalette, } from './appearance' afterEach(() => { @@ -35,6 +37,58 @@ describe('resolveDesktopAppearanceTheme', () => { }) }) +describe('resolveTerminalThemePalette', () => { + const fallbackPalette = { ...TERMINAL_DARK_THEME, background: '#111111' } + const lightPalette = { ...TERMINAL_LIGHT_THEME, background: '#fafafa' } + const darkPalette = { ...TERMINAL_DARK_THEME, background: '#222222' } + const profile = { + id: 'iterm2:ocean', + name: 'Ocean', + source: 'iterm2' as const, + palette: fallbackPalette, + lightPalette, + darkPalette, + } + + it('uses an imported profile palette matching Sim appearance', () => { + expect(resolveTerminalThemePalette(profile, 'light')).toBe(lightPalette) + expect(resolveTerminalThemePalette(profile, 'dark')).toBe(darkPalette) + }) + + it('falls back to the source palette when a profile has no mode-specific colors', () => { + const legacyProfile = { ...profile, lightPalette: undefined, darkPalette: undefined } + expect(resolveTerminalThemePalette(legacyProfile, 'light')).toBe(fallbackPalette) + expect(resolveTerminalThemePalette(legacyProfile, 'dark')).toBe(fallbackPalette) + }) + + it('keeps built-in Sim themes unchanged', () => { + expect(resolveTerminalThemePalette('light', 'dark')).toBe(TERMINAL_LIGHT_THEME) + expect(resolveTerminalThemePalette('dark', 'light')).toBe(TERMINAL_DARK_THEME) + }) +}) + +describe('refreshSelectedTerminalProfile', () => { + const storedProfile = { + id: 'iterm2:ocean', + name: 'Ocean', + source: 'iterm2' as const, + palette: { ...TERMINAL_DARK_THEME, background: '#111111' }, + } + const refreshedProfile = { + ...storedProfile, + palette: { ...storedProfile.palette, background: '#222222' }, + } + + it('uses newly discovered colors for the active source profile', () => { + expect(refreshSelectedTerminalProfile([refreshedProfile], storedProfile)).toBe(refreshedProfile) + }) + + it('keeps built-in and unavailable profile selections unchanged', () => { + expect(refreshSelectedTerminalProfile([refreshedProfile], 'app')).toBe('app') + expect(refreshSelectedTerminalProfile([], storedProfile)).toBe(storedProfile) + }) +}) + describe('loadDesktopTerminalAppearance', () => { it('returns a cached profile selection without waiting for source discovery', async () => { const selectedProfile = { diff --git a/apps/sim/lib/desktop/appearance.ts b/apps/sim/lib/desktop/appearance.ts index 2702c3c0983..00caae6e4bf 100644 --- a/apps/sim/lib/desktop/appearance.ts +++ b/apps/sim/lib/desktop/appearance.ts @@ -4,7 +4,10 @@ import { isDesktopAppearanceTheme, isDesktopZoomPercent, isTerminalAppearanceTheme, + TERMINAL_DARK_THEME, + TERMINAL_LIGHT_THEME, type TerminalAppearanceTheme, + type TerminalThemePalette, type TerminalThemeProfile, } from '@sim/desktop-bridge' import { getDesktopBridge } from '@/lib/desktop' @@ -66,6 +69,15 @@ export function withSelectedProfile( : profiles } +/** Replaces a persisted profile snapshot with freshly discovered source colors. */ +export function refreshSelectedTerminalProfile( + profiles: TerminalThemeProfile[], + theme: TerminalAppearanceTheme +): TerminalAppearanceTheme { + if (typeof theme === 'string') return theme + return profiles.find(({ id }) => id === theme.id) ?? theme +} + /** * Resolves `app` against next-themes' raw or resolved value. `system` stays * meaningful for browser CDP; terminal callers treat it as the light fallback @@ -78,3 +90,22 @@ export function resolveDesktopAppearanceTheme( if (preference !== 'app') return preference return appTheme === 'light' || appTheme === 'dark' || appTheme === 'system' ? appTheme : 'system' } + +/** + * Resolves built-in and imported terminal palettes against Sim's live + * appearance. Imported profiles always follow the app appearance — they carry + * their own colors, so there is no separate preference to pin them to. + */ +export function resolveTerminalThemePalette( + theme: TerminalAppearanceTheme, + appTheme: string | undefined +): TerminalThemePalette { + if (typeof theme !== 'string') { + return resolveDesktopAppearanceTheme('app', appTheme) === 'dark' + ? (theme.darkPalette ?? theme.palette) + : (theme.lightPalette ?? theme.palette) + } + return resolveDesktopAppearanceTheme(theme, appTheme) === 'dark' + ? TERMINAL_DARK_THEME + : TERMINAL_LIGHT_THEME +} diff --git a/apps/sim/lib/desktop/index.ts b/apps/sim/lib/desktop/index.ts index 1d31fb67cbf..f902743f021 100644 --- a/apps/sim/lib/desktop/index.ts +++ b/apps/sim/lib/desktop/index.ts @@ -55,6 +55,16 @@ export function hasDesktopSettings(): boolean { return isDesktopApp() } +/** + * True when an internal link must navigate the current view rather than open a + * second one. The shell has no tab strip, so its window-open policy routes a + * same-origin `window.open` to a full new Sim window — where a browser would + * have added a background tab, the desktop app throws up another window. + */ +export function prefersInPlaceNavigation(): boolean { + return isDesktopApp() +} + /** * The device switches for the browser and terminal, cached because the chat UI * reads availability synchronously while the shell only answers over async diff --git a/apps/sim/lib/terminal/transport.test.ts b/apps/sim/lib/terminal/transport.test.ts index b1ad29b544c..637f8b95c5a 100644 --- a/apps/sim/lib/terminal/transport.test.ts +++ b/apps/sim/lib/terminal/transport.test.ts @@ -11,6 +11,7 @@ const { markScopeSuspended, migrateStoreScope, nativeMigrateScope, + nativeReorderTerminal, onCommand, onData, onDefaultZoomChanged, @@ -38,6 +39,7 @@ const { markScopeSuspended: vi.fn(), migrateStoreScope: vi.fn(), nativeMigrateScope: vi.fn(), + nativeReorderTerminal: vi.fn(), onCommand: vi.fn(), onData: vi.fn(() => vi.fn()), onDefaultZoomChanged: vi.fn(() => vi.fn()), @@ -68,6 +70,7 @@ vi.mock('@/lib/desktop', () => ({ onTabs, onScopeSuspended, openTerminal: vi.fn(), + reorderTerminal: nativeReorderTerminal, resize: vi.fn(), start: vi.fn(), switchTerminal: vi.fn(), @@ -99,6 +102,7 @@ import { onTerminalData, onTerminalDefaultZoomChanged, onTerminalShortcutCommand, + reorderTerminal, suspendTerminalScope, writeToTerminal, } from '@/lib/terminal/transport' @@ -119,6 +123,7 @@ describe('terminal transport chat scopes', () => { markScopeSuspended.mockClear() migrateStoreScope.mockClear() nativeMigrateScope.mockReset() + nativeReorderTerminal.mockReset() write.mockClear() }) @@ -179,6 +184,12 @@ describe('terminal transport chat scopes', () => { expect(write).toHaveBeenCalledWith('same-id', 'ls\r', 'chat-b') }) + it('forwards terminal tab reordering with the explicit chat scope', async () => { + await reorderTerminal('terminal-b', 2, 'chat-b') + + expect(nativeReorderTerminal).toHaveBeenCalledWith('terminal-b', 2, 'chat-b') + }) + it('clears retained terminal output in the explicit chat scope', async () => { await expect(clearTerminalScrollback('same-id', 'chat-b')).resolves.toBe(true) diff --git a/apps/sim/lib/terminal/transport.ts b/apps/sim/lib/terminal/transport.ts index f48c02534ab..260aaa53b5e 100644 --- a/apps/sim/lib/terminal/transport.ts +++ b/apps/sim/lib/terminal/transport.ts @@ -153,14 +153,19 @@ export function onTerminalData( } /** - * Tells the desktop app whether a terminal owns keyboard focus. Menu - * accelerators are global, so Cmd-W has to know whether the user is typing in - * a shell before it decides what to close. + * Tells the desktop app whether the visible terminal resource owns shortcut + * routing. Menu accelerators are global, so this claim must survive transient + * focus moves through the panel chrome. */ export function reportTerminalFocused(focused: boolean, scopeId = currentTerminalScopeId()): void { bridge()?.setFocused(focused, scopeId) } +/** Tells the shell which terminal panel is visibly open for tab shortcuts. */ +export function reportTerminalVisible(visible: boolean, scopeId = currentTerminalScopeId()): void { + bridge()?.setVisible?.(visible, scopeId) +} + /** Subscribes to menu shortcuts routed to one focused terminal scope. */ export function onTerminalShortcutCommand( callback: (command: TerminalShortcutCommand) => void, @@ -252,8 +257,10 @@ export function resizeTerminal( export async function openTerminal( cwd?: string, scopeId = currentTerminalScopeId() -): Promise { - await bridge()?.openTerminal(cwd, scopeId) +): Promise { + const terminal = bridge() + if (!terminal) throw new Error('The Sim desktop terminal is unavailable.') + return terminal.openTerminal(cwd, scopeId) } export async function switchTerminal( @@ -263,6 +270,15 @@ export async function switchTerminal( await bridge()?.switchTerminal(terminalId, scopeId) } +/** Moves a terminal tab when the installed shell supports ordering. */ +export async function reorderTerminal( + terminalId: string, + targetIndex: number, + scopeId = currentTerminalScopeId() +): Promise { + await bridge()?.reorderTerminal?.(terminalId, targetIndex, scopeId) +} + export async function closeTerminal( terminalId: string, scopeId = currentTerminalScopeId() diff --git a/apps/sim/stores/browser-session/store.test.ts b/apps/sim/stores/browser-session/store.test.ts index 6cf466cad74..31f9c10822b 100644 --- a/apps/sim/stores/browser-session/store.test.ts +++ b/apps/sim/stores/browser-session/store.test.ts @@ -6,6 +6,10 @@ function resetStore(): void { pageState: null, tabs: [], activeTabId: null, + automationTabId: null, + automationActive: false, + automationNeedsAttention: false, + agentRunIds: [], sessionAlive: true, suspended: false, } @@ -72,6 +76,96 @@ describe('browser session store', () => { expect(getBrowserSession('chat-test').sessionAlive).toBe(false) }) + it('reorders tabs optimistically without changing the active page', () => { + const store = useBrowserSessionStore.getState() + store.setTabsState({ + scopeId: 'chat-test', + activeTabId: '2', + tabs: [ + { + tabId: '1', + title: 'One', + url: 'https://one.example', + loading: false, + active: false, + pinned: false, + }, + { + tabId: '2', + title: 'Two', + url: 'https://two.example', + loading: false, + active: true, + pinned: false, + }, + ], + }) + + store.reorderTab('chat-test', '2', 0) + + expect(getBrowserSession('chat-test').tabs.map((tab) => tab.tabId)).toEqual(['2', '1']) + expect(getBrowserSession('chat-test').activeTabId).toBe('2') + expect(getBrowserSession('chat-test').pageState?.tabId).toBe('2') + }) + + it('retains a settled tab title when opening a new tab pushes a temporary blank title', () => { + const store = useBrowserSessionStore.getState() + store.setTabsState({ + scopeId: 'chat-test', + activeTabId: '1', + tabs: [ + { + tabId: '1', + title: 'Example docs', + url: 'https://example.com/docs', + loading: false, + active: true, + pinned: false, + }, + ], + }) + + // Electron publishes the new active page before its following full-list + // push. The new id is not in the renderer's old list yet. + store.setPageState({ + tabId: '2', + scopeId: 'chat-test', + title: '', + url: '', + loading: false, + canGoBack: false, + canGoForward: false, + }) + + store.setTabsState({ + scopeId: 'chat-test', + activeTabId: '2', + tabs: [ + { + tabId: '1', + title: '', + url: 'https://example.com/docs', + loading: false, + active: false, + pinned: false, + }, + { + tabId: '2', + title: '', + url: '', + loading: false, + active: true, + pinned: false, + }, + ], + }) + + expect(getBrowserSession('chat-test').tabs).toMatchObject([ + { tabId: '1', title: 'Example docs', active: false }, + { tabId: '2', title: '', active: true }, + ]) + }) + it('keeps overlapping tab ids isolated while chats switch', () => { const store = useBrowserSessionStore.getState() store.activateScope('chat-a') @@ -147,6 +241,85 @@ describe('browser session store', () => { expect(getBrowserSession('chat-1').pageState?.url).toBe('https://pending.example') }) + it('keeps browser-agent activity across tool gaps and clears it by exact run', () => { + const store = useBrowserSessionStore.getState() + + store.setTabsState({ + scopeId: 'chat-test', + activeTabId: 'tab-1', + tabs: [ + { + tabId: 'tab-1', + title: 'Current page', + url: 'https://example.com', + loading: false, + active: true, + pinned: false, + }, + ], + }) + + store.setAgentRunActive('chat-test', 'browser-run-1', true) + store.setAgentRunActive('chat-test', 'browser-run-2', true) + expect(getBrowserSession('chat-test').agentRunIds).toEqual(['browser-run-1', 'browser-run-2']) + expect(getBrowserSession('chat-test').automationTabId).toBe('tab-1') + + store.setTabsState({ + scopeId: 'chat-test', + activeTabId: 'tab-1', + automationTabId: 'tab-1', + automationActive: true, + tabs: getBrowserSession('chat-test').tabs, + }) + store.setTabsState({ + scopeId: 'chat-test', + activeTabId: 'tab-1', + automationTabId: null, + automationActive: false, + tabs: getBrowserSession('chat-test').tabs, + }) + expect(getBrowserSession('chat-test').automationTabId).toBe('tab-1') + + store.setAgentRunActive('ignored-after-migration', 'browser-run-1', false) + expect(getBrowserSession('chat-test').agentRunIds).toEqual(['browser-run-2']) + + store.clearAgentRuns('chat-test') + expect(getBrowserSession('chat-test').agentRunIds).toEqual([]) + expect(getBrowserSession('chat-test').automationTabId).toBeNull() + }) + + it('hard-settles an old stream without clearing a newer browser run', () => { + const store = useBrowserSessionStore.getState() + store.setTabsState({ + scopeId: 'chat-test', + activeTabId: 'tab-1', + automationTabId: 'tab-1', + automationActive: true, + automationNeedsAttention: true, + tabs: [ + { + tabId: 'tab-1', + title: 'Current page', + url: 'https://example.com', + loading: false, + active: true, + pinned: false, + }, + ], + }) + store.setAgentRunActive('chat-test', 'browser-run-old', true) + store.setAgentRunActive('chat-test', 'browser-run-new', true) + + store.clearAgentRunIds(['browser-run-old'], { hardResetScopeIds: ['chat-test'] }) + + expect(getBrowserSession('chat-test')).toMatchObject({ + agentRunIds: ['browser-run-new'], + automationTabId: 'tab-1', + automationActive: false, + automationNeedsAttention: false, + }) + }) + it('replaces a pristine durable bucket created before pending migration finishes', () => { const store = useBrowserSessionStore.getState() store.setPageState({ diff --git a/apps/sim/stores/browser-session/store.ts b/apps/sim/stores/browser-session/store.ts index fba79ecd93e..19afa105071 100644 --- a/apps/sim/stores/browser-session/store.ts +++ b/apps/sim/stores/browser-session/store.ts @@ -14,6 +14,11 @@ export interface BrowserSessionData { /** All live tabs in this browser scope. */ tabs: BrowserTabState[] activeTabId: string | null + automationTabId: string | null + automationActive: boolean + automationNeedsAttention: boolean + /** Browser-subagent spans currently working in this chat scope. */ + agentRunIds: string[] /** False after this chat's browser session ends; true again when a new one starts. */ sessionAlive: boolean /** Live views were administratively stopped while the restart descriptor was retained. */ @@ -29,6 +34,13 @@ interface BrowserSessionState { suspendScope: (scopeId: string) => void setPageState: (state: BrowserPageState) => void setTabsState: (state: BrowserTabsState) => void + setAgentRunActive: (scopeId: string, runId: string, active: boolean) => void + clearAgentRuns: (scopeId: string) => void + clearAgentRunIds: ( + runIds: readonly string[], + options?: { hardResetScopeIds?: readonly string[] } + ) => void + reorderTab: (scopeId: string, tabId: string, targetIndex: number) => void setSessionAlive: (alive: boolean, scopeId: string) => void } @@ -37,6 +49,10 @@ function createInitialSession(): BrowserSessionData { pageState: null, tabs: [], activeTabId: null, + automationTabId: null, + automationActive: false, + automationNeedsAttention: false, + agentRunIds: [], sessionAlive: true, suspended: false, } @@ -54,7 +70,11 @@ function isPristineSession(session: BrowserSessionData): boolean { !session.suspended && session.pageState === null && session.tabs.length === 0 && - session.activeTabId === null + session.activeTabId === null && + session.automationTabId === null && + !session.automationActive && + !session.automationNeedsAttention && + session.agentRunIds.length === 0 ) } @@ -74,6 +94,31 @@ function tabsEqual(a: BrowserTabState[], b: BrowserTabState[]): boolean { return a.length === b.length && a.every((tab, index) => tabFieldsEqual(tab, b[index])) } +/** + * A full native tab-list push can briefly report an empty title for a settled + * background WebContents even though its richer page-state push already gave + * us the title. Keep that known title only while the tab remains on the same + * URL and is not loading; navigation is still free to replace it. + */ +function retainSettledTabTitles( + currentTabs: BrowserTabState[], + incomingTabs: BrowserTabState[] +): BrowserTabState[] { + const currentById = new Map(currentTabs.map((tab) => [tab.tabId, tab])) + return incomingTabs.map((incoming) => { + const current = currentById.get(incoming.tabId) + if ( + incoming.title.trim() === '' && + !incoming.loading && + current?.url === incoming.url && + current.title.trim() !== '' + ) { + return { ...incoming, title: current.title } + } + return incoming + }) +} + function pageStateEqual(a: BrowserPageState | null, b: BrowserPageState | null): boolean { if (a === b) return true if (!a || !b) return false @@ -118,6 +163,10 @@ export const useBrowserSessionStore = create()( current.pageState === null && current.tabs.length === 0 && current.activeTabId === null && + current.automationTabId === null && + !current.automationActive && + !current.automationNeedsAttention && + current.agentRunIds.length === 0 && !current.sessionAlive ) { return current @@ -127,6 +176,10 @@ export const useBrowserSessionStore = create()( pageState: null, tabs: [], activeTabId: null, + automationTabId: null, + automationActive: false, + automationNeedsAttention: false, + agentRunIds: [], sessionAlive: false, suspended: true, } @@ -173,8 +226,17 @@ export const useBrowserSessionStore = create()( const { scopeId } = tabsState return withSession(state, scopeId, (current) => { if (current.suspended) return current - const tabs = tabsEqual(current.tabs, tabsState.tabs) ? current.tabs : tabsState.tabs + const incomingTabs = retainSettledTabTitles(current.tabs, tabsState.tabs) + const tabs = tabsEqual(current.tabs, incomingTabs) ? current.tabs : incomingTabs const activeTab = tabs.find((tab) => tab.tabId === tabsState.activeTabId) + const reportedAutomationTabId = tabsState.automationTabId ?? null + const automationTabId = + reportedAutomationTabId ?? + (current.agentRunIds.length > 0 + ? tabs.some((tab) => tab.tabId === current.automationTabId) + ? current.automationTabId + : tabsState.activeTabId + : null) const hasCurrentPageState = current.pageState?.tabId !== undefined && current.pageState.tabId === tabsState.activeTabId @@ -195,6 +257,9 @@ export const useBrowserSessionStore = create()( if ( tabs === current.tabs && tabsState.activeTabId === current.activeTabId && + automationTabId === current.automationTabId && + (tabsState.automationActive ?? false) === current.automationActive && + (tabsState.automationNeedsAttention ?? false) === current.automationNeedsAttention && sessionAlive === current.sessionAlive && pageState === current.pageState ) { @@ -204,11 +269,108 @@ export const useBrowserSessionStore = create()( ...current, tabs, activeTabId: tabsState.activeTabId, + automationTabId, + automationActive: tabsState.automationActive ?? false, + automationNeedsAttention: tabsState.automationNeedsAttention ?? false, sessionAlive, pageState, } }) }), + setAgentRunActive: (scopeId, runId, active) => + set((state) => { + if (!runId) return state + if (!active) { + let changed = false + const sessions = Object.fromEntries( + Object.entries(state.sessions).map(([id, session]) => { + if (!session.agentRunIds.includes(runId)) return [id, session] + changed = true + const agentRunIds = session.agentRunIds.filter((entry) => entry !== runId) + return [ + id, + { + ...session, + agentRunIds, + automationTabId: + agentRunIds.length === 0 && !session.automationActive + ? null + : session.automationTabId, + }, + ] + }) + ) + return changed ? { sessions } : state + } + return withSession(state, scopeId, (current) => { + if (current.suspended || current.agentRunIds.includes(runId)) return current + return { + ...current, + agentRunIds: [...current.agentRunIds, runId], + automationTabId: current.automationTabId ?? current.activeTabId, + } + }) + }), + clearAgentRuns: (scopeId) => + set((state) => + withSession(state, scopeId, (current) => + current.agentRunIds.length === 0 + ? current + : { + ...current, + agentRunIds: [], + automationTabId: current.automationActive ? current.automationTabId : null, + } + ) + ), + clearAgentRunIds: (runIds, options) => + set((state) => { + const ids = new Set(runIds) + const hardResetScopes = new Set(options?.hardResetScopeIds ?? []) + if (ids.size === 0 && hardResetScopes.size === 0) return {} + let changed = false + const sessions = Object.fromEntries( + Object.entries(state.sessions).map(([id, session]) => { + const agentRunIds = session.agentRunIds.filter((runId) => !ids.has(runId)) + const hardResetActivity = hardResetScopes.has(id) + if (agentRunIds.length === session.agentRunIds.length && !hardResetActivity) { + return [id, session] + } + changed = true + return [ + id, + { + ...session, + agentRunIds, + ...(hardResetActivity + ? { automationActive: false, automationNeedsAttention: false } + : {}), + automationTabId: + agentRunIds.length === 0 && (hardResetActivity || !session.automationActive) + ? null + : session.automationTabId, + }, + ] + }) + ) + return changed ? { sessions } : {} + }), + reorderTab: (scopeId, tabId, targetIndex) => + set((state) => + withSession(state, scopeId, (current) => { + const currentIndex = current.tabs.findIndex((tab) => tab.tabId === tabId) + if (currentIndex < 0 || !Number.isFinite(targetIndex)) return current + const nextIndex = Math.max( + 0, + Math.min(current.tabs.length - 1, Math.trunc(targetIndex)) + ) + if (currentIndex === nextIndex) return current + const tabs = [...current.tabs] + const [tab] = tabs.splice(currentIndex, 1) + tabs.splice(nextIndex, 0, tab) + return { ...current, tabs } + }) + ), setSessionAlive: (alive, scopeId) => set((state) => { return withSession(state, scopeId, (current) => { @@ -220,7 +382,11 @@ export const useBrowserSessionStore = create()( !current.sessionAlive && current.pageState === null && current.tabs.length === 0 && - current.activeTabId === null + current.activeTabId === null && + current.automationTabId === null && + !current.automationActive && + !current.automationNeedsAttention && + current.agentRunIds.length === 0 ) { return current } @@ -230,6 +396,10 @@ export const useBrowserSessionStore = create()( pageState: null, tabs: [], activeTabId: null, + automationTabId: null, + automationActive: false, + automationNeedsAttention: false, + agentRunIds: [], } }) }), diff --git a/apps/sim/stores/copilot-terminal/store.test.ts b/apps/sim/stores/copilot-terminal/store.test.ts index 9683d5e7763..f6a557b1fbe 100644 --- a/apps/sim/stores/copilot-terminal/store.test.ts +++ b/apps/sim/stores/copilot-terminal/store.test.ts @@ -31,14 +31,10 @@ function activateTestScope() { describe('copilot terminal store', () => { beforeEach(() => { - const session = { - tabs: { tabs: [], activeTerminalId: null }, - agentCommandIds: [], - suspended: false, - } useCopilotTerminalStore.setState({ activeScopeId: null, sessions: {}, + settledAgentCommandIds: [], }) }) @@ -135,13 +131,47 @@ describe('copilot terminal store', () => { expect(useCopilotTerminalStore.getState().activeScopeId).toBe('chat-b') expect(getCopilotTerminalSession('chat-b').tabs.tabs[0].title).toBe('B') - expect(getCopilotTerminalSession('chat-b').agentCommandIds).toEqual([]) - expect(getCopilotTerminalSession('chat-a').agentCommandIds).toEqual(['tool-a']) + expect(getCopilotTerminalSession('chat-b').agentCommandTerminalIds).toEqual({}) + expect(getCopilotTerminalSession('chat-a').agentCommandTerminalIds).toEqual({ + 'tool-a': 't1', + }) store.activateScope('chat-a') expect(useCopilotTerminalStore.getState().activeScopeId).toBe('chat-a') expect(getCopilotTerminalSession('chat-a').tabs.tabs[0].title).toBe('A') - expect(getCopilotTerminalSession('chat-a').agentCommandIds).toEqual(['tool-a']) + expect(getCopilotTerminalSession('chat-a').agentCommandTerminalIds).toEqual({ + 'tool-a': 't1', + }) + }) + + it('tracks each running command on its exact terminal and clears closed targets', () => { + const store = activateTestScope() + store.setTabs(tabsState([tab(), tab({ terminalId: 't2', title: 'two', active: false })], 't1')) + store.applyCommandEvent({ + scopeId: TEST_SCOPE, + terminalId: 't1', + phase: 'start', + command: 'bun test', + toolCallId: 'tool-a', + }) + store.applyCommandEvent({ + scopeId: TEST_SCOPE, + terminalId: 't2', + phase: 'start', + command: 'bun dev', + toolCallId: 'tool-b', + }) + + expect(getCopilotTerminalSession(TEST_SCOPE).agentCommandTerminalIds).toEqual({ + 'tool-a': 't1', + 'tool-b': 't2', + }) + + store.setTabs(tabsState([tab({ terminalId: 't2', title: 'two' })], 't2')) + + expect(getCopilotTerminalSession(TEST_SCOPE).agentCommandTerminalIds).toEqual({ + 'tool-b': 't2', + }) }) it('moves pending terminals onto the resolved chat id', () => { @@ -199,11 +229,64 @@ describe('copilot terminal store', () => { expect(getCopilotTerminalSession('chat-a')).toEqual({ tabs: { tabs: [], activeTerminalId: null }, - agentCommandIds: [], + agentCommandTerminalIds: {}, + activityResetEpoch: 1, suspended: true, }) }) + it('stale-settles exact commands without resetting a newer command in the same chat', () => { + const store = useCopilotTerminalStore.getState() + store.activateScope(TEST_SCOPE) + store.applyCommandEvent({ + scopeId: TEST_SCOPE, + terminalId: 't1', + phase: 'start', + command: 'old', + toolCallId: 'tool-old', + }) + store.applyCommandEvent({ + scopeId: TEST_SCOPE, + terminalId: 't1', + phase: 'start', + command: 'new', + toolCallId: 'tool-new', + }) + + store.clearAgentCommands(TEST_SCOPE, ['tool-old'], { hardResetActivity: false }) + + expect(getCopilotTerminalSession(TEST_SCOPE)).toMatchObject({ + agentCommandTerminalIds: { 'tool-new': 't1' }, + activityResetEpoch: 0, + }) + }) + + it('ignores a delayed native start after its stream hard-settles', () => { + const store = useCopilotTerminalStore.getState() + store.activateScope(TEST_SCOPE) + + store.clearAgentCommands(TEST_SCOPE, ['tool-late'], { hardResetActivity: true }) + expect(getCopilotTerminalSession(TEST_SCOPE).activityResetEpoch).toBe(1) + store.applyCommandEvent({ + scopeId: TEST_SCOPE, + terminalId: 't1', + phase: 'start', + command: 'late', + toolCallId: 'tool-late', + }) + + expect(getCopilotTerminalSession(TEST_SCOPE).agentCommandTerminalIds).toEqual({}) + + store.applyCommandEvent({ + scopeId: TEST_SCOPE, + terminalId: 't1', + phase: 'end', + command: 'late', + toolCallId: 'tool-late', + }) + expect(useCopilotTerminalStore.getState().settledAgentCommandIds).not.toContain('tool-late') + }) + it('clears suspension on explicit activation, including the already-active scope', () => { const store = useCopilotTerminalStore.getState() store.activateScope('chat-a') diff --git a/apps/sim/stores/copilot-terminal/store.ts b/apps/sim/stores/copilot-terminal/store.ts index 96ba030a2fd..119fb73e388 100644 --- a/apps/sim/stores/copilot-terminal/store.ts +++ b/apps/sim/stores/copilot-terminal/store.ts @@ -15,8 +15,10 @@ import { export interface CopilotTerminalSessionData { tabs: TerminalTabsState - /** Tool call ids whose commands the agent is currently running. */ - agentCommandIds: string[] + /** Exact terminal targeted by each currently running agent command. */ + agentCommandTerminalIds: Record + /** Remount key for immediately settling activity chrome at a stream boundary. */ + activityResetEpoch: number /** Live PTYs were stopped while the restart descriptor was retained. */ suspended: boolean } @@ -32,18 +34,28 @@ export interface CopilotTerminalSessionData { interface CopilotTerminalState { activeScopeId: string | null sessions: Record + /** Recently hard-settled tool ids whose delayed native starts must be ignored. */ + settledAgentCommandIds: string[] activateScope: (scopeId: string) => void migrateScope: (fromScopeId: string, toScopeId: string) => void discardScope: (scopeId: string) => void suspendScope: (scopeId: string) => void setTabs: (tabs: ScopedTerminalTabsState) => void applyCommandEvent: (event: ScopedTerminalCommandEvent) => void + clearAgentCommands: ( + scopeId: string, + toolCallIds: readonly string[], + options: { hardResetActivity: boolean } + ) => void } +const MAX_SETTLED_AGENT_COMMAND_IDS = 256 + function createInitialSession(): CopilotTerminalSessionData { return { tabs: { tabs: [], activeTerminalId: null }, - agentCommandIds: [], + agentCommandTerminalIds: {}, + activityResetEpoch: 0, suspended: false, } } @@ -60,7 +72,7 @@ function isPristineSession(session: CopilotTerminalSessionData): boolean { !session.suspended && session.tabs.tabs.length === 0 && session.tabs.activeTerminalId === null && - session.agentCommandIds.length === 0 + Object.keys(session.agentCommandTerminalIds).length === 0 ) } @@ -72,6 +84,7 @@ function isPristineSession(session: CopilotTerminalSessionData): boolean { */ function tabsEqual(a: TerminalTabsState, b: TerminalTabsState): boolean { if (a.activeTerminalId !== b.activeTerminalId) return false + if ((a.agentActiveTerminalId ?? null) !== (b.agentActiveTerminalId ?? null)) return false if (a.tabs.length !== b.tabs.length) return false return a.tabs.every((tab, index) => tabEqual(tab, b.tabs[index])) } @@ -104,6 +117,7 @@ export const useCopilotTerminalStore = create()( (set) => ({ activeScopeId: null, sessions: {}, + settledAgentCommandIds: [], activateScope: (scopeId) => set((state) => activateScopedSession(state, scopeId, createInitialSession)), migrateScope: (fromScopeId, toScopeId) => @@ -116,39 +130,109 @@ export const useCopilotTerminalStore = create()( current.suspended && current.tabs.tabs.length === 0 && current.tabs.activeTerminalId === null && - current.agentCommandIds.length === 0 + Object.keys(current.agentCommandTerminalIds).length === 0 ) { return current } return { tabs: { tabs: [], activeTerminalId: null }, - agentCommandIds: [], + agentCommandTerminalIds: {}, + activityResetEpoch: (current.activityResetEpoch ?? 0) + 1, suspended: true, } }) ), setTabs: (tabs) => set((state) => { - return withSession(state, tabs.scopeId, (current) => - current.suspended || tabsEqual(current.tabs, tabs) ? current : { ...current, tabs } - ) + return withSession(state, tabs.scopeId, (current) => { + if (current.suspended) return current + const liveTerminalIds = new Set(tabs.tabs.map((tab) => tab.terminalId)) + const agentCommandTerminalIds = Object.fromEntries( + Object.entries(current.agentCommandTerminalIds).filter(([, terminalId]) => + liveTerminalIds.has(terminalId) + ) + ) + const commandsUnchanged = + Object.keys(agentCommandTerminalIds).length === + Object.keys(current.agentCommandTerminalIds).length + const nextTabs = tabsEqual(current.tabs, tabs) ? current.tabs : tabs + if (nextTabs === current.tabs && commandsUnchanged) return current + return { ...current, tabs: nextTabs, agentCommandTerminalIds } + }) }), applyCommandEvent: (event) => set((state) => { const toolCallId = event.toolCallId if (!toolCallId) return {} - return withSession(state, event.scopeId, (current) => { + const settledIds = state.settledAgentCommandIds ?? [] + const wasHardSettled = settledIds.includes(toolCallId) + if (event.phase === 'start' && wasHardSettled) return {} + const sessionUpdate = withSession(state, event.scopeId, (current) => { if (current.suspended) return current - const agentCommandIds = - event.phase === 'start' - ? current.agentCommandIds.includes(toolCallId) - ? current.agentCommandIds - : [...current.agentCommandIds, toolCallId] - : current.agentCommandIds.filter((id) => id !== toolCallId) - return agentCommandIds === current.agentCommandIds - ? current - : { ...current, agentCommandIds } + if (event.phase === 'start') { + if (current.agentCommandTerminalIds[toolCallId] === event.terminalId) return current + return { + ...current, + agentCommandTerminalIds: { + ...current.agentCommandTerminalIds, + [toolCallId]: event.terminalId, + }, + } + } + if (!(toolCallId in current.agentCommandTerminalIds)) return current + const { [toolCallId]: _finished, ...agentCommandTerminalIds } = + current.agentCommandTerminalIds + return { ...current, agentCommandTerminalIds } }) + if (!wasHardSettled) return sessionUpdate + return { + ...sessionUpdate, + settledAgentCommandIds: settledIds.filter((settledId) => settledId !== toolCallId), + } + }), + clearAgentCommands: (scopeId, toolCallIds, options) => + set((state) => { + const ids = new Set(toolCallIds) + const previousSettledIds = state.settledAgentCommandIds ?? [] + const settledAgentCommandIds = [ + ...previousSettledIds.filter((toolCallId) => !ids.has(toolCallId)), + ...ids, + ].slice(-MAX_SETTLED_AGENT_COMMAND_IDS) + let changed = false + const sessions = Object.fromEntries( + Object.entries(state.sessions).map(([id, session]) => { + const agentCommandTerminalIds = Object.fromEntries( + Object.entries(session.agentCommandTerminalIds).filter( + ([toolCallId]) => !ids.has(toolCallId) + ) + ) + const commandsChanged = + Object.keys(agentCommandTerminalIds).length !== + Object.keys(session.agentCommandTerminalIds).length + const resetActivity = options.hardResetActivity && (id === scopeId || commandsChanged) + if (!commandsChanged && !resetActivity) return [id, session] + changed = true + return [ + id, + { + ...session, + agentCommandTerminalIds, + ...(resetActivity + ? { activityResetEpoch: (session.activityResetEpoch ?? 0) + 1 } + : {}), + }, + ] + }) + ) + const settledChanged = + settledAgentCommandIds.length !== previousSettledIds.length || + settledAgentCommandIds.some( + (toolCallId, index) => toolCallId !== previousSettledIds[index] + ) + return { + ...(changed ? { sessions } : {}), + ...(settledChanged ? { settledAgentCommandIds } : {}), + } }), }), { name: 'copilot-terminal-store' } diff --git a/bun.lock b/bun.lock index 6fdf0020645..f951e9d18e3 100644 --- a/bun.lock +++ b/bun.lock @@ -1,6 +1,5 @@ { "lockfileVersion": 1, - "configVersion": 0, "workspaces": { "": { "name": "simstudio", diff --git a/packages/browser-protocol/src/index.ts b/packages/browser-protocol/src/index.ts index 4517ecddf16..5158764dd0b 100644 --- a/packages/browser-protocol/src/index.ts +++ b/packages/browser-protocol/src/index.ts @@ -136,7 +136,7 @@ export interface BrowserPanelSnapshot { /** * Browser-chrome commands from the panel header (URL bar, back/forward, - * reload) plus `takeover-done`, sent by the Done chip on the chat's + * reload) plus `takeover-done`, sent by the question card on the chat's * `browser_request_takeover` tool row when the user finishes a * hand-control-back request. Page interactions need no protocol — the user * acts on the real embedded page directly, and its right-click menu is native @@ -161,6 +161,8 @@ export interface BrowserPanelAction { url?: string /** Stable tab id for `duplicate-tab`, `switch-tab`, and `close-tab`. */ tabId?: string + /** Optional free-text instruction submitted with `takeover-done`. */ + takeoverResponse?: string } /** Live state of the active page, pushed to the panel header. */ @@ -225,6 +227,12 @@ export interface BrowserTabState { export interface BrowserTabsState { tabs: BrowserTabState[] activeTabId: string | null + /** Tab currently driven by the agent when it differs from the user's visible tab. */ + automationTabId?: string | null + /** True while a browser tool is actively driving that tab. */ + automationActive?: boolean + /** True while automation is paused for the user on this tab. */ + automationNeedsAttention?: boolean /** Chat scope that owns this tab set. */ scopeId: string } diff --git a/packages/desktop-bridge/src/index.ts b/packages/desktop-bridge/src/index.ts index f6c3b759c1d..425a65669e7 100644 --- a/packages/desktop-bridge/src/index.ts +++ b/packages/desktop-bridge/src/index.ts @@ -76,6 +76,12 @@ export interface SimDesktopTerminalApi { /** Open an additional terminal and make it active. */ openTerminal(cwd: string | undefined, scopeId: string): Promise switchTerminal(terminalId: string, scopeId: string): Promise + /** Move a terminal to its final position. Optional for older installed shells. */ + reorderTerminal?( + terminalId: string, + targetIndex: number, + scopeId: string + ): Promise closeTerminal(terminalId: string, scopeId: string): Promise getTabs(scopeId: string): Promise /** Makes a chat's terminal group the renderer-visible group. */ @@ -99,11 +105,12 @@ export interface SimDesktopTerminalApi { /** Forget retained output for one terminal. */ clearScrollback(terminalId: string, scopeId: string): Promise /** - * Reports whether the terminal panel owns keyboard focus, so global menu - * accelerators can tell a Cmd-W meant for a terminal from one meant for the - * window. + * Reports whether the visible terminal panel owns resource shortcuts, so a + * transient DOM blur cannot turn Cmd-W into a window-level command. */ setFocused(focused: boolean, scopeId: string): void + /** Reports whether this renderer is currently displaying the terminal resource. */ + setVisible?(visible: boolean, scopeId: string): void /** * The user finishing a handoff — the hand-back chip on the waiting tool row. */ @@ -141,8 +148,17 @@ export interface SimDesktopBrowserAgentApi { params: Record, scopeId: string ): Promise - /** Browser-chrome commands from the panel (URL bar, back, reload, takeover Done). */ + /** Cancel one exact in-flight tool. Optional for compatibility with older shells. */ + cancelTool?(toolCallId: string, scopeId: string): Promise + /** Cancel the currently active tool in a scope after renderer state was lost. */ + cancelActiveTool?(scopeId: string): Promise + /** Browser-chrome commands from the panel (URL bar, back, reload, takeover hand-back). */ panelAction(action: BrowserPanelAction, scopeId: string): void + /** + * Create and activate a blank tab, returning the authoritative list. + * Optional for compatibility with installed shells that predate acknowledged tab creation. + */ + openTab?(scopeId: string): Promise /** Makes a chat's browser tab set the renderer-visible set. */ activateScope(scopeId: string): Promise /** Materializes a lazily activated chat's persisted tabs without showing its panel. */ @@ -731,7 +747,15 @@ export interface TerminalSelectedProfile { id: string name: string source: TerminalThemeSource + /** + * Palette used when the source does not provide appearance-specific colors. + * Ignored once both `lightPalette` and `darkPalette` are present. + */ palette: TerminalThemePalette + /** Optional palette used while Sim is in light appearance. */ + lightPalette?: TerminalThemePalette + /** Optional palette used while Sim is in dark appearance. */ + darkPalette?: TerminalThemePalette } export type TerminalThemeProfile = TerminalSelectedProfile @@ -744,41 +768,60 @@ const TERMINAL_THEME_PALETTE_KEYS: readonly (keyof TerminalThemePalette)[] = [ ...TERMINAL_THEME_ANSI_KEYS, ] +const TERMINAL_THEME_OPTIONAL_PALETTE_KEYS = ['cursorAccent', 'selectionForeground'] as const + const TERMINAL_THEME_COLOR_PATTERN = /^#[0-9a-f]{6}$/i +function isTerminalThemeColor(value: unknown): value is string { + return typeof value === 'string' && TERMINAL_THEME_COLOR_PATTERN.test(value) +} + +function isTerminalThemePalette(value: unknown): value is TerminalThemePalette { + if (typeof value !== 'object' || value === null) return false + const palette = value as Partial + return ( + TERMINAL_THEME_PALETTE_KEYS.every((key) => isTerminalThemeColor(palette[key])) && + TERMINAL_THEME_OPTIONAL_PALETTE_KEYS.every( + (key) => palette[key] === undefined || isTerminalThemeColor(palette[key]) + ) + ) +} + export function isTerminalSelectedProfile(value: unknown): value is TerminalSelectedProfile { if (typeof value !== 'object' || value === null) return false const candidate = value as Partial - if ( - typeof candidate.id !== 'string' || - candidate.id.length === 0 || - candidate.id.length > 300 || - typeof candidate.name !== 'string' || - candidate.name.length === 0 || - candidate.name.length > 200 || - (candidate.source !== 'terminal' && candidate.source !== 'iterm2') || - typeof candidate.palette !== 'object' || - candidate.palette === null - ) { - return false - } - if ( - !TERMINAL_THEME_PALETTE_KEYS.every( - (key) => - typeof candidate.palette?.[key] === 'string' && - TERMINAL_THEME_COLOR_PATTERN.test(candidate.palette[key]) - ) - ) { - return false - } - return (['cursorAccent', 'selectionForeground'] as const).every( - (key) => - candidate.palette?.[key] === undefined || - (typeof candidate.palette[key] === 'string' && - TERMINAL_THEME_COLOR_PATTERN.test(candidate.palette[key])) + return ( + typeof candidate.id === 'string' && + candidate.id.length > 0 && + candidate.id.length <= 300 && + typeof candidate.name === 'string' && + candidate.name.length > 0 && + candidate.name.length <= 200 && + (candidate.source === 'terminal' || candidate.source === 'iterm2') && + isTerminalThemePalette(candidate.palette) && + (candidate.lightPalette === undefined || isTerminalThemePalette(candidate.lightPalette)) && + (candidate.darkPalette === undefined || isTerminalThemePalette(candidate.darkPalette)) ) } +/** + * Copies only the known profile fields, so untrusted source output and stored + * config never carry extra keys. The single definition of a profile's shape — + * new palette slots are added here rather than at each call site. + */ +export function cloneTerminalSelectedProfile( + profile: TerminalSelectedProfile +): TerminalSelectedProfile { + return { + id: profile.id, + name: profile.name, + source: profile.source, + palette: { ...profile.palette }, + ...(profile.lightPalette ? { lightPalette: { ...profile.lightPalette } } : {}), + ...(profile.darkPalette ? { darkPalette: { ...profile.darkPalette } } : {}), + } +} + export interface DesktopPreferences { notificationsEnabled: boolean notificationSounds: boolean diff --git a/packages/emcn/src/components/index.ts b/packages/emcn/src/components/index.ts index de3ce13c392..1b23439bf8b 100644 --- a/packages/emcn/src/components/index.ts +++ b/packages/emcn/src/components/index.ts @@ -184,7 +184,9 @@ export { TabStrip, type TabStripItem, type TabStripProps, + type TabStripSelectionSource, tabDropIndex, + tabStripWheelPosition, } from './tab-strip/tab-strip' export { Table, diff --git a/packages/emcn/src/components/tab-strip/tab-strip.dom.test.tsx b/packages/emcn/src/components/tab-strip/tab-strip.dom.test.tsx new file mode 100644 index 00000000000..ccdec5a762a --- /dev/null +++ b/packages/emcn/src/components/tab-strip/tab-strip.dom.test.tsx @@ -0,0 +1,186 @@ +/** + * @vitest-environment jsdom + */ +import { act, type ReactNode } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { TabStrip, type TabStripItem } from './tab-strip' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount(ui: ReactNode): void { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render(ui)) +} + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +const tabs: TabStripItem[] = [ + { id: 'pinned', title: 'Pinned', pinned: true }, + { id: 'one', title: 'One', active: true }, + { id: 'two', title: 'Two' }, +] + +function renderStrip(items: TabStripItem[], onSelect = vi.fn(), onClose = vi.fn()): ReactNode { + return {}} /> +} + +function tabButton(id: string): HTMLButtonElement { + const button = container?.querySelector(`[data-tab-strip-button="${id}"]`) + if (!button) throw new Error(`Missing tab button ${id}`) + return button +} + +function scrollRow(): HTMLDivElement { + const row = container?.querySelector('.overflow-x-auto') + if (!row) throw new Error('Missing scrolling tab row') + return row +} + +describe('TabStrip interactions', () => { + it('uses one keyboard tab stop and exposes tab semantics', () => { + mount(renderStrip(tabs)) + + expect(container?.querySelector('[role="tablist"]')).not.toBeNull() + expect(tabButton('one').getAttribute('aria-selected')).toBe('true') + expect(tabButton('one').tabIndex).toBe(0) + expect(tabButton('two').tabIndex).toBe(-1) + expect(container?.querySelector('[aria-label="Close Two"]')?.tabIndex).toBe( + -1 + ) + }) + + it('cycles, jumps, and closes from the keyboard', () => { + const onSelect = vi.fn() + const onClose = vi.fn() + mount(renderStrip(tabs, onSelect, onClose)) + + act(() => { + tabButton('one').dispatchEvent( + new KeyboardEvent('keydown', { key: 'ArrowRight', bubbles: true, cancelable: true }) + ) + }) + expect(onSelect).toHaveBeenCalledWith('two', 'keyboard') + expect(document.activeElement).toBe(tabButton('two')) + + act(() => { + tabButton('two').dispatchEvent( + new KeyboardEvent('keydown', { key: 'Home', bubbles: true, cancelable: true }) + ) + }) + expect(onSelect).toHaveBeenLastCalledWith('pinned', 'keyboard') + + act(() => { + tabButton('two').dispatchEvent( + new KeyboardEvent('keydown', { key: 'Delete', bubbles: true, cancelable: true }) + ) + }) + expect(onClose).toHaveBeenCalledWith('two') + }) + + it('identifies pointer selection separately from keyboard navigation', () => { + const onSelect = vi.fn() + mount(renderStrip(tabs, onSelect)) + + act(() => tabButton('two').click()) + + expect(onSelect).toHaveBeenCalledWith('two', 'pointer') + }) + + it('closes an unpinned tab with the middle mouse button', () => { + const onClose = vi.fn() + mount(renderStrip(tabs, vi.fn(), onClose)) + + act(() => { + tabButton('two').parentElement?.dispatchEvent( + new MouseEvent('auxclick', { button: 1, bubbles: true, cancelable: true }) + ) + }) + + expect(onClose).toHaveBeenCalledWith('two') + }) + + it('forwards a wheel gesture from the new-tab button to the scrolling row', () => { + mount(renderStrip(tabs)) + const row = scrollRow() + Object.defineProperties(row, { + clientWidth: { configurable: true, value: 100 }, + scrollWidth: { configurable: true, value: 400 }, + scrollLeft: { configurable: true, value: 0, writable: true }, + }) + const newTab = container?.querySelector('[aria-label="New tab"]') + const event = new WheelEvent('wheel', { deltaY: 80, bubbles: true, cancelable: true }) + + act(() => newTab?.dispatchEvent(event)) + + expect(row.scrollLeft).toBe(80) + expect(event.defaultPrevented).toBe(true) + }) + + it('keeps pinned tabs out of the scrolling lane', () => { + mount(renderStrip(tabs)) + + expect(scrollRow().contains(tabButton('pinned'))).toBe(false) + expect(scrollRow().contains(tabButton('one'))).toBe(true) + }) + + it('shows background activity without marking that tab selected', () => { + mount(renderStrip(tabs.map((tab) => ({ ...tab, attention: tab.id === 'two' })))) + + expect(tabButton('two').querySelector('[aria-label="Background activity"]')).not.toBeNull() + expect(tabButton('two').getAttribute('aria-selected')).toBe('false') + expect(tabButton('one').querySelector('[aria-label="Background activity"]')).toBeNull() + }) + + it('does not reserve phantom space after a pointer close', () => { + const regularTabs = tabs.filter((tab) => !tab.pinned) + mount(renderStrip(regularTabs)) + + act(() => { + container + ?.querySelector('[aria-label="Close Two"]') + ?.dispatchEvent(new MouseEvent('click', { bubbles: true })) + root?.render(renderStrip(regularTabs.slice(0, 1))) + }) + + expect(scrollRow().querySelector('[data-tab-width-lock]')).toBeNull() + }) + + it('puts a new tab in its final layout immediately', () => { + const regularTabs = tabs.filter((tab) => !tab.pinned) + mount(renderStrip(regularTabs.slice(0, 1))) + + act(() => root?.render(renderStrip(regularTabs))) + + const openedTab = scrollRow().querySelector('[data-tab-strip-item="two"]') + expect(openedTab?.style.width).toBe('') + expect(openedTab?.style.minWidth).toBe('') + }) + + it('reveals a newly active offscreen tab', () => { + mount(renderStrip(tabs)) + const row = scrollRow() + Object.defineProperties(row, { + clientWidth: { configurable: true, value: 100 }, + scrollWidth: { configurable: true, value: 300 }, + scrollLeft: { configurable: true, value: 0, writable: true }, + }) + row.getBoundingClientRect = () => ({ left: 0, right: 100, width: 100 }) as DOMRect + const second = tabButton('two').parentElement as HTMLDivElement + second.getBoundingClientRect = () => ({ left: 180, right: 280, width: 100 }) as DOMRect + row.scrollTo = vi.fn() + + act(() => root?.render(renderStrip(tabs.map((tab) => ({ ...tab, active: tab.id === 'two' }))))) + + expect(row.scrollTo).toHaveBeenCalledWith({ left: 180, behavior: 'smooth' }) + }) +}) diff --git a/packages/emcn/src/components/tab-strip/tab-strip.test.ts b/packages/emcn/src/components/tab-strip/tab-strip.test.ts index 370185fcc7d..c501ce8cea6 100644 --- a/packages/emcn/src/components/tab-strip/tab-strip.test.ts +++ b/packages/emcn/src/components/tab-strip/tab-strip.test.ts @@ -1,18 +1,37 @@ import { readFileSync } from 'node:fs' import { describe, expect, it } from 'vitest' -import { isTabTitleTruncated, type TabStripItem, tabDropIndex } from './tab-strip' +import { + isTabTitleTruncated, + type TabStripItem, + tabDropIndex, + tabStripWheelPosition, +} from './tab-strip' describe('isTabTitleTruncated', () => { it('shows title help only after a meaningful amount of text is clipped', () => { expect(isTabTitleTruncated({ clientWidth: 100, scrollWidth: 140 })).toBe(true) - expect(isTabTitleTruncated({ clientWidth: 100, scrollWidth: 131 })).toBe(false) - expect(isTabTitleTruncated({ clientWidth: 160, scrollWidth: 199 })).toBe(false) + expect(isTabTitleTruncated({ clientWidth: 100, scrollWidth: 108 })).toBe(true) + expect(isTabTitleTruncated({ clientWidth: 160, scrollWidth: 167 })).toBe(false) expect(isTabTitleTruncated({ clientWidth: 160, scrollWidth: 200 })).toBe(true) expect(isTabTitleTruncated({ clientWidth: 100, scrollWidth: 100 })).toBe(false) expect(isTabTitleTruncated({ clientWidth: 120, scrollWidth: 80 })).toBe(false) }) }) +describe('tabStripWheelPosition', () => { + it('uses native horizontal deltas and falls back to vertical wheel movement', () => { + expect(tabStripWheelPosition(20, 500, 200, 100, 40)).toBe(120) + expect(tabStripWheelPosition(20, 500, 200, 0, 100)).toBe(120) + }) + + it('clamps at each edge and declines gestures that cannot move', () => { + expect(tabStripWheelPosition(280, 500, 200, 0, 50)).toBe(300) + expect(tabStripWheelPosition(300, 500, 200, 0, 50)).toBeNull() + expect(tabStripWheelPosition(0, 500, 200, -50, 0)).toBeNull() + expect(tabStripWheelPosition(0, 200, 200, 10, 0)).toBeNull() + }) +}) + describe('tabDropIndex', () => { const tabs: TabStripItem[] = [ { id: 'pinned-1', title: 'pinned-1', pinned: true }, @@ -103,6 +122,6 @@ describe('tab strip vertical overflow', () => { expect(negativeBottomMargins).toHaveLength(1) const scrollRow = markup.match(/className='([^']*overflow-x-auto[^']*)'/)?.[1] ?? '' - expect(scrollRow).toContain('-mb-px') + expect(scrollRow).not.toContain('-mb-px') }) }) diff --git a/packages/emcn/src/components/tab-strip/tab-strip.tsx b/packages/emcn/src/components/tab-strip/tab-strip.tsx index f6f6e4e90e5..f373a518cea 100644 --- a/packages/emcn/src/components/tab-strip/tab-strip.tsx +++ b/packages/emcn/src/components/tab-strip/tab-strip.tsx @@ -1,19 +1,29 @@ 'use client' import { + forwardRef, type DragEvent as ReactDragEvent, + type KeyboardEvent as ReactKeyboardEvent, type MouseEvent as ReactMouseEvent, type ReactNode, useCallback, + useEffect, useLayoutEffect, + useMemo, useRef, useState, } from 'react' +import { AnimatePresence, motion, useReducedMotion } from 'framer-motion' import { Plus, X } from '../../icons' import { cn } from '../../lib/cn' import { Button } from '../button/button' import { Tooltip } from '../tooltip/tooltip' +const DRAG_EDGE_ZONE = 40 +const DRAG_SCROLL_SPEED = 8 +const TITLE_TOOLTIP_HIDDEN_PX = 8 +const TAB_TRANSITION = { duration: 0.1, ease: [0.2, 0, 0, 1] as const } + /** One tab in a {@link TabStrip}. */ export interface TabStripItem { id: string @@ -37,11 +47,16 @@ export interface TabStripItem { * when the title is actually cut off. */ tooltip?: string + /** Shows that background work is happening in a tab the user is not viewing. */ + attention?: boolean } +/** How a tab selection was initiated. */ +export type TabStripSelectionSource = 'pointer' | 'keyboard' + export interface TabStripProps { tabs: TabStripItem[] - onSelect: (id: string) => void + onSelect: (id: string, source?: TabStripSelectionSource) => void /** Omit to make tabs uncloseable. Never offered for a pinned tab. */ onClose?: (id: string) => void /** Omit to hide the new-tab button. */ @@ -64,14 +79,30 @@ export interface TabStripProps { /** * Whether a title is clipped enough to be worth a tooltip. A couple of hidden - * pixels is not, and a tooltip on every tab is noise. + * pixels is not, but a tab should not lose a meaningful part of its identity + * before it explains itself. */ export function isTabTitleTruncated( element: Pick ): boolean { const hiddenWidth = element.scrollWidth - element.clientWidth - const tooltipThreshold = Math.max(32, element.clientWidth * 0.25) - return hiddenWidth >= tooltipThreshold + return hiddenWidth >= TITLE_TOOLTIP_HIDDEN_PX +} + +/** Final horizontal position for a wheel gesture, or null when it cannot move the strip. */ +export function tabStripWheelPosition( + scrollLeft: number, + scrollWidth: number, + clientWidth: number, + deltaX: number, + deltaY: number +): number | null { + const maxScrollLeft = Math.max(0, scrollWidth - clientWidth) + if (maxScrollLeft === 0) return null + const delta = Math.abs(deltaX) >= Math.abs(deltaY) ? deltaX : deltaY + if (delta === 0) return null + const next = Math.max(0, Math.min(maxScrollLeft, scrollLeft + delta)) + return next === scrollLeft ? null : next } /** @@ -100,35 +131,38 @@ export function tabDropIndex( interface TabProps { tab: TabStripItem - index: number - onSelect: (id: string) => void + onSelect: (id: string, source?: TabStripSelectionSource) => void onClose?: (id: string) => void onContextMenu?: (event: ReactMouseEvent, id: string) => void + onKeyDown: (event: ReactKeyboardEvent, id: string) => void draggable: boolean dragging: boolean + focusable: boolean showDropBefore: boolean showDropAfter: boolean + reduceMotion: boolean onDragStart: (event: ReactDragEvent, id: string) => void - onDragOver: (event: ReactDragEvent, index: number) => void - onDragLeave: (event: ReactDragEvent) => void onDragEnd: () => void } -function Tab({ - tab, - index, - onSelect, - onClose, - onContextMenu, - draggable, - dragging, - showDropBefore, - showDropAfter, - onDragStart, - onDragOver, - onDragLeave, - onDragEnd, -}: TabProps) { +const Tab = forwardRef(function Tab( + { + tab, + onSelect, + onClose, + onContextMenu, + onKeyDown, + draggable, + dragging, + focusable, + showDropBefore, + showDropAfter, + reduceMotion, + onDragStart, + onDragEnd, + }, + ref +) { const titleRef = useRef(null) const [titleTruncated, setTitleTruncated] = useState(false) const closeable = Boolean(onClose) && !tab.pinned @@ -145,7 +179,12 @@ function Tab({ }, [tab.title]) return ( -
onDragStart(event, tab.id)} - onDragOver={(event) => onDragOver(event, index)} - onDragLeave={onDragLeave} - onDragEnd={onDragEnd} + onDragStartCapture={(event) => onDragStart(event, tab.id)} + onDragEndCapture={onDragEnd} onContextMenu={(event) => onContextMenu?.(event, tab.id)} + onAuxClick={(event) => { + if (event.button !== 1 || !closeable) return + event.preventDefault() + onClose?.(tab.id) + }} > {showDropBefore && (
@@ -177,16 +220,20 @@ function Tab({ type='button' variant='subtle' size='sm' - aria-current={tab.active ? 'page' : undefined} + role='tab' + aria-selected={Boolean(tab.active)} aria-label={tab.pinned ? tab.title : undefined} + data-tab-strip-button={tab.id} + tabIndex={focusable ? 0 : -1} className={cn( 'h-[30px] w-full select-none rounded-b-none border border-transparent border-b-0 bg-transparent py-0 text-caption', tab.pinned ? 'justify-center px-0' : 'justify-start gap-1.5 px-2', - closeable && !tab.pinned && 'pr-7', + closeable && !tab.pinned && 'pr-8', tab.active && 'hover-hover:!border-[var(--border)] hover-hover:!bg-[var(--bg)] hover-hover:!text-[var(--text-primary)] hover-hover:!brightness-100 hover-hover:!opacity-100 relative z-10 border-[var(--border)] bg-[var(--bg)] text-[var(--text-primary)] transition-none' )} - onClick={() => onSelect(tab.id)} + onClick={() => onSelect(tab.id, 'pointer')} + onKeyDown={(event) => onKeyDown(event, tab.id)} > {tab.icon} {!tab.pinned && ( @@ -194,6 +241,15 @@ function Tab({ {tab.title} )} + {tab.attention && !tab.active && ( + + )} {(tab.tooltip || tab.pinned || titleTruncated) && ( @@ -206,9 +262,12 @@ function Tab({ variant='ghost-secondary' size='sm' aria-label={`Close ${tab.title}`} + tabIndex={-1} className={cn( - 'absolute top-[5px] right-1 z-20 size-[20px] p-0 transition-opacity', - tab.active ? 'opacity-100' : 'opacity-0 group-hover:opacity-100' + 'absolute top-[3px] right-0.5 z-20 size-[24px] p-0 transition-opacity', + tab.active + ? 'opacity-100' + : 'opacity-0 group-focus-within:opacity-100 group-hover:opacity-100' )} onClick={(event) => { event.stopPropagation() @@ -218,9 +277,9 @@ function Tab({ )} -
+ ) -} +}) /** * Chrome-style tab strip, shared by every panel that hosts multiple live @@ -245,18 +304,119 @@ export function TabStrip({ children, }: TabStripProps) { const atLimit = maxTabs !== undefined && tabs.length >= maxTabs + const stripRef = useRef(null) + const scrollNodeRef = useRef(null) const draggedIdRef = useRef(null) const dropTargetIndexRef = useRef(null) + const autoScrollRafRef = useRef(null) + const autoScrollDirectionRef = useRef(0) const [draggedId, setDraggedId] = useState(null) const [dropTargetIndex, setDropTargetIndex] = useState(null) + const [canScrollLeft, setCanScrollLeft] = useState(false) + const [canScrollRight, setCanScrollRight] = useState(false) + const reduceMotion = useReducedMotion() ?? false const reorderable = Boolean(onReorder) + const pinnedTabs = useMemo(() => tabs.filter((tab) => tab.pinned), [tabs]) + const regularTabs = useMemo(() => tabs.filter((tab) => !tab.pinned), [tabs]) + const activeRegularId = regularTabs.find((tab) => tab.active)?.id ?? null + const regularTabOrder = regularTabs.map((tab) => tab.id).join('\u0000') + const activeIndex = tabs.findIndex((tab) => tab.active) + + const updateOverflow = useCallback(() => { + const node = scrollNodeRef.current + if (!node) { + setCanScrollLeft(false) + setCanScrollRight(false) + return + } + const maxScrollLeft = Math.max(0, node.scrollWidth - node.clientWidth) + setCanScrollLeft(node.scrollLeft > 1) + setCanScrollRight(node.scrollLeft < maxScrollLeft - 1) + }, []) + + const stopAutoScroll = useCallback(() => { + if (autoScrollRafRef.current !== null) cancelAnimationFrame(autoScrollRafRef.current) + autoScrollRafRef.current = null + autoScrollDirectionRef.current = 0 + }, []) const resetDrag = useCallback(() => { + stopAutoScroll() draggedIdRef.current = null dropTargetIndexRef.current = null setDraggedId(null) setDropTargetIndex(null) - }, []) + }, [stopAutoScroll]) + + useEffect(() => resetDrag, [resetDrag]) + + const revealActiveTab = useCallback(() => { + const node = scrollNodeRef.current + if (!node || !activeRegularId) return + const element = Array.from(node.querySelectorAll('[data-tab-strip-item]')).find( + (candidate) => candidate.dataset.tabStripItem === activeRegularId + ) + if (!element) return + const tabRect = element.getBoundingClientRect() + const nodeRect = node.getBoundingClientRect() + const tabLeft = tabRect.left - nodeRect.left + node.scrollLeft + const tabRight = tabLeft + tabRect.width + const nextLeft = + tabLeft < node.scrollLeft + ? tabLeft + : tabRight > node.scrollLeft + node.clientWidth + ? tabRight - node.clientWidth + : null + if (nextLeft === null) return + const reduceMotion = window.matchMedia?.('(prefers-reduced-motion: reduce)').matches ?? false + node.scrollTo({ left: nextLeft, behavior: reduceMotion ? 'auto' : 'smooth' }) + }, [activeRegularId, regularTabOrder]) + + useLayoutEffect(() => { + revealActiveTab() + }, [revealActiveTab]) + + useLayoutEffect(() => { + const node = scrollNodeRef.current + if (!node) return + const updateLayout = () => { + updateOverflow() + revealActiveTab() + } + updateLayout() + node.addEventListener('scroll', updateOverflow, { passive: true }) + if (typeof ResizeObserver === 'undefined') { + return () => node.removeEventListener('scroll', updateOverflow) + } + const observer = new ResizeObserver(updateLayout) + observer.observe(node) + return () => { + observer.disconnect() + node.removeEventListener('scroll', updateOverflow) + } + }, [regularTabs.length, revealActiveTab, updateOverflow]) + + useEffect(() => { + const strip = stripRef.current + if (!strip) return + const handleWheel = (event: WheelEvent) => { + const node = scrollNodeRef.current + if (!node) return + const next = tabStripWheelPosition( + node.scrollLeft, + node.scrollWidth, + node.clientWidth, + event.deltaX, + event.deltaY + ) + if (next === null) return + node.scrollLeft = next + updateOverflow() + event.preventDefault() + } + strip.addEventListener('wheel', handleWheel, { passive: false }) + return () => strip.removeEventListener('wheel', handleWheel) + }, [updateOverflow]) const handleDragStart = useCallback( (event: ReactDragEvent, id: string) => { @@ -280,19 +440,58 @@ export function TabStrip({ [reorderable, onTabDragStart] ) + const startEdgeScroll = useCallback( + (clientX: number) => { + const node = scrollNodeRef.current + if (!node) return + const dragged = tabs.find((tab) => tab.id === draggedIdRef.current) + if (dragged?.pinned) { + stopAutoScroll() + return + } + const rect = node.getBoundingClientRect() + const direction = + clientX < rect.left + DRAG_EDGE_ZONE ? -1 : clientX > rect.right - DRAG_EDGE_ZONE ? 1 : 0 + if (direction !== 0 && autoScrollDirectionRef.current === direction) return + stopAutoScroll() + if (direction === 0) return + autoScrollDirectionRef.current = direction + const tick = () => { + const before = node.scrollLeft + node.scrollLeft += direction * DRAG_SCROLL_SPEED + updateOverflow() + if (node.scrollLeft === before) { + autoScrollRafRef.current = null + autoScrollDirectionRef.current = 0 + return + } + autoScrollRafRef.current = requestAnimationFrame(tick) + } + autoScrollRafRef.current = requestAnimationFrame(tick) + }, + [stopAutoScroll, tabs, updateOverflow] + ) + const handleDragOver = useCallback( - (event: ReactDragEvent, index: number) => { + (event: ReactDragEvent) => { const id = draggedIdRef.current if (!reorderable || !id) return event.preventDefault() - event.dataTransfer.dropEffect = 'move' - const rect = event.currentTarget.getBoundingClientRect() - const gapIndex = event.clientX < rect.left + rect.width / 2 ? index : index + 1 - const targetIndex = tabDropIndex(tabs, id, gapIndex) + const strip = stripRef.current + if (!strip) return + const elements = Array.from(strip.querySelectorAll('[data-tab-strip-item]')) + const gapIndex = elements.findIndex((element) => { + const rect = element.getBoundingClientRect() + return event.clientX < rect.left + rect.width / 2 + }) + const resolvedGapIndex = gapIndex < 0 ? elements.length : gapIndex + const targetIndex = tabDropIndex(tabs, id, resolvedGapIndex) + event.dataTransfer.dropEffect = targetIndex === null ? 'none' : 'move' dropTargetIndexRef.current = targetIndex setDropTargetIndex(targetIndex) + startEdgeScroll(event.clientX) }, - [reorderable, tabs] + [reorderable, startEdgeScroll, tabs] ) const handleDrop = useCallback( @@ -308,8 +507,91 @@ export function TabStrip({ const draggedIndex = tabs.findIndex((tab) => tab.id === draggedId) + const focusTab = useCallback((id: string) => { + const strip = stripRef.current + const button = strip + ? Array.from(strip.querySelectorAll('[data-tab-strip-button]')).find( + (candidate) => candidate.dataset.tabStripButton === id + ) + : null + button?.focus() + }, []) + + const handleTabKeyDown = useCallback( + (event: ReactKeyboardEvent, id: string) => { + const index = tabs.findIndex((tab) => tab.id === id) + if (index < 0) return + let target: TabStripItem | undefined + switch (event.key) { + case 'ArrowLeft': + target = tabs[(index - 1 + tabs.length) % tabs.length] + break + case 'ArrowRight': + target = tabs[(index + 1) % tabs.length] + break + case 'Home': + target = tabs[0] + break + case 'End': + target = tabs[tabs.length - 1] + break + case 'Delete': + if (onClose && !tabs[index].pinned) { + event.preventDefault() + onClose(id) + } + return + default: + return + } + if (!target) return + event.preventDefault() + onSelect(target.id, 'keyboard') + focusTab(target.id) + }, + [focusTab, onClose, onSelect, tabs] + ) + + const renderTab = (tab: TabStripItem) => { + const index = tabs.findIndex((candidate) => candidate.id === tab.id) + return ( + = 0 && draggedIndex > index} + showDropAfter={dropTargetIndex === index && draggedIndex >= 0 && draggedIndex < index} + reduceMotion={reduceMotion} + onSelect={onSelect} + {...(onClose ? { onClose } : {})} + {...(onTabContextMenu ? { onContextMenu: onTabContextMenu } : {})} + onKeyDown={handleTabKeyDown} + onDragStart={handleDragStart} + onDragEnd={resetDrag} + /> + ) + } + return ( -
+
{ + if ( + event.relatedTarget instanceof Node && + event.currentTarget.contains(event.relatedTarget) + ) { + return + } + stopAutoScroll() + dropTargetIndexRef.current = null + setDropTargetIndex(null) + }} + onDrop={handleDrop} + > {/* The row is sized by its tabs rather than filling the strip, so the new-tab button that follows sits beside the last tab instead of against the far @@ -326,39 +608,33 @@ export function TabStrip({ row down instead keeps the tabs flush inside it, so there is nothing to scroll. */}
{ - if (draggedIdRef.current) event.preventDefault() - }} - onDrop={handleDrop} + role='tablist' + aria-label='Tabs' + className='-mb-px flex min-w-0 shrink items-end gap-0.5' > - {tabs.map((tab, index) => ( - = 0 && draggedIndex > index} - showDropAfter={dropTargetIndex === index && draggedIndex >= 0 && draggedIndex < index} - onSelect={onSelect} - {...(onClose ? { onClose } : {})} - {...(onTabContextMenu ? { onContextMenu: onTabContextMenu } : {})} - onDragStart={handleDragStart} - onDragOver={handleDragOver} - onDragLeave={(event) => { - if ( - event.relatedTarget instanceof Node && - event.currentTarget.contains(event.relatedTarget) - ) { - return - } - dropTargetIndexRef.current = null - setDropTargetIndex(null) - }} - onDragEnd={resetDrag} - /> - ))} + {pinnedTabs.length > 0 && ( +
+ + {pinnedTabs.map(renderTab)} + +
+ )} +
+
+ + {regularTabs.map(renderTab)} + +
+ {canScrollLeft && ( +
+ )} + {canScrollRight && ( +
+ )} +
{onNew && ( @@ -368,7 +644,7 @@ export function TabStrip({ variant='ghost-secondary' size='sm' aria-label={newTabLabel} - className='mb-px size-[28px] shrink-0 p-0' + className='mb-px size-[30px] shrink-0 p-0' disabled={atLimit} onClick={onNew} > diff --git a/packages/terminal-protocol/src/index.ts b/packages/terminal-protocol/src/index.ts index 65441bf31a6..e1fcc101bee 100644 --- a/packages/terminal-protocol/src/index.ts +++ b/packages/terminal-protocol/src/index.ts @@ -332,6 +332,8 @@ export interface TerminalPanesResult { export interface TerminalTabsState { tabs: TerminalTabState[] activeTerminalId: string | null + /** Terminal currently driven by the agent when it differs from the user's visible terminal. */ + agentActiveTerminalId?: string | null } /** A tab strip crossing the desktop bridge, tagged with its owning chat. */