From 444fd8badd265ddde33babf2a21bf3f527334e98 Mon Sep 17 00:00:00 2001 From: Dara Adedeji <76637177+SunkenInTime@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:33:09 -0400 Subject: [PATCH 01/15] fix(mobile): keep Android markdown icons aligned with text (#11079) --- .../t3-markdown-text/android/build.gradle | 12 +++++ .../T3MarkdownTextSelectionModule.kt | 14 ++++-- .../MarkdownSelectionCopyTest.kt | 48 +++++++++++++++++++ .../src/NativeMarkdownBlock.ios.tsx | 35 ++++++++++---- .../src/NativeMarkdownSelectableText.ios.tsx | 4 ++ .../src/SelectableMarkdownText.ios.tsx | 6 ++- .../src/nativeMarkdownText.ts | 14 ++++-- .../mobile/src/lib/nativeMarkdownText.test.ts | 6 +++ docs/user/composer.md | 3 ++ 9 files changed, 120 insertions(+), 22 deletions(-) create mode 100644 apps/mobile/modules/t3-markdown-text/android/src/test/java/expo/modules/t3markdowntext/MarkdownSelectionCopyTest.kt diff --git a/apps/mobile/modules/t3-markdown-text/android/build.gradle b/apps/mobile/modules/t3-markdown-text/android/build.gradle index 13584a00be42..5b7e372006f3 100644 --- a/apps/mobile/modules/t3-markdown-text/android/build.gradle +++ b/apps/mobile/modules/t3-markdown-text/android/build.gradle @@ -8,6 +8,10 @@ android { namespace 'expo.modules.t3markdowntext' compileSdk rootProject.ext.compileSdkVersion + testOptions { + unitTests.includeAndroidResources = true + } + defaultConfig { minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion @@ -17,4 +21,12 @@ android { dependencies { implementation project(':expo-modules-core') implementation 'com.facebook.react:react-android' + testImplementation 'junit:junit:4.13.2' + testImplementation 'org.robolectric:robolectric:4.16.1' +} + +tasks.withType(Test).configureEach { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(21) + } } diff --git a/apps/mobile/modules/t3-markdown-text/android/src/main/java/expo/modules/t3markdowntext/T3MarkdownTextSelectionModule.kt b/apps/mobile/modules/t3-markdown-text/android/src/main/java/expo/modules/t3markdowntext/T3MarkdownTextSelectionModule.kt index af8675831f2d..c68b8e641aab 100644 --- a/apps/mobile/modules/t3-markdown-text/android/src/main/java/expo/modules/t3markdowntext/T3MarkdownTextSelectionModule.kt +++ b/apps/mobile/modules/t3-markdown-text/android/src/main/java/expo/modules/t3markdowntext/T3MarkdownTextSelectionModule.kt @@ -18,19 +18,23 @@ import kotlin.math.min private const val OBJECT_REPLACEMENT_CHARACTER = "\uFFFC" -private fun copyTextWithoutInlineImages( +internal fun copyTextWithoutInlineImages( text: CharSequence, start: Int, end: Int ): String { if (text !is Spanned) return text.subSequence(start, end).toString() + fun isInlineImage(index: Int): Boolean = + index >= 0 && text[index].toString() == OBJECT_REPLACEMENT_CHARACTER && + text.getSpans(index, index + 1, ReplacementSpan::class.java).isNotEmpty() + return buildString { for (index in start until end) { - val isInlineImage = - text[index].toString() == OBJECT_REPLACEMENT_CHARACTER && - text.getSpans(index, index + 1, ReplacementSpan::class.java).isNotEmpty() - if (!isInlineImage) append(text[index]) + // The renderer inserts one NBSP after each image to keep its label on the same line. + // Inspect the original text even when selection starts after the image. + val isIconSpacer = text[index] == '\u00A0' && isInlineImage(index - 1) + if (!isInlineImage(index) && !isIconSpacer) append(text[index]) } } } diff --git a/apps/mobile/modules/t3-markdown-text/android/src/test/java/expo/modules/t3markdowntext/MarkdownSelectionCopyTest.kt b/apps/mobile/modules/t3-markdown-text/android/src/test/java/expo/modules/t3markdowntext/MarkdownSelectionCopyTest.kt new file mode 100644 index 000000000000..da9012ee1665 --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/android/src/test/java/expo/modules/t3markdowntext/MarkdownSelectionCopyTest.kt @@ -0,0 +1,48 @@ +package expo.modules.t3markdowntext + +import android.graphics.drawable.ColorDrawable +import android.text.SpannableString +import android.text.Spanned +import android.text.style.ImageSpan +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [36], manifest = Config.NONE) +class MarkdownSelectionCopyTest { + private fun withIcon(value: String): SpannableString = SpannableString(value).apply { + val index = value.indexOf('\uFFFC') + setSpan(ImageSpan(ColorDrawable()), index, index + 1, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE) + } + + @Test + fun removesIconAndInjectedSpacer() { + val text = withIcon("\uFFFC\u00A0main.go:12 starts the server.") + assertEquals("main.go:12 starts the server.", copyTextWithoutInlineImages(text, 0, text.length)) + } + + @Test + fun removesSpacerWhenSelectionStartsAfterIcon() { + val text = withIcon("\uFFFC\u00A0main.go:12 starts the server.") + assertEquals("main.go:12", copyTextWithoutInlineImages(text, 1, 12)) + } + + @Test + fun preservesAuthoredWhitespaceAndLiteralObjectCharacters() { + val text = withIcon("before\u00A0 \uFFFC\u00A0\u00A0 main.go after\u00A0\uFFFC\u00A0") + assertEquals( + "before\u00A0 \u00A0 main.go after\u00A0\uFFFC\u00A0", + copyTextWithoutInlineImages(text, 0, text.length) + ) + } + + @Test + fun preservesTextWithoutImageSpans() { + val text = "\uFFFC\u00A0main.go" + assertEquals(text, copyTextWithoutInlineImages(text, 0, text.length)) + assertEquals(text, copyTextWithoutInlineImages(SpannableString(text), 0, text.length)) + } +} diff --git a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.ios.tsx b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.ios.tsx index 348a3c489a2c..3aa7e1cf4e46 100644 --- a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.ios.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.ios.tsx @@ -4,7 +4,11 @@ import type { MarkdownNode } from "react-native-nitro-markdown/headless"; import { CopyTextButton } from "./CopyTextButton"; import { MarkdownTextPrimitive } from "./MarkdownTextPrimitive"; -import { nativeMarkdownDocumentRuns, nativeMarkdownListItemBlocks } from "./nativeMarkdownText"; +import { + nativeMarkdownBlockSpacing, + nativeMarkdownDocumentRuns, + nativeMarkdownListItemBlocks, +} from "./nativeMarkdownText"; import { NativeMarkdownSelectableText } from "./NativeMarkdownSelectableText.ios"; import type { MarkdownCodeHighlighter, @@ -595,17 +599,28 @@ export function NativeMarkdownBlock(props: { switch (props.node.type) { case "document": return ( - + {(props.node.children ?? []).map((child, index) => ( - + style={{ + paddingTop: + Platform.OS === "android" + ? nativeMarkdownBlockSpacing(props.node.children?.[index - 1], child) + : index > 0 + ? 8 + : 0, + }} + > + + ))} ); diff --git a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.ios.tsx b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.ios.tsx index a5c6cf540f1c..7e1eaf88f546 100644 --- a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.ios.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.ios.tsx @@ -222,6 +222,10 @@ export function NativeMarkdownSelectableText(props: { } } + if (Platform.OS === "android" && (run.fileIcon || linkIcon)) { + text = `\u00A0${text}`; + } + return { key: `${signature}:${occurrence}`, run, text, linkIcon }; }); // T3MarkdownText only rebuilds its attributed string during native layout. A diff --git a/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.ios.tsx b/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.ios.tsx index 2a231c603584..b7768ffbae03 100644 --- a/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.ios.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.ios.tsx @@ -1,5 +1,5 @@ import { useMemo } from "react"; -import { View } from "react-native"; +import { Platform, View } from "react-native"; import { parseMarkdownWithOptions } from "react-native-nitro-markdown/headless"; import { @@ -84,8 +84,10 @@ export function SelectableMarkdownText({ the resulting single-line width instead of reflowing it. */} {chunks.map((chunk, index) => { + // Android inline images drift when one Text mixes paragraph and list + // spacer line heights. Keep those layouts in separate native blocks. const content = - chunk.kind === "rich" ? ( + chunk.kind === "rich" || Platform.OS === "android" ? ( 0) { - const previous = children[index - 1]; - appendSpacer( - runs, - child.type === "heading" ? 20 : previous?.type === "heading" ? 10 : 12, - ); + appendSpacer(runs, nativeMarkdownBlockSpacing(children[index - 1], child)); } appendDocumentBlock(runs, child, depth); } diff --git a/apps/mobile/src/lib/nativeMarkdownText.test.ts b/apps/mobile/src/lib/nativeMarkdownText.test.ts index c3951c8d81f0..1091aeb65a6c 100644 --- a/apps/mobile/src/lib/nativeMarkdownText.test.ts +++ b/apps/mobile/src/lib/nativeMarkdownText.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test"; import type { MarkdownNode } from "react-native-nitro-markdown/headless"; import { + nativeMarkdownBlockSpacing, nativeMarkdownChunkSpacing, nativeMarkdownDocumentChunks, nativeMarkdownDocumentRuns, @@ -381,6 +382,11 @@ describe("nativeMarkdownDocumentRuns", () => { .filter((run) => run.role === "spacer") .map((run) => run.spacing), ).toEqual([20, 10, 12]); + expect( + node.children?.map((child, index) => + nativeMarkdownBlockSpacing(node.children?.[index - 1], child), + ), + ).toEqual([0, 20, 10, 12]); }); it("renders tight list items whose inline nodes are direct children", () => { diff --git a/docs/user/composer.md b/docs/user/composer.md index 4a8df5333664..7e388d4eb34f 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -6,6 +6,9 @@ include a skill when the task needs more context. Messages can contain up to 120,000 characters. Longer drafts stay in the composer so you can shorten them or split them into several messages. +On Android, long-press message text to select within a paragraph or list item. +Use the message’s copy button to copy the whole message. + ## Attach files Attach up to eight files per message. Images can be up to 10 MB; other files can From 3836890e4484406813997259efc69065b25ce698 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 10 Sep 2026 10:35:34 -0700 Subject: [PATCH 02/15] Revert "fix(mobile): keep Android markdown icons aligned with text" (#11098) --- .../t3-markdown-text/android/build.gradle | 12 ----- .../T3MarkdownTextSelectionModule.kt | 14 ++---- .../MarkdownSelectionCopyTest.kt | 48 ------------------- .../src/NativeMarkdownBlock.ios.tsx | 35 ++++---------- .../src/NativeMarkdownSelectableText.ios.tsx | 4 -- .../src/SelectableMarkdownText.ios.tsx | 6 +-- .../src/nativeMarkdownText.ts | 14 ++---- .../mobile/src/lib/nativeMarkdownText.test.ts | 6 --- docs/user/composer.md | 3 -- 9 files changed, 22 insertions(+), 120 deletions(-) delete mode 100644 apps/mobile/modules/t3-markdown-text/android/src/test/java/expo/modules/t3markdowntext/MarkdownSelectionCopyTest.kt diff --git a/apps/mobile/modules/t3-markdown-text/android/build.gradle b/apps/mobile/modules/t3-markdown-text/android/build.gradle index 5b7e372006f3..13584a00be42 100644 --- a/apps/mobile/modules/t3-markdown-text/android/build.gradle +++ b/apps/mobile/modules/t3-markdown-text/android/build.gradle @@ -8,10 +8,6 @@ android { namespace 'expo.modules.t3markdowntext' compileSdk rootProject.ext.compileSdkVersion - testOptions { - unitTests.includeAndroidResources = true - } - defaultConfig { minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion @@ -21,12 +17,4 @@ android { dependencies { implementation project(':expo-modules-core') implementation 'com.facebook.react:react-android' - testImplementation 'junit:junit:4.13.2' - testImplementation 'org.robolectric:robolectric:4.16.1' -} - -tasks.withType(Test).configureEach { - javaLauncher = javaToolchains.launcherFor { - languageVersion = JavaLanguageVersion.of(21) - } } diff --git a/apps/mobile/modules/t3-markdown-text/android/src/main/java/expo/modules/t3markdowntext/T3MarkdownTextSelectionModule.kt b/apps/mobile/modules/t3-markdown-text/android/src/main/java/expo/modules/t3markdowntext/T3MarkdownTextSelectionModule.kt index c68b8e641aab..af8675831f2d 100644 --- a/apps/mobile/modules/t3-markdown-text/android/src/main/java/expo/modules/t3markdowntext/T3MarkdownTextSelectionModule.kt +++ b/apps/mobile/modules/t3-markdown-text/android/src/main/java/expo/modules/t3markdowntext/T3MarkdownTextSelectionModule.kt @@ -18,23 +18,19 @@ import kotlin.math.min private const val OBJECT_REPLACEMENT_CHARACTER = "\uFFFC" -internal fun copyTextWithoutInlineImages( +private fun copyTextWithoutInlineImages( text: CharSequence, start: Int, end: Int ): String { if (text !is Spanned) return text.subSequence(start, end).toString() - fun isInlineImage(index: Int): Boolean = - index >= 0 && text[index].toString() == OBJECT_REPLACEMENT_CHARACTER && - text.getSpans(index, index + 1, ReplacementSpan::class.java).isNotEmpty() - return buildString { for (index in start until end) { - // The renderer inserts one NBSP after each image to keep its label on the same line. - // Inspect the original text even when selection starts after the image. - val isIconSpacer = text[index] == '\u00A0' && isInlineImage(index - 1) - if (!isInlineImage(index) && !isIconSpacer) append(text[index]) + val isInlineImage = + text[index].toString() == OBJECT_REPLACEMENT_CHARACTER && + text.getSpans(index, index + 1, ReplacementSpan::class.java).isNotEmpty() + if (!isInlineImage) append(text[index]) } } } diff --git a/apps/mobile/modules/t3-markdown-text/android/src/test/java/expo/modules/t3markdowntext/MarkdownSelectionCopyTest.kt b/apps/mobile/modules/t3-markdown-text/android/src/test/java/expo/modules/t3markdowntext/MarkdownSelectionCopyTest.kt deleted file mode 100644 index da9012ee1665..000000000000 --- a/apps/mobile/modules/t3-markdown-text/android/src/test/java/expo/modules/t3markdowntext/MarkdownSelectionCopyTest.kt +++ /dev/null @@ -1,48 +0,0 @@ -package expo.modules.t3markdowntext - -import android.graphics.drawable.ColorDrawable -import android.text.SpannableString -import android.text.Spanned -import android.text.style.ImageSpan -import org.junit.Assert.assertEquals -import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -import org.robolectric.annotation.Config - -@RunWith(RobolectricTestRunner::class) -@Config(sdk = [36], manifest = Config.NONE) -class MarkdownSelectionCopyTest { - private fun withIcon(value: String): SpannableString = SpannableString(value).apply { - val index = value.indexOf('\uFFFC') - setSpan(ImageSpan(ColorDrawable()), index, index + 1, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE) - } - - @Test - fun removesIconAndInjectedSpacer() { - val text = withIcon("\uFFFC\u00A0main.go:12 starts the server.") - assertEquals("main.go:12 starts the server.", copyTextWithoutInlineImages(text, 0, text.length)) - } - - @Test - fun removesSpacerWhenSelectionStartsAfterIcon() { - val text = withIcon("\uFFFC\u00A0main.go:12 starts the server.") - assertEquals("main.go:12", copyTextWithoutInlineImages(text, 1, 12)) - } - - @Test - fun preservesAuthoredWhitespaceAndLiteralObjectCharacters() { - val text = withIcon("before\u00A0 \uFFFC\u00A0\u00A0 main.go after\u00A0\uFFFC\u00A0") - assertEquals( - "before\u00A0 \u00A0 main.go after\u00A0\uFFFC\u00A0", - copyTextWithoutInlineImages(text, 0, text.length) - ) - } - - @Test - fun preservesTextWithoutImageSpans() { - val text = "\uFFFC\u00A0main.go" - assertEquals(text, copyTextWithoutInlineImages(text, 0, text.length)) - assertEquals(text, copyTextWithoutInlineImages(SpannableString(text), 0, text.length)) - } -} diff --git a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.ios.tsx b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.ios.tsx index 3aa7e1cf4e46..348a3c489a2c 100644 --- a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.ios.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.ios.tsx @@ -4,11 +4,7 @@ import type { MarkdownNode } from "react-native-nitro-markdown/headless"; import { CopyTextButton } from "./CopyTextButton"; import { MarkdownTextPrimitive } from "./MarkdownTextPrimitive"; -import { - nativeMarkdownBlockSpacing, - nativeMarkdownDocumentRuns, - nativeMarkdownListItemBlocks, -} from "./nativeMarkdownText"; +import { nativeMarkdownDocumentRuns, nativeMarkdownListItemBlocks } from "./nativeMarkdownText"; import { NativeMarkdownSelectableText } from "./NativeMarkdownSelectableText.ios"; import type { MarkdownCodeHighlighter, @@ -599,28 +595,17 @@ export function NativeMarkdownBlock(props: { switch (props.node.type) { case "document": return ( - + {(props.node.children ?? []).map((child, index) => ( - 0 - ? 8 - : 0, - }} - > - - + node={child} + skills={props.skills} + textStyle={props.textStyle} + highlightCode={props.highlightCode} + onLinkPress={props.onLinkPress} + depth={depth} + /> ))} ); diff --git a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.ios.tsx b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.ios.tsx index 7e1eaf88f546..a5c6cf540f1c 100644 --- a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.ios.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownSelectableText.ios.tsx @@ -222,10 +222,6 @@ export function NativeMarkdownSelectableText(props: { } } - if (Platform.OS === "android" && (run.fileIcon || linkIcon)) { - text = `\u00A0${text}`; - } - return { key: `${signature}:${occurrence}`, run, text, linkIcon }; }); // T3MarkdownText only rebuilds its attributed string during native layout. A diff --git a/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.ios.tsx b/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.ios.tsx index b7768ffbae03..2a231c603584 100644 --- a/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.ios.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.ios.tsx @@ -1,5 +1,5 @@ import { useMemo } from "react"; -import { Platform, View } from "react-native"; +import { View } from "react-native"; import { parseMarkdownWithOptions } from "react-native-nitro-markdown/headless"; import { @@ -84,10 +84,8 @@ export function SelectableMarkdownText({ the resulting single-line width instead of reflowing it. */} {chunks.map((chunk, index) => { - // Android inline images drift when one Text mixes paragraph and list - // spacer line heights. Keep those layouts in separate native blocks. const content = - chunk.kind === "rich" || Platform.OS === "android" ? ( + chunk.kind === "rich" ? ( 0) { - appendSpacer(runs, nativeMarkdownBlockSpacing(children[index - 1], child)); + const previous = children[index - 1]; + appendSpacer( + runs, + child.type === "heading" ? 20 : previous?.type === "heading" ? 10 : 12, + ); } appendDocumentBlock(runs, child, depth); } diff --git a/apps/mobile/src/lib/nativeMarkdownText.test.ts b/apps/mobile/src/lib/nativeMarkdownText.test.ts index 1091aeb65a6c..c3951c8d81f0 100644 --- a/apps/mobile/src/lib/nativeMarkdownText.test.ts +++ b/apps/mobile/src/lib/nativeMarkdownText.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vite-plus/test"; import type { MarkdownNode } from "react-native-nitro-markdown/headless"; import { - nativeMarkdownBlockSpacing, nativeMarkdownChunkSpacing, nativeMarkdownDocumentChunks, nativeMarkdownDocumentRuns, @@ -382,11 +381,6 @@ describe("nativeMarkdownDocumentRuns", () => { .filter((run) => run.role === "spacer") .map((run) => run.spacing), ).toEqual([20, 10, 12]); - expect( - node.children?.map((child, index) => - nativeMarkdownBlockSpacing(node.children?.[index - 1], child), - ), - ).toEqual([0, 20, 10, 12]); }); it("renders tight list items whose inline nodes are direct children", () => { diff --git a/docs/user/composer.md b/docs/user/composer.md index 7e388d4eb34f..4a8df5333664 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -6,9 +6,6 @@ include a skill when the task needs more context. Messages can contain up to 120,000 characters. Longer drafts stay in the composer so you can shorten them or split them into several messages. -On Android, long-press message text to select within a paragraph or list item. -Use the message’s copy button to copy the whole message. - ## Attach files Attach up to eight files per message. Images can be up to 10 MB; other files can From e784975bbec7c54ed75fc6da3d4ffd4038fae041 Mon Sep 17 00:00:00 2001 From: maria Date: Thu, 10 Sep 2026 15:24:25 -0300 Subject: [PATCH 03/15] fix(ui): simplify multiple linked pull request badges (#11104) --- .../src/features/threads/thread-list-v2-items.tsx | 14 ++++++++++---- apps/mobile/src/state/thread-pr-presentation.ts | 9 +++++++-- apps/mobile/src/state/use-thread-pr.test.ts | 2 +- apps/web/src/components/Sidebar.tsx | 2 +- apps/web/src/components/ThreadStatusIndicators.tsx | 9 +++++---- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index 834da0af9a6a..67a917c0079c 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -871,16 +871,22 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { )} {pr ? ( - {pr.kind === "stack" ? ( + {pr.kind === "stack" || pr.others > 0 ? ( ) : null} @@ -896,7 +902,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { )} style={{ fontFamily: MONO_FONT }} > - {pr.kind === "stack" ? pr.label : `#${pr.label}`} + {pr.kind === "stack" || pr.others > 0 ? pr.label : `#${pr.label}`} ) : null} diff --git a/apps/mobile/src/state/thread-pr-presentation.ts b/apps/mobile/src/state/thread-pr-presentation.ts index dd21e2d70ec4..48fe3abf5f66 100644 --- a/apps/mobile/src/state/thread-pr-presentation.ts +++ b/apps/mobile/src/state/thread-pr-presentation.ts @@ -17,11 +17,12 @@ export interface ThreadPrPresentation { readonly number: number; readonly state: ThreadPr["state"] | null; readonly kind: "pull-request" | "stack"; + readonly others: number; readonly isDraft: boolean; /** Provider-side last activity, bounding when a terminal state landed. */ readonly updatedAt: string | null; readonly url: string; - /** Compact pull request number label, e.g. "3774". */ + /** Compact pull request number or linked count, e.g. "3774" or "+2". */ readonly label: string; /** Full, provider-aware label for assistive technologies. */ readonly accessibilityLabel: string; @@ -42,6 +43,7 @@ export function presentThreadPr( const isDraft = pr.state === "open" && pr.isDraft === true; return { kind: "pull-request", + others: 0, number: pr.number, state: pr.state, isDraft, @@ -66,9 +68,12 @@ export function presentThreadLinkedPullRequests( const label = badge.kind === "stack" ? String(badge.layers) - : `${link.number}${badge.others > 0 ? ` +${badge.others}` : ""}`; + : badge.others > 0 + ? `+${badge.others}` + : String(link.number); return { kind: badge.kind, + others: badge.kind === "pull-request" ? badge.others : 0, number: link.number, state, isDraft, diff --git a/apps/mobile/src/state/use-thread-pr.test.ts b/apps/mobile/src/state/use-thread-pr.test.ts index 330e31cb0171..5fdb90e19c65 100644 --- a/apps/mobile/src/state/use-thread-pr.test.ts +++ b/apps/mobile/src/state/use-thread-pr.test.ts @@ -87,7 +87,7 @@ describe("presentThreadLinkedPullRequests", () => { it("counts unrelated links without labelling them a stack", () => { expect(presentThreadLinkedPullRequests([linkedPr(1), linkedPr(2)])).toMatchObject({ kind: "pull-request", - label: "1 +1", + label: "+1", }); }); diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 79573635212e..72a119cb76ba 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -1485,7 +1485,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { ); - // Stacks show their layer count; unrelated links show the current PR and a remainder count. + // Stacks show their layer count; unrelated links show only the remainder count. // Plain clicks open T3; individual PR links also support opening the host in a new tab. const prBadgeShape = supportsMultiplePullRequests ? resolveThreadPullRequestBadge(thread.pullRequests) diff --git a/apps/web/src/components/ThreadStatusIndicators.tsx b/apps/web/src/components/ThreadStatusIndicators.tsx index 6a73a805fb67..2d724a726506 100644 --- a/apps/web/src/components/ThreadStatusIndicators.tsx +++ b/apps/web/src/components/ThreadStatusIndicators.tsx @@ -174,10 +174,11 @@ export function ThreadPullRequestBadgeControl({ const content = ( <> - {isStack ? badge.layers : number} - {badge?.kind === "pull-request" && badge.others > 0 ? ( - +{badge.others} - ) : null} + {isStack + ? badge.layers + : badge?.kind === "pull-request" && badge.others > 0 + ? `+${badge.others}` + : number} ); return ( From 0882431e0fc0fabe950743dd44b674a07a584bc9 Mon Sep 17 00:00:00 2001 From: maria Date: Thu, 10 Sep 2026 15:25:31 -0300 Subject: [PATCH 04/15] fix(preview): return to pip when closing the right panel (#11102) --- apps/web/src/components/ChatView.tsx | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index f1b5afba94e9..597b7333ce28 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -4293,10 +4293,21 @@ export default function ChatView(props: ChatViewProps) { supportsPullRequests, threadDetailLoading, ]); + const closePreviewPanel = useCallback(() => { + if (activeThreadRef) { + if (activeRightPanelSurface?.kind === "preview" && activeRightPanelSurface.resourceId) { + usePreviewMiniPlayerStore + .getState() + .open(activeThreadRef, activeRightPanelSurface.resourceId); + } + setMaximizedRightPanelThreadKey(null); + useRightPanelStore.getState().close(activeThreadRef); + } + }, [activeRightPanelSurface, activeThreadRef]); const togglePreviewPanel = useCallback(() => { if (!activeThreadRef || !isPreviewSupportedInRuntime()) return; if (previewPanelOpen) { - useRightPanelStore.getState().close(activeThreadRef); + closePreviewPanel(); return; } const activeTabId = activePreviewState.activeTabId; @@ -4305,13 +4316,13 @@ export default function ChatView(props: ChatViewProps) { } else { createBrowserSurface(); } - }, [activePreviewState.activeTabId, activeThreadRef, createBrowserSurface, previewPanelOpen]); - const closePreviewPanel = useCallback(() => { - if (activeThreadRef) { - setMaximizedRightPanelThreadKey(null); - useRightPanelStore.getState().close(activeThreadRef); - } - }, [activeThreadRef]); + }, [ + activePreviewState.activeTabId, + activeThreadRef, + closePreviewPanel, + createBrowserSurface, + previewPanelOpen, + ]); const addTerminalSurface = useCallback(() => { if (!activeThreadRef || !activeThreadId || !activeProject) return; const cwd = gitCwd ?? activeProject.workspaceRoot; From 0527ddf06defc88169c245a0fe53635d41d66469 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 10 Sep 2026 11:34:55 -0700 Subject: [PATCH 05/15] fix: quiet settled threads and simplify PR badges (#11101) --- .../src/features/threads/thread-list-items.tsx | 4 ++-- .../src/features/threads/thread-list-v2-items.tsx | 8 +------- apps/mobile/src/state/thread-pr-presentation.ts | 10 +++++++--- apps/mobile/src/state/use-thread-pr.test.ts | 4 +++- apps/web/src/components/Sidebar.tsx | 10 ++++++---- apps/web/src/components/ThreadStatusIndicators.tsx | 13 +++++++------ 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/apps/mobile/src/features/threads/thread-list-items.tsx b/apps/mobile/src/features/threads/thread-list-items.tsx index 0ec50c674451..a4da32ef7018 100644 --- a/apps/mobile/src/features/threads/thread-list-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-items.tsx @@ -44,11 +44,11 @@ export const THREAD_LIST_COMPACT_INSET = HOME_HORIZONTAL_INSET; const SIDEBAR_ROW_RADIUS = 12; function pullRequestTintColor( - pr: Pick, + pr: Pick, colorScheme: "light" | "dark", ) { const dark = colorScheme === "dark"; - if (pr.state === "open" && pr.isDraft === true) { + if (pr.others > 0 || (pr.state === "open" && pr.isDraft === true)) { return dark ? "#a1a1aa" : "#71717a"; } switch (pr.state) { diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index 67a917c0079c..937ca5f909a8 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -880,13 +880,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { ? materialYouStyleLayoutActive ? "accent-thread-selected-foreground" : "accent-user-bubble-foreground" - : pr.kind === "stack" || pr.isDraft || pr.state === null - ? "accent-foreground-muted" - : pr.state === "open" - ? "accent-adaptive-emerald-600-400" - : pr.state === "merged" - ? "accent-adaptive-violet-600-400" - : "accent-foreground-muted" + : "accent-foreground-muted" } /> ) : null} diff --git a/apps/mobile/src/state/thread-pr-presentation.ts b/apps/mobile/src/state/thread-pr-presentation.ts index 48fe3abf5f66..fc310d070acc 100644 --- a/apps/mobile/src/state/thread-pr-presentation.ts +++ b/apps/mobile/src/state/thread-pr-presentation.ts @@ -65,11 +65,12 @@ export function presentThreadLinkedPullRequests( const snapshot = link.snapshot; const state = badge.kind === "stack" ? badge.state : (snapshot?.state ?? null); const isDraft = snapshot?.isDraft === true && state === "open"; + const linkedCount = badge.kind === "pull-request" && badge.others > 0 ? badge.others + 1 : null; const label = badge.kind === "stack" ? String(badge.layers) - : badge.others > 0 - ? `+${badge.others}` + : linkedCount !== null + ? `+${linkedCount}` : String(link.number); return { kind: badge.kind, @@ -84,7 +85,10 @@ export function presentThreadLinkedPullRequests( badge.kind === "stack" ? `${badge.layers} pull requests in stack, ${state ?? "status pending"}` : `#${link.number} pull request ${state === null ? "status pending" : isDraft ? "draft" : state}${badge.others > 0 ? `, ${badge.others} more linked` : ""}`, - textClassName: state === null || isDraft ? "text-foreground-muted" : PR_STATE_TEXT_CLASS[state], + textClassName: + linkedCount !== null || state === null || isDraft + ? "text-foreground-muted" + : PR_STATE_TEXT_CLASS[state], }; } diff --git a/apps/mobile/src/state/use-thread-pr.test.ts b/apps/mobile/src/state/use-thread-pr.test.ts index 5fdb90e19c65..e36bd7d81434 100644 --- a/apps/mobile/src/state/use-thread-pr.test.ts +++ b/apps/mobile/src/state/use-thread-pr.test.ts @@ -87,7 +87,9 @@ describe("presentThreadLinkedPullRequests", () => { it("counts unrelated links without labelling them a stack", () => { expect(presentThreadLinkedPullRequests([linkedPr(1), linkedPr(2)])).toMatchObject({ kind: "pull-request", - label: "+1", + label: "+2", + others: 1, + textClassName: "text-foreground-muted", }); }); diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 72a119cb76ba..b1583a346b56 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -1387,6 +1387,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { // content; surface is reserved for interaction (hover, multi-select, route). const rowSurfaceClassName = cn( "group/sidebar-row relative w-full cursor-pointer overflow-hidden rounded-md text-left outline-none select-none", + variantAction === "unsettle" && "[&:not(:hover):not(:focus-within)_*]:text-secondary-label/70", props.isActive ? "bg-sidebar-row-active text-sidebar-foreground" : isSelected @@ -1469,7 +1470,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { : "text-foreground/90", ) : cn( - "truncate group-hover/sidebar-row:text-foreground", + "truncate group-focus-within/sidebar-row:text-foreground group-hover/sidebar-row:text-foreground", shouldRecede ? "text-secondary-label/70" : props.isActive || isWoke @@ -1485,7 +1486,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { ); - // Stacks show their layer count; unrelated links show only the remainder count. + // Stacks show their layer count; multiple unrelated links show their total count. // Plain clicks open T3; individual PR links also support opening the host in a new tab. const prBadgeShape = supportsMultiplePullRequests ? resolveThreadPullRequestBadge(thread.pullRequests) @@ -1597,8 +1598,8 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { {props.project ? : null} @@ -1617,6 +1618,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { the time/jump label yields to the settle affordance. */} {prBadge} {prBadge && + variantAction !== "unsettle" && pr && (supportsMultiplePullRequests ? visibleThreadPullRequests(thread.pullRequests).length === 0 diff --git a/apps/web/src/components/ThreadStatusIndicators.tsx b/apps/web/src/components/ThreadStatusIndicators.tsx index 2d724a726506..e948aa4091ee 100644 --- a/apps/web/src/components/ThreadStatusIndicators.tsx +++ b/apps/web/src/components/ThreadStatusIndicators.tsx @@ -154,6 +154,7 @@ export function ThreadPullRequestBadgeControl({ onOpenPullRequest: (event: MouseEvent) => void; }) { const isStack = badge?.kind === "stack"; + const linkedCount = badge?.kind === "pull-request" && badge.others > 0 ? badge.others + 1 : null; if (!isStack && (number === undefined || url === undefined)) return null; const label = isStack ? `Stack of ${badge.layers} pull requests, ${badge.state}` @@ -169,16 +170,16 @@ export function ThreadPullRequestBadgeControl({ "text-xs tabular-nums", variant === "ghost" && "font-normal text-xs! active:scale-100 [--control-icon-color:currentColor]", - isStack ? PR_STATE_COLOR_CLASS[badge.state] : (status?.colorClass ?? "text-muted-foreground"), + linkedCount !== null + ? "text-secondary-label" + : isStack + ? PR_STATE_COLOR_CLASS[badge.state] + : (status?.colorClass ?? "text-muted-foreground"), ); const content = ( <> - {isStack - ? badge.layers - : badge?.kind === "pull-request" && badge.others > 0 - ? `+${badge.others}` - : number} + {isStack ? badge.layers : linkedCount !== null ? `+${linkedCount}` : number} ); return ( From f814983c262b42bd79247bae377a709925c70d63 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 10 Sep 2026 11:34:56 -0700 Subject: [PATCH 06/15] fix(web): emphasize primary pull request actions (#11105) --- .../pullRequest/PullRequestDetailPanel.tsx | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/apps/web/src/components/pullRequest/PullRequestDetailPanel.tsx b/apps/web/src/components/pullRequest/PullRequestDetailPanel.tsx index 54ad13980236..236d7cd86ad9 100644 --- a/apps/web/src/components/pullRequest/PullRequestDetailPanel.tsx +++ b/apps/web/src/components/pullRequest/PullRequestDetailPanel.tsx @@ -1728,7 +1728,7 @@ export function PullRequestDetailPanel({ @@ -1777,7 +1774,7 @@ export function PullRequestDetailPanel({ } /> @@ -199,7 +199,7 @@ export function SidebarProviderUpdatePill() { className="relative z-[1] mr-1 [--control-icon-color:currentColor] rounded-md text-inherit opacity-70 hover:bg-transparent hover:opacity-100" onClick={() => startExit(displayedView.key, null, displayedView.key)} > - + } /> From 3997b9a3a4ee6b43ee768a9b372ff3dd52fd840e Mon Sep 17 00:00:00 2001 From: Henry Zhang <113233555+caezium@users.noreply.github.com> Date: Fri, 11 Sep 2026 02:37:15 +0800 Subject: [PATCH 09/15] fix(web): align floating browser preview corners (#10915) --- .../preview/ThreadPreviewMiniPlayer.tsx | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index 9e59e56a24f0..4384019abbad 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -44,6 +44,8 @@ interface Props { readonly bottomInset: number; } +const PREVIEW_MINI_PLAYER_CORNER_RADIUS = 12; + // Invisible grab zones straddling each edge; the cursor is the only affordance. const RESIZE_HANDLES: ReadonlyArray<{ readonly direction: BrowserViewportResizeDirection; @@ -59,6 +61,10 @@ const RESIZE_HANDLES: ReadonlyArray<{ { direction: "southeast", className: "-bottom-2 -right-2 size-4 cursor-nwse-resize" }, ]; +/** + * Floats the thread's browser surface over chat. Native clipping and the DOM + * frame use the same radius so their separately composited edges stay aligned. + */ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props) { const containerRef = useRef(null); const gestureRef = useRef(null); @@ -193,7 +199,13 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props aria-label="Floating browser preview" data-preview-mini-player={tabId} className="pointer-events-none absolute select-none" - style={{ left: frame.x, top: frame.y, width: frame.width, height: frame.height }} + style={{ + left: frame.x, + top: frame.y, + width: frame.width, + height: frame.height, + borderRadius: PREVIEW_MINI_PLAYER_CORNER_RADIUS, + }} >
-
+
-
+
{!desktopOverlay?.hasWebContents ? ( -
+
Reconnecting preview…
) : null} From 60eff99f25dbfc9799043ad979537e2da1aea590 Mon Sep 17 00:00:00 2001 From: oliver <97427849+flamboh@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:37:35 -0700 Subject: [PATCH 10/15] fix(web): save PR body edits with Cmd/Ctrl+Enter (#10660) --- .../pullRequest/PullRequestMarkdownEditor.tsx | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/apps/web/src/components/pullRequest/PullRequestMarkdownEditor.tsx b/apps/web/src/components/pullRequest/PullRequestMarkdownEditor.tsx index fde29d023774..a92418bbb862 100644 --- a/apps/web/src/components/pullRequest/PullRequestMarkdownEditor.tsx +++ b/apps/web/src/components/pullRequest/PullRequestMarkdownEditor.tsx @@ -55,11 +55,24 @@ export function PullRequestMarkdownEditor({ setDraft(value); } const empty = draft.trim().length === 0; + const saveDisabled = saving || (empty && !allowEmpty); return (
{ + if (event.nativeEvent.isComposing || event.keyCode === 229) return; + if ( + event.key === "Enter" && + (event.metaKey || event.ctrlKey) && + !event.shiftKey && + !event.altKey + ) { + event.preventDefault(); + event.stopPropagation(); + if (!saveDisabled && !event.repeat) onSave(draft); + return; + } if (event.key !== "Escape" || saving) return; event.preventDefault(); onCancel(); @@ -106,12 +119,7 @@ export function PullRequestMarkdownEditor({ -
From 502131adff7494d1a1797b4fe518870f7e8431fd Mon Sep 17 00:00:00 2001 From: maria Date: Thu, 10 Sep 2026 15:38:35 -0300 Subject: [PATCH 11/15] fix(web): collapse a tool call by clicking its expanded label (#11017) Co-authored-by: Claude Opus 5 (1M context) --- .../components/chat/MessagesTimeline.test.tsx | 49 +++++++++++++++++++ .../src/components/chat/MessagesTimeline.tsx | 14 +++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/chat/MessagesTimeline.test.tsx b/apps/web/src/components/chat/MessagesTimeline.test.tsx index 655156048c1d..c47402f4204b 100644 --- a/apps/web/src/components/chat/MessagesTimeline.test.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.test.tsx @@ -1764,4 +1764,53 @@ describe("MessagesTimeline", () => { expect(markup).toContain("lucide-circle-alert"); expect(markup).toContain("text-destructive"); }); + + it("only withholds an expanded tool-call label click while text is selected", async () => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal("requestAnimationFrame", () => 0); + vi.stubGlobal("cancelAnimationFrame", () => {}); + let renderer: ReactTestRenderer | undefined; + try { + await act(() => { + renderer = create( + , + ); + }); + await act(() => renderer!.root.findByProps({ "aria-expanded": false }).props.onClick()); + const label = renderer!.root.findAll( + (node) => node.type === "span" && String(node.props.className).includes("select-text"), + )[0]; + const stopPropagation = vi.fn(); + // Only the click that ends a selection may be withheld from the row + // toggle; the plain click has to reach it so the label can collapse. + for (const isCollapsed of [false, true]) { + label!.props.onClick({ + currentTarget: { ownerDocument: { getSelection: () => ({ isCollapsed }) } }, + stopPropagation, + }); + } + expect(stopPropagation).toHaveBeenCalledTimes(1); + } finally { + await act(() => renderer?.unmount()); + } + }); }); diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 7764f64ea4b6..f685d511fd4b 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -3172,6 +3172,18 @@ function workEntryIconName(workEntry: TimelineWorkEntry): WorkEntryIconName { const stopRowToggle = (e: { stopPropagation: () => void }) => e.stopPropagation(); +/** + * Click handler for expanded row labels, which turn text selection back on. + * Only a click that ends a real selection is withheld from the row toggle, so + * an ordinary click on the label still bubbles and collapses the row it opened. + */ +const stopRowToggleWhileSelectingText = (e: MouseEvent) => { + const selection = e.currentTarget.ownerDocument.getSelection(); + if (selection && !selection.isCollapsed) { + e.stopPropagation(); + } +}; + /** * A1 spawn CTA: one anchored row per workflow run (or per-turn direct-spawn * batch). Live status is derived from the shared agent panel model at render @@ -3407,7 +3419,7 @@ const PlainWorkEntryRow = memo(function PlainWorkEntryRow(props: { expanded ? "whitespace-pre-wrap break-words select-text" : "truncate", headingClass, )} - onClick={expanded ? stopRowToggle : undefined} + onClick={expanded ? stopRowToggleWhileSelectingText : undefined} onPointerDown={expanded ? stopRowToggle : undefined} > {previewText} From dca7b59bea7e8a4a24e75b290027496a523b0ca7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 10 Sep 2026 11:59:32 -0700 Subject: [PATCH 12/15] feat(devices): add simulator and emulator support (#10677) Co-authored-by: Claude Fable 5 --- .../src/features/threads/thread-work-log.tsx | 10 +- apps/mobile/src/lib/threadActivity.ts | 2 +- apps/server/src/auth/RpcAuthorization.ts | 8 + apps/server/src/device/AgentDeviceShim.ts | 35 + apps/server/src/device/DeviceActions.test.ts | 283 +++++++ apps/server/src/device/DeviceActions.ts | 568 +++++++++++++ apps/server/src/device/DeviceHost.ts | 102 +++ apps/server/src/device/DeviceHubProxy.test.ts | 141 ++++ apps/server/src/device/DeviceHubProxy.ts | 224 +++++ apps/server/src/device/DeviceService.test.ts | 337 ++++++++ apps/server/src/device/DeviceService.ts | 764 ++++++++++++++++++ .../server/src/device/DeviceToolchain.test.ts | 39 + apps/server/src/device/DeviceToolchain.ts | 223 +++++ .../server/src/device/LocalDeviceHost.test.ts | 114 +++ apps/server/src/device/LocalDeviceHost.ts | 697 ++++++++++++++++ apps/server/src/mcp/McpDeviceToolkit.test.ts | 127 +++ apps/server/src/mcp/McpHttpServer.ts | 172 ++++ apps/server/src/mcp/McpInvocationContext.ts | 2 +- .../server/src/mcp/McpProviderSession.test.ts | 30 + apps/server/src/mcp/McpProviderSession.ts | 27 +- .../server/src/mcp/McpSessionRegistry.test.ts | 22 +- apps/server/src/mcp/McpSessionRegistry.ts | 15 +- .../src/mcp/toolkits/device/handlers.test.ts | 43 + .../src/mcp/toolkits/device/handlers.ts | 212 +++++ apps/server/src/mcp/toolkits/device/tools.ts | 98 +++ .../provider/CodexDeveloperInstructions.ts | 39 +- .../src/provider/Drivers/AntigravityDriver.ts | 3 +- .../src/provider/Layers/AntigravityAdapter.ts | 3 + .../src/provider/Layers/ClaudeAdapter.ts | 2 +- .../src/provider/Layers/CodexAdapter.ts | 7 +- .../provider/Layers/CodexSessionRuntime.ts | 35 +- .../src/provider/Layers/CursorAdapter.ts | 9 +- .../server/src/provider/Layers/GrokAdapter.ts | 9 +- .../src/provider/Layers/OpenCodeAdapter.ts | 7 +- .../provider/Layers/ProviderService.test.ts | 42 +- .../src/provider/Layers/ProviderService.ts | 91 ++- .../src/provider/acp/AntigravityAcpSupport.ts | 2 + apps/server/src/server.test.ts | 18 + apps/server/src/server.ts | 11 +- apps/server/src/ws.ts | 36 + apps/web/src/components/ChatView.tsx | 75 ++ .../src/components/RightPanelTabs.test.tsx | 2 + apps/web/src/components/RightPanelTabs.tsx | 31 + .../components/chat/MessagesTimeline.logic.ts | 2 +- .../src/components/chat/MessagesTimeline.tsx | 6 + .../web/src/components/device/DevicePanel.tsx | 473 +++++++++++ .../web/src/components/device/DeviceSetup.tsx | 307 +++++++ .../device/DeviceStreamView.test.tsx | 58 ++ .../components/device/DeviceStreamView.tsx | 327 ++++++++ .../components/device/DeviceToolsPanel.tsx | 722 +++++++++++++++++ .../components/device/deviceHubApi.test.ts | 44 + .../web/src/components/device/deviceHubApi.ts | 248 ++++++ .../components/device/deviceStream.test.ts | 180 +++++ .../web/src/components/device/deviceStream.ts | 697 ++++++++++++++++ .../components/preview/PreviewEmptyState.tsx | 9 +- .../preview/PreviewLocalServerCard.tsx | 18 +- .../settings/IntegrationsSettings.test.tsx | 62 +- .../settings/IntegrationsSettings.tsx | 123 +++ .../src/components/settings/settingsSearch.ts | 21 + apps/web/src/components/ui/discovery-list.tsx | 37 + apps/web/src/rightPanelStore.ts | 12 +- apps/web/src/routes/_chat.pull-requests.tsx | 2 + apps/web/src/state/device.ts | 70 ++ apps/web/vite.config.ts | 9 +- docs/README.md | 2 + docs/internals/devices.md | 87 ++ docs/user/devices.md | 63 ++ packages/client-runtime/package.json | 8 + packages/client-runtime/src/rpc/client.ts | 1 + packages/client-runtime/src/state/device.ts | 68 ++ .../src/state/deviceHubAccess.ts | 72 ++ .../src/work-log/presentation.test.ts | 63 ++ .../src/work-log/presentation.ts | 17 +- packages/contracts/src/device.ts | 516 ++++++++++++ packages/contracts/src/index.ts | 1 + packages/contracts/src/rpc.ts | 78 ++ packages/contracts/src/settings.ts | 19 + 77 files changed, 9041 insertions(+), 98 deletions(-) create mode 100644 apps/server/src/device/AgentDeviceShim.ts create mode 100644 apps/server/src/device/DeviceActions.test.ts create mode 100644 apps/server/src/device/DeviceActions.ts create mode 100644 apps/server/src/device/DeviceHost.ts create mode 100644 apps/server/src/device/DeviceHubProxy.test.ts create mode 100644 apps/server/src/device/DeviceHubProxy.ts create mode 100644 apps/server/src/device/DeviceService.test.ts create mode 100644 apps/server/src/device/DeviceService.ts create mode 100644 apps/server/src/device/DeviceToolchain.test.ts create mode 100644 apps/server/src/device/DeviceToolchain.ts create mode 100644 apps/server/src/device/LocalDeviceHost.test.ts create mode 100644 apps/server/src/device/LocalDeviceHost.ts create mode 100644 apps/server/src/mcp/McpDeviceToolkit.test.ts create mode 100644 apps/server/src/mcp/McpProviderSession.test.ts create mode 100644 apps/server/src/mcp/toolkits/device/handlers.test.ts create mode 100644 apps/server/src/mcp/toolkits/device/handlers.ts create mode 100644 apps/server/src/mcp/toolkits/device/tools.ts create mode 100644 apps/web/src/components/device/DevicePanel.tsx create mode 100644 apps/web/src/components/device/DeviceSetup.tsx create mode 100644 apps/web/src/components/device/DeviceStreamView.test.tsx create mode 100644 apps/web/src/components/device/DeviceStreamView.tsx create mode 100644 apps/web/src/components/device/DeviceToolsPanel.tsx create mode 100644 apps/web/src/components/device/deviceHubApi.test.ts create mode 100644 apps/web/src/components/device/deviceHubApi.ts create mode 100644 apps/web/src/components/device/deviceStream.test.ts create mode 100644 apps/web/src/components/device/deviceStream.ts create mode 100644 apps/web/src/components/ui/discovery-list.tsx create mode 100644 apps/web/src/state/device.ts create mode 100644 docs/internals/devices.md create mode 100644 docs/user/devices.md create mode 100644 packages/client-runtime/src/state/device.ts create mode 100644 packages/client-runtime/src/state/deviceHubAccess.ts create mode 100644 packages/contracts/src/device.ts diff --git a/apps/mobile/src/features/threads/thread-work-log.tsx b/apps/mobile/src/features/threads/thread-work-log.tsx index ab1c69beffa5..593a49755376 100644 --- a/apps/mobile/src/features/threads/thread-work-log.tsx +++ b/apps/mobile/src/features/threads/thread-work-log.tsx @@ -76,7 +76,7 @@ export const THREAD_DISCLOSURE_TRANSITION_MS = 180; const WORK_LOG_LAYOUT_TRANSITION = LinearTransition.duration(THREAD_DISCLOSURE_TRANSITION_MS); const WORK_LOG_DETAIL_ENTER_TRANSITION = FadeIn.duration(140); const WORK_LOG_DETAIL_EXIT_TRANSITION = FadeOut.duration(120); -type WorkContentIcon = AppSymbolName | "browser" | "t3-code" | "pull-request"; +type WorkContentIcon = AppSymbolName | "browser" | "device" | "t3-code" | "pull-request"; function WorkLogIcon(props: { readonly icon: WorkContentIcon; @@ -97,7 +97,9 @@ function WorkLogIcon(props: { ? "arrow.triangle.pull" : props.icon === "browser" ? { ios: "globe", android: "public" } - : props.icon + : props.icon === "device" + ? { ios: "iphone", android: "smartphone" } + : props.icon } size={14} weight="medium" @@ -899,7 +901,7 @@ export function ThreadWorkGroupToggle(props: { readonly iconSubtleColor: import("react-native").ColorValue; readonly summary: string; readonly summaryKind: ToolGroupSummaryKind; - readonly summaryToolIcon?: "browser" | "t3-code" | "pull-request"; + readonly summaryToolIcon?: "browser" | "device" | "t3-code" | "pull-request"; readonly themeAppearance: "light" | "dark"; readonly toolSurface?: import("@t3tools/contracts").ToolActivitySurface; readonly toolIcon?: ToolActivityIcon; @@ -1248,6 +1250,8 @@ function toolGroupSummarySymbolName(kind: ToolGroupSummaryKind): AppSymbolName { return { ios: "square.and.pencil", android: "edit" }; case "command": return { ios: "terminal", android: "terminal" }; + case "device": + return { ios: "iphone", android: "smartphone" }; case "browser": case "search": return { ios: "globe", android: "public" }; diff --git a/apps/mobile/src/lib/threadActivity.ts b/apps/mobile/src/lib/threadActivity.ts index 7f0241f949c3..c7d27af11d5e 100644 --- a/apps/mobile/src/lib/threadActivity.ts +++ b/apps/mobile/src/lib/threadActivity.ts @@ -167,7 +167,7 @@ export type ThreadFeedEntry = readonly summaryKind: ToolGroupSummaryKind; readonly toolSurface?: WorkLogEntry["toolSurface"]; readonly toolIcon?: WorkLogEntry["toolIcon"]; - readonly summaryToolIcon?: "browser" | "t3-code" | "pull-request"; + readonly summaryToolIcon?: "browser" | "device" | "t3-code" | "pull-request"; readonly hasFailure: boolean; readonly live: boolean; readonly shimmer: boolean; diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index a63b07f0adef..97afa4f40775 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -144,6 +144,14 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.previewAutomationFocusHost]: AuthOrchestrationOperateScope, [WS_METHODS.subscribePreviewEvents]: AuthOrchestrationReadScope, [WS_METHODS.subscribeDiscoveredLocalServers]: AuthOrchestrationReadScope, + [WS_METHODS.deviceConfigure]: AuthOrchestrationOperateScope, + [WS_METHODS.deviceList]: AuthOrchestrationReadScope, + [WS_METHODS.deviceOpen]: AuthOrchestrationOperateScope, + [WS_METHODS.deviceClose]: AuthOrchestrationOperateScope, + [WS_METHODS.deviceShutdown]: AuthOrchestrationOperateScope, + [WS_METHODS.deviceDetail]: AuthOrchestrationReadScope, + [WS_METHODS.deviceAction]: AuthOrchestrationOperateScope, + [WS_METHODS.subscribeDeviceState]: AuthOrchestrationReadScope, [WS_METHODS.subscribeServerConfig]: AuthOrchestrationReadScope, [WS_METHODS.subscribeServerLifecycle]: AuthOrchestrationReadScope, [WS_METHODS.subscribeAuthAccess]: AuthAccessReadScope, diff --git a/apps/server/src/device/AgentDeviceShim.ts b/apps/server/src/device/AgentDeviceShim.ts new file mode 100644 index 000000000000..c2a28127e31d --- /dev/null +++ b/apps/server/src/device/AgentDeviceShim.ts @@ -0,0 +1,35 @@ +/** + * A directory holding an `agent-device` launcher that runs the pinned install + * with the server's Node. Prepended to provider subprocess PATHs so the agent + * types `agent-device …` and gets the version the injected instructions were + * written for, regardless of what is or is not globally installed. + */ +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +const SHIM_DIR = "device/bin"; + +export const ensureAgentDeviceShim = Effect.fn("AgentDeviceShim.ensure")(function* (input: { + readonly entryPath: string; + readonly stateDir: string; +}) { + const { entryPath } = input; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcessPlatform; + const shimDir = path.join(input.stateDir, SHIM_DIR); + yield* fs.makeDirectory(shimDir, { recursive: true }); + const node = process.execPath; + if (platform === "win32") { + const script = `@echo off\r\n"${node}" "${entryPath}" %*\r\n`; + yield* fs.writeFileString(path.join(shimDir, "agent-device.cmd"), script); + } else { + const script = `#!/bin/sh\nexec "${node}" "${entryPath}" "$@"\n`; + const shimPath = path.join(shimDir, "agent-device"); + yield* fs.writeFileString(shimPath, script); + yield* fs.chmod(shimPath, 0o755); + } + return shimDir; +}); diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts new file mode 100644 index 000000000000..ed6b5a87335d --- /dev/null +++ b/apps/server/src/device/DeviceActions.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from "@effect/vitest"; +import type { DeviceActionInput } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import { readDeviceDetail, runDeviceAction, supportsAction } from "./DeviceActions.ts"; +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Call = { command: string; args: ReadonlyArray; stdin?: string }; + +const makeReady = ( + respond: (call: Call) => { stdout?: string; stderr?: string; code?: number } = () => ({}), + helpers: DeviceHostReady["helpers"] = { + serveSimAxSettings: "/hub/simax/serve-sim-ax-settings", + serveSimCli: "/hub/serve-sim.js", + }, +) => { + const calls: Call[] = []; + const ready: DeviceHostReady = { + hub: { origin: "http://127.0.0.1:1" }, + helpers, + run: (command, args, options) => { + const call = { command, args, ...(options?.stdin ? { stdin: options.stdin } : {}) }; + calls.push(call); + const result = respond(call); + return Effect.succeed({ + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + code: result.code ?? 0, + }); + }, + }; + return { ready, calls }; +}; + +const udid = "SIM-1"; + +describe("supportsAction", () => { + it("advertises platform-specific toggles", () => { + const toggle = (setting: Extract["setting"]) => + ({ type: "setToggle", deviceId: udid, setting, value: true }) as const; + expect(supportsAction("ios", toggle("voiceOver"))).toBe(true); + expect(supportsAction("android", toggle("voiceOver"))).toBe(false); + expect(supportsAction("android", toggle("networkEnabled"))).toBe(true); + expect(supportsAction("ios", toggle("networkEnabled"))).toBe(false); + expect(supportsAction("ios", { type: "setLiquidGlass", deviceId: udid, value: "clear" })).toBe( + true, + ); + expect( + supportsAction("android", { type: "setLiquidGlass", deviceId: udid, value: "clear" }), + ).toBe(false); + expect( + supportsAction("android", { type: "setOrientation", deviceId: udid, value: "portrait" }), + ).toBe(true); + expect( + supportsAction("ios", { type: "setOrientation", deviceId: udid, value: "portrait" }), + ).toBe(false); + }); +}); + +describe("runDeviceAction", () => { + it.effect("maps shared text sizes onto simctl content-size categories", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "setTextSize", + deviceId: udid, + value: "extra-large", + }); + expect(calls).toEqual([ + { command: "xcrun", args: ["simctl", "ui", udid, "content_size", "accessibility-large"] }, + ]); + }), + ); + + it.effect("maps shared text sizes onto Android font_scale", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "android", { + type: "setTextSize", + deviceId: "emulator-5554", + value: "large", + }); + expect(calls[0]?.args).toEqual([ + "-s", + "emulator-5554", + "shell", + "settings", + "put", + "system", + "font_scale", + "1.15", + ]); + }), + ); + + it.effect( + "rotates emulators through the accelerometer and physical devices through the lock", + () => + Effect.gen(function* () { + const emulator = makeReady(); + yield* runDeviceAction(emulator.ready, "android", { + type: "setOrientation", + deviceId: "emulator-5554", + value: "landscape_left", + }); + expect(emulator.calls.at(-1)?.args).toEqual([ + "-s", + "emulator-5554", + "emu", + "sensor", + "set", + "acceleration", + "9.81:0:0", + ]); + const phone = makeReady(); + yield* runDeviceAction(phone.ready, "android", { + type: "setOrientation", + deviceId: "R5CT1234", + value: "landscape_right", + }); + expect(phone.calls).toEqual([ + { + command: "adb", + args: ["-s", "R5CT1234", "shell", "cmd", "window", "user-rotation", "lock", "3"], + }, + ]); + }), + ); + + it.effect("runs accessibility toggles through the bundled helper via simctl spawn", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "setToggle", + deviceId: udid, + setting: "voiceOver", + value: true, + }); + expect(calls).toEqual([ + { + command: "xcrun", + args: [ + "simctl", + "spawn", + udid, + "/hub/simax/serve-sim-ax-settings", + "set", + "voiceover", + "on", + ], + }, + ]); + }), + ); + + it.effect("fails clearly when the helper is missing", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({}), { serveSimAxSettings: null, serveSimCli: null }); + const error = yield* Effect.flip( + runDeviceAction(ready, "ios", { + type: "setColorFilter", + deviceId: udid, + value: "grayscale", + }), + ); + expect(error._tag).toBe("DeviceActionUnavailableError"); + expect(error.message).toContain("requires a helper"); + }), + ); + + it.effect("rejects actions the platform does not support without running anything", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + const error = yield* Effect.flip( + runDeviceAction(ready, "android", { + type: "sendPush", + deviceId: "emulator-5554", + appId: "com.example", + payload: "hi", + }), + ); + expect(error.message).toContain("not supported on android"); + expect(calls).toEqual([]); + }), + ); + + it.effect("surfaces non-zero exit codes as operation errors", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({ code: 1, stderr: "Invalid device: SIM-1" })); + const error = yield* Effect.flip( + runDeviceAction(ready, "ios", { type: "setAppearance", deviceId: udid, value: "dark" }), + ); + expect(error.operation).toBe("appearance"); + expect(error.message).toContain("exit code 1"); + expect(error.message).not.toContain("Invalid device: SIM-1"); + expect(error._tag === "DeviceOperationError" && error.cause).toMatchObject({ + stderr: "Invalid device: SIM-1", + }); + }), + ); + + it.effect("wraps a bare push string in an APNs alert and feeds it on stdin", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "sendPush", + deviceId: udid, + appId: "com.example.app", + payload: "Hello", + }); + expect(calls[0]?.args).toEqual(["simctl", "push", udid, "com.example.app", "-"]); + expect(calls[0]?.stdin).toBe('{"aps":{"alert":"Hello"}}'); + }), + ); +}); + +describe("readDeviceDetail", () => { + it.effect("reads iOS settings from simctl and the accessibility helper", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("ui SIM-1 appearance")) return { stdout: "dark\n" }; + if (key.endsWith("ui SIM-1 content_size")) return { stdout: "extra-extra-large\n" }; + if (key.endsWith("ui SIM-1 increase_contrast")) return { stdout: "enabled\n" }; + if (key.includes("serve-sim-ax-settings status")) { + return { + stdout: + '{"reduce-motion":"on","reduce-transparency":"off","show-borders":"off","voiceover":"off","liquid-glass":"tinted","color-filter":"grayscale"}', + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "large", + increaseContrast: true, + reduceMotion: true, + reduceTransparency: false, + showBorders: false, + voiceOver: false, + liquidGlass: "tinted", + colorFilter: "grayscale", + }); + }), + ); + + it.effect("degrades unreadable values to unknown instead of failing", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({ code: 1, stderr: "boom" })); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({}); + expect(detail.foregroundApp).toBeNull(); + }), + ); + + it.effect("reads Android settings and the focused package", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("cmd uimode night")) return { stdout: "Night mode: yes\n" }; + if (key.endsWith("font_scale")) return { stdout: "0.85\n" }; + if (key.endsWith("animator_duration_scale")) return { stdout: "0\n" }; + if (key.endsWith("wifi_on")) return { stdout: "1\n" }; + if (key.endsWith("dumpsys window")) { + return { + stdout: + " mFocusedApp=ActivityRecord{155579877 u0 com.example.app/.MainActivity t15}\n mCurrentFocus=Window{1a2b u0 com.example.app/com.example.app.MainActivity}\n", + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "android", "emulator-5554"); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "small", + reduceMotion: true, + networkEnabled: true, + }); + expect(detail.foregroundApp).toEqual({ id: "com.example.app" }); + }), + ); +}); diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts new file mode 100644 index 000000000000..12781b31b4b5 --- /dev/null +++ b/apps/server/src/device/DeviceActions.ts @@ -0,0 +1,568 @@ +/** + * Device settings and one-shot actions, run by the server against the host's + * toolchain instead of through serve-sim's shell-exec channel. + * + * serve-sim's preview drives its "Simulator" panel by sending shell commands + * over a token-gated socket. Proxying that would hand any environment + * session arbitrary command execution on the host, so T3 runs the same + * underlying commands itself, typed per action: `xcrun simctl ui` and + * `simctl privacy` for iOS, the `serve-sim-ax-settings` helper that serve-sim + * bundles for the accessibility toggles, and `adb shell` for Android. + * + * Each platform advertises which actions it supports; the panel hides the + * rest rather than showing controls that cannot work. + */ +import { + type DeviceActionInput, + type DeviceActionType, + type DeviceForegroundApp, + DeviceOperationError, + DeviceActionUnavailableError, + type DeviceOrientation, + type DevicePlatform, + type DeviceSettings, + type DeviceTextSize, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Runner = DeviceHostReady["run"]; + +const decodeAxStatus = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +); +const encodePushPayload = Schema.encodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +); + +const IOS_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setLiquidGlass", + "setColorFilter", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", + "sendPush", +]); + +const ANDROID_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setOrientation", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", +]); + +/** Toggle settings each platform can actually flip. */ +const IOS_TOGGLES = new Set([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", +]); +const ANDROID_TOGGLES = new Set(["reduceMotion", "networkEnabled"]); + +export const supportsAction = (platform: DevicePlatform, input: DeviceActionInput): boolean => { + const actions = platform === "ios" ? IOS_ACTIONS : ANDROID_ACTIONS; + if (!actions.has(input.type)) return false; + if (input.type === "setToggle") { + return (platform === "ios" ? IOS_TOGGLES : ANDROID_TOGGLES).has(input.setting); + } + return true; +}; + +const ok = (operation: string) => (result: { code: number; stderr: string; stdout: string }) => + result.code === 0 + ? Effect.succeed(result.stdout) + : Effect.fail( + new DeviceOperationError({ + operation, + reason: "command_failed", + exitCode: result.code, + cause: result, + }), + ); + +// iOS text-size categories in ascending order; the four shared steps index +// into it. `default` is what a fresh simulator reports ("large"). +const IOS_TEXT_SIZES: Record = { + small: "small", + default: "large", + large: "extra-extra-large", + "extra-large": "accessibility-large", +}; +const ANDROID_TEXT_SIZES: Record = { + small: "0.85", + default: "1.0", + large: "1.15", + "extra-large": "1.3", +}; + +const textSizeFromIos = (category: string): DeviceTextSize | undefined => { + const entry = (Object.entries(IOS_TEXT_SIZES) as Array<[DeviceTextSize, string]>).find( + ([, value]) => value === category, + ); + if (entry) return entry[0]; + if (category.startsWith("accessibility")) return "extra-large"; + if (category.includes("extra")) return "large"; + return category === "extra-small" || category === "small" || category === "medium" + ? "small" + : "default"; +}; + +const textSizeFromAndroid = (scale: number): DeviceTextSize => { + if (scale <= 0.9) return "small"; + if (scale >= 1.25) return "extra-large"; + if (scale >= 1.1) return "large"; + return "default"; +}; + +const IOS_TOGGLE_OPTIONS: Record = { + reduceMotion: "reduce-motion", + increaseContrast: "increase-contrast", + reduceTransparency: "reduce-transparency", + showBorders: "show-borders", + voiceOver: "voiceover", +}; + +// serve-sim permission names -> the TCC service or simctl privacy service. +const IOS_TCC_SERVICES: Record = { + camera: "camera", + microphone: "microphone", + photos: "photos", + contacts: "contacts", + calendar: "calendar", + reminders: "reminders", + motion: "motion", + "media-library": "media-library", + faceid: "faceid", +}; + +const ANDROID_PERMISSIONS: Record> = { + camera: ["android.permission.CAMERA"], + microphone: ["android.permission.RECORD_AUDIO"], + photos: ["android.permission.READ_MEDIA_IMAGES", "android.permission.READ_EXTERNAL_STORAGE"], + contacts: ["android.permission.READ_CONTACTS", "android.permission.WRITE_CONTACTS"], + calendar: ["android.permission.READ_CALENDAR", "android.permission.WRITE_CALENDAR"], + location: [ + "android.permission.ACCESS_FINE_LOCATION", + "android.permission.ACCESS_COARSE_LOCATION", + ], + notifications: ["android.permission.POST_NOTIFICATIONS"], + motion: ["android.permission.ACTIVITY_RECOGNITION"], +}; + +// Gravity vector (x:y:z) that makes the emulator report each orientation, +// and the window-manager rotation index for the same. +const ANDROID_GRAVITY: Record = { + portrait: "0:9.81:0", + landscape_left: "9.81:0:0", + portrait_upside_down: "0:-9.81:0", + landscape_right: "-9.81:0:0", +}; +const ANDROID_ROTATION: Record = { + portrait: "0", + landscape_left: "1", + portrait_upside_down: "2", + landscape_right: "3", +}; + +export const runDeviceAction = Effect.fn("DeviceActions.run")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + input: DeviceActionInput, +) { + if (!supportsAction(platform, input)) { + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform, + reason: "unsupported", + }); + } + if (platform === "ios") return yield* runIos(ready, input); + return yield* runAndroid(ready.run, input); +}); + +const simctl = (run: Runner, udid: string, args: ReadonlyArray, operation: string) => + run("xcrun", ["simctl", ...args.slice(0, 1), udid, ...args.slice(1)]).pipe( + Effect.flatMap(ok(operation)), + ); + +const axSettings = (ready: DeviceHostReady, udid: string, args: ReadonlyArray) => + Effect.gen(function* () { + const helper = ready.helpers.serveSimAxSettings; + if (!helper) { + return yield* new DeviceActionUnavailableError({ + operation: "accessibility", + platform: "ios", + reason: "helper_missing", + }); + } + return yield* ready + .run("xcrun", ["simctl", "spawn", udid, helper, ...args]) + .pipe(Effect.flatMap(ok("accessibility"))); + }); + +const runIos = Effect.fn("DeviceActions.runIos")(function* ( + ready: DeviceHostReady, + input: DeviceActionInput, +) { + const udid = input.deviceId; + const { run } = ready; + switch (input.type) { + case "setAppearance": + yield* simctl(run, udid, ["ui", "appearance", input.value], "appearance"); + return; + case "setTextSize": + yield* simctl(run, udid, ["ui", "content_size", IOS_TEXT_SIZES[input.value]], "text size"); + return; + case "setToggle": + if (input.setting === "increaseContrast") { + yield* simctl( + run, + udid, + ["ui", "increase_contrast", input.value ? "enabled" : "disabled"], + "increase contrast", + ); + return; + } + yield* axSettings(ready, udid, [ + "set", + IOS_TOGGLE_OPTIONS[input.setting]!, + input.value ? "on" : "off", + ]); + return; + case "setLiquidGlass": + yield* axSettings(ready, udid, ["set", "liquid-glass", input.value]); + return; + case "setColorFilter": + yield* axSettings(ready, udid, ["set", "color-filter", input.value]); + return; + case "setLocation": + yield* simctl( + run, + udid, + ["location", "set", `${input.latitude},${input.longitude}`], + "location", + ); + return; + case "clearLocation": + yield* simctl(run, udid, ["location", "clear"], "location"); + return; + case "setPermission": { + if (input.permission === "notifications") { + // simctl has no notification permission verb; serve-sim's CLI edits + // the BulletinBoard plist for it. + yield* serveSimPermissions(ready, udid, input); + return; + } + const service = + input.permission === "location" ? "location" : IOS_TCC_SERVICES[input.permission]; + if (!service) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "unsupported", + }); + yield* simctl(run, udid, ["privacy", input.decision, service, input.appId], "permission"); + return; + } + case "openUrl": + yield* simctl(run, udid, ["openurl", input.url], "open url"); + return; + case "launchApp": + yield* simctl(run, udid, ["launch", input.appId], "launch"); + return; + case "terminateApp": + yield* simctl(run, udid, ["terminate", input.appId], "terminate"); + return; + case "sendPush": { + const payload = + typeof input.payload === "string" ? { aps: { alert: input.payload } } : input.payload; + const encoded = yield* encodePushPayload(payload).pipe( + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "push", reason: "invalid_payload", cause }), + ), + ); + yield* run("xcrun", ["simctl", "push", udid, input.appId, "-"], { stdin: encoded }).pipe( + Effect.flatMap(ok("push")), + ); + return; + } + case "shake": + case "setOrientation": + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "ios", + reason: "unsupported", + }); + } +}); + +const serveSimPermissions = ( + ready: DeviceHostReady, + udid: string, + input: Extract, +) => + Effect.gen(function* () { + const cli = ready.helpers.serveSimCli; + if (!cli) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "helper_missing", + }); + yield* ready + .run(process.execPath, [ + cli, + "permissions", + input.decision, + input.permission, + input.appId, + "-d", + udid, + ]) + .pipe(Effect.flatMap(ok("permission"))); + }); + +const adb = (run: Runner, serial: string, args: ReadonlyArray, operation: string) => + run("adb", ["-s", serial, ...args]).pipe(Effect.flatMap(ok(operation))); + +const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( + run: Runner, + input: DeviceActionInput, +) { + const serial = input.deviceId; + const shell = (args: ReadonlyArray, operation: string) => + adb(run, serial, ["shell", ...args], operation); + switch (input.type) { + case "setAppearance": + yield* shell(["cmd", "uimode", "night", input.value === "dark" ? "yes" : "no"], "appearance"); + return; + case "setTextSize": + yield* shell( + ["settings", "put", "system", "font_scale", ANDROID_TEXT_SIZES[input.value]], + "text size", + ); + return; + case "setToggle": + if (input.setting === "networkEnabled") { + const state = input.value ? "enable" : "disable"; + yield* shell(["svc", "wifi", state], "network"); + yield* shell(["svc", "data", state], "network"); + return; + } + if (input.setting === "reduceMotion") { + const scale = input.value ? "0" : "1"; + for (const key of [ + "animator_duration_scale", + "transition_animation_scale", + "window_animation_scale", + ]) { + yield* shell(["settings", "put", "global", key, scale], "reduce motion"); + } + return; + } + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); + case "setOrientation": { + // `user-rotation lock` only rotates window content on recent images; + // the display the encoder captures stays put. Tilting the emulator's + // accelerometer rotates it for real, so that is used whenever the + // target is an emulator. Physical devices get the lock. + if (serial.startsWith("emulator-")) { + yield* shell(["settings", "put", "system", "accelerometer_rotation", "1"], "orientation"); + yield* shell(["cmd", "window", "user-rotation", "free"], "orientation"); + yield* adb( + run, + serial, + ["emu", "sensor", "set", "acceleration", ANDROID_GRAVITY[input.value]], + "orientation", + ); + return; + } + yield* shell( + ["cmd", "window", "user-rotation", "lock", ANDROID_ROTATION[input.value]], + "orientation", + ); + return; + } + case "setLocation": + yield* adb( + run, + serial, + ["emu", "geo", "fix", String(input.longitude), String(input.latitude)], + "location", + ); + return; + case "clearLocation": + // The emulator has no "clear"; leaving the fix in place is the closest + // behavior, so this is a no-op that still refreshes the reading. + return; + case "setPermission": { + const permissions = ANDROID_PERMISSIONS[input.permission]; + if (!permissions) { + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "android", + reason: "unsupported", + }); + } + const verb = input.decision === "grant" ? "grant" : "revoke"; + for (const permission of permissions) { + // Not every app declares every permission in a group; ignore those. + yield* shell(["pm", verb, input.appId, permission], "permission").pipe(Effect.ignore); + } + return; + } + case "openUrl": + yield* shell( + ["am", "start", "-a", "android.intent.action.VIEW", "-d", input.url], + "open url", + ); + return; + case "launchApp": + yield* shell( + ["monkey", "-p", input.appId, "-c", "android.intent.category.LAUNCHER", "1"], + "launch", + ); + return; + case "terminateApp": + yield* shell(["am", "force-stop", input.appId], "terminate"); + return; + case "setLiquidGlass": + case "setColorFilter": + case "shake": + case "sendPush": + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); + } +}); + +/** Read the current settings and foreground app. Errors degrade to unknowns. */ +export const readDeviceDetail = Effect.fn("DeviceActions.readDetail")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + deviceId: string, +): Effect.fn.Return<{ settings: DeviceSettings; foregroundApp: DeviceForegroundApp | null }> { + return platform === "ios" + ? yield* readIos(ready, deviceId) + : yield* readAndroid(ready.run, deviceId); +}); + +const quiet = (effect: Effect.Effect) => + effect.pipe(Effect.orElseSucceed((): A | undefined => undefined)); + +const readIos = Effect.fn("DeviceActions.readIos")(function* ( + ready: DeviceHostReady, + udid: string, +) { + const { run } = ready; + const uiValue = (option: string) => + quiet( + simctl(run, udid, ["ui", option], option).pipe(Effect.map((out) => out.trim().toLowerCase())), + ); + const [appearance, contentSize, contrast, axStatus] = yield* Effect.all( + [ + uiValue("appearance"), + uiValue("content_size"), + uiValue("increase_contrast"), + quiet(axSettings(ready, udid, ["status"]).pipe(Effect.flatMap(decodeAxStatus))), + ], + { concurrency: 4 }, + ); + const onOff = (value: string | undefined) => + value === "on" ? true : value === "off" ? false : undefined; + const settings: DeviceSettings = { + ...(appearance === "light" || appearance === "dark" ? { appearance } : {}), + ...(contentSize ? { textSize: textSizeFromIos(contentSize) } : {}), + ...(contrast ? { increaseContrast: contrast === "enabled" } : {}), + ...(axStatus + ? { + ...(onOff(axStatus["reduce-motion"]) === undefined + ? {} + : { reduceMotion: onOff(axStatus["reduce-motion"]) }), + ...(onOff(axStatus["reduce-transparency"]) === undefined + ? {} + : { reduceTransparency: onOff(axStatus["reduce-transparency"]) }), + ...(onOff(axStatus["show-borders"]) === undefined + ? {} + : { showBorders: onOff(axStatus["show-borders"]) }), + ...(onOff(axStatus.voiceover) === undefined + ? {} + : { voiceOver: onOff(axStatus.voiceover) }), + ...(axStatus["liquid-glass"] === "clear" || axStatus["liquid-glass"] === "tinted" + ? { liquidGlass: axStatus["liquid-glass"] } + : {}), + ...(isColorFilter(axStatus["color-filter"]) + ? { colorFilter: axStatus["color-filter"] } + : {}), + } + : {}), + }; + return { settings, foregroundApp: null }; +}); + +const isColorFilter = (value: unknown): value is DeviceSettings["colorFilter"] & string => + value === "none" || + value === "grayscale" || + value === "red-green" || + value === "green-red" || + value === "blue-yellow"; + +const readAndroid = Effect.fn("DeviceActions.readAndroid")(function* (run: Runner, serial: string) { + const shell = (args: ReadonlyArray) => + quiet( + adb(run, serial, ["shell", ...args], args[0] ?? "shell").pipe(Effect.map((s) => s.trim())), + ); + const [night, fontScale, animator, wifi, focus] = yield* Effect.all( + [ + shell(["cmd", "uimode", "night"]), + shell(["settings", "get", "system", "font_scale"]), + shell(["settings", "get", "global", "animator_duration_scale"]), + shell(["settings", "get", "global", "wifi_on"]), + // `dumpsys window windows` stopped printing the focus on API 36; the + // unfiltered dump still does. + shell(["dumpsys", "window"]), + ], + { concurrency: 5 }, + ); + const scale = fontScale && fontScale !== "null" ? Number(fontScale) : Number.NaN; + const focused = focus?.match(/m(?:CurrentFocus|FocusedApp)=\w+\{[^ ]+ u\d+ ([^/ ]+)\//); + const settings: DeviceSettings = { + ...(night?.includes("yes") + ? { appearance: "dark" } + : night?.includes("no") + ? { appearance: "light" } + : {}), + ...(Number.isFinite(scale) ? { textSize: textSizeFromAndroid(scale) } : {}), + ...(animator !== undefined && animator !== "null" + ? { reduceMotion: Number(animator) === 0 } + : {}), + ...(wifi === "1" || wifi === "0" ? { networkEnabled: wifi === "1" } : {}), + }; + return { + settings, + foregroundApp: focused ? { id: focused[1]! } : null, + }; +}); diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts new file mode 100644 index 000000000000..8d1bafbc5dfc --- /dev/null +++ b/apps/server/src/device/DeviceHost.ts @@ -0,0 +1,102 @@ +/** + * A device host is a machine with simulators or emulators on it. The service + * layer only ever talks to this interface, so a future SSH or cloud host slots + * in beside `LocalDeviceHost` without touching discovery, the proxy, or the + * MCP tools. + * + * Every ready host presents a loopback origin where expo-device-hub answers. + * Hosts add an agent-device daemon endpoint only after agent access is granted. + * For the local host both run on this machine; a remote host would forward + * them here. + */ +import type { + DeviceHostId, + DeviceHostSummary, + DevicePlatform, + DevicePlatformAvailability, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +export class DeviceHostError extends Schema.TaggedError()("DeviceHostError", { + hostId: Schema.String, + step: Schema.String, + cause: Schema.Defect(), +}) { + override get message(): string { + return `Device host ${this.hostId} failed while ${this.step}.`; + } +} + +export class DeviceHostTimeoutError extends Schema.TaggedError()( + "DeviceHostTimeoutError", + { hostId: Schema.String, timeoutMs: Schema.Number }, +) { + override get message(): string { + return `Device host ${this.hostId} did not start agent tools within ${this.timeoutMs} ms.`; + } +} + +export interface DeviceHubEndpoint { + /** Loopback origin of expo-device-hub, e.g. `http://127.0.0.1:3400`. */ + readonly origin: string; +} + +export interface AgentDeviceEndpoint { + readonly baseUrl: string; + readonly token: string; + /** Absolute path of the agent-device entry script for the provider PATH shim. */ + readonly entryPath: string; +} + +export interface DeviceHostReady { + readonly hub: DeviceHubEndpoint; + /** + * Runs a host command (`xcrun`, `adb`, or a helper bundled with the hub) + * where the devices live. On the local host this is a plain spawn; a + * remote host would run it over its transport. + */ + readonly run: ( + command: string, + args: ReadonlyArray, + options?: { readonly timeoutMs?: number; readonly stdin?: string }, + ) => Effect.Effect<{ readonly stdout: string; readonly stderr: string; readonly code: number }>; + /** Absolute paths of helper binaries vendored with the hub, when present. */ + readonly helpers: { + readonly serveSimAxSettings: string | null; + readonly serveSimCli: string | null; + }; +} + +export interface DeviceHostAgentReady extends DeviceHostReady { + readonly agentDevice: AgentDeviceEndpoint; +} + +export class DeviceHost extends Context.Service< + DeviceHost, + { + readonly id: DeviceHostId; + readonly summary: Effect.Effect; + readonly platformAvailability: ( + platform: DevicePlatform, + ) => Effect.Effect; + /** + * Installs tools on first use and starts the helper processes. Idempotent: + * concurrent callers share one start, and a ready host returns immediately. + */ + readonly ensureReady: ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ) => Effect.Effect; + /** Installs and starts agent-device after the user grants agent access. */ + readonly ensureAgentReady: ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ) => Effect.Effect; + /** Current endpoints when already running, without starting anything. */ + readonly current: Effect.Effect; + /** Stops only agent-device. Manual viewing through the hub stays available. */ + readonly stopAgent: Effect.Effect; + /** Stops helpers. Devices themselves keep running; the user owns those. */ + readonly stop: Effect.Effect; + } +>()("t3/device/DeviceHost") {} diff --git a/apps/server/src/device/DeviceHubProxy.test.ts b/apps/server/src/device/DeviceHubProxy.test.ts new file mode 100644 index 000000000000..0f039274e207 --- /dev/null +++ b/apps/server/src/device/DeviceHubProxy.test.ts @@ -0,0 +1,141 @@ +import { afterEach, describe, expect, it } from "vite-plus/test"; +import { + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + AuthSessionId, + LOCAL_DEVICE_HOST_ID, + type AuthEnvironmentScope, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { HttpClient, HttpClientResponse, HttpRouter } from "effect/unstable/http"; +import { + EnvironmentAuth, + ServerAuthMissingCredentialError, + ServerAuthSessionCredentialValidationError, + type ServerAuthCredentialError, + type ServerAuthInternalError, +} from "../auth/EnvironmentAuth.ts"; +import { DeviceService } from "./DeviceService.ts"; +import { deviceHubProxyRouteLayer } from "./DeviceHubProxy.ts"; + +const disposers: Array<() => Promise> = []; +afterEach(async () => { + for (const dispose of disposers.splice(0)) await dispose(); +}); + +const fixture = ( + scopes: ReadonlyArray, + fail = false, + authError?: ServerAuthCredentialError | ServerAuthInternalError, +) => { + let finalized = 0; + const requests: string[] = []; + const client = HttpClient.make((request, _url, signal) => + Effect.gen(function* () { + requests.push(request.url); + signal.addEventListener("abort", () => { + finalized++; + }); + if (fail) return yield* Effect.die(new Error("upstream failed")); + return HttpClientResponse.fromWeb(request, new Response("frame")); + }), + ); + const { handler, dispose } = HttpRouter.toWebHandler( + deviceHubProxyRouteLayer.pipe( + Layer.provideMerge( + Layer.succeed(EnvironmentAuth, { + authenticateWebSocketUpgrade: () => + authError + ? Effect.fail(authError) + : Effect.succeed({ + sessionId: AuthSessionId.make("test"), + subject: "test", + method: "bearer-access-token", + scopes, + }), + } as unknown as EnvironmentAuth["Service"]), + ), + Layer.provideMerge( + Layer.succeed(DeviceService, { + currentReadiness: () => + Effect.succeed({ hostId: LOCAL_DEVICE_HOST_ID, hub: { origin: "http://hub.test" } }), + } as DeviceService["Service"]), + ), + Layer.provideMerge(Layer.succeed(HttpClient.HttpClient, client)), + ), + { disableLogger: true }, + ); + disposers.push(dispose); + return { handler, requests, finalized: () => finalized }; +}; + +describe("device hub proxy", () => { + it("releases the upstream response after forwarding its body and strips tickets", async () => { + const { handler, requests, finalized } = fixture([AuthOrchestrationReadScope]); + const response = await handler( + new Request("http://t3.test/api/device-hub/api/devices?wsTicket=secret"), + ); + expect(response.status).toBe(200); + expect(await response.text()).toBe("frame"); + expect(requests).toEqual(["http://hub.test/api/devices"]); + expect(finalized()).toBe(1); + }); + + it("releases resources when upstream acquisition fails", async () => { + const { handler, finalized } = fixture([AuthOrchestrationReadScope], true); + const response = await handler(new Request("http://t3.test/api/device-hub/api/devices")); + expect(response.status).toBe(500); + expect(finalized()).toBe(1); + }); + + it.each(["/vendor/serve-sim/helper/ws", "/vendor/serve-emu/ws"])( + "rejects input socket %s for a read-only session", + async (path) => { + const { handler, requests } = fixture([AuthOrchestrationReadScope]); + const response = await handler( + new Request(`http://t3.test/api/device-hub${path}`, { headers: { upgrade: "websocket" } }), + ); + expect(response.status).toBe(403); + expect(requests).toEqual([]); + }, + ); + + it("requires operate scope for stream tuning", async () => { + const readOnly = fixture([AuthOrchestrationReadScope]); + const path = "http://t3.test/api/device-hub/vendor/serve-emu/api/stream-settings"; + expect((await readOnly.handler(new Request(path, { method: "POST" }))).status).toBe(403); + const operator = fixture([AuthOrchestrationOperateScope]); + const response = await operator.handler(new Request(path, { method: "POST" })); + expect(response.status).toBe(200); + await response.text(); + }); + + it("never forwards the vendor shell endpoint", async () => { + const { handler, requests } = fixture([AuthOrchestrationOperateScope]); + expect( + ( + await handler( + new Request("http://t3.test/api/device-hub/vendor/serve-sim/exec", { method: "POST" }), + ) + ).status, + ).toBe(404); + expect(requests).toEqual([]); + }); +}); + +it.each([ + [new ServerAuthMissingCredentialError({}), 401], + [ + new ServerAuthSessionCredentialValidationError({ + cause: new Error("private credential diagnostic"), + }), + 500, + ], +] as const)("translates authentication failure to HTTP %s", async (error, status) => { + const { handler, requests } = fixture([], false, error); + const response = await handler(new Request("http://t3.test/api/device-hub/api/devices")); + expect(response.status).toBe(status); + expect(await response.text()).not.toContain("private credential diagnostic"); + expect(requests).toEqual([]); +}); diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts new file mode 100644 index 000000000000..f8685727c882 --- /dev/null +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -0,0 +1,224 @@ +/** + * Same-origin proxy in front of expo-device-hub. + * + * The hub binds loopback and is never reachable directly: serve-sim exposes a + * shell-exec route and serve-emu's action routes are unauthenticated, so the + * only way to a device stream is through this route, which requires an + * environment session with read scope (operate scope for input and tuning). Reusing the T3 + * origin is also what makes remote connections work unchanged — Tailscale and + * T3 Connect already carry `/api/*` and WebSocket upgrades for the app itself. + * + * Only the routes the Device panel needs are forwarded. Anything under the + * hub's dashboard, exec, or WebRTC surface is rejected here. + */ +import { + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + type AuthEnvironmentScope, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import { + HttpClient, + HttpClientRequest, + HttpRouter, + HttpServerRequest, + HttpServerResponse, +} from "effect/unstable/http"; +import * as Socket from "effect/unstable/socket/Socket"; +import * as NodeSocket from "@effect/platform-node/NodeSocket"; + +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import { + failEnvironmentAuthInvalid, + failEnvironmentInternal, + failEnvironmentScopeRequired, +} from "../auth/http.ts"; +import * as DeviceService from "./DeviceService.ts"; + +const ALLOWED_PATHS: ReadonlyArray = [ + /^\/api\/devices$/, + /^\/vendor\/serve-sim\/api$/, + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/, + /^\/vendor\/serve-sim\/appstate$/, + /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/, + /^\/vendor\/serve-emu\/health$/, +]; + +/** Read paths are GET-only; only these accept other methods (screenshot captures, stream tuning). */ +const MUTABLE_PATHS: ReadonlyArray = [ + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/, +]; + +const ALLOWED_WS_PATHS: ReadonlyArray = [ + /^\/api\/devices\/ws$/, + /^\/vendor\/serve-sim\/helper\/ws$/, + /^\/vendor\/serve-emu\/ws$/, +]; + +/** Hop-by-hop and origin headers that must not cross the proxy. */ +const DROPPED_REQUEST_HEADERS = new Set([ + "host", + "connection", + "upgrade", + "sec-websocket-key", + "sec-websocket-version", + "sec-websocket-extensions", + "sec-websocket-protocol", + "cookie", + "authorization", + "dpop", + "content-length", + "accept-encoding", +]); + +const isWebSocketUpgrade = (request: HttpServerRequest.HttpServerRequest) => + request.headers.upgrade?.toLowerCase() === "websocket"; + +/** + * `` and WebSocket cannot set headers, so every proxied request + * authenticates the way the `/ws` upgrade does: a cookie for browser + * sessions, or a short-lived `wsTicket` minted over authenticated HTTP for + * bearer and DPoP clients. The upgrade authenticator already implements that + * fallback order, so it is used for plain requests as well. + */ +const authenticate = (requiredScope: AuthEnvironmentScope) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( + Effect.catch((error) => + Effect.gen(function* () { + if (EnvironmentAuth.isServerAuthCredentialError(error)) { + return yield* failEnvironmentAuthInvalid( + EnvironmentAuth.serverAuthCredentialReason(error), + EnvironmentAuth.serverAuthDpopFailureReason(error), + ); + } + return yield* failEnvironmentInternal("internal_error", error); + }), + ), + ); + if (!session.scopes.includes(requiredScope)) { + return yield* failEnvironmentScopeRequired(requiredScope); + } + }); + +const forwardHeaders = (request: HttpServerRequest.HttpServerRequest, origin: string) => { + const headers: Record = {}; + for (const [name, value] of Object.entries(request.headers)) { + if (DROPPED_REQUEST_HEADERS.has(name) || value === undefined) continue; + headers[name] = value; + } + // serve-emu refuses mutations whose Origin differs from the request origin. + if (request.headers.origin !== undefined) headers.origin = origin; + return headers; +}; + +/** + * Pipe a client WebSocket to the hub's with no framing changes. Frames are + * opaque: H.264 access units one way, input packets the other. + */ +const proxyWebSocket = Effect.fn("DeviceHubProxy.proxyWebSocket")(function* ( + request: HttpServerRequest.HttpServerRequest, + upstreamUrl: string, +) { + const client = yield* request.upgrade; + const upstream = yield* Socket.makeWebSocket(upstreamUrl, { + openTimeout: "10 seconds", + }).pipe(Effect.provide(NodeSocket.layerWebSocketConstructor)); + yield* Effect.scoped( + Effect.gen(function* () { + const writeToClient = yield* client.writer; + const writeToUpstream = yield* upstream.writer; + const downstream = upstream.runRaw((data) => writeToClient(data)); + const upstreamPump = client.runRaw((data) => writeToUpstream(data)); + // Whichever side closes first ends the other via scope teardown. + yield* Effect.raceFirst(downstream, upstreamPump); + }), + ).pipe(Effect.catchCause(() => Effect.void)); + return HttpServerResponse.empty(); +}); + +const proxyHttp = Effect.fn("DeviceHubProxy.proxyHttp")(function* ( + request: HttpServerRequest.HttpServerRequest, + upstreamUrl: string, + hubOrigin: string, +) { + const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient); + const method = request.method; + const upstreamRequest = HttpClientRequest.make(method)(upstreamUrl).pipe( + HttpClientRequest.setHeaders(forwardHeaders(request, hubOrigin)), + method === "GET" || method === "HEAD" + ? (self) => self + : HttpClientRequest.bodyStream(request.stream), + ); + const response = yield* httpClient.execute(upstreamRequest); + const headers: Record = {}; + for (const [name, value] of Object.entries(response.headers)) { + if (name === "content-encoding" || name === "transfer-encoding" || name === "connection") { + continue; + } + if (value !== undefined) headers[name] = value; + } + // Long-lived MJPEG and AVCC responses must not be buffered by compression. + headers["cache-control"] = "no-store, no-transform"; + return HttpServerResponse.stream(response.stream, { + status: response.status, + headers, + ...(headers["content-type"] ? { contentType: headers["content-type"] } : {}), + }); +}); + +const handler = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = HttpServerRequest.toURL(request); + if (Option.isNone(url)) { + return HttpServerResponse.text("Bad Request", { status: 400 }); + } + const hubPath = url.value.pathname.slice(DeviceService.DEVICE_HUB_ROUTE_PREFIX.length) || "/"; + const upgrade = isWebSocketUpgrade(request); + const allowed = (upgrade ? ALLOWED_WS_PATHS : ALLOWED_PATHS).some((pattern) => + pattern.test(hubPath), + ); + if (!allowed) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + const readOnly = request.method === "GET" || request.method === "HEAD"; + if (!upgrade && !readOnly && !MUTABLE_PATHS.some((pattern) => pattern.test(hubPath))) { + return HttpServerResponse.text("Method Not Allowed", { status: 405 }); + } + const controlsDevice = + (upgrade && hubPath !== "/api/devices/ws") || + (!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath)); + yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope); + const devices = yield* DeviceService.DeviceService; + const ready = yield* devices.currentReadiness(); + if (!ready) { + return HttpServerResponse.text("Device hub is not running", { status: 503 }); + } + // The hub runs in standalone mode at its origin root; the panel builds every + // stream and socket URL itself, so nothing depends on the hub knowing the + // T3 prefix. + // The ticket authenticates here and must not travel on to the hub. + const upstreamSearch = new URLSearchParams(url.value.search); + upstreamSearch.delete("wsTicket"); + const search = upstreamSearch.size > 0 ? `?${upstreamSearch.toString()}` : ""; + const upstreamPath = `${hubPath}${search}`; + if (upgrade) { + return yield* proxyWebSocket( + request, + `${ready.hub.origin.replace(/^http/, "ws")}${upstreamPath}`, + ); + } + return yield* proxyHttp(request, `${ready.hub.origin}${upstreamPath}`, ready.hub.origin); +}); + +export const deviceHubProxyRouteLayer = HttpRouter.add( + "*", + `${DeviceService.DEVICE_HUB_ROUTE_PREFIX}/*`, + handler, +); diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts new file mode 100644 index 000000000000..ed42be64eb02 --- /dev/null +++ b/apps/server/src/device/DeviceService.test.ts @@ -0,0 +1,337 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + DEFAULT_SERVER_SETTINGS, + DeviceId, + LOCAL_DEVICE_HOST_ID, + ThreadId, + type DeviceServiceState, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; +import { ServerSettingsService } from "../serverSettings.ts"; +import * as DeviceHost from "./DeviceHost.ts"; + +import { type DeviceService, make, stateStream } from "./DeviceService.ts"; + +const baseState: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: "/api/device-hub", + revision: 0, +}; + +describe("DeviceService.stateStream", () => { + it.effect("emits the current snapshot and then every published change", () => + Effect.gen(function* () { + const pubsub = yield* PubSub.unbounded(); + const current = yield* Ref.make(baseState); + const service: Pick = { + state: Ref.get(current), + subscribe: PubSub.subscribe(pubsub), + }; + + const collected = yield* stateStream(service as DeviceService["Service"]).pipe( + Stream.take(3), + Stream.runCollect, + Effect.forkChild, + ); + yield* Effect.yieldNow; + for (const revision of [1, 2]) { + const next = { ...baseState, revision, hostStatus: "ready" as const }; + yield* Ref.set(current, next); + yield* PubSub.publish(pubsub, next); + } + const seen = yield* Fiber.join(collected); + expect(seen.map((state) => state.revision)).toEqual([0, 1, 2]); + }), + ); +}); + +const fixture = Effect.fn("fixture")(function* ( + onBoot: Effect.Effect = Effect.void, + bootError?: string, + failListAfterShutdown = false, +) { + const settings = yield* Ref.make(DEFAULT_SERVER_SETTINGS); + const starts: string[] = []; + const agentStarts: string[] = []; + const agentStops: string[] = []; + const requests: string[] = []; + let booted = false; + let shutDown = false; + const ready: DeviceHost.DeviceHostReady = { + hub: { origin: "http://device.test" }, + helpers: { serveSimAxSettings: null, serveSimCli: null }, + run: () => Effect.succeed({ code: 0, stdout: "Pixel_API_35\n", stderr: "" }), + }; + const host: DeviceHost.DeviceHost["Service"] = { + id: LOCAL_DEVICE_HOST_ID, + summary: Effect.succeed({ + id: LOCAL_DEVICE_HOST_ID, + kind: "local", + label: "Test server", + platforms: [{ platform: "android", available: true }], + hubInstalled: true, + agentDeviceInstalled: false, + }), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: (onPhase) => + Effect.gen(function* () { + starts.push("start"); + yield* onPhase("starting"); + return ready; + }), + ensureAgentReady: (onPhase) => + Effect.gen(function* () { + agentStarts.push("start"); + yield* onPhase("starting"); + return { + ...ready, + agentDevice: { baseUrl: "http://agent.test", token: "test", entryPath: "/agent" }, + }; + }), + current: Effect.succeed(null), + stopAgent: Effect.sync(() => { + agentStops.push("stop"); + }), + stop: Effect.sync(() => { + starts.push("stop"); + }), + }; + const service = yield* make.pipe( + Effect.provideService(DeviceHost.DeviceHost, host), + Effect.provideService( + ServerSettingsService, + ServerSettingsService.of({ + start: Effect.void, + ready: Effect.void, + getSettings: Ref.get(settings), + updateSettings: (patch) => + Ref.updateAndGet(settings, (current) => ({ + ...current, + enableDeviceSupport: patch.enableDeviceSupport ?? current.enableDeviceSupport, + enableAgentDeviceAccess: + patch.enableAgentDeviceAccess ?? current.enableAgentDeviceAccess, + deviceOnboardingCompleted: + patch.deviceOnboardingCompleted ?? current.deviceOnboardingCompleted, + })), + streamChanges: Stream.empty, + subscribeChanges: Effect.succeed(Stream.empty), + }), + ), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.gen(function* () { + requests.push(request.url); + if (request.url.includes("/api/screenshot")) { + return HttpClientResponse.fromWeb( + request, + new Response(new Uint8Array([137, 80, 78, 71])), + ); + } + if (request.url.endsWith("/shutdown")) { + shutDown = true; + booted = false; + return HttpClientResponse.fromWeb(request, Response.json({ ok: true })); + } + if (shutDown && failListAfterShutdown) { + return HttpClientResponse.fromWeb( + request, + new Response("Discovery busy", { status: 503 }), + ); + } + if (request.url.endsWith("/boot")) { + yield* onBoot; + booted = true; + return HttpClientResponse.fromWeb( + request, + Response.json( + bootError ? { ok: false, error: bootError } : { ok: true, serial: "emulator-5554" }, + ), + ); + } + return HttpClientResponse.fromWeb( + request, + Response.json({ + simulators: [], + emulators: booted + ? [ + { + id: "emulator-5554", + name: "Pixel_API_35", + platform: "android", + version: "Android 15", + booted: true, + physical: false, + }, + ] + : [], + }), + ); + }), + ), + ), + ); + return { service, starts, agentStarts, agentStops, requests, settings }; +}); + +describe("device setup consent", () => { + it.effect("listing and provider startup do not start helpers before consent", () => + Effect.gen(function* () { + const { service, starts, requests } = yield* fixture(); + expect((yield* service.list).hostStatus).toBe("disabled"); + expect(yield* service.readinessIfSupported()).toBeNull(); + const readiness = yield* service.readiness().pipe(Effect.result); + expect(readiness._tag).toBe("Failure"); + expect(starts).toEqual([]); + expect(requests).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect( + "explicit setup discovers never-booted AVDs; disabling stops helpers and blocks agents", + () => + Effect.gen(function* () { + const { service, starts, settings } = yield* fixture(); + const state = yield* service.configure({ enabled: true }); + expect((yield* Ref.get(settings)).enableDeviceSupport).toBe(true); + expect(state.devices.map((device) => [device.id, device.booted])).toEqual([ + ["Pixel_API_35", false], + ]); + expect(starts).toEqual(["start"]); + const disabled = yield* service.configure({ enabled: false }); + expect(disabled.hostStatus).toBe("disabled"); + expect(disabled.devices).toEqual([]); + expect((yield* Ref.get(settings)).enableDeviceSupport).toBe(false); + expect(yield* service.readinessIfSupported()).toBeNull(); + expect(starts).toEqual(["start", "stop"]); + }).pipe(Effect.scoped), + ); + + it.effect("boots a stopped Android AVD and uses its emulator serial without duplicating it", () => + Effect.gen(function* () { + const { service, requests } = yield* fixture(); + yield* service.configure({ enabled: true }); + const session = yield* service.open({ + threadId: ThreadId.make("thread-1"), + deviceId: "Pixel_API_35", + platform: "android", + }); + expect(session.deviceId).toBe("emulator-5554"); + expect(requests.filter((url) => url.endsWith("/boot"))).toHaveLength(1); + const state = yield* service.state; + expect(state.devices.map((device) => device.id)).toEqual(["emulator-5554"]); + expect(state.bootingDevices).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect("installs agent support only after the separate agent permission", () => + Effect.gen(function* () { + const { service, agentStarts, agentStops, settings } = yield* fixture(); + yield* service.configure({ enabled: true }); + expect(agentStarts).toEqual([]); + expect(yield* service.agentReadinessIfSupported()).toBeNull(); + + yield* service.configure({ agentAccessEnabled: true }); + expect(agentStarts).toEqual(["start"]); + expect((yield* Ref.get(settings)).enableAgentDeviceAccess).toBe(true); + expect((yield* service.state).agentAccessEnabled).toBe(true); + + yield* service.configure({ agentAccessEnabled: false, onboardingCompleted: true }); + expect(agentStops).toEqual(["stop"]); + expect((yield* service.state).onboardingCompleted).toBe(true); + expect((yield* Ref.get(settings)).deviceOnboardingCompleted).toBe(true); + }).pipe(Effect.scoped), + ); +}); + +it.effect("publishes boot progress and does not restore sessions after support is disabled", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finish = yield* Deferred.make(); + const { service } = yield* fixture( + Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(finish))), + ); + yield* service.configure({ enabled: true }); + const opening = yield* service + .open({ threadId: ThreadId.make("thread-1"), deviceId: "Pixel_API_35", platform: "android" }) + .pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(started); + expect((yield* service.state).bootingDevices?.map((device) => device.name)).toEqual([ + "Pixel_API_35", + ]); + yield* service.configure({ enabled: false }); + yield* Deferred.succeed(finish, undefined); + expect((yield* Fiber.join(opening))._tag).toBe("Failure"); + const state = yield* service.state; + expect(state.hostStatus).toBe("disabled"); + expect(state.devices).toEqual([]); + expect(state.sessions).toEqual([]); + expect(state.bootingDevices).toEqual([]); + }).pipe(Effect.scoped), +); + +describe("device discovery after server restart", () => { + it.effect("captures an explicit device before any client lists devices", () => + Effect.gen(function* () { + const { service, settings, requests } = yield* fixture(); + yield* Ref.update(settings, (current) => ({ ...current, enableDeviceSupport: true })); + expect((yield* service.state).devices).toEqual([]); + const capture = yield* service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") }); + expect(capture.device.id).toBe("Pixel_API_35"); + expect(Array.from(capture.png)).toEqual([137, 80, 78, 71]); + expect(requests.some((url) => url.endsWith("/api/devices"))).toBe(true); + }).pipe(Effect.scoped), + ); +}); + +for (const [diagnostic, reason, message] of [ + ["Insufficient disk space at /private/user/path", "disk_space", "not enough free disk space"], + ["Timed out spawning /private/user/command", "timeout", "did not become ready in time"], + ["Unexpected failure: secret-token", "launch_failed", "could not start"], +] as const) { + it.effect(`normalizes boot failure: ${reason}`, () => + Effect.gen(function* () { + const { service } = yield* fixture(Effect.void, diagnostic); + yield* service.configure({ enabled: true }); + const error = yield* service + .open({ + threadId: ThreadId.make("boot-failure"), + deviceId: "Pixel_API_35", + platform: "android", + }) + .pipe(Effect.flip); + expect(error._tag).toBe("DeviceBootError"); + expect(error.message).toContain(message); + expect(error.message).not.toContain(diagnostic); + expect((yield* service.state).bootingDevices).toEqual([]); + }).pipe(Effect.scoped), + ); +} + +it.effect("keeps shutdown successful when subsequent discovery fails", () => + Effect.gen(function* () { + const { service } = yield* fixture(Effect.void, undefined, true); + yield* service.configure({ enabled: true }); + const threadId = ThreadId.make("shutdown-refresh"); + const session = yield* service.open({ + threadId, + deviceId: "Pixel_API_35", + platform: "android", + }); + yield* service.close({ threadId, deviceId: session.deviceId, shutdown: true }); + const state = yield* service.state; + expect(state.sessions).toEqual([]); + expect(state.devices.find((device) => device.id === session.deviceId)?.booted).toBe(false); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts new file mode 100644 index 000000000000..6cca6631bd7e --- /dev/null +++ b/apps/server/src/device/DeviceService.ts @@ -0,0 +1,764 @@ +/** + * Device discovery, per-thread device sessions, and the state stream clients + * render the Device panel from. + * + * Discovery and boot go through expo-device-hub's JSON API rather than + * shelling out to simctl and adb here: the hub already normalizes both + * platforms into one device shape and is the process that has to know a + * device is booted before it can stream it. Sessions are the server's own + * bookkeeping — which thread is looking at which device — so the panel and + * the `device_*` tools agree, and so a `device_open` from an agent surfaces in + * every connected client the way `preview_open` does. + */ +import { + type DeviceActionInput, + type DeviceCloseInput, + type DeviceConfigureInput, + type DeviceDetail, + type DeviceDetailInput, + type DeviceError, + type DeviceHostId, + type DeviceId, + DeviceBootError, + DeviceHostUnavailableError, + DeviceNotFoundError, + DeviceOperationError, + type DeviceOpenInput, + type DevicePlatform, + DevicePlatformUnavailableError, + type DeviceServiceState, + type DeviceSession, + type DeviceShutdownInput, + type DeviceSummary, + LOCAL_DEVICE_HOST_ID, + type ThreadId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as SynchronizedRef from "effect/SynchronizedRef"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +import * as ServerSettings from "../serverSettings.ts"; + +import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import * as LocalDeviceHost from "./LocalDeviceHost.ts"; + +/** Origin-relative prefix the hub is proxied under. See DeviceHubProxy. */ +export const DEVICE_HUB_ROUTE_PREFIX = "/api/device-hub"; + +const BOOT_TIMEOUT = Duration.minutes(3); +const SCREENSHOT_TIMEOUT = Duration.seconds(20); + +const HubDevice = Schema.Struct({ + id: Schema.String, + name: Schema.String, + version: Schema.String, + platform: Schema.Literals(["ios", "android"]), + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +const HubDeviceList = Schema.Struct({ + simulators: Schema.Array(HubDevice), + emulators: Schema.Array(HubDevice), + errors: Schema.optional(Schema.Array(Schema.Struct({ message: Schema.String }))), +}); +const HubActionResult = Schema.Struct({ + ok: Schema.Boolean, + id: Schema.optional(Schema.String), + serial: Schema.optional(Schema.String), + error: Schema.optional(Schema.String), +}); + +export interface DeviceScreenshot { + readonly device: DeviceSummary; + readonly png: Uint8Array; +} + +export interface DeviceReadiness extends DeviceHost.DeviceHostReady { + readonly hostId: DeviceHostId; +} + +export interface DeviceAgentReadiness extends DeviceReadiness { + readonly agentDevice: DeviceHost.AgentDeviceEndpoint; +} + +export class DeviceService extends Context.Service< + DeviceService, + { + readonly state: Effect.Effect; + readonly subscribe: Effect.Effect, never, Scope.Scope>; + readonly configure: ( + input: DeviceConfigureInput, + ) => Effect.Effect; + /** Refreshes devices only after device support has been enabled. */ + readonly list: Effect.Effect; + readonly open: (input: DeviceOpenInput) => Effect.Effect; + readonly close: (input: DeviceCloseInput) => Effect.Effect; + readonly shutdown: (input: DeviceShutdownInput) => Effect.Effect; + /** Current settings and foreground app for one device. */ + readonly detail: (input: DeviceDetailInput) => Effect.Effect; + /** Runs one action, then returns the refreshed detail. */ + readonly action: (input: DeviceActionInput) => Effect.Effect; + readonly screenshot: (input: { + readonly hostId?: DeviceHostId | undefined; + readonly deviceId: DeviceId; + }) => Effect.Effect; + /** Host endpoints for the proxy and the provider environment. */ + readonly readiness: (hostId?: DeviceHostId) => Effect.Effect; + /** + * `readiness` only when the host can run at least one platform; a machine + * with no simulator toolchain never installs or starts anything. + */ + readonly readinessIfSupported: ( + hostId?: DeviceHostId, + ) => Effect.Effect; + readonly agentReadinessIfSupported: ( + hostId?: DeviceHostId, + ) => Effect.Effect; + readonly currentReadiness: (hostId?: DeviceHostId) => Effect.Effect; + readonly sessionsForThread: (threadId: ThreadId) => Effect.Effect>; + } +>()("t3/device/DeviceService") {} + +interface ServiceState { + readonly state: DeviceServiceState; +} + +const vendorPrefix = (platform: DevicePlatform) => + platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + +export const make = Effect.gen(function* () { + const localHost = yield* DeviceHost.DeviceHost; + const settings = yield* ServerSettings.ServerSettingsService; + const lifecycleLock = yield* Semaphore.make(1); + const readDeviceSettings = settings.getSettings.pipe( + Effect.map((value) => ({ + enabled: value.enableDeviceSupport, + agentAccessEnabled: value.enableAgentDeviceAccess, + onboardingCompleted: value.deviceOnboardingCompleted, + })), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "settings", reason: "settings_failed", cause }), + ), + ); + const initialSettings = yield* readDeviceSettings; + const hosts: ReadonlyMap = new Map([ + [localHost.id, localHost], + ]); + const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); + const statePubSub = yield* PubSub.unbounded(); + const initialHosts = yield* Effect.forEach(hosts.values(), (host) => host.summary); + const stateRef = yield* SynchronizedRef.make({ + state: { + hosts: initialHosts, + hostStatus: initialSettings.enabled ? "idle" : "disabled", + devices: [], + sessions: [], + onboardingCompleted: initialSettings.onboardingCompleted, + agentAccessEnabled: initialSettings.agentAccessEnabled, + hubBasePath: DEVICE_HUB_ROUTE_PREFIX, + revision: 0, + }, + }); + + const publish = (update: (state: DeviceServiceState) => DeviceServiceState) => + SynchronizedRef.updateAndGetEffect(stateRef, ({ state }) => { + const next = { ...update(state), revision: state.revision + 1 }; + return PubSub.publish(statePubSub, next).pipe(Effect.as({ state: next })); + }).pipe(Effect.map(({ state }) => state)); + + const resolveHost = (hostId: DeviceHostId | undefined) => + Effect.gen(function* () { + const id = hostId ?? LOCAL_DEVICE_HOST_ID; + const host = hosts.get(id); + if (!host) { + return yield* new DeviceHostUnavailableError({ hostId: id, reason: "Unknown host." }); + } + return host; + }); + + const readiness: DeviceService["Service"]["readiness"] = Effect.fn("DeviceService.readiness")( + function* (hostId) { + const host = yield* resolveHost(hostId); + if (!(yield* readDeviceSettings).enabled) { + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: + "Device support is off. Enable it in the Device panel before installing or starting device tools.", + }); + } + const ready = yield* host + .ensureReady((phase) => + publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( + Effect.asVoid, + ), + ) + .pipe( + Effect.tapError((error) => + publish((state) => ({ + ...state, + hostStatus: "failed", + hostStatusDetail: error.message, + })), + ), + Effect.mapError( + (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), + ), + ); + yield* SynchronizedRef.get(stateRef).pipe( + Effect.flatMap(({ state }) => + state.hostStatus === "ready" + ? Effect.void + : publish((current) => ({ + ...current, + hostStatus: "ready", + hostStatusDetail: undefined, + })), + ), + ); + return { hostId: host.id, ...ready }; + }, + lifecycleLock.withPermit, + ); + + const readinessIfSupported: DeviceService["Service"]["readinessIfSupported"] = Effect.fn( + "DeviceService.readinessIfSupported", + )(function* (hostId) { + if (!(yield* readDeviceSettings).enabled) return null; + const host = yield* resolveHost(hostId); + const summary = yield* host.summary; + if (!summary.platforms.some((platform) => platform.available)) return null; + return yield* readiness(host.id); + }); + + const agentReadinessIfSupported: DeviceService["Service"]["agentReadinessIfSupported"] = + Effect.fn("DeviceService.agentReadinessIfSupported")(function* (hostId) { + const deviceSettings = yield* readDeviceSettings; + if (!deviceSettings.enabled || !deviceSettings.agentAccessEnabled) return null; + const host = yield* resolveHost(hostId); + const summary = yield* host.summary; + if (!summary.platforms.some((platform) => platform.available)) return null; + const ready = yield* host + .ensureAgentReady((phase) => + publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( + Effect.asVoid, + ), + ) + .pipe( + Effect.tapError((error) => + publish((state) => ({ + ...state, + hostStatus: "failed", + hostStatusDetail: error.message, + })), + ), + Effect.mapError( + (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), + ), + ); + const hostSummaries = yield* Effect.forEach(hosts.values(), (candidate) => candidate.summary); + yield* publish((state) => ({ ...state, hosts: hostSummaries, hostStatus: "ready" })); + return { hostId: host.id, ...ready }; + }, lifecycleLock.withPermit); + + const currentReadiness: DeviceService["Service"]["currentReadiness"] = (hostId) => + resolveHost(hostId).pipe( + Effect.flatMap((host) => + host.current.pipe(Effect.map((ready) => (ready ? { hostId: host.id, ...ready } : null))), + ), + Effect.orElseSucceed(() => null), + ); + + const hubJson = ( + request: HttpClientRequest.HttpClientRequest, + schema: Schema.Codec, + operation: string, + timeout: Duration.Input = Duration.seconds(15), + ) => + httpClient.execute(request).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), + Effect.scoped, + Effect.timeout(timeout), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation, + reason: "request_failed", + cause, + }), + ), + ); + + const fetchDevices = Effect.fn("DeviceService.fetchDevices")(function* (ready: DeviceReadiness) { + const list = yield* hubJson( + HttpClientRequest.get(`${ready.hub.origin}/api/devices`), + HubDeviceList, + "list", + ); + const toSummary = (device: typeof HubDevice.Type): DeviceSummary => ({ + hostId: ready.hostId, + id: device.id, + platform: device.platform, + name: device.name, + version: device.version, + booted: device.booted, + physical: device.physical, + }); + const devices = [...list.simulators, ...list.emulators].map(toSummary); + const host = yield* resolveHost(ready.hostId); + if ((yield* host.platformAvailability("android")).available) { + const avds = yield* ready.run("emulator", ["-list-avds"]); + if (avds.code !== 0) { + return yield* new DeviceOperationError({ + operation: "list", + reason: "command_failed", + exitCode: avds.code, + cause: avds, + }); + } + for (const name of avds.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean)) { + if (!devices.some((device) => device.platform === "android" && device.name === name)) { + devices.push({ + hostId: ready.hostId, + id: name, + name, + platform: "android", + version: "Android", + booted: false, + physical: false, + }); + } + } + } + return { devices, detail: list.errors?.map((error) => error.message).join("\n") || undefined }; + }); + + const refresh = Effect.fn("DeviceService.refresh")(function* (ready: DeviceReadiness) { + const { devices, detail } = yield* fetchDevices(ready); + const hostSummaries = yield* Effect.forEach(hosts.values(), (host) => host.summary); + return yield* lifecycleLock.withPermit( + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) + return (yield* SynchronizedRef.get(stateRef)).state; + return yield* publish((state) => ({ + ...state, + hosts: hostSummaries, + devices, + hostStatusDetail: detail, + })); + }), + ); + }); + + const list: DeviceService["Service"]["list"] = Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) return (yield* SynchronizedRef.get(stateRef)).state; + const ready = yield* readiness(); + return yield* refresh(ready); + }).pipe( + Effect.tapError((error) => + publish((state) => + state.hostStatus === "disabled" + ? state + : { ...state, hostStatus: "failed", hostStatusDetail: error.message }, + ), + ), + Effect.withSpan("DeviceService.list"), + ); + + const configure: DeviceService["Service"]["configure"] = Effect.fn("DeviceService.configure")( + function* (input) { + const currentSettings = yield* readDeviceSettings; + const nextEnabled = input.enabled ?? currentSettings.enabled; + const nextAgentAccess = input.agentAccessEnabled ?? currentSettings.agentAccessEnabled; + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + yield* settings + .updateSettings({ + ...(input.enabled === undefined ? {} : { enableDeviceSupport: input.enabled }), + ...(input.agentAccessEnabled === undefined + ? {} + : { enableAgentDeviceAccess: input.agentAccessEnabled }), + ...(input.onboardingCompleted === undefined + ? {} + : { deviceOnboardingCompleted: input.onboardingCompleted }), + }) + .pipe( + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "configure", + reason: "settings_failed", + cause, + }), + ), + ); + if (!nextEnabled) { + yield* Effect.forEach(hosts.values(), (host) => host.stop, { discard: true }); + } else if (input.agentAccessEnabled === false) { + yield* Effect.forEach(hosts.values(), (host) => host.stopAgent, { discard: true }); + } + yield* publish((state) => ({ + ...state, + hostStatus: nextEnabled ? "idle" : "disabled", + hostStatusDetail: undefined, + devices: nextEnabled ? state.devices : [], + sessions: nextEnabled ? state.sessions : [], + bootingDevices: nextEnabled ? state.bootingDevices : [], + agentAccessEnabled: nextAgentAccess, + onboardingCompleted: input.onboardingCompleted ?? state.onboardingCompleted, + })); + }), + ); + if (nextEnabled && nextAgentAccess && input.agentAccessEnabled === true) { + yield* agentReadinessIfSupported(); + } + return yield* list; + }, + ); + + const findDevice = ( + state: DeviceServiceState, + hostId: DeviceHostId, + deviceId: DeviceId, + ): DeviceSummary | undefined => + state.devices.find((device) => device.hostId === hostId && device.id === deviceId); + + const ensurePlatform = Effect.fn("DeviceService.ensurePlatform")(function* ( + host: DeviceHost.DeviceHost["Service"], + platform: DevicePlatform, + ) { + const availability = yield* host.platformAvailability(platform); + if (!availability.available) { + return yield* new DevicePlatformUnavailableError({ + hostId: host.id, + platform, + reason: availability.reason ?? "Platform toolchain missing.", + }); + } + }); + + /** + * Boot through the hub so its device list and the streaming helper both see + * the device come up. Android AVDs change id when they boot (AVD name to + * emulator serial), so the returned id is authoritative. + */ + const boot = Effect.fn("DeviceService.boot")(function* ( + ready: DeviceReadiness, + device: DeviceSummary, + ) { + const result = yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/boot`).pipe( + HttpClientRequest.bodyJson({ platform: device.platform, id: device.id, name: device.name }), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "boot", reason: "invalid_payload", cause }), + ), + Effect.flatMap((request) => hubJson(request, HubActionResult, "boot", BOOT_TIMEOUT)), + ); + if (!result.ok) { + return yield* new DeviceBootError({ + hostId: ready.hostId, + deviceId: device.id, + reason: /insufficient.*(?:disk|space)|not enough.*(?:disk|space)|no space left/i.test( + result.error ?? "", + ) + ? "disk_space" + : /timed? out|timeout/i.test(result.error ?? "") + ? "timeout" + : "launch_failed", + cause: result, + }); + } + if (device.platform === "ios") { + // Booting alone does not attach a serve-sim helper; the grid start + // does both and is idempotent for a booted simulator. + yield* HttpClientRequest.post( + `${ready.hub.origin}${vendorPrefix("ios")}/grid/api/start`, + ).pipe( + HttpClientRequest.bodyJson({ udid: device.id }), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "boot", reason: "invalid_payload", cause }), + ), + Effect.flatMap((request) => + hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT), + ), + ); + } + return result.serial ?? result.id ?? device.id; + }); + + const open: DeviceService["Service"]["open"] = Effect.fn("DeviceService.open")(function* (input) { + const host = yield* resolveHost(input.hostId); + yield* ensurePlatform(host, input.platform); + const ready = yield* readiness(host.id); + let state = yield* refresh(ready); + let device = findDevice(state, host.id, input.deviceId); + if (!device) { + return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: input.deviceId }); + } + if (!device.booted && input.boot !== false) { + const booting = { ...device, threadId: input.threadId }; + yield* publish((current) => ({ + ...current, + bootingDevices: [ + ...(current.bootingDevices ?? []).filter( + (entry) => entry.hostId !== booting.hostId || entry.id !== booting.id, + ), + booting, + ], + })); + const bootedId = yield* boot(ready, device).pipe( + Effect.ensuring( + publish((current) => ({ + ...current, + bootingDevices: (current.bootingDevices ?? []).filter( + (entry) => entry.hostId !== booting.hostId || entry.id !== booting.id, + ), + })), + ), + ); + state = yield* refresh(ready); + device = findDevice(state, host.id, bootedId) ?? findDevice(state, host.id, device.id); + if (!device) { + return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: bootedId }); + } + } else if (device.platform === "ios" && device.booted) { + // A simulator booted outside T3 has no helper attached yet. + yield* HttpClientRequest.post( + `${ready.hub.origin}${vendorPrefix("ios")}/grid/api/start`, + ).pipe( + HttpClientRequest.bodyJson({ udid: device.id }), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "open", reason: "invalid_payload", cause }), + ), + Effect.flatMap((request) => + hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT), + ), + ); + } + const openedAt = DateTime.formatIso(yield* DateTime.now); + const session: DeviceSession = { + threadId: input.threadId, + hostId: host.id, + deviceId: device.id, + platform: device.platform, + openedAt, + }; + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: "Device support was turned off while the device was opening.", + }); + yield* publish((current) => ({ + ...current, + sessions: [ + ...current.sessions.filter( + (existing) => + !( + existing.threadId === session.threadId && + existing.hostId === session.hostId && + existing.deviceId === session.deviceId + ), + ), + session, + ], + })); + }), + ); + return session; + }); + + const shutdownDevice = Effect.fn("DeviceService.shutdownDevice")(function* ( + hostId: DeviceHostId, + deviceId: DeviceId, + platform: DevicePlatform, + ) { + const ready = yield* readiness(hostId); + yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/shutdown`).pipe( + HttpClientRequest.bodyJson({ platform, id: deviceId }), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "shutdown", reason: "invalid_payload", cause }), + ), + Effect.flatMap((request) => hubJson(request, HubActionResult, "shutdown")), + Effect.flatMap((result) => + result.ok + ? Effect.void + : Effect.fail( + new DeviceOperationError({ + operation: "shutdown", + reason: "hub_rejected", + cause: result, + }), + ), + ), + ); + yield* publish((state) => ({ + ...state, + devices: state.devices.map((device) => + device.hostId === ready.hostId && device.id === deviceId + ? { ...device, booted: false } + : device, + ), + sessions: state.sessions.filter( + (session) => !(session.hostId === ready.hostId && session.deviceId === deviceId), + ), + })); + // Discovery can stall while an emulator saves its snapshot. A failed + // refresh must not turn an accepted shutdown into an action failure. + yield* refresh(ready).pipe( + Effect.catch((cause) => + Effect.logWarning("Device discovery unavailable after shutdown", { cause }), + ), + ); + }); + + const close: DeviceService["Service"]["close"] = Effect.fn("DeviceService.close")( + function* (input) { + const { state } = yield* SynchronizedRef.get(stateRef); + const closing = state.sessions.filter( + (session) => + session.threadId === input.threadId && + (input.deviceId === undefined || session.deviceId === input.deviceId), + ); + if (closing.length === 0) return; + yield* publish((current) => ({ + ...current, + sessions: current.sessions.filter((session) => !closing.includes(session)), + })); + if (input.shutdown) { + yield* Effect.forEach( + closing, + (session) => shutdownDevice(session.hostId, session.deviceId, session.platform), + { discard: true }, + ); + } + }, + ); + + const shutdown: DeviceService["Service"]["shutdown"] = Effect.fn("DeviceService.shutdown")( + function* (input) { + const host = yield* resolveHost(input.hostId); + yield* shutdownDevice(host.id, input.deviceId, input.platform); + // Sessions on a powered-off device are stale in every thread. + yield* publish((current) => ({ + ...current, + sessions: current.sessions.filter( + (session) => !(session.hostId === host.id && session.deviceId === input.deviceId), + ), + })); + }, + ); + + const screenshot: DeviceService["Service"]["screenshot"] = Effect.fn("DeviceService.screenshot")( + function* (input) { + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); + const url = `${ready.hub.origin}${vendorPrefix(device.platform)}/api/screenshot?device=${encodeURIComponent(device.id)}`; + const png = yield* httpClient.execute(HttpClientRequest.post(url)).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.arrayBuffer), + Effect.map((buffer) => new Uint8Array(buffer)), + Effect.scoped, + Effect.timeout(SCREENSHOT_TIMEOUT), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "screenshot", + reason: "request_failed", + cause, + }), + ), + ); + return { device, png }; + }, + ); + + const resolveDevice = Effect.fn("DeviceService.resolveDevice")(function* ( + hostId: DeviceHostId | undefined, + deviceId: DeviceId, + ) { + const host = yield* resolveHost(hostId); + const ready = yield* readiness(host.id); + const { state } = yield* SynchronizedRef.get(stateRef); + const device = + findDevice(state, host.id, deviceId) ?? findDevice(yield* refresh(ready), host.id, deviceId); + if (!device) return yield* new DeviceNotFoundError({ hostId: host.id, deviceId }); + return { ready, device }; + }); + + const detail: DeviceService["Service"]["detail"] = Effect.fn("DeviceService.detail")( + function* (input) { + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); + const read = yield* readDeviceDetail(ready, device.platform, device.id); + return { + hostId: ready.hostId, + deviceId: device.id, + settings: read.settings, + foregroundApp: read.foregroundApp, + readAt: DateTime.formatIso(yield* DateTime.now), + }; + }, + ); + + const action: DeviceService["Service"]["action"] = Effect.fn("DeviceService.action")( + function* (input) { + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); + yield* runDeviceAction(ready, device.platform, input); + return yield* detail({ hostId: ready.hostId, deviceId: device.id }); + }, + ); + + const sessionsForThread: DeviceService["Service"]["sessionsForThread"] = (threadId) => + SynchronizedRef.get(stateRef).pipe( + Effect.map(({ state }) => state.sessions.filter((session) => session.threadId === threadId)), + ); + + return DeviceService.of({ + state: SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)), + subscribe: PubSub.subscribe(statePubSub), + configure, + list, + open, + close, + shutdown, + detail, + action, + screenshot, + readiness, + readinessIfSupported, + agentReadinessIfSupported, + currentReadiness, + sessionsForThread, + }); +}); + +export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); + +/** State stream for WS subscribers: current snapshot first, then every change. */ +export const stateStream = (service: DeviceService["Service"]): Stream.Stream => + Stream.unwrap( + Effect.gen(function* () { + // Subscribe before reading the snapshot so no change between the two + // is lost; the scope lives as long as the stream does. + const subscription = yield* service.subscribe; + const initial = yield* service.state; + return Stream.concat(Stream.make(initial), Stream.fromSubscription(subscription)); + }), + ).pipe(Stream.scoped); diff --git a/apps/server/src/device/DeviceToolchain.test.ts b/apps/server/src/device/DeviceToolchain.test.ts new file mode 100644 index 000000000000..9f70ca91a8e1 --- /dev/null +++ b/apps/server/src/device/DeviceToolchain.test.ts @@ -0,0 +1,39 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import { ensureDeviceHub, isDeviceHubInstalled } from "./DeviceToolchain.ts"; + +it.effect("failed installation cleans staging and exposes only a safe failure message", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-install-" }); + const result = { + code: ChildProcessSpawner.ExitCode(1), + stdout: "", + stderr: "registry rejected https://private:credential@example.test/package", + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + const error = yield* ensureDeviceHub(baseDir).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, { + run: () => Effect.succeed(result), + }), + Effect.flip, + ); + expect(error.message).toBe( + "Installing expo-device-hub failed while running npm install (exit code 1).", + ); + expect(error.cause).toBe(result); + expect(yield* isDeviceHubInstalled(baseDir)).toBe(false); + expect(yield* fs.readDirectory(path.join(baseDir, "tools", "expo-device-hub"))).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts new file mode 100644 index 000000000000..e6b43cd81ee8 --- /dev/null +++ b/apps/server/src/device/DeviceToolchain.ts @@ -0,0 +1,223 @@ +/** + * Pinned installs of the two external tools device support is built on. + * + * `expo-device-hub` streams simulator and emulator screens and `agent-device` + * drives them. Each is npm-installed separately after its matching consent + * step into `/tools//` and executed from there with the + * server's own Node, never `npx`: an ephemeral + * npx cache would make every first `device_open` after a reboot depend on the + * registry, and the pinned versions are part of the contract the injected + * agent instructions describe. + * + * Install follows the pinned-runtime recipe: stage into a temp sibling, write a + * sentinel only after npm exits 0, then rename into place. npm extracts files + * before it finishes, so an entry file alone does not prove a usable tree. + */ +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import * as ProcessRunner from "../processRunner.ts"; + +const DEVICE_HUB_PACKAGE = "expo-device-hub"; +const DEVICE_HUB_VERSION = "0.9.0"; +const AGENT_DEVICE_PACKAGE = "agent-device"; +const AGENT_DEVICE_VERSION = "0.20.10"; + +const INSTALL_TIMEOUT = Duration.minutes(10); +const installLock = Semaphore.makeUnsafe(1); + +export interface DeviceToolPaths { + readonly installDir: string; + /** Absolute path of the tool's entry script, run with the server's Node. */ + readonly entryPath: string; + readonly sentinelPath: string; +} + +export interface DeviceToolchainPaths { + readonly hub: DeviceToolPaths; + readonly agentDevice: DeviceToolPaths; +} + +export class DeviceToolchainInstallError extends Schema.TaggedError()( + "DeviceToolchainInstallError", + { + tool: Schema.String, + step: Schema.String, + exitCode: Schema.optional(Schema.Number), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + const suffix = this.exitCode === undefined ? "" : ` (exit code ${this.exitCode})`; + return `Installing ${this.tool} failed while ${this.step}${suffix}.`; + } +} + +interface ToolSpec { + readonly name: string; + readonly version: string; + readonly entry: ReadonlyArray; +} + +const HUB_SPEC: ToolSpec = { + name: DEVICE_HUB_PACKAGE, + version: DEVICE_HUB_VERSION, + entry: ["dist", "server", "cli.mjs"], +}; + +const AGENT_DEVICE_SPEC: ToolSpec = { + name: AGENT_DEVICE_PACKAGE, + version: AGENT_DEVICE_VERSION, + entry: ["bin", "agent-device.mjs"], +}; + +const toolPaths = (path: Path.Path, baseDir: string, spec: ToolSpec): DeviceToolPaths => { + const installDir = path.join(baseDir, "tools", spec.name, spec.version); + return { + installDir, + entryPath: path.join(installDir, "node_modules", spec.name, ...spec.entry), + sentinelPath: path.join(installDir, ".install-complete"), + }; +}; + +const deviceToolchainPaths = (path: Path.Path, baseDir: string): DeviceToolchainPaths => ({ + hub: toolPaths(path, baseDir, HUB_SPEC), + agentDevice: toolPaths(path, baseDir, AGENT_DEVICE_SPEC), +}); + +/** Keep daemon state (daemon.json, sessions) in userdata, separate from tool installs. */ +export const agentDeviceStateDir = (path: Path.Path, stateDir: string): string => + path.join(stateDir, "device", "agent-device"); + +const isInstalled = Effect.fn("DeviceToolchain.isInstalled")(function* ( + fs: FileSystem.FileSystem, + paths: DeviceToolPaths, + version: string, +) { + const [entryExists, sentinel] = yield* Effect.all([ + fs.exists(paths.entryPath), + fs.readFileString(paths.sentinelPath).pipe(Effect.option), + ]).pipe(Effect.orElseSucceed(() => [false, Option.none()] as const)); + return entryExists && Option.isSome(sentinel) && sentinel.value.trim() === version; +}); + +const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( + spec: ToolSpec, + paths: DeviceToolPaths, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const runner = yield* ProcessRunner.ProcessRunner; + const fail = (step: string) => (cause: unknown) => + new DeviceToolchainInstallError({ tool: spec.name, step, cause }); + + if (yield* isInstalled(fs, paths, spec.version)) return paths; + + const parentDir = path.dirname(paths.installDir); + yield* fs + .remove(paths.installDir, { recursive: true, force: true }) + .pipe(Effect.mapError(fail("removing an incomplete install"))); + yield* fs + .makeDirectory(parentDir, { recursive: true }) + .pipe(Effect.mapError(fail("preparing the install directory"))); + const stagingDir = yield* fs + .makeTempDirectory({ directory: parentDir, prefix: ".staging-" }) + .pipe(Effect.mapError(fail("preparing the install directory"))); + + return yield* Effect.gen(function* () { + const installArgs = [ + "install", + "--prefix", + stagingDir, + "--no-fund", + "--no-audit", + `${spec.name}@${spec.version}`, + ]; + const result = yield* runner + .run({ command: "npm", args: installArgs, timeout: INSTALL_TIMEOUT }) + .pipe( + Effect.catchTags({ + ProcessSpawnError: (error) => + error.cause instanceof PlatformError.PlatformError && + error.cause.reason._tag === "NotFound" + ? runner.run({ + command: "pnpm", + args: ["--package=npm@11", "dlx", "npm", ...installArgs], + timeout: INSTALL_TIMEOUT, + }) + : Effect.fail(error), + }), + Effect.mapError(fail("running npm install")), + ); + if (result.code !== 0) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "running npm install", + exitCode: Number(result.code), + cause: result, + }); + } + const stagedEntry = path.join(stagingDir, "node_modules", spec.name, ...spec.entry); + if (!(yield* fs.exists(stagedEntry).pipe(Effect.orElseSucceed(() => false)))) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "verifying the installed entry point", + }); + } + yield* fs + .writeFileString(path.join(stagingDir, ".install-complete"), `${spec.version}\n`) + .pipe(Effect.mapError(fail("recording the completed install"))); + yield* fs.rename(stagingDir, paths.installDir).pipe( + Effect.catch((cause) => + // A concurrent server may have published the same version first. + isInstalled(fs, paths, spec.version).pipe( + Effect.flatMap((published) => + published ? Effect.void : Effect.fail(fail("publishing the install")(cause)), + ), + ), + ), + ); + return paths; + }).pipe( + Effect.ensuring(fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore)), + ); +}); + +const ensureTool = Effect.fn("DeviceToolchain.ensureTool")(function* ( + baseDir: string, + spec: ToolSpec, + select: (paths: DeviceToolchainPaths) => DeviceToolPaths, +) { + const path = yield* Path.Path; + const paths = deviceToolchainPaths(path, baseDir); + return yield* installLock.withPermit(installTool(spec, select(paths))); +}); + +export const ensureDeviceHub = (baseDir: string) => + ensureTool(baseDir, HUB_SPEC, (paths) => paths.hub); + +export const ensureAgentDevice = (baseDir: string) => + ensureTool(baseDir, AGENT_DEVICE_SPEC, (paths) => paths.agentDevice); + +const isToolInstalled = Effect.fn("DeviceToolchain.isToolInstalled")(function* ( + baseDir: string, + spec: ToolSpec, + select: (paths: DeviceToolchainPaths) => DeviceToolPaths, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const paths = deviceToolchainPaths(path, baseDir); + return yield* isInstalled(fs, select(paths), spec.version); +}); + +export const isDeviceHubInstalled = (baseDir: string) => + isToolInstalled(baseDir, HUB_SPEC, (paths) => paths.hub); + +export const isAgentDeviceInstalled = (baseDir: string) => + isToolInstalled(baseDir, AGENT_DEVICE_SPEC, (paths) => paths.agentDevice); diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts new file mode 100644 index 000000000000..17d274e73529 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as NodePath from "@effect/platform-node/NodePath"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as FileSystem from "effect/FileSystem"; + +import * as LocalDeviceHost from "./LocalDeviceHost.ts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; +import { HttpClient } from "effect/unstable/http"; +import * as NetService from "@t3tools/shared/Net"; +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; + +const diagnose = (files: ReadonlyArray, environment: NodeJS.ProcessEnv) => + LocalDeviceHost.__testing.platformReason("android").pipe( + Effect.provideService(HostProcessEnvironment, environment), + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService( + FileSystem.FileSystem, + FileSystem.makeNoop({ + exists: (file) => Effect.succeed(files.includes(file)), + }), + ), + Effect.provide(NodePath.layer), + ); + +describe("Android SDK availability", () => { + it.effect("explains that adb alone is insufficient to launch an emulator", () => + Effect.gen(function* () { + const reason = yield* diagnose(["/sdk/platform-tools/adb"], { ANDROID_HOME: "/sdk" }); + expect(reason).toContain("Android Emulator is missing"); + }), + ); + + it.effect("identifies command-line tools required by the device hub", () => + Effect.gen(function* () { + const reason = yield* diagnose(["/sdk/platform-tools/adb", "/sdk/emulator/emulator"], { + ANDROID_HOME: "/sdk", + }); + expect(reason).toContain("Command-line Tools (latest)"); + }), + ); + + it.effect("discovers the standard macOS SDK without ANDROID_HOME", () => + Effect.gen(function* () { + const root = "/test/home/Library/Android/sdk"; + const reason = yield* diagnose( + [ + `${root}/platform-tools/adb`, + `${root}/emulator/emulator`, + `${root}/cmdline-tools/latest/bin/avdmanager`, + ], + { HOME: "/test/home" }, + ); + expect(reason).toBeNull(); + }), + ); + + it.effect("reports an absent SDK without running or installing tools", () => + Effect.gen(function* () { + expect(yield* diagnose([], { HOME: "/test/home" })).toContain("Android SDK was not found"); + }), + ); +}); + +it.effect("puts detected Android tools on the helper PATH without losing existing commands", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const environment = LocalDeviceHost.__testing.deviceHostEnvironment( + { PATH: "/usr/bin", HOME: "/test/home" }, + "/sdk", + "darwin", + path, + ); + expect(environment.PATH).toBe("/sdk/platform-tools:/sdk/emulator:/usr/bin"); + expect(environment.ANDROID_HOME).toBe("/sdk"); + expect(environment.HOME).toBe("/test/home"); + }).pipe(Effect.provide(NodePath.layer)), +); + +it.effect( + "constructs and inspects an unconfigured host without installing or starting helpers", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-consent-" }); + const host = yield* LocalDeviceHost.make().pipe( + Effect.provide(Layer.mergeAll(ServerConfig.layerTest(baseDir, baseDir), NetService.layer)), + Effect.provideService(HostProcessEnvironment, { HOME: baseDir, PATH: "" }), + Effect.provideService(HostProcessPlatform, "linux"), + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.die(new Error("Host construction must not spawn processes")), + ), + ), + Effect.provideService(ProcessRunner.ProcessRunner, { + run: () => Effect.die(new Error("Host construction must not run commands")), + }), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make(() => + Effect.die(new Error("Host construction must not make network requests")), + ), + ), + ); + expect(yield* host.current).toBeNull(); + yield* host.stop; + expect(yield* fs.exists(`${baseDir}/tools`)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts new file mode 100644 index 000000000000..b24ebfbd98b6 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -0,0 +1,697 @@ +/** + * The device host that is this machine. + * + * Runs expo-device-hub as a supervised child on a loopback port and starts the + * agent-device daemon in HTTP mode under a T3-owned state directory. Both are + * lazy: the device service requires explicit setup consent before it calls + * ensureReady to install tools or start helper processes. + * + * The hub runs in its standalone mode (origin root). The T3 proxy strips its + * own prefix, and the Device panel derives stream and socket URLs from the + * prefix itself rather than from anything the hub prints. + */ +import { + type DeviceHostSummary, + type DevicePlatform, + type DevicePlatformAvailability, + LOCAL_DEVICE_HOST_ID, +} from "@t3tools/contracts"; +import { waitForHttpReady } from "@t3tools/shared/httpReadiness"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as NetService from "@t3tools/shared/Net"; +import { isCommandAvailable } from "@t3tools/shared/shell"; +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { HttpClient } from "effect/unstable/http"; +import * as ChildProcess from "effect/unstable/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import { + agentDeviceStateDir, + type DeviceToolPaths, + ensureAgentDevice, + ensureDeviceHub, + isAgentDeviceInstalled, + isDeviceHubInstalled, +} from "./DeviceToolchain.ts"; + +const HUB_READY_TIMEOUT_MS = 30_000; +const DAEMON_READY_TIMEOUT_MS = 30_000; +const DAEMON_POLL_MS = 100; +const HUB_RESTART_STABLE_UPTIME_MS = 60_000; +const HUB_RESTART_MAX_DELAY_MS = 30_000; + +/** + * Written beside the agent-device state so a server that dies without running + * its finalizers (SIGKILL, dev-runner restarts) does not leave a hub bound to + * a loopback port forever. The next start reads it, kills only a process that + * is still that hub, and replaces the file. + */ +const HubStateFile = Schema.Struct({ + pid: Schema.Int, + port: Schema.Int, + entryPath: Schema.String, +}); +const decodeHubStateFile = Schema.decodeUnknownEffect(Schema.fromJsonString(HubStateFile)); +const encodeHubStateFile = Schema.encodeUnknownEffect(Schema.fromJsonString(HubStateFile)); + +const AgentDeviceDaemonFile = Schema.Struct({ + httpPort: Schema.Int, + token: Schema.String, + pid: Schema.optional(Schema.Int), +}); +const decodeDaemonFile = Schema.decodeUnknownEffect(Schema.fromJsonString(AgentDeviceDaemonFile)); + +interface HubProcess { + readonly child: ChildProcessSpawner.ChildProcessHandle; + readonly scope: Scope.Closeable; + readonly origin: string; + readonly startedAtMillis: number; +} + +interface RunningHost { + readonly hub: HubProcess; + readonly agentDevice: DeviceHost.AgentDeviceEndpoint | null; + readonly helpers: DeviceHost.DeviceHostReady["helpers"]; +} + +const platformReason = Effect.fn("LocalDeviceHost.platformReason")(function* ( + platform: DevicePlatform, +): Effect.fn.Return { + const hostPlatform = yield* HostProcessPlatform; + if (platform === "ios") { + if (hostPlatform !== "darwin") return "iOS Simulators need macOS with Xcode."; + if (!(yield* isCommandAvailable("xcrun"))) return "Xcode command line tools were not found."; + return null; + } + const sdk = yield* androidSdk; + if (!sdk.root) + return "Android SDK was not found. Install it with Android Studio or set ANDROID_HOME to your SDK directory."; + if (!sdk.adb) + return `Android SDK Platform-Tools are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + if (!sdk.emulator) + return `Android Emulator is missing from ${sdk.root}. Install it in Android Studio's SDK Manager.`; + if (!sdk.avdmanager) + return `Android SDK Command-line Tools (latest) are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + return null; +}); + +/** Resolve the SDK once for both diagnostics and the environment passed to helpers. */ +const androidSdk = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const environment = yield* HostProcessEnvironment; + const platform = yield* HostProcessPlatform; + const home = environment.HOME ?? environment.USERPROFILE ?? ""; + const explicit = environment.ANDROID_HOME?.trim() || environment.ANDROID_SDK_ROOT?.trim(); + const candidates = explicit + ? [explicit] + : [ + path.join(home, "Library", "Android", "sdk"), + path.join(home, "Android", "Sdk"), + path.join( + environment.LOCALAPPDATA ?? path.join(home, "AppData", "Local"), + "Android", + "Sdk", + ), + ]; + if (!explicit) { + for (const directory of (environment.PATH ?? "").split(platform === "win32" ? ";" : ":")) { + if (!directory) continue; + const resolved = yield* fs + .realPath(path.join(directory, platform === "win32" ? "adb.exe" : "adb")) + .pipe(Effect.option); + if (resolved._tag === "Some") candidates.push(path.dirname(path.dirname(resolved.value))); + } + } + const exists = (file: string) => fs.exists(file).pipe(Effect.orElseSucceed(() => false)); + for (const root of candidates) { + const adb = yield* exists( + path.join(root, "platform-tools", platform === "win32" ? "adb.exe" : "adb"), + ); + const emulator = yield* exists( + path.join(root, "emulator", platform === "win32" ? "emulator.exe" : "emulator"), + ); + if (explicit || adb || emulator) { + const avdmanager = yield* exists( + path.join( + root, + "cmdline-tools", + "latest", + "bin", + platform === "win32" ? "avdmanager.bat" : "avdmanager", + ), + ); + return { root, adb, emulator, avdmanager }; + } + } + return { root: null, adb: false, emulator: false, avdmanager: false }; +}); + +const deviceHostEnvironment = ( + environment: NodeJS.ProcessEnv, + sdkRoot: string | null, + hostPlatform: NodeJS.Platform, + path: Path.Path, +): NodeJS.ProcessEnv => { + return sdkRoot + ? { + ...environment, + ANDROID_HOME: sdkRoot, + PATH: [ + path.join(sdkRoot, "platform-tools"), + path.join(sdkRoot, "emulator"), + environment.PATH ?? environment.Path ?? "", + ].join(hostPlatform === "win32" ? ";" : ":"), + } + : environment; +}; + +export const make = Effect.fn("LocalDeviceHost.make")(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const config = yield* ServerConfig.ServerConfig; + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + const net = yield* NetService.NetService; + const runner = yield* ProcessRunner.ProcessRunner; + const httpClient = yield* HttpClient.HttpClient; + const environment = yield* HostProcessEnvironment; + const hostPlatform = yield* HostProcessPlatform; + const sdk = yield* androidSdk; + const hostEnvironment = deviceHostEnvironment(environment, sdk.root, hostPlatform, path); + const startLock = yield* Semaphore.make(1); + const runningRef = yield* Ref.make(null); + const restartDelayRef = yield* Ref.make(0); + const hostId = LOCAL_DEVICE_HOST_ID; + + const platformAvailability = Effect.fn("LocalDeviceHost.platformAvailability")(function* ( + platform: DevicePlatform, + ): Effect.fn.Return { + const reason = yield* platformReason(platform).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + return reason === null ? { platform, available: true } : { platform, available: false, reason }; + }); + + const summary: Effect.Effect = Effect.gen(function* () { + const [platforms, hubInstalled, agentDeviceInstalled] = yield* Effect.all([ + Effect.all([platformAvailability("ios"), platformAvailability("android")]), + isDeviceHubInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ), + isAgentDeviceInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ), + ]); + return { + id: hostId, + kind: "local", + label: "This machine", + platforms, + hubInstalled, + agentDeviceInstalled, + }; + }); + + const hubEnvironment = (): NodeJS.ProcessEnv => ({ + ...hostEnvironment, + FORCE_COLOR: "0", + NO_COLOR: "1", + }); + + const stopHub = (hub: HubProcess | undefined) => + hub ? Scope.close(hub.scope, Exit.void).pipe(Effect.ignore) : Effect.void; + + const hubStatePath = () => path.join(agentDeviceStateDir(path, config.stateDir), "hub.json"); + + const isProcessAlive = (pid: number) => + Effect.sync(() => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }); + + /** + * A hub left behind by a previous server is identified by pid plus the + * command line's entry path, so a recycled pid belonging to something else + * is never touched. + */ + const reapStaleHub = Effect.gen(function* () { + const previous = yield* fs + .readFileString(hubStatePath()) + .pipe(Effect.flatMap(decodeHubStateFile), Effect.option); + if (previous._tag === "None") return; + const alive = yield* isProcessAlive(previous.value.pid); + if (alive) { + const commandLine = yield* runner + .run({ + command: "ps", + args: ["-o", "command=", "-p", String(previous.value.pid)], + timeout: Duration.seconds(5), + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.map((result) => result.stdout), + Effect.orElseSucceed(() => ""), + ); + if (commandLine.includes(previous.value.entryPath)) { + yield* Effect.logWarning("Stopping a device hub left behind by a previous server", { + pid: previous.value.pid, + port: previous.value.port, + }); + yield* Effect.sync(() => { + try { + process.kill(previous.value.pid, "SIGTERM"); + } catch { + // Already gone. + } + }); + } + } + yield* fs.remove(hubStatePath(), { force: true }).pipe(Effect.ignore); + }).pipe(Effect.catchCause(() => Effect.void)); + + const recordHub = (hub: HubProcess, hubTool: DeviceToolPaths) => + encodeHubStateFile({ + pid: Number(hub.child.pid), + port: Number(new URL(hub.origin).port), + entryPath: hubTool.entryPath, + }).pipe( + Effect.flatMap((json) => fs.writeFileString(hubStatePath(), json)), + Effect.ignore, + ); + + const spawnHub = Effect.fn("LocalDeviceHost.spawnHub")(function* ( + hubTool: DeviceToolPaths, + ): Effect.fn.Return { + yield* reapStaleHub; + yield* fs + .makeDirectory(agentDeviceStateDir(path, config.stateDir), { recursive: true }) + .pipe(Effect.ignore); + const port = yield* net.reserveLoopbackPort("127.0.0.1").pipe( + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId, + step: "reserving a port for the device hub", + cause, + }), + ), + ); + const origin = `http://127.0.0.1:${port}`; + const scope = yield* Scope.make("sequential"); + const child = yield* spawner + .spawn( + ChildProcess.make( + process.execPath, + [ + hubTool.entryPath, + "--port", + String(port), + "--host", + "127.0.0.1", + "--hide-sidebar", + "--hide-boot-device", + ], + { + detached: false, + shell: false, + stdout: "pipe", + stderr: "pipe", + env: hubEnvironment(), + }, + ), + ) + .pipe( + Effect.provideService(Scope.Scope, scope), + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId, + step: "starting the device hub", + cause, + }), + ), + ); + const startedAtMillis = yield* Clock.currentTimeMillis; + const hub: HubProcess = { child, scope, origin, startedAtMillis }; + yield* Effect.forkIn(observeHubOutput(hub), scope); + yield* waitForHttpReady({ + baseUrl: origin, + path: "/readyz", + timeoutMs: HUB_READY_TIMEOUT_MS, + makeError: (info) => + new DeviceHost.DeviceHostError({ + hostId, + step: "waiting for the device hub to answer", + cause: info.cause, + }), + }).pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), + Effect.tapError(() => stopHub(hub)), + ); + yield* recordHub(hub, hubTool); + yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); + return hub; + }); + + const observeHubOutput = (hub: HubProcess) => + hub.child.all.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.map((line) => line.trim()), + Stream.filter((line) => line.length > 0), + Stream.runForEach((line) => + Effect.logDebug("Device hub output", { pid: Number(hub.child.pid), output: line }), + ), + Effect.catchCause(() => Effect.void), + ); + + /** + * Restart the hub when it dies under us, with the same doubling backoff the + * relay connector uses so a hub that crashes on boot cannot spin. + */ + const superviseHub = (hub: HubProcess, hubTool: DeviceToolPaths): Effect.Effect => + Effect.gen(function* () { + yield* Effect.result(hub.child.exitCode); + const running = yield* Ref.get(runningRef); + if (running?.hub.child.pid !== hub.child.pid) return; + const uptime = (yield* Clock.currentTimeMillis) - hub.startedAtMillis; + const delay = yield* Ref.modify(restartDelayRef, (current) => { + if (uptime >= HUB_RESTART_STABLE_UPTIME_MS) return [0, 0]; + const next = current === 0 ? 1_000 : Math.min(current * 2, HUB_RESTART_MAX_DELAY_MS); + return [current, next]; + }); + yield* Effect.logWarning("Device hub exited; restarting", { + pid: Number(hub.child.pid), + delayMs: delay, + }); + yield* Effect.sleep(Duration.millis(delay)); + yield* startLock.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(runningRef); + if (current?.hub.child.pid !== hub.child.pid) return; + const replacement = yield* spawnHub(hubTool); + yield* Ref.set(runningRef, { ...current, hub: replacement }); + yield* Effect.forkDetach(superviseHub(replacement, hubTool)); + }), + ); + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("Device hub supervisor failed", { cause })), + ); + + const daemonFilePath = () => path.join(agentDeviceStateDir(path, config.stateDir), "daemon.json"); + + const readDaemonFile = Effect.fn("LocalDeviceHost.readDaemonFile")(function* () { + const raw = yield* fs.readFileString(daemonFilePath()); + return yield* decodeDaemonFile(raw); + }); + + /** + * agent-device auto-starts its daemon on any command. A trivial `devices` + * call in HTTP mode is the documented way to bring it up; its output is the + * daemon.json this reads back. + */ + const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* ( + agentTool: DeviceToolPaths, + ): Effect.fn.Return { + const stateDir = agentDeviceStateDir(path, config.stateDir); + yield* fs.makeDirectory(stateDir, { recursive: true }).pipe(Effect.ignore); + const existing = yield* readDaemonFile().pipe(Effect.option); + const daemonEnvironment: NodeJS.ProcessEnv = { + ...hostEnvironment, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: "http", + // The daemon idles out after five minutes by default; the server owns + // its lifetime here and stops it explicitly. + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", + AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", + FORCE_COLOR: "0", + NO_COLOR: "1", + }; + const toEndpoint = ( + file: typeof AgentDeviceDaemonFile.Type, + ): DeviceHost.AgentDeviceEndpoint => ({ + baseUrl: `http://127.0.0.1:${file.httpPort}`, + token: file.token, + entryPath: agentTool.entryPath, + }); + if (existing._tag === "Some") { + const alive = yield* HttpClient.withScope(httpClient) + .get(`http://127.0.0.1:${existing.value.httpPort}/health`) + .pipe( + Effect.timeout(Duration.seconds(2)), + Effect.flatMap((response) => + response.arrayBuffer.pipe(Effect.as(response.status === 200)), + ), + Effect.scoped, + Effect.orElseSucceed(() => false), + ); + if (alive) return toEndpoint(existing.value); + yield* fs.remove(daemonFilePath(), { force: true }).pipe(Effect.ignore); + } + // There is no `daemon start`; the first command in a state dir spawns the + // daemon and blocks until it answers. `devices` is the cheapest one. + yield* runner + .run({ + command: process.execPath, + args: [agentTool.entryPath, "devices", "--json"], + env: daemonEnvironment, + timeout: Duration.millis(DAEMON_READY_TIMEOUT_MS), + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.ignore); + const deadline = (yield* Clock.currentTimeMillis) + DAEMON_READY_TIMEOUT_MS; + while (true) { + const file = yield* readDaemonFile().pipe(Effect.option); + if (file._tag === "Some") return toEndpoint(file.value); + if ((yield* Clock.currentTimeMillis) > deadline) { + return yield* new DeviceHost.DeviceHostTimeoutError({ + hostId, + timeoutMs: DAEMON_READY_TIMEOUT_MS, + }); + } + yield* Effect.sleep(Duration.millis(DAEMON_POLL_MS)); + } + }); + + const stopAgentDeviceDaemon = (agentTool: DeviceToolPaths | null) => + agentTool + ? runner + .run({ + command: process.execPath, + args: [ + agentTool.entryPath, + "daemon", + "stop", + "--state-dir", + agentDeviceStateDir(path, config.stateDir), + ], + env: { ...hostEnvironment, AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" }, + timeout: Duration.seconds(10), + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.ignore) + : Effect.void; + + let agentToolRef: DeviceToolPaths | null = null; + + const ensureHubReady = Effect.fn("LocalDeviceHost.ensureHubReady")(function* ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ): Effect.fn.Return { + const running = yield* Ref.get(runningRef); + if (running) { + const alive = yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)); + if (alive) return running; + yield* Ref.set(runningRef, null); + } + const installed = yield* isDeviceHubInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + if (!installed) yield* onPhase("installing"); + const hubTool = yield* ensureDeviceHub(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId, + step: "installing device support", + cause, + }), + ), + ); + yield* onPhase("starting"); + const hub = yield* spawnHub(hubTool); + const candidate = helperPaths(hubTool); + const [axExists, cliExists] = yield* Effect.all([ + fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), + fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), + ]); + const next: RunningHost = { + hub, + agentDevice: null, + helpers: { + serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, + serveSimCli: cliExists ? candidate.serveSimCli : null, + }, + }; + yield* Ref.set(runningRef, next); + yield* Ref.set(restartDelayRef, 0); + yield* Effect.forkDetach(superviseHub(hub, hubTool)); + return next; + }); + + const ensureReady: DeviceHost.DeviceHost["Service"]["ensureReady"] = (onPhase) => + startLock.withPermits(1)(ensureHubReady(onPhase).pipe(Effect.map(toReady))); + + const ensureAgentReady: DeviceHost.DeviceHost["Service"]["ensureAgentReady"] = (onPhase) => + startLock.withPermits(1)( + Effect.gen(function* (): Generator< + Effect.Effect, + DeviceHost.DeviceHostAgentReady + > { + const running = yield* ensureHubReady(onPhase); + if (running.agentDevice) return { ...toReady(running), agentDevice: running.agentDevice }; + const installed = yield* isAgentDeviceInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + if (!installed) yield* onPhase("installing"); + const agentTool = yield* ensureAgentDevice(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId, + step: "installing agent tools", + cause, + }), + ), + ); + agentToolRef = agentTool; + yield* onPhase("starting"); + const agentDevice = yield* startAgentDeviceDaemon(agentTool); + const next = { ...running, agentDevice }; + yield* Ref.set(runningRef, next); + return { ...toReady(next), agentDevice }; + }), + ); + + const helperPaths = (hubTool: DeviceToolPaths) => { + const serveSimDist = path.join( + hubTool.installDir, + "node_modules", + "expo-device-hub", + "vendor", + "serve-sim", + "dist", + ); + return { + serveSimAxSettings: path.join(serveSimDist, "simax", "serve-sim-ax-settings"), + serveSimCli: path.join(serveSimDist, "serve-sim.js"), + }; + }; + + const run: DeviceHost.DeviceHostReady["run"] = (command, args, options) => + runner + .run({ + command: + command === "emulator" && sdk.root + ? path.join( + sdk.root, + "emulator", + hostPlatform === "win32" ? "emulator.exe" : "emulator", + ) + : command, + args, + env: hostEnvironment, + timeout: Duration.millis(options?.timeoutMs ?? 20_000), + timeoutBehavior: "timedOutResult", + ...(options?.stdin === undefined ? {} : { stdin: options.stdin }), + }) + .pipe( + Effect.map((result) => ({ + stdout: result.stdout, + stderr: result.stderr, + code: Number(result.code), + })), + Effect.catch((cause) => Effect.succeed({ stdout: "", stderr: String(cause), code: 127 })), + ); + + const toReady = (running: RunningHost): DeviceHost.DeviceHostReady => ({ + hub: { origin: running.hub.origin } satisfies DeviceHost.DeviceHubEndpoint, + run, + helpers: running.helpers, + }); + + const current: DeviceHost.DeviceHost["Service"]["current"] = Ref.get(runningRef).pipe( + Effect.map((running) => (running ? toReady(running) : null)), + ); + + const stopAgent: DeviceHost.DeviceHost["Service"]["stopAgent"] = startLock.withPermits(1)( + Effect.gen(function* () { + yield* stopAgentDeviceDaemon(agentToolRef); + yield* Ref.update(runningRef, (running) => + running ? { ...running, agentDevice: null } : running, + ); + }), + ); + + const stop: DeviceHost.DeviceHost["Service"]["stop"] = startLock.withPermits(1)( + Effect.gen(function* () { + const running = yield* Ref.getAndSet(runningRef, null); + yield* stopHub(running?.hub); + yield* fs.remove(hubStatePath(), { force: true }).pipe(Effect.ignore); + yield* stopAgentDeviceDaemon(agentToolRef); + }), + ); + + // Never leave the hub or daemon behind when the server's scope closes. + yield* Effect.addFinalizer(() => stop); + + const host: DeviceHost.DeviceHost["Service"] = { + id: hostId, + summary, + platformAvailability, + ensureReady, + ensureAgentReady, + current, + stopAgent, + stop, + }; + return host; +}); + +export const layer = Layer.effect(DeviceHost.DeviceHost, make()); + +/** Exposed for tests. */ +export const __testing = { + AgentDeviceDaemonFile, + androidSdk, + platformReason, + deviceHostEnvironment, +}; diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts new file mode 100644 index 000000000000..083ac0c034fe --- /dev/null +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -0,0 +1,127 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { McpSchema, McpServer } from "effect/unstable/ai"; + +import * as DeviceService from "../device/DeviceService.ts"; +import * as McpHttpServer from "./McpHttpServer.ts"; +import * as McpInvocationContext from "./McpInvocationContext.ts"; + +const environmentId = EnvironmentId.make("environment-device-test"); +const threadId = ThreadId.make("thread-device-test"); +const invocation = (capabilities: ReadonlyArray) => ({ + environmentId, + threadId, + providerSessionId: "provider-session-device-test", + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(capabilities), + issuedAt: 1, +}); +const client = McpSchema.McpServerClient.of({ + clientId: 1, + clientCapabilities: {}, + clientInfo: { name: "mcp-test", version: "1.0.0" }, + protocolVersion: "2025-06-18", + initializePayload: { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "mcp-test", version: "1.0.0" }, + }, + getClient: Effect.die("unused"), +}); + +const device = { + hostId: "local", + id: "UDID-1", + platform: "ios" as const, + name: "iPhone 17 Pro", + version: "iOS 27.0", + booted: true, + physical: false, +}; +const state = { + hosts: [ + { + id: "local", + kind: "local" as const, + label: "This machine", + platforms: [ + { platform: "ios" as const, available: true }, + { platform: "android" as const, available: false, reason: "No SDK" }, + ], + hubInstalled: true, + agentDeviceInstalled: true, + }, + ], + hostStatus: "ready" as const, + devices: [device], + sessions: [], + onboardingCompleted: true, + agentAccessEnabled: true, + hubBasePath: "/api/device-hub", + revision: 1, +}; +const png = new Uint8Array(24); +new DataView(png.buffer).setUint32(0, 0x89504e47); +new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); +new DataView(png.buffer).setUint32(12, 0x49484452); +new DataView(png.buffer).setUint32(16, 1206); +new DataView(png.buffer).setUint32(20, 2622); + +const DeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ + state: Effect.succeed(state), + list: Effect.succeed(state), + open: (input) => + Effect.succeed({ + threadId: input.threadId, + hostId: "local", + deviceId: input.deviceId, + platform: input.platform, + openedAt: "2026-09-08T00:00:00.000Z", + }), + sessionsForThread: () => Effect.succeed([]), + screenshot: () => Effect.succeed({ device, png }), + close: () => Effect.void, +}); + +const TestLayer = McpHttpServer.DeviceToolkitRegistrationLive.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provideMerge(DeviceServiceMock), + Layer.provide(NodeServices.layer), +); + +it.effect("registers the device tools and returns the screenshot as image content", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const names = server.tools.map(({ tool }) => tool.name).toSorted(); + expect(names).toEqual(["device_close", "device_list", "device_open", "device_screenshot"]); + + const callWith = (capabilities: ReadonlyArray) => + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation(capabilities)); + + const opened = yield* server + .callTool({ name: "device_open", arguments: { platform: "ios" } }) + .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(opened.isError).toBe(false); + const openedContent = opened.structuredContent as { quickStart: string }; + expect(openedContent.quickStart).toContain("--udid UDID-1"); + + const shot = yield* server + .callTool({ name: "device_screenshot", arguments: { deviceId: "UDID-1" } }) + .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(shot.isError).toBe(false); + expect(shot.content.map((entry) => entry.type)).toEqual(["text", "image"]); + expect(shot.structuredContent).toMatchObject({ + screenshot: { mimeType: "image/png", width: 1206, height: 2622 }, + }); + + const denied = yield* server + .callTool({ name: "device_list", arguments: {} }) + .pipe(callWith(["preview"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(denied.isError).toBe(true); + }), + ).pipe(Effect.provide(TestLayer)), +); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 3556a57befdc..bf7cf0520668 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -16,6 +16,7 @@ import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstab import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; +import * as DeviceService from "../device/DeviceService.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as McpSessionRegistry from "./McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts"; @@ -30,6 +31,15 @@ import { } from "./toolkits/preview/tools.ts"; import { PullRequestsToolkitHandlersLive } from "./toolkits/pullRequests/handlers.ts"; import { PullRequestsToolkit } from "./toolkits/pullRequests/tools.ts"; +import { + DeviceScreenshotToolkitHandlersLive, + DeviceStandardToolkitHandlersLive, +} from "./toolkits/device/handlers.ts"; +import { + DeviceScreenshotTool, + DeviceScreenshotToolkit, + DeviceStandardToolkit, +} from "./toolkits/device/tools.ts"; const unauthorized = HttpServerResponse.jsonUnsafe( { @@ -426,6 +436,154 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot }); }); +interface ImageToolResult { + readonly screenshot: { + readonly mimeType: "image/png"; + readonly data: string; + readonly width: number; + readonly height: number; + }; + readonly [key: string]: unknown; +} + +/** + * Failures surface only their tag: the remote message may carry renderer or + * device output the agent should not see, and the tag is what it can act on. + */ +const imageToolFailure = + (toolName: string, operation: string, failureText: string) => + (cause: Cause.Cause) => { + if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { + return Effect.failCause(cause).pipe(Effect.orDie); + } + const failures = cause.reasons.filter(Cause.isFailReason); + const firstFailure = failures[0]?.error; + const errorTag = + typeof firstFailure === "object" && + firstFailure !== null && + "_tag" in firstFailure && + typeof firstFailure._tag === "string" + ? firstFailure._tag + : `${toolName}Error`; + const result = new McpSchema.CallToolResult({ + isError: true, + structuredContent: { + error: { + _tag: errorTag, + operation, + failureCount: failures.length, + }, + }, + content: [{ type: "text", text: failureText }], + }); + return Effect.logWarning(`${toolName} failed`, { + operation, + errorTag, + failureCount: failures.length, + }).pipe(Effect.as(result)); + }; + +/** + * `McpServer.toolkit` serializes every result as JSON text, which is the + * wrong shape for a screenshot: the model needs image content. Tools whose + * result carries a `screenshot` field are registered by hand so the PNG goes + * out as an image block and the rest of the payload as JSON metadata. + */ +const registerImageTool = ( + tool: T, + handle: (payload: Tool.Parameters) => Effect.Effect<{ readonly encodedResult: unknown }, E, R>, + provide: ( + effect: Effect.Effect<{ readonly encodedResult: unknown }, E, R>, + ) => Effect.Effect< + { readonly encodedResult: unknown }, + E, + McpInvocationContext.McpInvocationContext + >, + operation: string, + failureText: string, +) => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + yield* server.addTool({ + tool: new McpSchema.Tool({ + name: tool.name, + description: Tool.getDescription(tool), + inputSchema: Tool.getJsonSchema(tool), + annotations: { + ...Context.getOption(tool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(tool.annotations, Tool.Readonly), + destructiveHint: Context.get(tool.annotations, Tool.Destructive), + idempotentHint: Context.get(tool.annotations, Tool.Idempotent), + openWorldHint: Context.get(tool.annotations, Tool.OpenWorld), + }, + }), + annotations: tool.annotations, + handle: (payload) => + Effect.withFiber((fiber) => { + const invocation = Context.getUnsafe( + fiber.context, + McpInvocationContext.McpInvocationContext, + ); + return provide(handle(payload as Tool.Parameters)).pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.matchCauseEffect({ + onFailure: imageToolFailure(tool.name, operation, failureText), + onSuccess: ({ encodedResult }) => { + const { screenshot, ...rest } = encodedResult as ImageToolResult; + const includeImage = + (payload as { readonly includeImage?: boolean } | undefined)?.includeImage !== + false; + const metadata = { + ...rest, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + }; + return Effect.succeed( + new McpSchema.CallToolResult({ + isError: false, + structuredContent: metadata, + content: [ + { type: "text", text: JSON.stringify(metadata) }, + ...(includeImage + ? [ + { + type: "image" as const, + data: new Uint8Array(Buffer.from(screenshot.data, "base64")), + mimeType: screenshot.mimeType, + }, + ] + : []), + ], + }), + ); + }, + }), + ); + }), + }); + }); + +const registerDeviceScreenshot = Effect.fn("McpHttpServer.registerDeviceScreenshot")(function* () { + const devices = yield* DeviceService.DeviceService; + const built = yield* DeviceScreenshotToolkit; + yield* registerImageTool( + DeviceScreenshotTool, + (payload) => + built + .handle("device_screenshot", payload) + .pipe(Stream.unwrap, Stream.run(Sink.last()), Effect.flatMap(Effect.fromOption)), + (effect) => effect.pipe(Effect.provideService(DeviceService.DeviceService, devices)), + "screenshot", + "Device screenshot failed.", + ); +}); + const PreviewStandardToolkitRegistrationLive = McpServer.toolkit(PreviewStandardToolkit).pipe( Layer.provide(PreviewStandardToolkitHandlersLive), ); @@ -443,6 +601,19 @@ export const PullRequestsToolkitRegistrationLive = McpServer.toolkit(PullRequest Layer.provide(PullRequestsToolkitHandlersLive), ); +const DeviceStandardToolkitRegistrationLive = McpServer.toolkit(DeviceStandardToolkit).pipe( + Layer.provide(DeviceStandardToolkitHandlersLive), +); + +const DeviceScreenshotRegistrationLive = Layer.effectDiscard(registerDeviceScreenshot()).pipe( + Layer.provide(DeviceScreenshotToolkitHandlersLive), +); + +export const DeviceToolkitRegistrationLive = Layer.mergeAll( + DeviceStandardToolkitRegistrationLive, + DeviceScreenshotRegistrationLive, +); + const McpTransportLive = McpServer.layerHttp({ name: "T3 Code", version: packageJson.version, @@ -453,4 +624,5 @@ const McpTransportLive = McpServer.layerHttp({ export const layer = Layer.mergeAll( PreviewToolkitRegistrationLive, PullRequestsToolkitRegistrationLive, + DeviceToolkitRegistrationLive, ).pipe(Layer.provideMerge(McpTransportLive)); diff --git a/apps/server/src/mcp/McpInvocationContext.ts b/apps/server/src/mcp/McpInvocationContext.ts index 0c0a0ab68ae9..eddfa7270a77 100644 --- a/apps/server/src/mcp/McpInvocationContext.ts +++ b/apps/server/src/mcp/McpInvocationContext.ts @@ -8,7 +8,7 @@ import { import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; -export type McpCapability = "preview" | "pull-requests"; +export type McpCapability = "preview" | "device" | "pull-requests"; export interface McpInvocationScope { readonly environmentId: EnvironmentId; diff --git a/apps/server/src/mcp/McpProviderSession.test.ts b/apps/server/src/mcp/McpProviderSession.test.ts new file mode 100644 index 000000000000..1c3c3af5091e --- /dev/null +++ b/apps/server/src/mcp/McpProviderSession.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vite-plus/test"; +import { withAgentDeviceEnvironment } from "./McpProviderSession.ts"; + +describe("device CLI environment", () => { + it("preserves provider credentials and commands while routing devices to the owned daemon", () => { + const environment = withAgentDeviceEnvironment( + { PATH: "/provider/bin:/usr/bin", PROVIDER_KEY: "fixture" }, + { + agentDeviceEnvironment: { + PATH: "/t3/device/bin", + PATH_SEPARATOR: ":", + AGENT_DEVICE_DAEMON_BASE_URL: "http://127.0.0.1:9000", + AGENT_DEVICE_DAEMON_AUTH_TOKEN: "fixture-device", + }, + }, + ); + expect(environment).toEqual({ + PATH: "/t3/device/bin:/provider/bin:/usr/bin", + PROVIDER_KEY: "fixture", + AGENT_DEVICE_DAEMON_BASE_URL: "http://127.0.0.1:9000", + AGENT_DEVICE_DAEMON_AUTH_TOKEN: "fixture-device", + }); + }); + + it("does not grant CLI access when device access was not supplied", () => { + const environment = { PATH: "/usr/bin", PROVIDER_KEY: "fixture" }; + expect(withAgentDeviceEnvironment(environment, undefined)).toBe(environment); + expect(withAgentDeviceEnvironment(environment, {})).toBe(environment); + }); +}); diff --git a/apps/server/src/mcp/McpProviderSession.ts b/apps/server/src/mcp/McpProviderSession.ts index 61c3ac1e0b20..4019cb37bd04 100644 --- a/apps/server/src/mcp/McpProviderSession.ts +++ b/apps/server/src/mcp/McpProviderSession.ts @@ -7,8 +7,31 @@ export interface McpProviderSessionConfig { readonly providerInstanceId: ProviderInstanceId; readonly endpoint: string; readonly authorizationHeader: string; - /** Whether the credential grants the preview (browser) toolkit; the pull request toolkit always is. */ - readonly preview: boolean; + /** Capabilities the credential grants ("preview", "device"). */ + readonly capabilities: ReadonlySet; + /** + * Set when the session may drive devices. Adapters spread this into the + * provider subprocess environment so the `agent-device` CLI is on PATH and + * already pointed at the server's daemon; the agent never handles a token. + */ + readonly agentDeviceEnvironment?: Readonly>; +} + +/** Provider env with the device variables applied over `base`, or `base` untouched. */ +export function withAgentDeviceEnvironment( + base: NodeJS.ProcessEnv, + config: Pick | undefined, +): NodeJS.ProcessEnv { + const extra = config?.agentDeviceEnvironment; + if (!extra) return base; + const separator = extra.PATH_SEPARATOR ?? ":"; + const basePath = base.PATH ?? base.Path; + const { PATH: shimDir, PATH_SEPARATOR: _separator, ...rest } = extra; + return { + ...base, + ...rest, + ...(shimDir ? { PATH: basePath ? `${shimDir}${separator}${basePath}` : shimDir } : {}), + }; } const sessionsByThread = new Map(); diff --git a/apps/server/src/mcp/McpSessionRegistry.test.ts b/apps/server/src/mcp/McpSessionRegistry.test.ts index 2e9749a04062..a1e333bd07b0 100644 --- a/apps/server/src/mcp/McpSessionRegistry.test.ts +++ b/apps/server/src/mcp/McpSessionRegistry.test.ts @@ -39,7 +39,7 @@ it.effect("stores only a token hash, resolves the bearer token, and revokes by t const issued = yield* registry.issue({ threadId, providerInstanceId: ProviderInstanceId.make("codex"), - preview: true, + capabilities: new Set(["preview"]), }); expect(issued.config.endpoint).toBe("http://127.0.0.1:43123/mcp"); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); @@ -55,18 +55,23 @@ it.effect("stores only a token hash, resolves the bearer token, and revokes by t }), ); -it.effect("always grants pull-requests and gates preview on the request", () => +it.effect("always grants pull-requests and gates browser and device access independently", () => Effect.gen(function* () { const registry = yield* makeRegistry(() => 1_000); const withPreview = yield* registry.issue({ threadId: ThreadId.make("thread-preview"), providerInstanceId: ProviderInstanceId.make("codex"), - preview: true, + capabilities: new Set(["preview"]), }); const withoutPreview = yield* registry.issue({ threadId: ThreadId.make("thread-no-preview"), providerInstanceId: ProviderInstanceId.make("codex"), - preview: false, + capabilities: new Set(), + }); + const withDevice = yield* registry.issue({ + threadId: ThreadId.make("thread-device"), + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(["device"]), }); const capabilitiesOf = (issued: typeof withPreview) => registry @@ -75,6 +80,7 @@ it.effect("always grants pull-requests and gates preview on the request", () => expect(yield* capabilitiesOf(withPreview)).toEqual(["preview", "pull-requests"]); expect(yield* capabilitiesOf(withoutPreview)).toEqual(["pull-requests"]); + expect(yield* capabilitiesOf(withDevice)).toEqual(["device", "pull-requests"]); }), ); @@ -92,7 +98,7 @@ it.effect("builds MCP endpoints from the bound server host", () => const issued = yield* registry.issue({ threadId: ThreadId.make(`thread-${hostname}`), providerInstanceId: ProviderInstanceId.make("codex"), - preview: true, + capabilities: new Set(["preview"]), }); expect(issued.config.endpoint).toBe(expectedEndpoint); } @@ -106,7 +112,7 @@ it.effect("expires credentials once their session stops showing signs of life", const issued = yield* registry.issue({ threadId: ThreadId.make("thread-2"), providerInstanceId: ProviderInstanceId.make("claude"), - preview: true, + capabilities: new Set(["preview"]), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); timestamp += 101; @@ -122,7 +128,7 @@ it.effect("keeps a credential alive across turns that never touch an MCP tool", const issued = yield* registry.issue({ threadId, providerInstanceId: ProviderInstanceId.make("claude"), - preview: true, + capabilities: new Set(["preview"]), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); @@ -144,7 +150,7 @@ it.effect("does not keep credentials of other threads alive", () => const issued = yield* registry.issue({ threadId: ThreadId.make("thread-4"), providerInstanceId: ProviderInstanceId.make("codex"), - preview: true, + capabilities: new Set(["preview"]), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index 130f6dce582b..383bfbfec390 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -14,11 +14,7 @@ import * as McpProviderSession from "./McpProviderSession.ts"; export interface McpCredentialRequest { readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId; - /** - * Whether the credential may drive the user's browser. The pull request - * toolkit is always granted: it only touches the thread's own links. - */ - readonly preview: boolean; + readonly capabilities: ReadonlySet; } export interface McpIssuedCredential { @@ -133,9 +129,10 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( threadId: ThreadId.make(request.threadId), providerSessionId, providerInstanceId: ProviderInstanceId.make(request.providerInstanceId), - capabilities: new Set( - request.preview ? ["pull-requests", "preview"] : ["pull-requests"], - ), + capabilities: new Set([ + "pull-requests", + ...request.capabilities, + ]), issuedAt, }; yield* SynchronizedRef.update(state, ({ records }) => { @@ -151,7 +148,7 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( providerInstanceId: scope.providerInstanceId, endpoint, authorizationHeader: `Bearer ${rawToken}`, - preview: request.preview, + capabilities: scope.capabilities, }, }; }, diff --git a/apps/server/src/mcp/toolkits/device/handlers.test.ts b/apps/server/src/mcp/toolkits/device/handlers.test.ts new file mode 100644 index 000000000000..e37b2a8b22c2 --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/handlers.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { agentDeviceQuickStart, agentDeviceTargetArgs, pngDimensions } from "./handlers.ts"; + +const device = { + hostId: "local", + id: "ABCD-1234", + platform: "ios" as const, + name: "iPhone 17 Pro", + version: "iOS 27.0", + booted: true, + physical: false, +}; + +describe("device tool helpers", () => { + it("pins agent-device commands to the device by platform-specific flag", () => { + expect(agentDeviceTargetArgs(device)).toEqual(["--platform", "ios", "--udid", "ABCD-1234"]); + expect(agentDeviceTargetArgs({ ...device, platform: "android", id: "emulator-5554" })).toEqual([ + "--platform", + "android", + "--serial", + "emulator-5554", + ]); + }); + + it("writes the quick start around the pinned target", () => { + const text = agentDeviceQuickStart(device); + expect(text).toContain("agent-device snapshot -i --platform ios --udid ABCD-1234"); + expect(text).toContain("iPhone 17 Pro (iOS 27.0)"); + expect(text).toContain("XCTest runner"); + }); + + it("reads PNG dimensions from the IHDR chunk", () => { + const png = new Uint8Array(24); + new DataView(png.buffer).setUint32(0, 0x89504e47); + new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); + new DataView(png.buffer).setUint32(12, 0x49484452); + new DataView(png.buffer).setUint32(16, 1179); + new DataView(png.buffer).setUint32(20, 2556); + expect(pngDimensions(png)).toEqual({ width: 1179, height: 2556 }); + expect(pngDimensions(new Uint8Array([1, 2, 3]))).toEqual({ width: 0, height: 0 }); + }); +}); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts new file mode 100644 index 000000000000..bab7391af24d --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -0,0 +1,212 @@ +import { + type DeviceError, + type DeviceHostId, + type DeviceId, + type DevicePlatform, + type DeviceSummary, + DeviceToolUnavailableError, + LOCAL_DEVICE_HOST_ID, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import * as DeviceService from "../../../device/DeviceService.ts"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import { DeviceScreenshotToolkit, DeviceStandardToolkit, DeviceToolkit } from "./tools.ts"; + +/** The flags that pin every agent-device command to one device. */ +export function agentDeviceTargetArgs(device: DeviceSummary): ReadonlyArray { + return device.platform === "ios" + ? ["--platform", "ios", "--udid", device.id] + : ["--platform", "android", "--serial", device.id]; +} + +/** + * Just-in-time guidance returned from `device_open`. This is the one place + * the agent learns how to drive the device, so it lives with the tool result + * rather than in the always-on prompt block; threads that never open a device + * never pay for it. + */ +export function agentDeviceQuickStart(device: DeviceSummary): string { + const target = agentDeviceTargetArgs(device).join(" "); + const platformNotes = + device.platform === "ios" + ? "First use builds an XCTest runner and can take a couple of minutes; later commands are fast." + : "The Android snapshot helper installs itself on first use."; + return [ + `The user is watching ${device.name} (${device.version}) in the Device panel.`, + `Drive it with the agent-device CLI, which is on PATH and already connected to this environment. Always pass ${target}.`, + "Typical loop:", + ` agent-device open ${target} # or: open `, + ` agent-device snapshot -i ${target} # accessibility tree with @eN refs`, + ` agent-device click @e3 ${target}`, + ` agent-device fill @e5 "text" ${target}`, + ` agent-device screenshot /tmp/shot.png ${target} # or call device_screenshot`, + ` agent-device install ${target}`, + "Prefer snapshot refs over coordinates. Run `agent-device help` for workflow guides and `agent-device --help` for flags.", + "Do not call simctl, adb, xcrun, or serve-sim directly while these tools are attached; use agent-device.", + platformNotes, + ].join("\n"); +} + +const requireDeviceAccess = McpInvocationContext.requireMcpCapability("device").pipe( + Effect.mapError( + () => + new DeviceToolUnavailableError({ + reason: "Agent device access is turned off for this environment.", + }), + ), +); + +const pickDevice = ( + devices: ReadonlyArray, + input: { + readonly deviceId?: DeviceId | undefined; + readonly platform?: DevicePlatform | undefined; + readonly hostId?: DeviceHostId | undefined; + }, +): Effect.Effect => + Effect.gen(function* () { + const hostId = input.hostId ?? LOCAL_DEVICE_HOST_ID; + if (input.deviceId !== undefined) { + const match = devices.find( + (device) => device.hostId === hostId && device.id === input.deviceId, + ); + if (match) return match; + return yield* new DeviceToolUnavailableError({ + reason: `No device ${input.deviceId} on host ${hostId}. Call device_list for current ids.`, + }); + } + const candidates = devices.filter( + (device) => + device.hostId === hostId && + (input.platform === undefined || device.platform === input.platform), + ); + if (candidates.length === 0) { + return yield* new DeviceToolUnavailableError({ + reason: + input.platform === undefined + ? "No simulators or emulators were found. Call device_list to see why." + : `No ${input.platform} devices were found on host ${hostId}. Call device_list to see why.`, + }); + } + const platforms = new Set(candidates.map((device) => device.platform)); + if (input.platform === undefined && platforms.size > 1) { + return yield* new DeviceToolUnavailableError({ + reason: "Both iOS and Android devices are available; pass platform or deviceId.", + }); + } + return candidates.find((device) => device.booted) ?? candidates[0]!; + }); + +const toolError = (error: DeviceError | DeviceToolUnavailableError) => error; + +const handlers = { + device_list: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const state = yield* devices.list; + if (state.hostStatus === "disabled") { + return yield* new DeviceToolUnavailableError({ + reason: + "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.", + }); + } + const hostId = input?.hostId; + const open = state.sessions + .filter((session) => session.threadId === scope.threadId) + .map((session) => ({ hostId: session.hostId, deviceId: session.deviceId })); + return { + hosts: hostId ? state.hosts.filter((host) => host.id === hostId) : state.hosts, + devices: hostId + ? state.devices.filter((device) => device.hostId === hostId) + : state.devices, + open, + }; + }).pipe(Effect.mapError(toolError)), + device_open: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const state = yield* devices.list; + if (state.hostStatus === "disabled") { + return yield* new DeviceToolUnavailableError({ + reason: + "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.", + }); + } + const target = yield* pickDevice(state.devices, input); + const session = yield* devices.open({ + threadId: scope.threadId, + hostId: target.hostId, + deviceId: target.id, + platform: target.platform, + }); + const after = yield* devices.state; + const device = + after.devices.find( + (candidate) => candidate.hostId === session.hostId && candidate.id === session.deviceId, + ) ?? target; + return { + device, + agentDevice: { command: "agent-device", targetArgs: agentDeviceTargetArgs(device) }, + quickStart: agentDeviceQuickStart(device), + }; + }).pipe(Effect.mapError(toolError)), + device_screenshot: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const sessions = yield* devices.sessionsForThread(scope.threadId); + const target = + input.deviceId !== undefined + ? { hostId: input.hostId ?? LOCAL_DEVICE_HOST_ID, deviceId: input.deviceId } + : sessions.at(-1); + if (!target) { + return yield* new DeviceToolUnavailableError({ + reason: "No device is open in this thread. Call device_open first.", + }); + } + const shot = yield* devices.screenshot(target); + return { + device: shot.device, + screenshot: { + mimeType: "image/png" as const, + data: Buffer.from(shot.png).toString("base64"), + ...pngDimensions(shot.png), + }, + }; + }).pipe(Effect.mapError(toolError)), + device_close: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + yield* devices.close({ + threadId: scope.threadId, + ...(input.deviceId === undefined ? {} : { deviceId: input.deviceId }), + ...(input.shutdown === undefined ? {} : { shutdown: input.shutdown }), + }); + return {}; + }).pipe(Effect.mapError(toolError)), +} satisfies Parameters[0]; + +/** Width and height from the IHDR chunk; a PNG that lacks one reports 0×0. */ +export function pngDimensions(png: Uint8Array): { width: number; height: number } { + if (png.length < 24) return { width: 0, height: 0 }; + const view = new DataView(png.buffer, png.byteOffset, png.byteLength); + const isPng = + view.getUint32(0) === 0x89504e47 && + view.getUint32(4) === 0x0d0a1a0a && + view.getUint32(12) === 0x49484452; + return isPng + ? { width: view.getUint32(16), height: view.getUint32(20) } + : { width: 0, height: 0 }; +} + +const { device_screenshot, ...standardHandlers } = handlers; + +export const DeviceStandardToolkitHandlersLive = DeviceStandardToolkit.toLayer(standardHandlers); + +export const DeviceScreenshotToolkitHandlersLive = DeviceScreenshotToolkit.toLayer({ + device_screenshot, +}); diff --git a/apps/server/src/mcp/toolkits/device/tools.ts b/apps/server/src/mcp/toolkits/device/tools.ts new file mode 100644 index 000000000000..e98d6ab64829 --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/tools.ts @@ -0,0 +1,98 @@ +import { + DeviceToolCloseInput, + DeviceToolError, + DeviceToolListResult, + DeviceToolOpenInput, + DeviceToolOpenResult, + DeviceToolScreenshotResult, + DeviceToolTargetInput, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import { Tool, Toolkit } from "effect/unstable/ai"; + +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as DeviceService from "../../../device/DeviceService.ts"; + +const dependencies = [McpInvocationContext.McpInvocationContext, DeviceService.DeviceService]; + +/** + * Deliberately a small surface: lifecycle, visibility for the user, and one + * image-returning verb. Driving the device (taps, typing, install, logs) + * happens through the preconfigured `agent-device` CLI, which has the + * semantic snapshot model agents need and stays current with its own + * releases. Wrapping its commands here would only lag behind it. + */ +const DeviceListTool = Tool.make("device_list", { + description: + "List iOS Simulators and Android Emulators on this environment's device hosts, which platforms each host can run, and which devices are already open in this thread's Device panel. Call this before device_open when you do not know a device id.", + // An empty struct serializes as `anyOf [object, array]`, which some + // providers reject and then drop every tool on the server with it. + parameters: Schema.Struct({ + hostId: Schema.optional( + Schema.String.annotate({ description: "Limit to one device host. Defaults to all hosts." }), + ), + }), + success: DeviceToolListResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "List devices") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, false); + +const DeviceOpenTool = Tool.make("device_open", { + description: + "Open a simulator or emulator for this thread: boots it if needed, starts its live stream, and shows it in the user's Device panel so they can watch. Returns the agent-device CLI invocation pinned to the device; drive the device with that CLI afterwards.", + parameters: DeviceToolOpenInput, + success: DeviceToolOpenResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Open device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceScreenshotTool = Tool.make("device_screenshot", { + description: + "Capture the current screen of an open device as a PNG image. Use it to see what the user sees; for taps and text use the agent-device CLI.", + parameters: DeviceToolTargetInput, + success: DeviceToolScreenshotResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Screenshot device") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +const DeviceCloseTool = Tool.make("device_close", { + description: + "Remove a device from this thread's Device panel. Pass shutdown=true to also power the simulator or emulator off.", + parameters: DeviceToolCloseInput, + success: Schema.Record(Schema.String, Schema.Never).annotate({ + description: "The device was closed.", + }), + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Close device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, true) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceStandardToolkit = Toolkit.make(DeviceListTool, DeviceOpenTool, DeviceCloseTool); + +export const DeviceScreenshotToolkit = Toolkit.make(DeviceScreenshotTool); + +export const DeviceToolkit = Toolkit.make( + DeviceListTool, + DeviceOpenTool, + DeviceScreenshotTool, + DeviceCloseTool, +); diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index 1c2439a9ad9a..85784d21ca4b 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -12,18 +12,39 @@ For browser work, first call \`preview_status\`. If no automation-capable previe Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Use an alternative browser system only when the T3 preview tools are absent, the user explicitly requests another browser, or \`preview_open\` returns an explicit unsupported/unavailable error. A failed T3 preview tool call should be inspected and retried with corrected arguments when the error is actionable. `; +const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = ` + +## T3 Code devices + +The \`t3-code\` MCP server also exposes \`device_*\` tools for iOS Simulators and Android Emulators on this environment. For mobile verification, call \`device_list\`, then \`device_open\` so the user can watch the device in their Device panel; its result explains how to drive the device. Driving happens through the \`agent-device\` CLI, which is on PATH and already connected: prefer \`agent-device snapshot -i\` refs over coordinates, and use \`device_screenshot\` when you need to see the screen. Do not call simctl, adb, xcrun, or serve-sim directly while these tools are present. If \`device_list\` reports a platform as unavailable, say so instead of trying another route. +`; + +export interface T3CodeToolAvailability { + readonly browser: boolean; + readonly device: boolean; +} + +const normalizeAvailability = ( + availability: boolean | T3CodeToolAvailability, +): T3CodeToolAvailability => + typeof availability === "boolean" ? { browser: availability, device: false } : availability; + /** - * The browser block is omitted entirely when the preview tools aren't attached. - * Describing `preview_*` tools that aren't in the turn's tool list would be + * Each block is omitted entirely when its tools aren't attached. Describing + * `preview_*` or `device_*` tools that aren't in the turn's tool list would be * worse than saying nothing: the instructions actively steer the model away - * from Playwright and agent-browser, so leaving them in would talk it out of - * the only browser automation it still has. + * from Playwright, agent-browser, and raw simctl/adb, so leaving them in would + * talk it out of the only automation it still has. */ -const browserToolInstructions = (browserToolsAvailable: boolean): string => - browserToolsAvailable ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : ""; +const browserToolInstructions = (availability: boolean | T3CodeToolAvailability): string => { + const tools = normalizeAvailability(availability); + return `${tools.browser ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : ""}${ + tools.device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : "" + }`; +}; const codexPlanModeDeveloperInstructions = ( - browserToolsAvailable: boolean, + browserToolsAvailable: boolean | T3CodeToolAvailability, ): string => `# Plan Mode (Conversational) You work in 3 phases, and you should *chat your way* to a great plan before finalizing it. A great plan is very detailed-intent- and implementation-wise-so that it can be handed to another engineer or agent to be implemented right away. It must be **decision complete**, where the implementer does not need to make any decisions. @@ -156,7 +177,7 @@ ${browserToolInstructions(browserToolsAvailable)} `; const codexDefaultModeDeveloperInstructions = ( - browserToolsAvailable: boolean, + browserToolsAvailable: boolean | T3CodeToolAvailability, ): string => `# Collaboration Mode: Default You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active. @@ -184,7 +205,7 @@ export function buildCodexDeveloperInstructions( * it from the session's actual MCP configuration rather than re-reading the * setting, so the prompt cannot claim tools the turn doesn't have. */ - browserToolsAvailable = true, + browserToolsAvailable: boolean | T3CodeToolAvailability = true, ): string { const base = interactionMode === "plan" diff --git a/apps/server/src/provider/Drivers/AntigravityDriver.ts b/apps/server/src/provider/Drivers/AntigravityDriver.ts index 65a8c97fe668..0082f3cbdc30 100644 --- a/apps/server/src/provider/Drivers/AntigravityDriver.ts +++ b/apps/server/src/provider/Drivers/AntigravityDriver.ts @@ -1,3 +1,4 @@ +import { withAgentDeviceEnvironment } from "../../mcp/McpProviderSession.ts"; import { AntigravitySettings, ProviderDriverKind, ProviderSetupError } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Crypto from "effect/Crypto"; @@ -162,7 +163,7 @@ export const AntigravityDriver: ProviderDriver 0 ? { extraArgs } : {}), ...(mcpSession diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 5c3d1f08e2d0..b43755736ca3 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -2274,7 +2274,10 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ...(mcpSession ? { environment: { - ...(options?.environment ?? process.env), + ...McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), T3_MCP_BEARER_TOKEN: mcpSession.authorizationHeader.replace(/^Bearer\s+/, ""), }, appServerArgs: [ @@ -2283,7 +2286,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( "-c", 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', ], - browserToolsAvailable: mcpSession.preview, + mcpCapabilities: mcpSession.capabilities, } : {}), }; diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 2ec77ecb7000..bd9a6b6f34a8 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -39,7 +39,10 @@ import * as EffectCodexSchema from "effect-codex-app-server/schema"; import { buildCodexInitializeParams } from "./CodexProvider.ts"; import { codexSessionAppServerArgs } from "./codexLaunchArgs.ts"; import { expandHomePath } from "../../pathExpansion.ts"; -import { buildCodexDeveloperInstructions } from "../CodexDeveloperInstructions.ts"; +import { + buildCodexDeveloperInstructions, + type T3CodeToolAvailability, +} from "../CodexDeveloperInstructions.ts"; const decodeV2TurnStartResponse = Schema.decodeUnknownEffect(EffectCodexSchema.V2TurnStartResponse); const PROVIDER = ProviderDriverKind.make("codex"); @@ -66,6 +69,16 @@ export function hasConfiguredMcpServer(appServerArgs: ReadonlyArray | un return appServerArgs?.some((argument) => argument.includes("mcp_servers.")) === true; } +function configuredMcpToolAvailability( + appServerArgs: ReadonlyArray | undefined, + mcpCapabilities: ReadonlySet | undefined, +): T3CodeToolAvailability { + if (!hasConfiguredMcpServer(appServerArgs)) return { browser: false, device: false }; + // Callers predating the capability set attached the browser toolkit only. + if (mcpCapabilities === undefined) return { browser: true, device: false }; + return { browser: mcpCapabilities.has("preview"), device: mcpCapabilities.has("device") }; +} + export const CodexResumeCursorSchema = Schema.Struct({ threadId: Schema.String, }); @@ -166,13 +179,8 @@ export interface CodexSessionRuntimeOptions { readonly serviceTier?: CodexServiceTier | undefined; readonly resumeCursor?: CodexResumeCursor; readonly appServerArgs?: ReadonlyArray; - /** - * Whether the attached `t3-code` MCP server exposes the preview tools. The - * server is attached for every session now (the pull request toolkit is - * always on), so its presence in `appServerArgs` no longer implies browser - * access; the credential's own capability decides the developer prompt. - */ - readonly browserToolsAvailable?: boolean; + /** Capabilities the session's `t3-code` MCP credential grants; drives the prompt blocks. */ + readonly mcpCapabilities?: ReadonlySet; } export interface CodexSessionRuntimeSendTurnInput { @@ -576,7 +584,7 @@ function buildCodexCollaborationMode(input: { readonly interactionMode?: ProviderInteractionMode; readonly model?: string; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; - readonly browserToolsAvailable?: boolean; + readonly browserToolsAvailable?: boolean | T3CodeToolAvailability; }): EffectCodexSchema.V2TurnStartParams__CollaborationMode | undefined { if (input.interactionMode === undefined) { return undefined; @@ -610,7 +618,7 @@ export function buildTurnStartParams(input: { readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; readonly interactionMode?: ProviderInteractionMode; /** Defaults to true so callers that predate the agent-access gate are unchanged. */ - readonly browserToolsAvailable?: boolean; + readonly browserToolsAvailable?: boolean | T3CodeToolAvailability; }): Effect.Effect< CodexTurnStartParamsWithCollaborationMode, CodexErrors.CodexAppServerProtocolParseError @@ -2359,9 +2367,10 @@ export const makeCodexSessionRuntime = ( // Derived from the session's own credential rather than the // setting, so the prompt describes the tools this turn actually // has even if the setting changed after the session started. - browserToolsAvailable: - hasConfiguredMcpServer(options.appServerArgs) && - (options.browserToolsAvailable ?? true), + browserToolsAvailable: configuredMcpToolAvailability( + options.appServerArgs, + options.mcpCapabilities, + ), }); const rawResponse = yield* client.raw.request("turn/start", params); const response = yield* decodeV2TurnStartResponse(rawResponse).pipe( diff --git a/apps/server/src/provider/Layers/CursorAdapter.ts b/apps/server/src/provider/Layers/CursorAdapter.ts index 1a77f964aac5..b19d46112fde 100644 --- a/apps/server/src/provider/Layers/CursorAdapter.ts +++ b/apps/server/src/provider/Layers/CursorAdapter.ts @@ -543,7 +543,14 @@ export function makeCursorAdapter( const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const acp = yield* makeCursorAcpRuntime({ cursorSettings: effectiveCursorSettings, - ...(options?.environment ? { environment: options.environment } : {}), + ...(options?.environment || mcpSession?.agentDeviceEnvironment + ? { + environment: McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), + } + : {}), childProcessSpawner, cwd, runtimeMode: input.runtimeMode, diff --git a/apps/server/src/provider/Layers/GrokAdapter.ts b/apps/server/src/provider/Layers/GrokAdapter.ts index 25188adcffcc..a2f78a0d72d1 100644 --- a/apps/server/src/provider/Layers/GrokAdapter.ts +++ b/apps/server/src/provider/Layers/GrokAdapter.ts @@ -989,7 +989,14 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const acp = yield* makeGrokAcpRuntime({ grokSettings, - ...(options?.environment ? { environment: options.environment } : {}), + ...(options?.environment || mcpSession?.agentDeviceEnvironment + ? { + environment: McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), + } + : {}), childProcessSpawner, cwd, runtimeMode: input.runtimeMode, diff --git a/apps/server/src/provider/Layers/OpenCodeAdapter.ts b/apps/server/src/provider/Layers/OpenCodeAdapter.ts index 742ee9b86d6a..312a9494bebc 100644 --- a/apps/server/src/provider/Layers/OpenCodeAdapter.ts +++ b/apps/server/src/provider/Layers/OpenCodeAdapter.ts @@ -2827,19 +2827,22 @@ export function makeOpenCodeAdapter( // The runtime binds the server's lifetime to the Scope.Scope // we provide below — closing `sessionScope` kills the child // process automatically. No manual `server.close()` needed. + const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const server = yield* openCodeRuntime.connectToOpenCodeServer({ binaryPath, directory, serverUrl, ...(serverPassword ? { serverPassword } : {}), - ...(options?.environment ? { environment: options.environment } : {}), + environment: McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), }); const client = openCodeRuntime.createOpenCodeSdkClient({ baseUrl: server.url, directory, ...(server.serverPassword ? { serverPassword: server.serverPassword } : {}), }); - const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); if (mcpSession && !server.external) { yield* runOpenCodeSdk("mcp.add", () => client.mcp.add({ diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 1ce1a396796f..9a3fcb8d65f4 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -4800,12 +4800,14 @@ describe("agent browser access", () => { const projectId = ProjectId.make("project-browser-access"); const startSessionWith = ( - enableAgentBrowserAccess: boolean, + access: boolean | { readonly browser: boolean; readonly device: boolean }, threadId: ThreadId, projectOverride?: boolean, ) => Effect.gen(function* () { - const issued: Array<{ readonly threadId: ThreadId; readonly preview: boolean }> = []; + const enableAgentBrowserAccess = typeof access === "boolean" ? access : access.browser; + const enableAgentDeviceAccess = typeof access === "boolean" ? access : access.device; + const issued: Array<{ threadId: ThreadId; capabilities: ReadonlyArray }> = []; const codex = makeFakeCodexAdapter(); const providerAdapterLayer = Layer.succeed( ProviderAdapterRegistry.ProviderAdapterRegistry, @@ -4864,7 +4866,10 @@ describe("agent browser access", () => { const providerLayer = makeProviderServiceLive({ issueMcpCredential: (request) => Effect.sync(() => { - issued.push({ threadId: request.threadId, preview: request.preview }); + issued.push({ + threadId: request.threadId, + capabilities: [...request.capabilities].toSorted(), + }); return undefined; }), }).pipe( @@ -4874,6 +4879,7 @@ describe("agent browser access", () => { Layer.provide( ServerSettings.ServerSettingsService.layerTest({ enableAgentBrowserAccess, + enableAgentDeviceAccess, projectAgentBrowserAccessOverrides: projectOverride === undefined ? {} : { [projectId]: projectOverride }, }), @@ -4910,7 +4916,7 @@ describe("agent browser access", () => { const issued = yield* startSessionWith(false, threadId); - assert.deepEqual(issued, [{ threadId, preview: false }]); + assert.deepEqual(issued, [{ threadId, capabilities: ["pull-requests"] }]); }).pipe(Effect.provide(NodeServices.layer)), ); @@ -4920,23 +4926,43 @@ describe("agent browser access", () => { const issued = yield* startSessionWith(true, threadId); - assert.deepEqual(issued, [{ threadId, preview: true }]); + assert.deepEqual(issued, [ + { threadId, capabilities: ["device", "preview", "pull-requests"] }, + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("drops only the preview capability when browser access alone is off", () => + Effect.gen(function* () { + const threadId = asThreadId("thread-browser-off-device-on"); + + const issued = yield* startSessionWith({ browser: false, device: true }, threadId); + + assert.deepEqual(issued, [{ threadId, capabilities: ["device", "pull-requests"] }]); }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("issues a credential without preview when the project disables browser access", () => Effect.gen(function* () { const threadId = asThreadId("thread-project-browser-off"); + const issued = yield* startSessionWith({ browser: true, device: false }, threadId, false); + assert.deepEqual(issued, [{ threadId, capabilities: ["pull-requests"] }]); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a project browser override leaves device access alone", () => + Effect.gen(function* () { + const threadId = asThreadId("thread-project-browser-off-device-on"); const issued = yield* startSessionWith(true, threadId, false); - assert.deepEqual(issued, [{ threadId, preview: false }]); + assert.deepEqual(issued, [{ threadId, capabilities: ["device", "pull-requests"] }]); }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("requests an MCP credential when the project overrides browser access to on", () => Effect.gen(function* () { const threadId = asThreadId("thread-project-browser-on"); - const issued = yield* startSessionWith(false, threadId, true); - assert.deepEqual(issued, [{ threadId, preview: true }]); + const issued = yield* startSessionWith({ browser: false, device: false }, threadId, true); + assert.deepEqual(issued, [{ threadId, capabilities: ["preview", "pull-requests"] }]); }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 5b9059bfa643..f2d94e735b0d 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -34,6 +34,7 @@ import { type ProviderSession, } from "@t3tools/contracts"; import { expandAssistantCitationsForProvider } from "@t3tools/shared/assistantCitations"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { causeErrorTag } from "@t3tools/shared/observability"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { resolveProjectAgentBrowserAccess } from "@t3tools/shared/serverSettings"; @@ -43,6 +44,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -52,6 +54,9 @@ import * as Stream from "effect/Stream"; import { appendUserInputAttachmentPaths } from "../userInputAttachments.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import * as ServerConfig from "../../config.ts"; +import { ensureAgentDeviceShim } from "../../device/AgentDeviceShim.ts"; +import * as DeviceService from "../../device/DeviceService.ts"; +import type * as McpInvocationContext from "../../mcp/McpInvocationContext.ts"; import { increment, providerMetricAttributes, @@ -249,6 +254,8 @@ export interface ProviderServiceLiveOptions { * test see whether a credential was requested at all. */ readonly issueMcpCredential?: typeof McpSessionRegistry.issueActiveMcpCredential; + /** Overrides the device host lookup used to build the agent-device environment. */ + readonly deviceReadiness?: DeviceService.DeviceService["Service"]["agentReadinessIfSupported"]; } interface TurnAnalyticsMetadata { @@ -477,7 +484,16 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ); const issueMcpCredential = options?.issueMcpCredential ?? McpSessionRegistry.issueActiveMcpCredential; + const deviceReadiness = + options?.deviceReadiness ?? + (() => + Effect.serviceOption(DeviceService.DeviceService).pipe( + Effect.flatMap((service) => + Option.isSome(service) ? service.value.agentReadinessIfSupported() : Effect.succeed(null), + ), + )); const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; const runtimeEventPubSub = yield* PubSub.unbounded(); const pendingCompactions = new Map(); const timedOutNativeCompactions = new Set(); @@ -879,22 +895,75 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), ); + const agentDeviceAccessEnabled = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.enableAgentDeviceAccess), + Effect.catch((cause) => + Effect.logWarning( + "Could not read server settings; withholding agent device access for this session.", + { cause }, + ).pipe(Effect.as(false)), + ), + ); + + const agentAccessCapabilities = Effect.fn("ProviderService.agentAccessCapabilities")(function* ( + threadId: ThreadId, + ) { + const capabilities = new Set(["pull-requests"]); + if (yield* agentBrowserAccessEnabled(threadId)) capabilities.add("preview"); + if (yield* agentDeviceAccessEnabled) capabilities.add("device"); + return capabilities; + }); + /** - * Attach the `t3-code` MCP server to the session that is about to start. - * - * Every session gets a credential: the pull request toolkit is always on, - * since it only registers links on the session's own thread. Browser access - * is a capability on that credential, so turning the setting off withholds - * the preview tools without taking the server away. `issueActiveMcpCredential` - * revokes the thread's previous token first, which matters because a session - * restart (runtime mode, cwd, model) re-prepares without stopping. + * Starting a session with device access also brings the device host up, so + * the `agent-device` CLI is on the provider's PATH from its first turn. The + * environment is fixed at spawn time, so a host started later by + * `device_open` could not reach an already-running agent. Tools install once + * and the host is idempotent, so this is cheap after the first session; + * a host that fails to start withholds only the CLI, not the MCP tools. */ + const hostPlatform = yield* HostProcessPlatform; + const agentDeviceEnvironment = Effect.gen(function* () { + const readiness = yield* deviceReadiness().pipe( + Effect.catch((cause) => + Effect.logWarning("Device host unavailable; starting session without agent-device", { + cause, + }).pipe(Effect.as(null)), + ), + ); + if (!readiness) return undefined; + const shimDir = yield* ensureAgentDeviceShim({ + entryPath: readiness.agentDevice.entryPath, + stateDir: serverConfig.stateDir, + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, pathService), + Effect.orElseSucceed(() => undefined), + ); + if (!shimDir) return undefined; + return { + PATH: shimDir, + PATH_SEPARATOR: hostPlatform === "win32" ? ";" : ":", + AGENT_DEVICE_DAEMON_BASE_URL: readiness.agentDevice.baseUrl, + AGENT_DEVICE_DAEMON_AUTH_TOKEN: readiness.agentDevice.token, + AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", + } satisfies Record; + }); + const prepareMcpSession = (threadId: ThreadId, providerInstanceId: ProviderInstanceId) => Effect.gen(function* () { - const preview = yield* agentBrowserAccessEnabled(threadId); - const credential = yield* issueMcpCredential({ threadId, providerInstanceId, preview }); + const capabilities = yield* agentAccessCapabilities(threadId); + const credential = yield* issueMcpCredential({ threadId, providerInstanceId, capabilities }); if (credential) { - yield* Effect.sync(() => McpProviderSession.setMcpProviderSession(credential.config)); + const deviceEnvironment = capabilities.has("device") + ? yield* agentDeviceEnvironment + : undefined; + yield* Effect.sync(() => + McpProviderSession.setMcpProviderSession({ + ...credential.config, + ...(deviceEnvironment ? { agentDeviceEnvironment: deviceEnvironment } : {}), + }), + ); } return credential; }); diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.ts index 6e625127cb18..17b73e552006 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.ts @@ -35,6 +35,8 @@ export interface AntigravityAcpRuntimeInput extends Omit< | "transformSessionUpdate" | "transformStdout" > { + /** Device CLI environment supplied for this provider session. */ + readonly agentDeviceEnvironment?: Readonly>; readonly childProcessSpawner: ChildProcessSpawner.ChildProcessSpawner["Service"]; readonly onAuthorizationUrl?: (url: string) => Effect.Effect; /** diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 01f2dd96b18f..62412ac25853 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -5,6 +5,7 @@ import * as NodeCrypto from "node:crypto"; import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { + type DeviceServiceState, AuthAccessTokenType, AuthStandardClientScopes, AuthEnvironmentBootstrapTokenType, @@ -97,6 +98,7 @@ const encodeTestJson = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unk import * as BackgroundPolicy from "./background/BackgroundPolicy.ts"; import * as ServerConfig from "./config.ts"; +import * as DeviceService from "./device/DeviceService.ts"; import { HTTP_ROUTER_CONFIG, makeRoutesLayer } from "./server.ts"; import { isThreadDetailEvent, @@ -805,6 +807,11 @@ const buildAppUnderTest = (options?: { listBindings: () => Effect.succeed([]), ...options?.layers?.providerSessionDirectory, }), + Layer.mock(DeviceService.DeviceService)({ + state: Effect.succeed(EMPTY_DEVICE_STATE), + currentReadiness: () => Effect.succeed(null), + sessionsForThread: () => Effect.succeed([]), + }), ), ), Layer.provide( @@ -1660,6 +1667,17 @@ const NodeHttpServerTestWithWsDeflate = HttpServer.layerTestClient.pipe( ), ); +const EMPTY_DEVICE_STATE: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: DeviceService.DEVICE_HUB_ROUTE_PREFIX, + revision: 0, +}; + it.layer(NodeServices.layer)("server router seam", (it) => { it.effect("parks HTTP ingress until command readiness", () => Effect.gen(function* () { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 3831763a3eac..e3b42a2637a1 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -57,6 +57,8 @@ import * as TerminalManager from "./terminal/Manager.ts"; import * as McpHttpServer from "./mcp/McpHttpServer.ts"; import * as McpSessionRegistry from "./mcp/McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; +import * as DeviceService from "./device/DeviceService.ts"; +import { deviceHubProxyRouteLayer } from "./device/DeviceHubProxy.ts"; import * as PreviewManager from "./preview/Manager.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as ProcessRunner from "./processRunner.ts"; @@ -386,6 +388,12 @@ const PreviewLayerLive = Layer.empty.pipe( Layer.provideMerge(PortScannerLayerLive), ); +const DeviceLayerLive = DeviceService.layer.pipe( + Layer.provide(ServerSettingsLayerLive), + Layer.provide(ProcessRunner.layer), + Layer.provide(NetService.layer), +); + const WorkspaceEntriesLayerLive = WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer)); const WorkspaceFileSystemLayerLive = WorkspaceFileSystem.layer.pipe( @@ -469,7 +477,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(GitLayerLive), Layer.provideMerge(VcsLayerLive), Layer.provideMerge(ProviderRuntimeLayerLive), - Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive)), + Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive, DeviceLayerLive)), Layer.provideMerge(PersistenceLayerLive), // Both read a user-owned file out of the state directory and stream changes // to clients; neither depends on the other. @@ -553,6 +561,7 @@ export const makeRoutesLayer = Layer.mergeAll( otlpTracesProxyRouteLayer, assetRouteLayer, attachmentUploadRouteLayer, + deviceHubProxyRouteLayer, staticAndDevRouteLayer, websocketRpcRouteLayer, ), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 5e4ca0a18db9..71a8ed19d905 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -114,6 +114,7 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import * as ServerSettings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; +import * as DeviceService from "./device/DeviceService.ts"; import * as PreviewManager from "./preview/Manager.ts"; import { issueAssetUrl } from "./assets/AssetAccess.ts"; import { deletePendingAttachment, issueAttachmentUploadUrl } from "./assets/AttachmentUpload.ts"; @@ -536,6 +537,7 @@ const makeWsRpcLayer = ( const vcsStatusBroadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; const terminalManager = yield* TerminalManager.TerminalManager; const previewManager = yield* PreviewManager.PreviewManager; + const deviceService = yield* DeviceService.DeviceService; const portDiscovery = yield* PortScanner.PortDiscovery; const providerRegistry = yield* ProviderRegistry.ProviderRegistry; const providerService = yield* ProviderService.ProviderService; @@ -2764,6 +2766,40 @@ const makeWsRpcLayer = ( observeRpcStream(WS_METHODS.subscribePreviewEvents, previewManager.events, { "rpc.aggregate": "preview", }), + [WS_METHODS.deviceConfigure]: (input) => + observeRpcEffect(WS_METHODS.deviceConfigure, deviceService.configure(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceList]: (_input) => + observeRpcEffect(WS_METHODS.deviceList, deviceService.list, { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceOpen]: (input) => + observeRpcEffect(WS_METHODS.deviceOpen, deviceService.open(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceClose]: (input) => + observeRpcEffect(WS_METHODS.deviceClose, deviceService.close(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceShutdown]: (input) => + observeRpcEffect(WS_METHODS.deviceShutdown, deviceService.shutdown(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceDetail]: (input) => + observeRpcEffect(WS_METHODS.deviceDetail, deviceService.detail(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceAction]: (input) => + observeRpcEffect(WS_METHODS.deviceAction, deviceService.action(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.subscribeDeviceState]: (_input) => + observeRpcStream( + WS_METHODS.subscribeDeviceState, + DeviceService.stateStream(deviceService), + { "rpc.aggregate": "device" }, + ), [WS_METHODS.subscribeDiscoveredLocalServers]: (input) => observeRpcStream( WS_METHODS.subscribeDiscoveredLocalServers, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 597b7333ce28..225d91310014 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -205,6 +205,10 @@ import { RightPanelTabs } from "./RightPanelTabs"; import { AgentsPanel } from "./AgentsPanel"; import { LinkPullRequestDialogHost } from "./pullRequest/LinkPullRequestDialog"; import { ThreadPullRequestsPanel } from "./pullRequest/ThreadPullRequestsPanel"; +import { useDeviceState } from "~/state/device"; +import { DeviceSetup } from "./device/DeviceSetup"; +import { Dialog } from "./ui/dialog"; +import { WizardPopup } from "./ui/wizard"; import { deriveAgentPanelModel, foldSubagentActivities, @@ -556,6 +560,9 @@ const PreviewPanel = lazy(() => import("./preview/PreviewPanel").then((module) => ({ default: module.PreviewPanel })), ); const DiffPanel = lazy(() => import("./DiffPanel")); +const DevicePanel = lazy(() => + import("./device/DevicePanel").then((module) => ({ default: module.DevicePanel })), +); const FilePreviewPanel = lazy(() => import("./files/FilePreviewPanel")); const EMPTY_PENDING_FILE_SURFACE_IDS: ReadonlySet = new Set(); const TYPE_TO_FOCUS_EDITABLE_SELECTOR = [ @@ -4136,6 +4143,34 @@ export default function ChatView(props: ChatViewProps) { if (!activeThreadRef || !supportsThreadPullRequests) return; useRightPanelStore.getState().open(activeThreadRef, "pull-requests"); }, [activeThreadRef, supportsThreadPullRequests]); + const { state: deviceState } = useDeviceState(activeThreadRef?.environmentId ?? null); + const [deviceSetupThread, setDeviceSetupThread] = useState(null); + const addDeviceSurface = useCallback(() => { + if (!activeThreadRef) return; + if (!deviceState.onboardingCompleted || deviceState.hostStatus === "disabled") { + setDeviceSetupThread(activeThreadRef); + return; + } + useRightPanelStore.getState().open(activeThreadRef, "device"); + }, [activeThreadRef, deviceState.onboardingCompleted, deviceState.hostStatus]); + // An agent's `device_open` surfaces in every client the same way a + // `preview_open` does: the thread starts a device or gains a session and the panel + // opens on it. Closing the last session leaves the tab in place so the + // user keeps their picker; only new sessions raise the panel. + const threadDeviceSessionCount = activeThreadRef + ? deviceState.sessions.filter((session) => session.threadId === activeThreadRef.threadId) + .length + + (deviceState.bootingDevices?.filter((device) => device.threadId === activeThreadRef.threadId) + .length ?? 0) + : 0; + const previousDeviceSessionCount = useRef(threadDeviceSessionCount); + useEffect(() => { + const previous = previousDeviceSessionCount.current; + previousDeviceSessionCount.current = threadDeviceSessionCount; + if (!activeThreadRef || threadDeviceSessionCount <= previous) return; + if (shouldUseRightPanelSheet) return; + useRightPanelStore.getState().open(activeThreadRef, "device"); + }, [activeThreadRef, shouldUseRightPanelSheet, threadDeviceSessionCount]); const openFileSurface = useCallback( (relativePath: string) => { if (!activeThreadRef || !activeProject) return; @@ -8113,6 +8148,19 @@ export default function ChatView(props: ChatViewProps) { environmentId={activeThreadRef?.environmentId ?? null} threadId={activeThreadRef?.threadId ?? null} /> + ) : renderedRightPanelSurface?.kind === "device" ? ( + + { + closeRightPanelSurface(renderedRightPanelSurface); + useRightPanelStore.getState().show(activeThreadRef); + }} + /> + ) : (renderedRightPanelSurface?.kind === "files" || renderedRightPanelSurface?.kind === "file") && ((activeProject && activeWorkspaceRoot) || @@ -8168,6 +8216,29 @@ export default function ChatView(props: ChatViewProps) { return (
+ { + if (!open) setDeviceSetupThread(null); + }} + > + + {activeThreadRef ? ( + { + useRightPanelStore.getState().open(activeThreadRef, "device"); + setDeviceSetupThread(null); + }} + /> + ) : null} + + {rightPanelControlsAtRoot ? panelLayoutControls : null}
{rightPanelContent} @@ -8718,6 +8791,7 @@ export default function ChatView(props: ChatViewProps) { onAddPullRequest={addPullRequestSurface} onAddPullRequests={addPullRequestsSurface} onAddAgents={addAgentsSurface} + onAddDevice={addDeviceSurface} browserAvailable={isPreviewSupportedInRuntime()} terminalAvailable={activeProject !== null} diffAvailable={isServerThread && isGitRepo} @@ -8725,6 +8799,7 @@ export default function ChatView(props: ChatViewProps) { pullRequestAvailable={pullRequestSurfaceAvailable} pullRequestsAvailable={isServerThread && supportsThreadPullRequests} agentsAvailable + deviceAvailable={activeThreadRef !== null} liveAgentCount={agentPanelModel.liveCount} > {rightPanelContent} diff --git a/apps/web/src/components/RightPanelTabs.test.tsx b/apps/web/src/components/RightPanelTabs.test.tsx index 077d3680e45a..dfaaf171c429 100644 --- a/apps/web/src/components/RightPanelTabs.test.tsx +++ b/apps/web/src/components/RightPanelTabs.test.tsx @@ -123,6 +123,7 @@ function renderTabs( onAddDiff={() => undefined} onAddFiles={() => undefined} onAddAgents={() => undefined} + onAddDevice={() => undefined} liveAgentCount={0} browserAvailable terminalAvailable={false} @@ -131,6 +132,7 @@ function renderTabs( pullRequestAvailable={false} pullRequestsAvailable={false} agentsAvailable={false} + deviceAvailable={false} >
content
, diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index d4ba544d8588..546c7c5ad45a 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -15,6 +15,7 @@ import type { import { getTerminalLabel } from "@t3tools/shared/terminalLabels"; import { Bot, + Smartphone, ChevronDown, ChevronLeft, ChevronRight, @@ -114,6 +115,7 @@ interface RightPanelTabsProps { onAddPullRequest: () => void; onAddPullRequests: () => void; onAddAgents: () => void; + onAddDevice: () => void; browserAvailable: boolean; terminalAvailable: boolean; diffAvailable: boolean; @@ -121,6 +123,7 @@ interface RightPanelTabsProps { pullRequestAvailable: boolean; pullRequestsAvailable: boolean; agentsAvailable: boolean; + deviceAvailable: boolean; pullRequestStatusSeeds?: Readonly>; /** Running + waiting subagents; badges the Agents card in the empty state. */ liveAgentCount: number; @@ -151,6 +154,7 @@ const SURFACE_DISABLED_REASONS = { pullRequest: "This thread's branch has no pull request yet.", pullRequests: "Linked pull requests are only available for server threads.", agents: "Agents are only available from a thread.", + device: "Devices are only available from a thread.", } as const; /** Overlays that must win over the launcher's letter shortcuts. */ @@ -174,6 +178,7 @@ const SURFACE_UNAVAILABLE_HINTS = { pullRequest: "No pull request on this branch yet.", pullRequests: "Available for server threads.", agents: "Available from a thread.", + device: "Available from a thread.", } as const; type TabContextMenuAction = @@ -312,6 +317,7 @@ function RightPanelEmptyState(props: { onAddPullRequest: () => void; onAddPullRequests: () => void; onAddAgents: () => void; + onAddDevice: () => void; browserAvailable: boolean; terminalAvailable: boolean; diffAvailable: boolean; @@ -319,6 +325,7 @@ function RightPanelEmptyState(props: { pullRequestAvailable: boolean; pullRequestsAvailable: boolean; agentsAvailable: boolean; + deviceAvailable: boolean; liveAgentCount: number; }) { // -1 means no highlight: it only appears on hover or arrow use. @@ -395,6 +402,16 @@ function RightPanelEmptyState(props: { onClick: props.onAddAgents, badgeCount: props.liveAgentCount, }, + { + label: "Device", + description: "Watch an iOS Simulator or Android Emulator.", + icon: Smartphone, + shortcut: "M", + available: props.deviceAvailable, + disabledReason: SURFACE_UNAVAILABLE_HINTS.device, + onClick: props.onAddDevice, + badgeCount: 0, + }, ] as const; type SurfaceAction = (typeof actions)[number]; @@ -630,6 +647,8 @@ function surfaceTitle( return "Pull requests"; case "agents": return "Agents"; + case "device": + return "Device"; case "preview": { const snapshot = surface.resourceId ? sessions[surface.resourceId] : null; if (!snapshot || snapshot.navStatus._tag === "Idle") return "Browser"; @@ -713,6 +732,8 @@ function SurfaceIcon({ return ; case "agents": return ; + case "device": + return ; } } @@ -906,6 +927,14 @@ export function RightPanelTabs(props: RightPanelTabsProps) { disabledReason: SURFACE_DISABLED_REASONS.agents, onClick: props.onAddAgents, }, + { + label: "Device", + icon: Smartphone, + shortcut: "M", + available: props.deviceAvailable, + disabledReason: SURFACE_DISABLED_REASONS.device, + onClick: props.onAddDevice, + }, ] as const; const handleAddSurfaceMenuKeyDown = (event: ReactKeyboardEvent) => { @@ -1344,6 +1373,7 @@ export function RightPanelTabs(props: RightPanelTabsProps) { onAddPullRequest={props.onAddPullRequest} onAddPullRequests={props.onAddPullRequests} onAddAgents={props.onAddAgents} + onAddDevice={props.onAddDevice} browserAvailable={props.browserAvailable} terminalAvailable={props.terminalAvailable} diffAvailable={props.diffAvailable} @@ -1351,6 +1381,7 @@ export function RightPanelTabs(props: RightPanelTabsProps) { pullRequestAvailable={props.pullRequestAvailable} pullRequestsAvailable={props.pullRequestsAvailable} agentsAvailable={props.agentsAvailable} + deviceAvailable={props.deviceAvailable} liveAgentCount={props.liveAgentCount} /> ) : ( diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index 0082c7dd227c..866cbd0201b5 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -337,7 +337,7 @@ export type MessagesTimelineRow = summaryKind: ToolGroupSummaryKind; toolSurface?: WorkLogEntry["toolSurface"]; toolIcon?: WorkLogEntry["toolIcon"]; - summaryToolIcon?: "browser" | "t3-code" | "pull-request"; + summaryToolIcon?: "browser" | "device" | "t3-code" | "pull-request"; hasFailure: boolean; } | { diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index f685d511fd4b..3544e87817bd 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -97,6 +97,7 @@ import { MousePointerClickIcon, PaintbrushIcon, SearchIcon, + SmartphoneIcon, SquarePenIcon, TerminalIcon, Undo2Icon, @@ -2239,6 +2240,8 @@ function toolGroupSummaryIconName( return "terminal"; case "browser": return "browser"; + case "device": + return "device"; case "search": return "globe"; case "code-search": @@ -2800,6 +2803,7 @@ type WorkEntryIconName = | "check" | "circle-alert" | "computer" + | "device" | "eye" | "globe" | "hammer" @@ -3022,6 +3026,8 @@ function WorkEntryIcon({ name, className }: { name: WorkEntryIconName; className return ; case "computer": return ; + case "device": + return ; case "t3-code": return ; case "check": diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx new file mode 100644 index 000000000000..d94913b03dff --- /dev/null +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -0,0 +1,473 @@ +import type { + DevicePlatform, + DeviceServiceState, + DeviceSummary, + ScopedThreadRef, +} from "@t3tools/contracts"; +import { + ChevronLeft, + Circle, + Home, + Power, + RotateCcw, + SlidersHorizontal, + Smartphone, + Square, + X, +} from "lucide-react"; +import { useCallback, useEffect, useMemo, useState } from "react"; + +import { Button } from "~/components/ui/button"; +import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list"; +import { Dialog } from "~/components/ui/dialog"; +import { WizardPopup } from "~/components/ui/wizard"; +import { + Select, + SelectGroup, + SelectGroupLabel, + SelectItem, + SelectPopup, + SelectTrigger, + SelectValue, +} from "~/components/ui/select"; +import { Spinner } from "~/components/ui/spinner"; +import { Toggle } from "~/components/ui/toggle"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; +import { cn } from "~/lib/utils"; +import { deviceEnvironment, useDeviceHubAccess, useDeviceState } from "~/state/device"; +import { formatEnvironmentQueryError } from "~/state/query"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { DeviceStreamView, type DeviceStreamHandle } from "./DeviceStreamView"; +import { DeviceSetup } from "./DeviceSetup"; +import { DeviceToolsPanel } from "./DeviceToolsPanel"; +import { PreviewPanelShell, type PreviewPanelMode } from "../preview/PreviewPanelShell"; + +const NEW_DEVICE_VALUE = "__new__"; + +const platformLabel = (platform: DevicePlatform) => + platform === "ios" ? "iOS Simulators" : "Android Emulators"; + +const deviceKey = (device: Pick) => + `${device.hostId}\u0000${device.id}`; + +/** + * The Device right-panel surface: one open device (from the thread's device + * sessions) with a picker to switch or boot another. Booting and streaming are + * server-owned; this panel only asks and renders. + */ +export function DevicePanel(props: { + readonly mode: PreviewPanelMode; + readonly threadRef: ScopedThreadRef; + /** `null` renders the picker with nothing open. */ + readonly deviceId: string | null; + readonly visible: boolean; + readonly onDismissSetup: () => void; +}) { + const { environmentId, threadId } = props.threadRef; + const { state, loaded } = useDeviceState(environmentId); + const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); + const open = useAtomCommand(deviceEnvironment.open); + const close = useAtomCommand(deviceEnvironment.close); + const [operationError, setOperationError] = useState(null); + const [pendingDeviceKey, setPendingDeviceKey] = useState(null); + const [handle, setHandle] = useState(null); + const [toolsOpen, setToolsOpen] = useState(false); + const [axOverlay, setAxOverlay] = useState(false); + const access = useDeviceHubAccess(environmentId); + + const hostDisabled = state.hostStatus === "disabled"; + + // Opening setup never grants permission to install or start helpers. + useEffect(() => { + if (!props.visible || !loaded || hostDisabled) return; + void list({ environmentId, input: {} }); + }, [environmentId, list, loaded, props.visible, hostDisabled]); + + const sessions = useMemo( + () => state.sessions.filter((session) => session.threadId === threadId), + [state.sessions, threadId], + ); + const activeSession = + (props.deviceId + ? sessions.find((session) => session.deviceId === props.deviceId) + : undefined) ?? sessions.at(-1); + const activeDevice = activeSession + ? state.devices.find( + (device) => device.hostId === activeSession.hostId && device.id === activeSession.deviceId, + ) + : undefined; + + const grouped = useMemo(() => groupDevices(state), [state]); + + const selectDevice = useCallback( + async (value: string) => { + if (value === NEW_DEVICE_VALUE) return; + const device = state.devices.find((candidate) => deviceKey(candidate) === value); + if (!device) return; + setOperationError(null); + setPendingDeviceKey(value); + try { + const result = await open({ + environmentId, + input: { + threadId, + hostId: device.hostId, + deviceId: device.id, + platform: device.platform, + }, + }); + if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); + } finally { + setPendingDeviceKey(null); + } + }, + [environmentId, open, state.devices, threadId], + ); + + const closeActive = useCallback( + (powerOff: boolean) => { + if (!activeSession) return; + setOperationError(null); + void close({ + environmentId, + input: { threadId, deviceId: activeSession.deviceId, shutdown: powerOff }, + }).then((result) => { + if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); + }); + }, + [activeSession, close, environmentId, threadId], + ); + + const bootingDevices = + state.bootingDevices?.filter((device) => device.threadId === threadId) ?? []; + const hostReady = state.hostStatus === "ready"; + const hostBusy = state.hostStatus === "installing" || state.hostStatus === "starting"; + const unavailablePlatforms = state.hosts.flatMap((host) => + host.platforms.filter((platform) => !platform.available), + ); + + if (loaded && (!state.onboardingCompleted || hostDisabled)) { + return ( + { + if (!isOpen) props.onDismissSetup(); + }} + > + + + + + ); + } + + return ( + +
+ + {activeDevice ? ( + <> + handle?.pressButton("home")} + disabled={!handle?.inputConnected} + > + + + {activeDevice.platform === "android" ? ( + <> + handle?.pressButton("back")} + disabled={!handle?.inputConnected} + > + + + handle?.pressButton("recents")} + disabled={!handle?.inputConnected} + > + + + + ) : ( + handle?.rotate()} + disabled={!handle?.inputConnected} + > + + + )} + setToolsOpen(Boolean(pressed))} + > + + + closeActive(true)}> + + + closeActive(false)}> + + + + ) : null} +
+ {hostReady && state.hostStatusDetail ? ( +
+ {state.hostStatusDetail} +
+ ) : null} + {bootingDevices.length > 0 ? ( +
+ Starting {bootingDevices.map((device) => device.name).join(", ")}… This can take a minute. +
+ ) : null} + {operationError ? ( +
+

{operationError}

+ +
+ ) : null} +
+ {activeDevice && activeSession ? ( + <> +
+ +
+ {toolsOpen ? ( + setToolsOpen(false)} + className="absolute inset-y-0 right-0 z-10 w-full max-w-72 border-l shadow-lg @[560px]:static @[560px]:w-72 @[560px]:shrink-0 @[560px]:shadow-none" + /> + ) : null} + + ) : ( +
+
+ {grouped.length === 0 || hostBusy || pendingDeviceKey ? ( + <> + {hostBusy || pendingDeviceKey ? ( + + ) : ( + + )} +

+ {state.hostStatus === "failed" + ? (state.hostStatusDetail ?? "The device hub failed to start.") + : pendingDeviceKey + ? "Booting device… this can take a minute." + : hostBusy + ? state.hostStatus === "installing" + ? "Installing device tools…" + : "Starting the device hub…" + : !loaded + ? "Connecting…" + : grouped.length === 0 + ? "No simulators or emulators were found on this environment." + : "Choose a device to open."} +

+ + ) : null} + {hostReady && grouped.length > 0 ? ( +
+ {grouped.map((group) => ( +
+
+ +

{platformLabel(group.platform)}

+
+ + {group.devices.map((device) => ( + + + + } + title={device.name} + description={`${device.version} · ${device.booted ? "Running" : "Stopped"}`} + disabled={pendingDeviceKey !== null} + aria-label={`${device.booted ? "Open" : "Start"} ${device.name}`} + onClick={() => void selectDevice(deviceKey(device))} + action={ + pendingDeviceKey === deviceKey(device) ? ( + + ) : ( + + {device.booted ? "Open" : "Start"} + + ) + } + /> + ))} + +
+ ))} +
+ ) : null} + {hostReady && + !state.devices.some((device) => device.platform === "android") && + !unavailablePlatforms.some((platform) => platform.platform === "android") ? ( +

+ No Android virtual devices found. Create one in Android Studio's Device Manager, + then refresh. +

+ ) : null} + {loaded && !hostBusy ? ( + + ) : null} + {unavailablePlatforms.length > 0 && hostReady ? ( +
    + {unavailablePlatforms.map((platform) => ( +
  • + {platform.platform === "ios" ? "iOS" : "Android"}: {platform.reason} +
    • + ))} +
+ ) : null} +
+
+ )} +
+ + ); +} + +function DeviceButton(props: { + readonly label: string; + readonly onClick: () => void; + readonly disabled?: boolean; + readonly children: React.ReactNode; +}) { + return ( + + + } + > + {props.children} + + {props.label} + + ); +} + +function groupDevices(state: DeviceServiceState) { + const groups: Array<{ platform: DevicePlatform; devices: DeviceSummary[] }> = []; + for (const platform of ["ios", "android"] as const) { + const devices = state.devices + .filter((device) => device.platform === platform) + .toSorted((a, b) => Number(b.booted) - Number(a.booted) || a.name.localeCompare(b.name)); + if (devices.length > 0) groups.push({ platform, devices }); + } + return groups; +} diff --git a/apps/web/src/components/device/DeviceSetup.tsx b/apps/web/src/components/device/DeviceSetup.tsx new file mode 100644 index 000000000000..67bf14181b21 --- /dev/null +++ b/apps/web/src/components/device/DeviceSetup.tsx @@ -0,0 +1,307 @@ +import type { DevicePlatform, DeviceServiceState, EnvironmentId } from "@t3tools/contracts"; +import { Check, CircleAlert } from "lucide-react"; +import { useState } from "react"; + +import { Button } from "~/components/ui/button"; +import { DialogClose } from "~/components/ui/dialog"; +import { WizardHeader, WizardPanel, WizardSteps, WizardFooter } from "~/components/ui/wizard"; +import { Spinner } from "~/components/ui/spinner"; +import { Switch } from "~/components/ui/switch"; +import { deviceEnvironment } from "~/state/device"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { cn } from "~/lib/utils"; + +const platformName = (platform: DevicePlatform) => (platform === "ios" ? "iOS" : "Android"); + +export const deviceHubDescription = + "Enable this environment to open simulators and emulators, whether they run here or on a remote device host."; +export const agentDeviceDescription = + "Allow new agent sessions in this environment to start and control local and remote devices, with required tools set up automatically."; + +export function platformSetupStatus(state: DeviceServiceState, platform: DevicePlatform) { + const availability = state.hosts + .flatMap((host) => host.platforms) + .find((candidate) => candidate.platform === platform); + if (!availability?.available) { + return { + ready: false, + message: availability?.reason ?? `${platformName(platform)} support was not detected.`, + }; + } + if ( + state.hostStatus === "ready" && + !state.devices.some((device) => device.platform === platform) + ) { + return { + ready: false, + message: + platform === "ios" + ? "Xcode is installed, but no iOS Simulator is available. Install a runtime in Xcode Settings → Components." + : "The Android SDK is installed, but no virtual device exists. Create one in Android Studio → Device Manager.", + }; + } + return { + ready: true, + message: + platform === "ios" + ? "Xcode and iOS Simulator are available." + : "The Android SDK and Emulator are available.", + }; +} + +export function DeviceSetup(props: { + readonly environmentId: EnvironmentId; + readonly state: DeviceServiceState; + readonly onComplete?: () => void; +}) { + const configure = useAtomCommand(deviceEnvironment.configure); + const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); + const [pending, setPending] = useState<"hub" | "check" | "agent" | "complete" | null>(null); + const [step, setStep] = useState(0); + const enabled = props.state.hostStatus !== "disabled"; + const busy = props.state.hostStatus === "installing" || props.state.hostStatus === "starting"; + + const update = async ( + kind: NonNullable, + input: { enabled?: boolean; agentAccessEnabled?: boolean; onboardingCompleted?: boolean }, + ) => { + setPending(kind); + try { + const result = await configure({ environmentId: props.environmentId, input }); + if (kind === "complete" && result._tag === "Success") props.onComplete?.(); + } finally { + setPending(null); + } + }; + + return ( + <> + + busy || pending !== null || requested > step} + /> + + + + {step === 0 ? ( +
+

Enable the device hub

+
+

{deviceHubDescription}

+ + void update("hub", { + enabled: Boolean(checked), + ...(checked ? {} : { agentAccessEnabled: false }), + }) + } + /> +
+ +
+ ) : null} + + {step === 1 ? ( +
+

Check simulator support

+ { + setPending("check"); + void list({ environmentId: props.environmentId, input: {} }).finally(() => + setPending(null), + ); + }} + /> +
+ ) : null} + + {step === 2 ? ( +
+

Allow agent control

+
+

{agentDeviceDescription}

+ + void update("agent", { agentAccessEnabled: Boolean(checked) }) + } + /> +
+ +

+ Leave this off to keep manual device controls without giving agents access. +

+
+ ) : null} + {props.state.hostStatus === "failed" && props.state.hostStatusDetail ? ( +

+ {props.state.hostStatusDetail} +

+ ) : null} + + + + {step === 0 ? ( + }>Cancel + ) : ( + + )} + {step < 2 ? ( + + ) : ( + + )} + + + ); +} + +export function DeviceHubSetupStatus({ + state, + pending, + compact = false, +}: { + readonly state: DeviceServiceState; + readonly pending: boolean; + readonly compact?: boolean; +}) { + if (!pending && state.hostStatus !== "ready") return null; + return ( +

+ {pending ? : } + {pending + ? state.hostStatus === "installing" + ? compact + ? "Installing…" + : "Installing device hub…" + : state.hostStatus === "starting" + ? compact + ? "Starting…" + : "Starting device hub…" + : compact + ? "Updating…" + : "Updating device hub…" + : "Device hub is ready."} +

+ ); +} + +function DevicePlatformSetup(props: { + readonly state: DeviceServiceState; + readonly checking: boolean; + readonly disabled: boolean; + readonly onCheck: () => void; +}) { + return ( +
+ + +

+ You can use either platform. Fixing a missing platform does not block the other one. +

+ +
+ ); +} + +export function AgentDeviceSetupStatus(props: { + readonly state: DeviceServiceState; + readonly pending: boolean; + readonly compact?: boolean; +}) { + if (props.pending) { + const label = + props.state.hostStatus === "installing" + ? props.compact + ? "Installing…" + : "Installing agent tools…" + : props.state.hostStatus === "starting" + ? props.compact + ? "Starting…" + : "Starting agent tools…" + : props.compact + ? "Updating…" + : "Updating agent access…"; + return ( +

+ + {label} +

+ ); + } + if ( + props.state.agentAccessEnabled && + props.state.hostStatus === "ready" && + props.state.hosts.some((host) => host.agentDeviceInstalled) + ) { + return ( +

+ + Agent tools are ready. +

+ ); + } + return null; +} + +export function PlatformStatus(props: { + readonly platform: string; + readonly status: { readonly ready: boolean; readonly message: string }; + readonly compact?: boolean; +}) { + const Icon = props.status.ready ? Check : CircleAlert; + return ( +
+ +
+

{props.platform}

+

+ {props.compact && props.status.ready ? "Ready" : props.status.message} +

+
+
+ ); +} diff --git a/apps/web/src/components/device/DeviceStreamView.test.tsx b/apps/web/src/components/device/DeviceStreamView.test.tsx new file mode 100644 index 000000000000..a46e5ed7b7b8 --- /dev/null +++ b/apps/web/src/components/device/DeviceStreamView.test.tsx @@ -0,0 +1,58 @@ +import { act } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { EnvironmentId } from "@t3tools/contracts"; +import { afterEach, expect, it, vi } from "vite-plus/test"; + +vi.mock("~/state/device", () => ({ + useDeviceHubAccess: () => access, + refreshDeviceHubAccess: vi.fn(), +})); +const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true }; +vi.mock("./deviceStream", () => ({ + createDeviceStreamClient: ( + _target: unknown, + _canvas: unknown, + events: { onMjpegFallback: (url: string) => void }, + ) => ({ + start: () => events.onMjpegFallback("http://test/stream.mjpeg"), + stop: vi.fn(), + }), +})); +import { DeviceStreamView } from "./DeviceStreamView"; +let renderer: ReactTestRenderer | undefined; +afterEach(async () => { + await act(async () => renderer?.unmount()); + vi.unstubAllGlobals(); +}); + +it("removes MJPEG requests while hidden and reconnects when shown", async () => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + disconnect() {} + }, + ); + const view = (visible: boolean) => ( + + ); + await act(async () => { + renderer = create(view(true), { + createNodeMock: () => ({ + style: { setProperty() {} }, + getBoundingClientRect: () => ({ width: 400, height: 800 }), + }), + }); + }); + expect(renderer!.root.findAllByType("img")).toHaveLength(1); + await act(async () => renderer!.update(view(false))); + expect(renderer!.root.findAllByType("img")).toHaveLength(0); + await act(async () => renderer!.update(view(true))); + expect(renderer!.root.findAllByType("img")).toHaveLength(1); +}); diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx new file mode 100644 index 000000000000..aed5c6019b9b --- /dev/null +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -0,0 +1,327 @@ +import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts"; +import { useEffect, useMemo, useRef, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device"; +import { Spinner } from "~/components/ui/spinner"; +import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi"; +import { + createDeviceStreamClient, + type DeviceHardwareButton, + type DeviceScreenSize, + type DeviceStreamClient, + type DeviceStreamStatus, +} from "./deviceStream"; + +const AX_POLL_INTERVAL_MS = 2_000; + +export interface DeviceStreamHandle { + readonly pressButton: (button: DeviceHardwareButton) => void; + readonly rotate: () => void; + /** False while the input socket is down; controls should disable. */ + readonly inputConnected: boolean; +} + +/** + * The live device screen. Pointer events map onto normalized coordinates in + * the displayed frame and go to the device; keyboard input is forwarded while + * the surface is focused. `visible=false` tears the stream down so a hidden + * panel decodes nothing. + */ +export function DeviceStreamView(props: { + readonly environmentId: EnvironmentId; + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly visible: boolean; + /** Draw accessibility element frames over the screen. */ + readonly axOverlay?: boolean; + readonly onHandle?: (handle: DeviceStreamHandle | null) => void; + readonly onScreen?: (screen: DeviceScreenSize | null) => void; +}) { + const access = useDeviceHubAccess(props.environmentId); + const canvasRef = useRef(null); + const clientRef = useRef(null); + const [status, setStatus] = useState("connecting"); + const [detail, setDetail] = useState(undefined); + const [screen, setScreen] = useState(null); + const [mjpegUrl, setMjpegUrl] = useState(null); + const [mjpegGeneration, setMjpegGeneration] = useState(0); + const [inputState, setInputState] = useState<{ connected: boolean; detail?: string }>({ + connected: false, + }); + const { onHandle, onScreen } = props; + + useEffect(() => { + const canvas = canvasRef.current; + if (!access || !canvas || !props.visible) { + setStatus("connecting"); + onHandle?.(null); + return; + } + const client = createDeviceStreamClient( + { platform: props.platform, deviceId: props.deviceId, access }, + canvas, + { + onStatus: (next, nextDetail) => { + setStatus(next); + setDetail(nextDetail); + }, + onScreen: (next) => { + setScreen(next); + onScreen?.(next); + }, + onUnauthorized: () => { + // A fresh ticket re-runs this effect through the access dependency. + refreshDeviceHubAccess(props.environmentId); + }, + onMjpegFallback: (url) => { + setMjpegUrl(url); + setMjpegGeneration((generation) => generation + 1); + }, + onInputConnected: (connected, detail) => { + setInputState({ connected, ...(detail ? { detail } : {}) }); + onHandle?.({ + pressButton: client.pressButton, + rotate: client.rotate, + inputConnected: connected, + }); + }, + }, + ); + clientRef.current = client; + setMjpegUrl(null); + setInputState({ connected: false }); + client.start(); + onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false }); + return () => { + client.stop(); + clientRef.current = null; + onHandle?.(null); + onScreen?.(null); + setScreen(null); + }; + }, [ + access, + onHandle, + onScreen, + props.deviceId, + props.environmentId, + props.platform, + props.visible, + ]); + + // Displayed aspect ratio (width / height) of the device as the user sees it. + const aspect = useMemo(() => { + if (!screen) return props.platform === "ios" ? 9 / 19.5 : 9 / 20; + const landscape = + screen.orientation === "landscape_left" || screen.orientation === "landscape_right"; + const w = landscape + ? Math.max(screen.width, screen.height) + : Math.min(screen.width, screen.height); + const h = landscape + ? Math.min(screen.width, screen.height) + : Math.max(screen.width, screen.height); + return w / h; + }, [props.platform, screen]); + + // The frame is the largest box at `aspect` that fits the container, so a + // narrow panel shows a shorter phone rather than a squeezed one. CSS + // `aspect-ratio` alone cannot do this: with the height pinned to 100% the + // width clamp wins and distorts the drawn frame. + const hostRef = useRef(null); + const [host, setHost] = useState({ width: 0, height: 0 }); + useEffect(() => { + const element = hostRef.current; + if (!element) return; + const update = () => { + const rect = element.getBoundingClientRect(); + setHost((current) => + current.width === rect.width && current.height === rect.height + ? current + : { width: rect.width, height: rect.height }, + ); + }; + update(); + const observer = new ResizeObserver(update); + observer.observe(element); + return () => observer.disconnect(); + }, []); + const frame = useMemo(() => { + if (host.width === 0 || host.height === 0) return { width: 0, height: 0 }; + const byHeight = { width: host.height * aspect, height: host.height }; + return byHeight.width <= host.width + ? byHeight + : { width: host.width, height: host.width / aspect }; + }, [aspect, host]); + + // serve-sim streams the raw framebuffer; rotate the display for a device + // that reports landscape while its frames stay portrait. + const rotation = useMemo(() => { + if (props.platform !== "ios" || !screen || screen.width > screen.height) return 0; + switch (screen.orientation) { + case "landscape_left": + return 90; + case "landscape_right": + return -90; + case "portrait_upside_down": + return 180; + default: + return 0; + } + }, [props.platform, screen]); + + // A sideways rotation draws the raw portrait frame into a landscape box: + // the media element takes the transposed size and is rotated about the + // box's center. + const sideways = rotation === 90 || rotation === -90; + const mediaStyle: React.CSSProperties = sideways + ? { + width: frame.height, + height: frame.width, + left: (frame.width - frame.height) / 2, + top: (frame.height - frame.width) / 2, + transform: `rotate(${rotation}deg)`, + } + : { + width: frame.width, + height: frame.height, + ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}), + }; + + // The accessibility tree is polled while the overlay is on; each poll is + // one JSON fetch, so there is nothing to repaint between polls. + const [axElements, setAxElements] = useState>([]); + useEffect(() => { + if (!props.axOverlay || !access || !props.visible) return; + const target = { access, platform: props.platform, deviceId: props.deviceId }; + let controller: AbortController | null = null; + let timer: ReturnType | null = null; + let stopped = false; + const poll = async () => { + controller = new AbortController(); + try { + const tree = await fetchDeviceAxTree(target, controller.signal); + if (!stopped) setAxElements(tree.elements); + } catch { + // Keep the last good tree; the next poll retries. + } + if (!stopped) timer = setTimeout(() => void poll(), AX_POLL_INTERVAL_MS); + }; + void poll(); + return () => { + stopped = true; + controller?.abort(); + if (timer) clearTimeout(timer); + setAxElements([]); + }; + }, [access, props.axOverlay, props.deviceId, props.platform, props.visible]); + + const pointerActive = useRef(false); + const normalizedPoint = (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + const x = (event.clientX - rect.left) / rect.width; + const y = (event.clientY - rect.top) / rect.height; + return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; + }; + + return ( +
{ + if (event.metaKey && !["r", "R"].includes(event.key)) return; + event.preventDefault(); + clientRef.current?.sendKey(event.nativeEvent, "down"); + }} + onKeyUp={(event) => { + clientRef.current?.sendKey(event.nativeEvent, "up"); + }} + > +
{ + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.parentElement as HTMLElement | null)?.focus(); + pointerActive.current = true; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("begin", x, y); + }} + onPointerMove={(event) => { + if (!pointerActive.current) return; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("move", x, y); + }} + onPointerUp={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + onPointerCancel={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + > + + {props.visible && access && mjpegUrl ? ( + + ) : null} + {axElements.length > 0 ? ( +
+ {axElements.map((element) => ( +
+ {element.label ? ( + + {element.label} + + ) : null} +
+ ))} +
+ ) : null} +
+ {status === "streaming" && !inputState.connected ? ( +
+ + Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting… + +
+ ) : null} + {status !== "streaming" ? ( +
+ {status === "connecting" ? : null} + {status === "error" ? (detail ?? "Stream failed.") : "Connecting to device…"} + {status === "connecting" && detail ? ( + {detail} + ) : null} +
+ ) : null} +
+ ); +} diff --git a/apps/web/src/components/device/DeviceToolsPanel.tsx b/apps/web/src/components/device/DeviceToolsPanel.tsx new file mode 100644 index 000000000000..7f3984257ecd --- /dev/null +++ b/apps/web/src/components/device/DeviceToolsPanel.tsx @@ -0,0 +1,722 @@ +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { + DeviceActionInput, + DeviceDetail, + DevicePermission, + DeviceSummary, + DeviceTextSize, + EnvironmentId, +} from "@t3tools/contracts"; +import { ChevronDown, X } from "lucide-react"; +import { useCallback, useEffect, useMemo, useState } from "react"; + +import { Button } from "~/components/ui/button"; +import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "~/components/ui/collapsible"; +import { Input } from "~/components/ui/input"; +import { + Select, + SelectItem, + SelectPopup, + SelectTrigger, + SelectValue, +} from "~/components/ui/select"; +import { Spinner } from "~/components/ui/spinner"; +import { Switch } from "~/components/ui/switch"; +import { Toggle, ToggleGroup } from "~/components/ui/toggle-group"; +import { cn } from "~/lib/utils"; +import { deviceEnvironment } from "~/state/device"; +import { formatEnvironmentQueryError } from "~/state/query"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { + type DeviceEventLogEntry, + type DeviceForegroundInfo, + subscribeDeviceEventLog, + subscribeDeviceForeground, +} from "./deviceHubApi"; + +type ActionBody = DeviceActionInput extends infer A + ? A extends { readonly type: string } + ? Omit + : never + : never; + +const TEXT_SIZES: ReadonlyArray<{ value: DeviceTextSize; label: string }> = [ + { value: "small", label: "Small" }, + { value: "default", label: "Default" }, + { value: "large", label: "Large" }, + { value: "extra-large", label: "Extra large" }, +]; + +const COLOR_FILTERS = [ + { value: "none", label: "None" }, + { value: "grayscale", label: "Grayscale" }, + { value: "red-green", label: "Red / green (protanopia)" }, + { value: "green-red", label: "Green / red (deuteranopia)" }, + { value: "blue-yellow", label: "Blue / yellow (tritanopia)" }, +] as const; + +const ORIENTATIONS = [ + { value: "portrait", label: "Portrait" }, + { value: "landscape_left", label: "Landscape left" }, + { value: "portrait_upside_down", label: "Upside down" }, + { value: "landscape_right", label: "Landscape right" }, +] as const; + +const IOS_PERMISSIONS: ReadonlyArray<{ value: DevicePermission; label: string }> = [ + { value: "camera", label: "Camera" }, + { value: "microphone", label: "Microphone" }, + { value: "photos", label: "Photos" }, + { value: "contacts", label: "Contacts" }, + { value: "calendar", label: "Calendar" }, + { value: "reminders", label: "Reminders" }, + { value: "location", label: "Location" }, + { value: "notifications", label: "Notifications" }, + { value: "motion", label: "Motion" }, + { value: "media-library", label: "Media library" }, + { value: "faceid", label: "Face ID" }, +]; + +const ANDROID_PERMISSIONS: ReadonlyArray<{ value: DevicePermission; label: string }> = [ + { value: "camera", label: "Camera" }, + { value: "microphone", label: "Microphone" }, + { value: "photos", label: "Photos" }, + { value: "contacts", label: "Contacts" }, + { value: "calendar", label: "Calendar" }, + { value: "location", label: "Location" }, + { value: "notifications", label: "Notifications" }, + { value: "motion", label: "Physical activity" }, +]; + +const LOCATION_PRESETS = [ + { label: "San Francisco", latitude: 37.7749, longitude: -122.4194 }, + { label: "New York", latitude: 40.7128, longitude: -74.006 }, + { label: "London", latitude: 51.5074, longitude: -0.1278 }, + { label: "Stockholm", latitude: 59.3293, longitude: 18.0686 }, + { label: "Tokyo", latitude: 35.6762, longitude: 139.6503 }, +] as const; + +/** + * The Tools drawer for one open device: current settings read from the device, + * one control per supported action, and the read-only feeds the hub exposes. + * Every change is a `device.action` round trip; the returned detail replaces + * local state so the controls never show a value the device did not confirm. + */ +export function DeviceToolsPanel(props: { + readonly environmentId: EnvironmentId; + readonly device: DeviceSummary; + readonly access: DeviceHubAccess | null; + readonly axOverlay: boolean; + readonly onAxOverlayChange: (enabled: boolean) => void; + readonly onClose: () => void; + readonly className?: string; +}) { + const { environmentId, device } = props; + const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false }); + const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false }); + const [detail, setDetail] = useState(null); + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + const [foreground, setForeground] = useState(undefined); + const isIos = device.platform === "ios"; + + const target = useMemo( + () => ({ hostId: device.hostId, deviceId: device.id }), + [device.hostId, device.id], + ); + + // The panel is keyed by device, so a mount is always a fresh device. + useEffect(() => { + let cancelled = false; + void readDetail({ environmentId, input: target }).then((result) => { + if (cancelled) return; + if (result._tag === "Success") setDetail(result.value); + else setError(formatEnvironmentQueryError(result.cause)); + }); + return () => { + cancelled = true; + }; + }, [environmentId, readDetail, target]); + + useEffect(() => { + if (!props.access) return; + return subscribeDeviceForeground( + { access: props.access, platform: device.platform, deviceId: device.id }, + setForeground, + ); + }, [device.id, device.platform, props.access]); + + const act = useCallback( + async (body: ActionBody) => { + setPending(true); + setError(null); + try { + const result = await runAction({ + environmentId, + input: { ...target, ...body } as DeviceActionInput, + }); + if (result._tag === "Success") setDetail(result.value); + else setError(formatEnvironmentQueryError(result.cause)); + } finally { + setPending(false); + } + }, + [environmentId, runAction, target], + ); + + const settings = detail?.settings; + const foregroundApp = foreground === undefined ? (detail?.foregroundApp ?? null) : foreground; + const disabled = pending || detail === null; + + return ( +
+
+ Tools + {pending ? : null} + +
+
+ {error ? ( +

{error}

+ ) : null} + {detail === null && !error ? ( +
+ Reading device settings… +
+ ) : null} + +
+ + {foregroundApp?.id ?? "—"} + + {foregroundApp ? ( +
+ + +
+ ) : null} + act({ type: "openUrl", url })} + /> + act({ type: "launchApp", appId })} + /> +
+ +
+ + { + const next = value[0]; + if (next === "light" || next === "dark") + void act({ type: "setAppearance", value: next }); + }} + > + Light + Dark + + + + act({ type: "setTextSize", value })} + /> + + {isIos ? ( + <> + + { + const next = value[0]; + if (next === "clear" || next === "tinted") { + void act({ type: "setLiquidGlass", value: next }); + } + }} + > + Clear + Tinted + + + + act({ type: "setColorFilter", value })} + /> + + + ) : ( + + act({ type: "setOrientation", value })} + /> + + )} + act({ type: "setToggle", setting: "reduceMotion", value })} + /> + {isIos ? ( + <> + act({ type: "setToggle", setting: "increaseContrast", value })} + /> + + act({ type: "setToggle", setting: "reduceTransparency", value }) + } + /> + act({ type: "setToggle", setting: "showBorders", value })} + /> + act({ type: "setToggle", setting: "voiceOver", value })} + /> + + ) : ( + act({ type: "setToggle", setting: "networkEnabled", value })} + /> + )} +
+ +
+ { + props.onAxOverlayChange(value); + return Promise.resolve(); + }} + /> +
+ + act({ type: "setLocation", latitude, longitude })} + onClear={() => act({ type: "clearLocation" })} + /> + + + act({ type: "setPermission", appId, permission, decision }) + } + /> + + {isIos ? ( +
+ + foregroundApp + ? act({ type: "sendPush", appId: foregroundApp.id, payload }) + : Promise.resolve() + } + /> + {!foregroundApp ? ( +

Open an app first.

+ ) : null} +
+ ) : null} + + {isIos && props.access ? : null} +
+
+ ); +} + +function Section(props: { readonly title: string; readonly children: React.ReactNode }) { + return ( +
+

+ {props.title} +

+ {props.children} +
+ ); +} + +function Row(props: { readonly label: string; readonly children: React.ReactNode }) { + return ( +
+ {props.label} +
{props.children}
+
+ ); +} + +function SwitchRow(props: { + readonly label: string; + readonly checked: boolean | undefined; + readonly disabled: boolean; + readonly onChange: (value: boolean) => Promise; +}) { + return ( + + void props.onChange(checked)} + /> + + ); +} + +function ChoiceSelect(props: { + readonly ariaLabel: string; + readonly value: V | null; + readonly options: ReadonlyArray<{ readonly value: V; readonly label: string }>; + readonly disabled: boolean; + readonly placeholder?: string; + readonly onChange: (value: V) => Promise; +}) { + const current = props.options.find((option) => option.value === props.value); + return ( + + ); +} + +function SubmitRow(props: { + readonly placeholder: string; + readonly action: string; + readonly disabled: boolean; + readonly onSubmit: (value: string) => Promise; +}) { + const [value, setValue] = useState(""); + const submit = () => { + const trimmed = value.trim(); + if (!trimmed) return; + void props.onSubmit(trimmed).then(() => setValue("")); + }; + return ( +
{ + event.preventDefault(); + submit(); + }} + > + setValue(event.target.value)} + /> + +
+ ); +} + +function LocationSection(props: { + readonly disabled: boolean; + readonly canClear: boolean; + readonly onSet: (latitude: number, longitude: number) => Promise; + readonly onClear: () => Promise; +}) { + const [latitude, setLatitude] = useState(""); + const [longitude, setLongitude] = useState(""); + const parsed = { latitude: Number(latitude), longitude: Number(longitude) }; + const valid = + latitude.trim() !== "" && + longitude.trim() !== "" && + Math.abs(parsed.latitude) <= 90 && + Math.abs(parsed.longitude) <= 180; + return ( +
+
+ setLatitude(event.target.value)} + /> + setLongitude(event.target.value)} + /> +
+
+ + value={null} + disabled={props.disabled} + onValueChange={(value) => { + const preset = LOCATION_PRESETS.find((candidate) => candidate.label === value); + if (!preset) return; + setLatitude(String(preset.latitude)); + setLongitude(String(preset.longitude)); + void props.onSet(preset.latitude, preset.longitude); + }} + > + + + Preset… + + + + {LOCATION_PRESETS.map((preset) => ( + + {preset.label} + + ))} + + + + {props.canClear ? ( + + ) : null} +
+
+ ); +} + +function PermissionsSection(props: { + readonly permissions: ReadonlyArray<{ value: DevicePermission; label: string }>; + readonly canReset: boolean; + readonly defaultAppId: string; + readonly disabled: boolean; + readonly onDecide: ( + appId: string, + permission: DevicePermission, + decision: "grant" | "revoke" | "reset", + ) => Promise; +}) { + const [appId, setAppId] = useState(""); + const [permission, setPermission] = useState("camera"); + const resolvedAppId = appId.trim() || props.defaultAppId; + const decide = (decision: "grant" | "revoke" | "reset") => + void props.onDecide(resolvedAppId, permission, decision); + return ( +
+ setAppId(event.target.value)} + /> +
+ { + setPermission(value); + return Promise.resolve(); + }} + /> + + + {props.canReset ? ( + + ) : null} +
+
+ ); +} + +const EVENT_LOG_LIMIT = 100; + +function EventLogSection(props: { + readonly access: DeviceHubAccess; + readonly device: DeviceSummary; +}) { + const [open, setOpen] = useState(false); + const [entries, setEntries] = useState>([]); + + useEffect(() => { + if (!open) return; + const unsubscribe = subscribeDeviceEventLog( + { access: props.access, platform: props.device.platform, deviceId: props.device.id }, + (incoming, reset) => { + setEntries((current) => { + const merged = reset ? [...incoming] : [...current, ...incoming]; + return merged.length > EVENT_LOG_LIMIT ? merged.slice(-EVENT_LOG_LIMIT) : merged; + }); + }, + ); + return () => { + unsubscribe(); + setEntries([]); + }; + }, [open, props.access, props.device.id, props.device.platform]); + + return ( + + + Event log + + + +
    + {entries.length === 0 ? ( +
  1. No events yet.
    2. + ) : ( + entries.map((entry) => ( +
  2. + + {entry.timestamp.slice(11, 19)} + + {entry.summary} +
    3. + )) + )} +
+ + + ); +} diff --git a/apps/web/src/components/device/deviceHubApi.test.ts b/apps/web/src/components/device/deviceHubApi.test.ts new file mode 100644 index 000000000000..52575f4a4224 --- /dev/null +++ b/apps/web/src/components/device/deviceHubApi.test.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { subscribeDeviceForeground } from "./deviceHubApi"; + +describe("foreground app events", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("clears the last app when it exits and ignores malformed events", () => { + let source: FakeEventSource; + class FakeEventSource { + onmessage: ((event: { data: string }) => void) | null = null; + addEventListener(_type: string, listener: (event: { data: string }) => void) { + this.onmessage = listener; + } + close = vi.fn(); + constructor() { + source = this; + } + } + vi.stubGlobal("EventSource", FakeEventSource); + const onChange = vi.fn(); + const stop = subscribeDeviceForeground( + { + platform: "ios", + deviceId: "test", + access: { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true }, + }, + onChange, + ); + const emit = (data: unknown) => source.onmessage?.({ data: JSON.stringify(data) }); + emit({ bundleId: "com.example.app", pid: 123 }); + emit({ bundleId: null }); + emit({ bundleId: "com.example.other" }); + emit({ bundleId: "" }); + emit({ other: "not app state" }); + expect(onChange.mock.calls.map(([app]) => app)).toEqual([ + { id: "com.example.app", pid: 123 }, + null, + { id: "com.example.other" }, + null, + ]); + stop(); + expect(source!.close).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/web/src/components/device/deviceHubApi.ts b/apps/web/src/components/device/deviceHubApi.ts new file mode 100644 index 000000000000..dbcf4fa0015c --- /dev/null +++ b/apps/web/src/components/device/deviceHubApi.ts @@ -0,0 +1,248 @@ +import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DevicePlatform } from "@t3tools/contracts"; + +/** + * Read-only hub endpoints the Tools drawer consumes directly: the accessibility + * tree, the foreground app, and the event log. Everything that changes device + * state goes through the `device.action` RPC instead, so this file never POSTs. + */ + +export interface DeviceAxElement { + readonly id: string; + readonly label: string; + readonly role: string; + /** Normalized to the displayed screen: 0..1 on both axes. */ + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface DeviceAxTree { + readonly elements: ReadonlyArray; + readonly errors: ReadonlyArray; +} + +export interface DeviceEventLogEntry { + readonly id: number; + readonly timestamp: string; + readonly kind: string; + readonly summary: string; +} + +export interface DeviceForegroundInfo { + readonly id: string; + readonly label?: string; + readonly pid?: number; + readonly isReactNative?: boolean; +} + +interface Target { + readonly access: DeviceHubAccess; + readonly platform: DevicePlatform; + readonly deviceId: string; +} + +const vendorBase = (target: Target) => + `${target.access.httpBase}${target.platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"}`; + +const hubUrl = (target: Target, path: string, params?: Record) => { + const search = params ? `?${new URLSearchParams(params).toString()}` : ""; + return withDeviceHubQuery(`${vendorBase(target)}${path}${search}`, target.access); +}; + +const fetchJson = async (target: Target, url: string, signal?: AbortSignal): Promise => { + const response = await fetch(url, { + cache: "no-store", + credentials: target.access.credentials ? "include" : "same-origin", + ...(signal ? { signal } : {}), + }); + if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); + return response.json(); +}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null; + +const numberOr = (value: unknown, fallback: number) => + typeof value === "number" && Number.isFinite(value) ? value : fallback; + +const AX_ELEMENT_LIMIT = 500; + +/** + * serve-sim's helper returns the native nested tree; the root node is the + * application covering the whole screen. Flatten it the way serve-sim's own + * overlay does: skip nodes with the root's frame, cap the count. + */ +const flattenIosAxTree = (roots: ReadonlyArray): ReadonlyArray => { + const first = roots[0]; + const rootFrame = isRecord(first) && isRecord(first.frame) ? first.frame : null; + const screenWidth = Math.max(1, numberOr(rootFrame?.width, 1)); + const screenHeight = Math.max(1, numberOr(rootFrame?.height, 1)); + const elements: DeviceAxElement[] = []; + const visit = (node: unknown, path: string) => { + if (elements.length >= AX_ELEMENT_LIMIT || !isRecord(node) || !isRecord(node.frame)) return; + const frame = node.frame; + const width = numberOr(frame.width, 0); + const height = numberOr(frame.height, 0); + const coversScreen = + Math.abs(width - screenWidth) < 0.5 && Math.abs(height - screenHeight) < 0.5; + if (!coversScreen && width > 0 && height > 0) { + elements.push({ + id: typeof node.AXUniqueId === "string" ? node.AXUniqueId : path, + label: typeof node.AXLabel === "string" ? node.AXLabel : "", + role: typeof node.type === "string" ? node.type : "", + x: numberOr(frame.x, 0) / screenWidth, + y: numberOr(frame.y, 0) / screenHeight, + width: width / screenWidth, + height: height / screenHeight, + }); + } + const children = Array.isArray(node.children) ? node.children : []; + children.forEach((child, index) => visit(child, `${path}.${index}`)); + }; + roots.forEach((root, index) => visit(root, String(index))); + return elements; +}; + +export async function fetchDeviceAxTree( + target: Target, + signal?: AbortSignal, +): Promise { + if (target.platform === "ios") { + const payload = await fetchJson( + target, + hubUrl(target, `/helper/${encodeURIComponent(target.deviceId)}/ax`), + signal, + ); + if (!Array.isArray(payload)) { + const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null; + return { elements: [], errors: [error ?? "Unexpected accessibility payload."] }; + } + return { elements: flattenIosAxTree(payload), errors: [] }; + } + const payload = await fetchJson( + target, + hubUrl(target, "/api/accessibility", { device: target.deviceId }), + signal, + ); + if (!isRecord(payload) || !Array.isArray(payload.nodes)) { + const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null; + return { elements: [], errors: [error ?? "Unexpected accessibility payload."] }; + } + // uiautomator reports pixel bounds; the first node is the full window. + const nodes = payload.nodes.filter( + (node): node is Record => isRecord(node) && isRecord(node.bounds), + ); + const root = nodes[0]?.bounds as Record | undefined; + const screenWidth = Math.max(1, numberOr(root?.right, 1)); + const screenHeight = Math.max(1, numberOr(root?.bottom, 1)); + // Layout containers span the whole window and would tint the entire + // screen; only nodes a user could point at are worth drawing. + const elements = nodes.slice(1).flatMap((node): DeviceAxElement[] => { + const bounds = node.bounds as Record; + const left = numberOr(bounds.left, 0); + const top = numberOr(bounds.top, 0); + const width = (numberOr(bounds.right, left) - left) / screenWidth; + const height = (numberOr(bounds.bottom, top) - top) / screenHeight; + const text = typeof node.text === "string" ? node.text : ""; + const description = typeof node.contentDescription === "string" ? node.contentDescription : ""; + const label = text || description; + if (width >= 0.95 && height >= 0.9) return []; + if (!label && node.clickable !== true) return []; + const className = typeof node.className === "string" ? node.className : ""; + return [ + { + id: String(node.id ?? ""), + label, + role: className.split(".").at(-1) ?? "", + x: left / screenWidth, + y: top / screenHeight, + width, + height, + }, + ]; + }); + return { elements, errors: [] }; +} + +const openEventSource = ( + target: Target, + url: string, + onMessage: (data: unknown) => void, +): (() => void) => { + const source = new EventSource(url, { withCredentials: target.access.credentials }); + source.addEventListener("message", (event) => { + try { + onMessage(JSON.parse(String(event.data))); + } catch { + // Keep-alive comments and malformed frames carry nothing to render. + } + }); + return () => source.close(); +}; + +/** iOS only: the frontmost app, pushed by serve-sim whenever it changes. */ +export function subscribeDeviceForeground( + target: Target, + onChange: (app: DeviceForegroundInfo | null) => void, +): () => void { + if (target.platform !== "ios") return () => {}; + return openEventSource( + target, + hubUrl(target, "/appstate", { device: target.deviceId }), + (data) => { + if (!isRecord(data)) return; + if (data.bundleId === null || data.bundleId === "") { + onChange(null); + return; + } + if (typeof data.bundleId !== "string") return; + onChange({ + id: data.bundleId, + ...(typeof data.pid === "number" ? { pid: data.pid } : {}), + ...(typeof data.isReactNative === "boolean" ? { isReactNative: data.isReactNative } : {}), + }); + }, + ); +} + +const toEventLogEntry = (raw: unknown): DeviceEventLogEntry | null => { + if (!isRecord(raw) || typeof raw.id !== "number") return null; + return { + id: raw.id, + timestamp: typeof raw.timestamp === "string" ? raw.timestamp : "", + kind: typeof raw.kind === "string" ? raw.kind : "", + summary: + typeof raw.summary === "string" ? raw.summary : typeof raw.msg === "string" ? raw.msg : "", + }; +}; + +/** + * iOS only: serve-sim's event log, seeded with recent history and then pushed + * live. Android's session recorder only tracks replayable gestures, which the + * user already sees themselves, so it is not surfaced. + */ +export function subscribeDeviceEventLog( + target: Target, + onEvents: (entries: ReadonlyArray, reset: boolean) => void, +): () => void { + if (target.platform !== "ios") return () => {}; + return openEventSource( + target, + hubUrl(target, "/api/event-log/events", { device: target.deviceId, limit: "100" }), + (data) => { + if (!isRecord(data)) return; + if (Array.isArray(data.events)) { + onEvents( + data.events.flatMap((raw) => toEventLogEntry(raw) ?? []), + true, + ); + return; + } + const entry = toEventLogEntry(data.event); + if (entry) onEvents([entry], false); + }, + ); +} diff --git a/apps/web/src/components/device/deviceStream.test.ts b/apps/web/src/components/device/deviceStream.test.ts new file mode 100644 index 000000000000..362bd44fedba --- /dev/null +++ b/apps/web/src/components/device/deviceStream.test.ts @@ -0,0 +1,180 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + createDeviceStreamClient, + AvccDemuxer, + avcCodecString, + parseSemuPacket, + scanAccessUnit, +} from "./deviceStream"; + +const envelope = (tag: number, payload: number[]) => { + const length = 1 + payload.length; + return [ + (length >>> 24) & 0xff, + (length >>> 16) & 0xff, + (length >>> 8) & 0xff, + length & 0xff, + tag, + ...payload, + ]; +}; + +describe("AvccDemuxer", () => { + it("reassembles envelopes split across reads", () => { + const demuxer = new AvccDemuxer(); + const bytes = new Uint8Array([ + ...envelope(1, [1, 0x64, 0x00, 0x1f]), + ...envelope(2, [9, 9, 9]), + ...envelope(0x7f, [0]), + ...envelope(3, [4]), + ]); + const first = demuxer.push(bytes.subarray(0, 7)); + const rest = demuxer.push(bytes.subarray(7)); + const chunks = [...first, ...rest]; + expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe", "delta"]); + expect(Array.from(chunks[0]!.payload)).toEqual([1, 0x64, 0x00, 0x1f]); + expect(Array.from(chunks[1]!.payload)).toEqual([9, 9, 9]); + }); + + it("derives the WebCodecs codec string from the avcC record", () => { + expect(avcCodecString(new Uint8Array([1, 0x64, 0x00, 0x1f]))).toBe("avc1.64001f"); + expect(avcCodecString(new Uint8Array([1]))).toBe("avc1.42E01E"); + }); +}); + +describe("serve-emu frames", () => { + it("strips the SEMU header and reads the keyframe flag and timestamp", () => { + const buffer = new ArrayBuffer(16 + 3); + const view = new DataView(buffer); + view.setUint32(0, 0x53454d55); + view.setUint8(4, 1); + view.setUint8(5, 1); + view.setBigUint64(8, 123456n); + new Uint8Array(buffer).set([7, 8, 9], 16); + const packet = parseSemuPacket(buffer); + expect(packet.isKey).toBe(true); + expect(packet.timestamp).toBe(123456); + expect(Array.from(packet.data)).toEqual([7, 8, 9]); + }); + + it("treats a frame without the header as raw data", () => { + const packet = parseSemuPacket(new Uint8Array([0, 0, 1, 0x65]).buffer); + expect(packet.isKey).toBeNull(); + expect(packet.data.length).toBe(4); + }); + + it("finds the SPS and IDR NAL units in an Annex-B access unit", () => { + const unit = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f, 0, 0, 1, 0x65, 0xaa]); + const scanned = scanAccessUnit(unit); + expect(scanned.isKey).toBe(true); + expect(scanned.sps && avcCodecString(scanned.sps)).toBe("avc1.64001f"); + expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false); + }); +}); + +describe("iOS input startup", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + const setup = () => { + vi.useFakeTimers(); + const sockets: FakeSocket[] = []; + class FakeSocket { + static OPEN = 1; + readyState = 1; + binaryType = ""; + onopen: (() => void) | null = null; + onmessage: ((event: { data: ArrayBuffer }) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = null; + send = vi.fn(); + close = vi.fn(); + constructor() { + sockets.push(this); + } + } + vi.stubGlobal("WebSocket", FakeSocket); + const signals: AbortSignal[] = []; + vi.stubGlobal( + "fetch", + vi.fn((_url, init: RequestInit) => { + const signal = init.signal!; + signals.push(signal); + return Promise.resolve( + new Response( + new ReadableStream({ + start(controller) { + signal.addEventListener("abort", () => controller.error(new Error("aborted"))); + }, + }), + ), + ); + }), + ); + const client = createDeviceStreamClient( + { + platform: "ios", + deviceId: "test-device", + access: { + httpBase: "http://test/api/device-hub", + wsBase: "ws://test/api/device-hub", + credentials: true, + query: {}, + }, + }, + { getContext: () => null } as unknown as HTMLCanvasElement, + { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }, + ); + return { client, sockets, signals }; + }; + + it("connects input when the MJPEG prime never produces a frame", async () => { + const { client, sockets, signals } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(2_000); + expect(signals[0]?.aborted).toBe(true); + expect(sockets).toHaveLength(1); + client.stop(); + }); + + it("aborts priming immediately when hidden without opening a socket later", async () => { + const { client, sockets, signals } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(0); + client.stop(); + expect(signals[0]?.aborted).toBe(true); + await vi.advanceTimersByTimeAsync(2_000); + expect(sockets).toHaveLength(0); + }); + + it.each([ + ["landscape_left", 0.7, 1 - 0.2], + ["landscape_right", 1 - 0.7, 0.2], + ])("maps %s touches back to the raw iOS framebuffer", async (orientation, x, y) => { + const { client, sockets } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(2_000); + const socket = sockets[0]!; + const json = new TextEncoder().encode(JSON.stringify({ width: 400, height: 800, orientation })); + const packet = new Uint8Array(1 + json.length); + packet[0] = 0x82; + packet.set(json, 1); + socket.onmessage?.({ data: packet.buffer }); + client.sendTouch("begin", 0.2, 0.7); + const sent = socket.send.mock.calls[0]![0] as Uint8Array; + expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({ + type: "begin", + x, + y, + }); + client.stop(); + }); +}); diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts new file mode 100644 index 000000000000..d6ae632202dd --- /dev/null +++ b/apps/web/src/components/device/deviceStream.ts @@ -0,0 +1,697 @@ +/** + * Framework-free client for expo-device-hub's per-device streams, reached + * through the T3 proxy. One class handles both platforms because the hub + * vendors two servers with different wire formats: + * + * - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed + * envelopes (`u32be length, u8 tag, payload`; tag 1 avcC description, + * 2 keyframe, 3 delta, 4 JPEG seed) decoded with WebCodecs; input goes over + * `helper/ws?device=` as `[tag][json]` packets. When WebCodecs is + * unavailable (plain-http remote origins) the MJPEG endpoint is used as an + * `` source instead. + * - Android (serve-emu): one WebSocket at `ws?device=&frame-meta=1` + * carries H.264 access units prefixed with a 16-byte "SEMU" header + * (magic, version, key flag, pts) and accepts JSON gestures upstream. + * + * The decoder only runs while frames arrive and the viewer is attached; a + * hidden panel calls `stop()` so an idle device costs nothing on the GPU. + */ +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DevicePlatform } from "@t3tools/contracts"; + +export type DeviceStreamStatus = "connecting" | "streaming" | "error"; + +export interface DeviceScreenSize { + readonly width: number; + readonly height: number; + readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right"; +} + +export interface DeviceStreamEvents { + readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; + readonly onScreen: (screen: DeviceScreenSize) => void; + /** The proxy rejected the credential; the owner should refresh access and reconnect. */ + readonly onUnauthorized: () => void; + /** + * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is + * unsupported); the owner should show this MJPEG URL in an `` instead of + * the canvas. + */ + readonly onMjpegFallback: (url: string) => void; + /** Whether touches and keys can currently reach the device. */ + readonly onInputConnected: (connected: boolean, detail?: string) => void; +} + +export interface DeviceStreamTarget { + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly access: DeviceHubAccess; +} + +export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; + +const RETRY_DELAY_MS = 1_000; +const FRAME_DURATION_US = 16_667; +const SEMU_MAGIC = 0x53454d55; +const SEMU_HEADER_BYTES = 16; +const SEMU_FLAG_KEY = 1; +const SOFT_DECODE_QUEUE = 8; + +// serve-sim binary WS message tags (browser -> helper). +const IOS_MSG_TOUCH = 0x03; +const IOS_MSG_BUTTON = 0x04; +const IOS_MSG_KEY = 0x06; +const IOS_MSG_ORIENTATION = 0x07; +const IOS_MSG_HARDWARE_KEYBOARD = 0x0d; +// helper -> browser. +const IOS_TAG_SCREEN_CONFIG = 0x82; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +const isWebCodecsSupported = (): boolean => + typeof globalThis !== "undefined" && + "VideoDecoder" in globalThis && + "EncodedVideoChunk" in globalThis; + +function taggedJson(tag: number, payload: unknown): Uint8Array { + const json = encoder.encode(JSON.stringify(payload)); + const out = new Uint8Array(1 + json.length); + out[0] = tag; + out.set(json, 1); + return out; +} + +/** Build the WebCodecs `avc1.PPCCLL` string from an avcC record or an SPS NAL. */ +export function avcCodecString(bytes: Uint8Array): string { + if (bytes.length < 4) return "avc1.42E01E"; + const hex = (byte: number) => byte.toString(16).padStart(2, "0"); + return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`; +} + +/** Split serve-emu's SEMU-framed message into metadata and the Annex-B payload. */ +export function parseSemuPacket(raw: ArrayBuffer): { + readonly data: Uint8Array; + readonly isKey: boolean | null; + readonly timestamp: number | null; +} { + const bytes = new Uint8Array(raw); + if (bytes.byteLength > SEMU_HEADER_BYTES) { + const view = new DataView(raw, 0, SEMU_HEADER_BYTES); + if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) { + const pts = view.getBigUint64(8, false); + return { + data: bytes.subarray(SEMU_HEADER_BYTES), + isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0, + timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, + }; + } + } + return { data: bytes, isKey: null, timestamp: null }; +} + +const isVideoSessionMessage = (text: string) => { + try { + const message = JSON.parse(text) as { type?: unknown }; + return message.type === "video-session"; + } catch { + return false; + } +}; + +/** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */ +export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { + let isKey = false; + let sps: Uint8Array | null = null; + const len = buf.length; + let i = 0; + while (i + 2 < len) { + if (buf[i] === 0 && buf[i + 1] === 0) { + let codeLen = 0; + if (buf[i + 2] === 1) codeLen = 3; + else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4; + if (codeLen) { + const nalType = buf[i + codeLen]! & 0x1f; + if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen); + if (nalType === 5) isKey = true; + i += codeLen + 1; + continue; + } + } + i++; + } + return { isKey, sps }; +} + +export type AvccChunk = { + readonly type: "description" | "keyframe" | "delta" | "seed"; + readonly payload: Uint8Array; +}; + +const AVCC_TAGS: Record = { + 1: "description", + 2: "keyframe", + 3: "delta", + 4: "seed", +}; + +/** Turns a fragmented AVCC byte stream into complete envelopes. */ +export class AvccDemuxer { + private buffer = new Uint8Array(64 * 1024); + private length = 0; + + push(bytes: Uint8Array): AvccChunk[] { + if (this.length + bytes.length > this.buffer.length) { + let capacity = this.buffer.length; + while (capacity < this.length + bytes.length) capacity *= 2; + const grown = new Uint8Array(capacity); + grown.set(this.buffer.subarray(0, this.length)); + this.buffer = grown; + } + this.buffer.set(bytes, this.length); + this.length += bytes.length; + + const chunks: AvccChunk[] = []; + let offset = 0; + while (this.length - offset >= 4) { + const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4); + const frameLength = view.getUint32(0, false); + if (this.length - offset - 4 < frameLength) break; + if (frameLength >= 1) { + const type = AVCC_TAGS[this.buffer[offset + 4]!]; + if (type) { + chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) }); + } + } + offset += 4 + frameLength; + } + if (offset > 0) { + this.buffer.copyWithin(0, offset, this.length); + this.length -= offset; + } + return chunks; + } + + reset(): void { + this.length = 0; + } +} + +export interface DeviceStreamClient { + readonly start: () => void; + readonly stop: () => void; + /** Normalized 0..1 coordinates in the displayed frame. */ + readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; + readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; + readonly pressButton: (button: DeviceHardwareButton) => void; + readonly rotate: () => void; +} + +const HID_USAGE_BY_CODE: Readonly> = { + Enter: 0x28, + Escape: 0x29, + Backspace: 0x2a, + Tab: 0x2b, + Space: 0x2c, + Minus: 0x2d, + Equal: 0x2e, + BracketLeft: 0x2f, + BracketRight: 0x30, + Backslash: 0x31, + Semicolon: 0x33, + Quote: 0x34, + Backquote: 0x35, + Comma: 0x36, + Period: 0x37, + Slash: 0x38, + Delete: 0x4c, + ArrowRight: 0x4f, + ArrowLeft: 0x50, + ArrowDown: 0x51, + ArrowUp: 0x52, + ControlLeft: 0xe0, + ShiftLeft: 0xe1, + AltLeft: 0xe2, + MetaLeft: 0xe3, + ControlRight: 0xe4, + ShiftRight: 0xe5, + AltRight: 0xe6, + MetaRight: 0xe7, +}; + +function hidUsageForCode(code: string): number | null { + if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65); + if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49); + if (code === "Digit0") return 0x27; + return HID_USAGE_BY_CODE[code] ?? null; +} + +const ANDROID_KEYCODE_BY_KEY: Readonly> = { + ArrowUp: 19, + ArrowDown: 20, + ArrowLeft: 21, + ArrowRight: 22, + Tab: 61, + Enter: 66, + Backspace: 67, + Delete: 112, + Home: 122, + End: 123, + PageUp: 92, + PageDown: 93, +}; + +const IOS_ORIENTATIONS: ReadonlyArray = [ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]; + +export function createDeviceStreamClient( + target: DeviceStreamTarget, + canvas: HTMLCanvasElement, + events: DeviceStreamEvents, +): DeviceStreamClient { + const { access, platform, deviceId } = target; + const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + const device = encodeURIComponent(deviceId); + const httpUrl = (path: string) => + withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access); + const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access); + const useWebCodecs = isWebCodecsSupported(); + + let stopped = true; + let socket: WebSocket | null = null; + let controller: AbortController | null = null; + const retryTimers = new Map<"video" | "input", ReturnType>(); + let primeController: AbortController | null = null; + let videoDecoder: VideoDecoder | null = null; + let timestamp = 0; + let awaitingKeyframe = true; + let screen: DeviceScreenSize | null = null; + let firstFrame = false; + let configuring = false; + let mjpeg = false; + + const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`); + + const fallBackToMjpeg = () => { + if (stopped || mjpeg) return; + mjpeg = true; + closeDecoder(); + events.onMjpegFallback(mjpegUrl()); + setStatus("streaming"); + }; + + const setStatus = (status: DeviceStreamStatus, detail?: string) => { + if (!stopped) events.onStatus(status, detail); + }; + + const paint = (source: CanvasImageSource, width: number, height: number) => { + if (stopped) return; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + if (platform === "android") { + screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; + events.onScreen(screen); + } + } + canvas.getContext("2d")?.drawImage(source, 0, 0, width, height); + if (!firstFrame) { + firstFrame = true; + setStatus("streaming"); + } + }; + + const closeDecoder = () => { + try { + videoDecoder?.close(); + } catch { + // Already closed. + } + videoDecoder = null; + awaitingKeyframe = true; + }; + + const makeDecoder = () => + new VideoDecoder({ + output: (frame) => { + try { + paint(frame, frame.displayWidth, frame.displayHeight); + } finally { + frame.close(); + } + }, + error: () => { + closeDecoder(); + requestKeyframe(); + }, + }); + + /** + * Resolves false when this browser cannot decode the stream's profile + * (simulators encode High 5.1, which headless and some hardware decoders + * reject). iOS then falls back to MJPEG; Android has no MJPEG. + */ + const configureDecoder = async (config: VideoDecoderConfig): Promise => { + const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; + const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); + if (stopped) return false; + if (!support.supported) { + setStatus("error", `This browser cannot decode ${config.codec}.`); + return false; + } + if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); + try { + videoDecoder.configure(full); + return true; + } catch (cause) { + setStatus("error", `Video decoder: ${(cause as Error).message}`); + return false; + } + }; + + const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => { + if (!videoDecoder || videoDecoder.state !== "configured") return; + if (awaitingKeyframe) { + if (!isKey) return; + awaitingKeyframe = false; + } + if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) { + closeDecoder(); + requestKeyframe(); + return; + } + try { + videoDecoder.decode( + new EncodedVideoChunk({ + type: isKey ? "key" : "delta", + timestamp: pts ?? timestamp, + data, + }), + ); + timestamp += FRAME_DURATION_US; + } catch { + closeDecoder(); + requestKeyframe(); + } + }; + + const requestKeyframe = () => { + if (platform === "android" && socket?.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ type: "reset-video", ack: false })); + } + }; + + const scheduleRetry = (channel: "video" | "input", run: () => void) => { + if (stopped || retryTimers.has(channel)) return; + retryTimers.set( + channel, + setTimeout(() => { + retryTimers.delete(channel); + run(); + }, RETRY_DELAY_MS), + ); + }; + + const handleUnauthorized = () => { + stop(); + events.onUnauthorized(); + }; + + // iOS video: fetch the AVCC body and demux into the decoder. + const readIosVideo = async () => { + const demuxer = new AvccDemuxer(); + controller = new AbortController(); + try { + const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), { + signal: controller.signal, + credentials: access.credentials ? "include" : "same-origin", + }); + if (response.status === 401 || response.status === 403) return handleUnauthorized(); + if (!response.ok || !response.body) throw new Error(`stream ${response.status}`); + const reader = response.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done || stopped) break; + for (const chunk of demuxer.push(value)) { + switch (chunk.type) { + case "seed": + void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" })) + .then((bitmap) => { + paint(bitmap, bitmap.width, bitmap.height); + bitmap.close(); + }) + .catch(() => {}); + break; + case "description": { + awaitingKeyframe = true; + const configured = await configureDecoder({ + codec: avcCodecString(chunk.payload), + description: chunk.payload, + }); + if (!configured) { + await reader.cancel().catch(() => {}); + fallBackToMjpeg(); + return; + } + break; + } + case "keyframe": + case "delta": + decode(chunk.type === "keyframe", chunk.payload); + break; + } + } + } + } catch (cause) { + if (stopped) return; + setStatus("connecting", (cause as Error).message); + } + if (!stopped) scheduleRetry("video", () => void readIosVideo()); + }; + + /** + * serve-sim's helper only accepts HID and pushes its screen config once + * screen capture is running, and the AVCC stream does not reliably start + * it. Touching the MJPEG endpoint does; one aborted request is enough. + */ + const primeIosHelper = async () => { + const controller = new AbortController(); + primeController = controller; + const timeout = setTimeout(() => controller.abort(), 2_000); + try { + const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), { + signal: controller.signal, + credentials: access.credentials ? "include" : "same-origin", + }); + if (response.status === 401 || response.status === 403) return handleUnauthorized(); + await response.body?.getReader().read(); + } catch { + // A failed prime just means the socket may take a retry to come up. + } finally { + clearTimeout(timeout); + controller.abort(); + if (primeController === controller) primeController = null; + } + }; + + // iOS input socket; also carries the screen config the helper pushes. + const connectIosInput = async () => { + if (stopped) return; + await primeIosHelper(); + if (stopped) return; + const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); + ws.binaryType = "arraybuffer"; + socket = ws; + ws.onopen = () => { + ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); + events.onInputConnected(true); + }; + ws.onmessage = (event) => { + if (!(event.data instanceof ArrayBuffer)) return; + const bytes = new Uint8Array(event.data); + if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; + try { + const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize; + if (config.width > 0 && config.height > 0) { + screen = config; + events.onScreen(config); + } + } catch { + // Ignore malformed config frames. + } + }; + ws.onclose = (event) => { + if (socket === ws) socket = null; + if (!stopped) { + events.onInputConnected( + false, + event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`), + ); + } + if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); + scheduleRetry("input", () => void connectIosInput()); + }; + ws.onerror = () => ws.close(); + }; + + // Android: one socket for video and input. + const connectAndroid = () => { + if (stopped) return; + const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); + ws.binaryType = "arraybuffer"; + socket = ws; + ws.onopen = () => { + setStatus("connecting"); + events.onInputConnected(true); + }; + ws.onmessage = (event) => { + if (typeof event.data === "string") { + // The encoder restarts at a new size when the device rotates; the + // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. + if (isVideoSessionMessage(event.data)) closeDecoder(); + return; + } + if (!(event.data instanceof ArrayBuffer)) return; + const packet = parseSemuPacket(event.data); + const needsScan = + packet.isKey === null || + (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured")); + const scanned = needsScan ? scanAccessUnit(packet.data) : null; + const isKey = packet.isKey ?? scanned?.isKey ?? false; + if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { + if (configuring) return; + configuring = true; + void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => { + configuring = false; + awaitingKeyframe = true; + if (configured) requestKeyframe(); + }); + return; + } + if (!videoDecoder || videoDecoder.state !== "configured") { + if (!isKey) requestKeyframe(); + return; + } + decode(isKey, packet.data, packet.timestamp); + }; + ws.onclose = (event) => { + if (socket === ws) socket = null; + closeDecoder(); + if (!stopped) events.onInputConnected(false, event.reason || `closed ${event.code}`); + if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); + if (!stopped) { + setStatus("connecting", event.reason || undefined); + scheduleRetry("input", connectAndroid); + } + }; + ws.onerror = () => ws.close(); + }; + + const start = () => { + if (!stopped) return; + stopped = false; + firstFrame = false; + events.onStatus("connecting"); + if (platform === "ios") { + void connectIosInput(); + if (useWebCodecs) void readIosVideo(); + else fallBackToMjpeg(); + } else if (useWebCodecs) { + connectAndroid(); + } else { + setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable)."); + } + }; + + const stop = () => { + if (stopped) return; + stopped = true; + mjpeg = false; + for (const timer of retryTimers.values()) clearTimeout(timer); + retryTimers.clear(); + primeController?.abort(); + primeController = null; + controller?.abort(); + controller = null; + socket?.close(); + socket = null; + closeDecoder(); + }; + + const send = (payload: Uint8Array | string) => { + if (socket?.readyState === WebSocket.OPEN) socket.send(payload); + }; + + const rawPoint = (x: number, y: number) => { + // serve-sim streams the raw framebuffer; rotated devices need input + // remapped into that raw space. + if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y }; + switch (screen.orientation) { + case "landscape_left": + return { x: y, y: 1 - x }; + case "landscape_right": + return { x: 1 - y, y: x }; + case "portrait_upside_down": + return { x: 1 - x, y: 1 - y }; + default: + return { x, y }; + } + }; + + return { + start, + stop, + sendTouch: (phase, x, y) => { + if (platform === "ios") { + send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); + return; + } + const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up"; + send(JSON.stringify({ type: "touch", action, x, y })); + }, + sendKey: (event, phase) => { + if (platform === "ios") { + const usage = hidUsageForCode(event.code); + if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage })); + return; + } + if (phase !== "down") return; + if (event.key === "Escape") return send(JSON.stringify({ type: "back" })); + const keycode = ANDROID_KEYCODE_BY_KEY[event.key]; + if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode })); + if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) { + send(JSON.stringify({ type: "text", text: event.key })); + } + }, + pressButton: (button) => { + if (platform === "ios") { + const name = + button === "home" + ? "home" + : button === "appSwitcher" + ? "app_switcher" + : button === "power" + ? "lock" + : null; + if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name })); + return; + } + const type = button === "appSwitcher" ? "recents" : button; + if (type === "home" || type === "back" || type === "recents" || type === "power") { + send(JSON.stringify({ type })); + } + }, + rotate: () => { + if (platform !== "ios") return; + const current = screen?.orientation ?? "portrait"; + const next = + IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; + send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); + }, + }; +} diff --git a/apps/web/src/components/preview/PreviewEmptyState.tsx b/apps/web/src/components/preview/PreviewEmptyState.tsx index 163849154000..35b4b2f13d64 100644 --- a/apps/web/src/components/preview/PreviewEmptyState.tsx +++ b/apps/web/src/components/preview/PreviewEmptyState.tsx @@ -3,6 +3,7 @@ import { Globe, History, RadioTower } from "lucide-react"; import type { BrowserHistoryEntry } from "~/browserHistoryStore"; import { Empty, EmptyDescription, EmptyMedia, EmptyTitle } from "~/components/ui/empty"; +import { DiscoveryList } from "../ui/discovery-list"; import { PreviewLocalServerCard } from "./PreviewLocalServerCard"; import { PreviewRecentUrlCard } from "./PreviewRecentUrlCard"; @@ -55,7 +56,7 @@ export function PreviewEmptyState({

Recently used

-
+ {recents.map((entry) => ( onRemoveRecent(entry.url)} /> ))} -
+
) : null} {servers.length > 0 ? ( @@ -74,7 +75,7 @@ export function PreviewEmptyState({

Local servers

-
+ {servers.map((server) => ( onOpenUrl(server.requestedUrl)} /> ))} -
+

Select a live local server to open it in this browser tab.

diff --git a/apps/web/src/components/preview/PreviewLocalServerCard.tsx b/apps/web/src/components/preview/PreviewLocalServerCard.tsx index 263cdb294f48..198d0495473c 100644 --- a/apps/web/src/components/preview/PreviewLocalServerCard.tsx +++ b/apps/web/src/components/preview/PreviewLocalServerCard.tsx @@ -1,4 +1,5 @@ import type { ScopedThreadRef } from "@t3tools/contracts"; +import { DiscoveryListRow } from "../ui/discovery-list"; import { PreviewFaviconIcon } from "./PreviewFaviconIcon"; import type { PreviewableServer } from "./useDiscoveredLocalServers"; @@ -12,19 +13,12 @@ interface Props { export function PreviewLocalServerCard({ threadRef, server, onOpen }: Props) { const subtitle = describeServer(server); return ( - + icon={} + title={subtitle} + description={`${server.host}:${server.port}`} + /> ); } diff --git a/apps/web/src/components/settings/IntegrationsSettings.test.tsx b/apps/web/src/components/settings/IntegrationsSettings.test.tsx index 5d185fee5824..9ce3b2126c0e 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.test.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.test.tsx @@ -1,4 +1,8 @@ -import { DEFAULT_CLIENT_SETTINGS, DEFAULT_UNIFIED_SETTINGS } from "@t3tools/contracts"; +import { + DEFAULT_CLIENT_SETTINGS, + DEFAULT_UNIFIED_SETTINGS, + type DeviceServiceState, +} from "@t3tools/contracts"; import { createMemoryHistory, createRootRoute, @@ -36,6 +40,7 @@ vi.mock("./settingsLayout", async (importOriginal) => ({ })); import { IntegrationsSettingsPanel } from "./IntegrationsSettings"; +import { platformSetupStatus } from "../device/DeviceSetup"; let renderer: ReactTestRenderer | undefined; @@ -74,4 +79,59 @@ describe("Integrations browser discovery", () => { await openSettings(); expect(listBrowserImportSources).not.toHaveBeenCalled(); }); + + it("places device settings directly after browser settings", async () => { + await openSettings(); + const sections = renderer!.root + .findAll((node) => node.type === "section") + .map((node) => node.props.id) + .filter(Boolean); + expect(sections.indexOf("devices")).toBeGreaterThan(sections.indexOf("browser")); + }); +}); + +const deviceState = (overrides: Partial = {}): DeviceServiceState => ({ + hosts: [ + { + id: "local", + kind: "local", + label: "This machine", + hubInstalled: false, + agentDeviceInstalled: false, + platforms: [ + { platform: "ios", available: true }, + { platform: "android", available: true }, + ], + }, + ], + hostStatus: "ready", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: "/api/device-hub", + revision: 0, + ...overrides, +}); + +describe("device setup guidance", () => { + it("directs users to install an iOS runtime and create an Android virtual device", () => { + expect(platformSetupStatus(deviceState(), "ios").message).toContain("Xcode Settings"); + expect(platformSetupStatus(deviceState(), "android").message).toContain("Device Manager"); + }); + + it("preserves a specific missing-tool explanation from the server", () => { + const state = deviceState({ + hosts: [ + { + ...deviceState().hosts[0]!, + platforms: [ + { platform: "ios", available: true }, + { platform: "android", available: false, reason: "Android Emulator is missing." }, + ], + }, + ], + }); + expect(platformSetupStatus(state, "android").message).toBe("Android Emulator is missing."); + }); }); diff --git a/apps/web/src/components/settings/IntegrationsSettings.tsx b/apps/web/src/components/settings/IntegrationsSettings.tsx index a7a3c7553540..a3f76f49cd5f 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.tsx @@ -39,10 +39,21 @@ import { MoreVertical, Plus as PlusIcon } from "lucide-react"; import { useCallback, useRef, useState } from "react"; import { ScreenRotationIcon } from "~/browser/ScreenRotationIcon"; +import { AnimatedHeight } from "~/components/AnimatedHeight"; import { resolveEnvironmentOptionLabel } from "~/components/BranchToolbar.logic"; import { previewBridge } from "~/components/preview/previewBridge"; import { cn, randomUUID } from "~/lib/utils"; import { useEnvironments, usePrimaryEnvironment } from "~/state/environments"; +import { deviceEnvironment, useDeviceState } from "~/state/device"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { + AgentDeviceSetupStatus, + DeviceHubSetupStatus, + PlatformStatus, + platformSetupStatus, + deviceHubDescription, + agentDeviceDescription, +} from "~/components/device/DeviceSetup"; import { isElectron } from "../../env"; import { Badge } from "../ui/badge"; @@ -571,6 +582,117 @@ function AgentBrowserAccessSetting() { ); } +function DeviceIntegrationSettings() { + const primaryEnvironment = usePrimaryEnvironment(); + const environmentId = primaryEnvironment?.environmentId ?? null; + const { state, loaded } = useDeviceState(environmentId); + const configure = useAtomCommand(deviceEnvironment.configure); + const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); + const [pending, setPending] = useState<"hub" | "check" | "agent" | null>(null); + const enabled = state.hostStatus !== "disabled"; + const busy = state.hostStatus === "installing" || state.hostStatus === "starting"; + const [platformsRevealed, setPlatformsRevealed] = useState(false); + // Keep diagnostics visible through subsequent agent setup and refresh phases. + if (platformsRevealed && !enabled) setPlatformsRevealed(false); + if (!platformsRevealed && state.hostStatus === "ready" && pending !== "hub") { + setPlatformsRevealed(true); + } + + const update = async ( + kind: NonNullable, + input: { enabled?: boolean; agentAccessEnabled?: boolean }, + ) => { + if (!environmentId) return; + setPending(kind); + try { + const result = await configure({ environmentId, input }); + if (result._tag === "Success" && input.enabled === true && !state.onboardingCompleted) { + await configure({ environmentId, input: { onboardingCompleted: true } }); + } + } finally { + setPending(null); + } + }; + + return ( + + + {pending === "hub" ? : null} + + void update("hub", { + enabled: Boolean(checked), + ...(checked ? {} : { agentAccessEnabled: false }), + }) + } + /> + + } + /> + + {platformsRevealed ? ( + + + +
+ } + control={ + + } + /> + ) : null} + + + {pending === "agent" ? : null} + + void update("agent", { agentAccessEnabled: Boolean(checked) }) + } + /> + + } + /> + {state.hostStatus === "failed" && state.hostStatusDetail ? ( +

+ {state.hostStatusDetail} +

+ ) : null} + + ); +} + function BrowserAutoShowFloatingPreviewSetting({ disabled }: { readonly disabled: boolean }) { const autoShow = useClientSettings((settings) => settings.browserAutoShowFloatingPreview); const updateSettings = useUpdatePrimarySettings(); @@ -1170,6 +1292,7 @@ export function IntegrationsSettingsPanel() { previewDefaults )} + ); } diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 5cf35a543392..24892b0e1e3b 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -402,6 +402,27 @@ export const SETTINGS_SEARCH_ITEMS = [ to: "/settings/projects", searchTerms: ["allow open drive preview tools sessions"], }, + { + id: "agent-device-access", + title: "Agent device access", + to: "/settings/integrations", + targetId: "devices", + searchTerms: ["allow simulator emulator ios android drive tools sessions"], + }, + { + id: "device-hub", + title: "Device hub", + to: "/settings/integrations", + targetId: "devices", + searchTerms: ["simulator emulator ios android install start"], + }, + { + id: "device-platform-support", + title: "Simulator support", + to: "/settings/integrations", + targetId: "devices", + searchTerms: ["xcode android studio sdk avd runtime"], + }, { id: "browser-profiles", title: "Browser profiles", diff --git a/apps/web/src/components/ui/discovery-list.tsx b/apps/web/src/components/ui/discovery-list.tsx new file mode 100644 index 000000000000..d03ca0edd5e0 --- /dev/null +++ b/apps/web/src/components/ui/discovery-list.tsx @@ -0,0 +1,37 @@ +import type { ComponentProps, ReactNode } from "react"; + +export function DiscoveryList({ children }: { readonly children: ReactNode }) { + return ( +
+ {children} +
+ ); +} + +export function DiscoveryListRow({ + icon, + title, + description, + action, + ...props +}: Omit, "title" | "children" | "className"> & { + readonly icon: ReactNode; + readonly title: ReactNode; + readonly description: ReactNode; + readonly action?: ReactNode; +}) { + return ( + + ); +} diff --git a/apps/web/src/rightPanelStore.ts b/apps/web/src/rightPanelStore.ts index 1719ae77725c..293733b5b3d9 100644 --- a/apps/web/src/rightPanelStore.ts +++ b/apps/web/src/rightPanelStore.ts @@ -24,6 +24,7 @@ const RIGHT_PANEL_KINDS = [ "files", "file", "preview", + "device", "terminal", "pull-request", "pull-requests", @@ -34,6 +35,12 @@ export type RightPanelKind = (typeof RIGHT_PANEL_KINDS)[number]; export type RightPanelSurface = | { id: `browser:${string}`; kind: "preview"; resourceId: string } | { id: "browser:new"; kind: "preview"; resourceId: null } + /** + * One Device tab per thread. The tab is the surface; which device it shows + * comes from the thread's server-side device sessions, so an agent opening a + * device from another client lands in the same tab. + */ + | { id: "device"; kind: "device" } | { id: `terminal:${string}`; kind: "terminal"; @@ -82,7 +89,8 @@ const RIGHT_PANEL_STORAGE_KEY = "t3code:right-panel-state:v2"; // v9 removed the "plan" surface kind (plans render inline in the transcript). // v10 keys pull-request surfaces by reference instead of a singleton tab. // v11 stops persisting the pull-request list's shared panel, so a restart opens the page fresh. -const RIGHT_PANEL_STORAGE_VERSION = 11; +// v12 adds the device surface. +const RIGHT_PANEL_STORAGE_VERSION = 12; /** A fixed workspace-level ref: each PR surface carries its own real environment. */ export const PULL_REQUESTS_PANEL_REF = scopeThreadRef( @@ -178,6 +186,8 @@ const singletonSurface = ( return { id: "pull-requests", kind }; case "agents": return { id: "agents", kind }; + case "device": + return { id: "device", kind }; } }; diff --git a/apps/web/src/routes/_chat.pull-requests.tsx b/apps/web/src/routes/_chat.pull-requests.tsx index 62bb23bb305f..24dcab132b86 100644 --- a/apps/web/src/routes/_chat.pull-requests.tsx +++ b/apps/web/src/routes/_chat.pull-requests.tsx @@ -1971,6 +1971,7 @@ function PullRequestsRouteView() { onAddPullRequest={() => undefined} onAddPullRequests={() => undefined} onAddAgents={() => undefined} + onAddDevice={() => undefined} browserAvailable={false} terminalAvailable={false} diffAvailable={false} @@ -1978,6 +1979,7 @@ function PullRequestsRouteView() { pullRequestAvailable={false} pullRequestsAvailable={false} agentsAvailable={false} + deviceAvailable={false} liveAgentCount={0} pullRequestStatusSeeds={listedPullRequestTabStatuses} > diff --git a/apps/web/src/state/device.ts b/apps/web/src/state/device.ts new file mode 100644 index 000000000000..b9c1adf86cdd --- /dev/null +++ b/apps/web/src/state/device.ts @@ -0,0 +1,70 @@ +import { useAtomValue } from "@effect/atom-react"; +import { createDeviceEnvironmentAtoms } from "@t3tools/client-runtime/state/device"; +import { + type DeviceHubAccess, + resolveDeviceHubAccess, +} from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DeviceServiceState, EnvironmentId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import { AsyncResult, Atom } from "effect/unstable/reactivity"; + +import { connectionAtomRuntime } from "../connection/runtime"; +import { appAtomRegistry } from "../rpc/atomRegistry"; +import { environmentSession } from "./session"; +import { useEnvironmentQuery } from "./query"; + +export const deviceEnvironment = createDeviceEnvironmentAtoms(connectionAtomRuntime); + +const EMPTY_DEVICE_STATE: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: "/api/device-hub", + revision: 0, +}; + +export function useDeviceState(environmentId: EnvironmentId | null): { + readonly state: DeviceServiceState; + readonly loaded: boolean; +} { + const query = useEnvironmentQuery( + environmentId === null ? null : deviceEnvironment.state({ environmentId, input: {} }), + ); + return { state: query.data ?? EMPTY_DEVICE_STATE, loaded: query.data !== undefined }; +} + +/** + * Hub access for one environment. Bearer and DPoP connections mint a ticket + * here; a stream that gets a 401 back refreshes this atom and reconnects. + * Keyed on the prepared connection so a re-pair produces new credentials. + */ +const deviceHubAccessAtom = Atom.family((environmentId: EnvironmentId) => + connectionAtomRuntime + .atom((get) => { + const prepared = Option.getOrNull( + get(environmentSession.preparedConnectionValueAtom(environmentId)), + ); + if (prepared === null) return Effect.never; + return resolveDeviceHubAccess({ prepared, hubBasePath: EMPTY_DEVICE_STATE.hubBasePath }); + }) + .pipe(Atom.setIdleTTL(60_000), Atom.withLabel(`device-hub-access:${environmentId}`)), +); + +export function useDeviceHubAccess(environmentId: EnvironmentId | null): DeviceHubAccess | null { + const result = useAtomValue( + environmentId === null ? EMPTY_ACCESS_ATOM : deviceHubAccessAtom(environmentId), + ); + return AsyncResult.isSuccess(result) ? result.value : null; +} + +const EMPTY_ACCESS_ATOM = Atom.make(AsyncResult.initial()).pipe( + Atom.withLabel("device-hub-access:empty"), +); + +export function refreshDeviceHubAccess(environmentId: EnvironmentId): void { + appAtomRegistry.refresh(deviceHubAccessAtom(environmentId)); +} diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 702c18840983..ec4f15fc7eeb 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -230,16 +230,17 @@ export default defineConfig(() => { ? { // One entry per shared prefix; the server's dev catch-all 404s the // same list, so the two sides cannot drift. `/ws` is the app's own - // socket — Vite's HMR socket is matched separately and exactly - // (path "/" plus a vite-hmr subprotocol), so the two upgrade - // handlers don't collide. + // socket and `/api` carries the device hub's stream sockets — + // Vite's HMR socket is matched separately and exactly (path "/" + // plus a vite-hmr subprotocol), so the upgrade handlers don't + // collide. proxy: Object.fromEntries( DEV_PROXIED_PATH_PREFIXES.map((prefix) => [ prefix, { target: devProxyTarget, changeOrigin: true, - ...(prefix === "/ws" ? { ws: true } : {}), + ...(prefix === "/ws" || prefix === "/api" ? { ws: true } : {}), }, ]), ), diff --git a/docs/README.md b/docs/README.md index 2d4809b7003a..4691e6f83c8e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ - [Keyboard shortcuts](./user/keybindings.md) - [SnapShots](./user/snap-shot.md) - [Import browser sessions](./user/browser-import.md) +- [Devices](./user/devices.md) - [Usage and limits](./user/usage.md) - [Product usage data](./user/telemetry.md) - [Remote access](./user/remote-access.md) @@ -46,6 +47,7 @@ source alone does not explain. Most code changes do not need an internal documen - [Mobile navigation](./internals/mobile-navigation.md) - [Mobile development lifecycle](./internals/mobile-development.md) - [Terminal runtime](./internals/terminal-runtime.md) +- [Devices](./internals/devices.md) - [Voice input](./internals/voice-input.md) ### Runbooks diff --git a/docs/internals/devices.md b/docs/internals/devices.md new file mode 100644 index 000000000000..49027022fdbc --- /dev/null +++ b/docs/internals/devices.md @@ -0,0 +1,87 @@ +# Devices + +The environment server owns simulators and emulators the way it owns +terminals: discovery, streaming, and agent access all run there, and every +client reaches them through the environment connection. This is what makes the +Device panel work over Tailscale and T3 Connect, and what will let a device +host on another machine slot in later. + +## Two external tools, one seam + +[expo-device-hub](../../apps/server/src/device/LocalDeviceHost.ts) streams and +[agent-device](../../apps/server/src/device/AgentDeviceShim.ts) drives. Each is +npm-installed at a pinned version into the T3 home after its matching Device +panel consent step. Manual setup installs and starts only expo-device-hub; +agent-device remains absent and stopped until agent access is granted. Both run +with the server's Node; `npx` would make the first `device_open` after a reboot +depend on the registry. The hub is a supervised child rather than an imported +middleware because serve-sim loads private CoreSimulator frameworks through a +native addon, and a crash there must not take the server down. + +Everything platform-specific sits behind +[`DeviceHost`](../../apps/server/src/device/DeviceHost.ts). The service, the +proxy, and the MCP tools only see a hub origin and an agent-device endpoint. +An SSH or cloud host would forward those two things to the server and change +nothing above it. + +## The hub is never exposed + +serve-sim has a shell-exec route whose token is readable from its own +unauthenticated `/api`, and serve-emu's action routes have no auth at all. The +hub binds loopback and the only way in is the +[proxy](../../apps/server/src/device/DeviceHubProxy.ts), which allowlists the +stream, config, and screenshot routes and authenticates every request as an +environment session. `` and `WebSocket` cannot carry headers, so the proxy +authenticates like the `/ws` upgrade: cookie, or a short-lived `wsTicket` that +bearer and DPoP clients mint over authenticated HTTP. The ticket is stripped +before the request reaches the hub. + +Stream responses carry `Cache-Control: no-transform`; the compression +middleware would otherwise buffer an MJPEG body that never ends. In browser dev, +the Vite proxy must forward WebSocket upgrades for `/api`, not only `/ws`. + +## Device settings never go through the hub + +serve-sim's preview drives its Tools panel by sending shell commands over that +same exec channel. Proxying it, even allowlisted, would hand any environment +session arbitrary command execution on the host, so T3 does not. The +[`device.action`](../../apps/server/src/device/DeviceActions.ts) RPC runs the +underlying `simctl`, `adb`, and serve-sim helper binaries itself through +`DeviceHostReady.run`, one typed action per control, and returns the settings +it reads back. The proxy allowlist grows only with read routes (accessibility +tree, foreground app, event log) and refuses non-GET methods everywhere except +screenshot capture and stream tuning. + +## Agents drive through the CLI + +The `device_*` toolkit is deliberately four tools: list, open, screenshot, and +close. Driving happens through the `agent-device` CLI, which has the semantic +snapshot model agents need and stays current with its own releases. T3 prepends +a shim directory to the provider's PATH and sets +`AGENT_DEVICE_DAEMON_BASE_URL` and `AGENT_DEVICE_DAEMON_AUTH_TOKEN` so the +agent never handles the endpoint or token. + +That environment is fixed when the provider subprocess spawns, so +[`prepareMcpSession`](../../apps/server/src/provider/Layers/ProviderService.ts) +starts agent-device only when device support and agent access have both been +enabled, the session has the `device` capability, and the machine can run at +least one platform. Starting it later from `device_open` would leave the +already-running agent without the CLI. + +How to drive a device is returned from `device_open`, not kept in an +always-loaded prompt or skill: it costs nothing in threads that never open a +device and cannot drift from the pinned CLI version. The always-on prompt block +is a few lines that point at the tools and forbid raw `simctl` and `adb`. + +## The viewer decodes both vendored protocols + +The hub vendors two streaming servers with different wire formats. iOS video is +an HTTP body of AVCC envelopes decoded with WebCodecs, with input on a separate +binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over +one WebSocket. [`deviceStream.ts`](../../apps/web/src/components/device/deviceStream.ts) +speaks both so one panel covers both platforms. + +Simulators encode H.264 High 5.1. Hardware decoders on some machines and all +headless browsers reject that profile, and WebCodecs is secure-context only, so +the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on +iOS. Android has no MJPEG; there the panel reports that it cannot decode. diff --git a/docs/user/devices.md b/docs/user/devices.md new file mode 100644 index 000000000000..51a0a578b607 --- /dev/null +++ b/docs/user/devices.md @@ -0,0 +1,63 @@ +# Devices + +The Device panel shows a live iOS Simulator or Android Emulator next to a +thread, so you can watch an agent verify mobile work and tap the device +yourself. Agents get the same device through `device_*` tools and the +`agent-device` command line, which T3 Code sets up for them. + +## Open a device + +Open the right panel in a project thread and choose **Device**. On first use, +the panel walks through three steps: starting the device hub, checking iOS and +Android support, and choosing whether agents may control devices. Opening the +panel alone does not download or start anything. If the hub is already +installed, the setup screen says so and reuses it. + +Choose a running device to watch it, or choose **Start** next to a stopped +device to boot it. The panel shows when you or an agent starts a device. +Turn off the device hub in **Settings → Integrations → Devices** to stop the +helper processes; simulators and emulators keep running until you power them +off. + +Simulators run on the machine that hosts the environment server. iOS needs +macOS with Xcode. Android needs the SDK Platform-Tools, Android Emulator, +and Command-line Tools (latest), plus a virtual device created in Android +Studio's Device Manager. T3 Code detects standard SDK locations; set +`ANDROID_HOME` for a custom location. The panel explains missing dependencies. +After installing them, restart the environment server and refresh devices. + +The screen is interactive: click and drag to touch, type while the screen is +focused, and use the toolbar for Home, Back, and Recents on Android, rotate on +iOS, and power off. Close the tab to stop watching; the device keeps running +unless you power it off. + +## Tools + +The toolbar's **Tools** button opens a drawer for the open device. It shows the +foreground app, and lets you switch light and dark mode, change text size, +flip accessibility settings, overlay the accessibility element frames on the +screen, set a fake location, and grant or revoke app permissions. iOS also +exposes Liquid Glass, color filters, VoiceOver, and sending a test push +notification; Android adds orientation and toggling the network. The drawer +only shows what the platform can do, and every control reflects the value read +back from the device after a change. + +## Agents and devices + +When an agent opens a device, the panel opens in web and desktop clients connected +to the thread. Mobile clients show device activity in the thread timeline. Agents drive the device through the `agent-device` command line. T3 +Code installs and starts it only after **Agent device access** is enabled. iOS +taps build a small test runner on first use, which takes a couple of minutes +once per server. Restart an existing agent session after granting access so it +receives the device CLI environment. + +To keep agents away from simulators, turn off **Agent device access** in +**Settings → Integrations → Devices**. This hides the device tools from agents +started from then on; your own Device panel is unaffected. + +## Remote connections + +The device stream goes through the environment server, so it works over the +local network, Tailscale, and T3 Connect. Live video needs a secure page +(HTTPS or localhost); on a plain-HTTP remote origin iOS falls back to a slower +still-image stream and Android cannot show video. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 2bacdfcb228a..d0bf4aad8e9e 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -131,6 +131,14 @@ "types": "./src/state/presentation.ts", "default": "./src/state/presentation.ts" }, + "./state/device": { + "types": "./src/state/device.ts", + "default": "./src/state/device.ts" + }, + "./state/deviceHubAccess": { + "types": "./src/state/deviceHubAccess.ts", + "default": "./src/state/deviceHubAccess.ts" + }, "./state/preview": { "types": "./src/state/preview.ts", "default": "./src/state/preview.ts" diff --git a/packages/client-runtime/src/rpc/client.ts b/packages/client-runtime/src/rpc/client.ts index 4ddfb9c4160e..cfabaa00c0b7 100644 --- a/packages/client-runtime/src/rpc/client.ts +++ b/packages/client-runtime/src/rpc/client.ts @@ -51,6 +51,7 @@ export type EnvironmentSubscriptionRpcTag = | typeof WS_METHODS.subscribeTerminalMetadata | typeof WS_METHODS.subscribePreviewEvents | typeof WS_METHODS.subscribeDiscoveredLocalServers + | typeof WS_METHODS.subscribeDeviceState | typeof WS_METHODS.subscribeResourceTelemetry | typeof WS_METHODS.pullRequestsSubscribeRefreshes | typeof WS_METHODS.previewAutomationConnect diff --git a/packages/client-runtime/src/state/device.ts b/packages/client-runtime/src/state/device.ts new file mode 100644 index 000000000000..df27f793720c --- /dev/null +++ b/packages/client-runtime/src/state/device.ts @@ -0,0 +1,68 @@ +import { WS_METHODS } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; + +import type { EnvironmentRegistry } from "../connection/registry.ts"; +import { + createAtomCommandScheduler, + createEnvironmentRpcCommand, + createEnvironmentRpcSubscriptionAtomFamily, +} from "./runtime.ts"; + +export function createDeviceEnvironmentAtoms( + runtime: Atom.AtomRuntime, +) { + const scheduler = createAtomCommandScheduler(); + const concurrency = { + mode: "serial" as const, + key: ({ environmentId }: { environmentId: string }) => environmentId, + }; + return { + /** Server-pushed device hosts, devices, and open sessions for one environment. */ + state: createEnvironmentRpcSubscriptionAtomFamily(runtime, { + label: "environment-data:device:state", + tag: WS_METHODS.subscribeDeviceState, + }), + configure: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:configure", + tag: WS_METHODS.deviceConfigure, + scheduler, + concurrency, + }), + list: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:list", + tag: WS_METHODS.deviceList, + scheduler, + concurrency, + }), + open: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:open", + tag: WS_METHODS.deviceOpen, + scheduler, + concurrency, + }), + close: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:close", + tag: WS_METHODS.deviceClose, + scheduler, + concurrency, + }), + shutdown: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:shutdown", + tag: WS_METHODS.deviceShutdown, + scheduler, + concurrency, + }), + detail: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:detail", + tag: WS_METHODS.deviceDetail, + scheduler, + concurrency, + }), + action: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:action", + tag: WS_METHODS.deviceAction, + scheduler, + concurrency, + }), + }; +} diff --git a/packages/client-runtime/src/state/deviceHubAccess.ts b/packages/client-runtime/src/state/deviceHubAccess.ts new file mode 100644 index 000000000000..328dea83ce7a --- /dev/null +++ b/packages/client-runtime/src/state/deviceHubAccess.ts @@ -0,0 +1,72 @@ +/** + * Credentials for the Device panel's media requests. + * + * The panel reaches simulator streams through `/api/device-hub/*` on the + * environment origin. ``, `EventSource`, and `WebSocket` cannot set + * bearer or DPoP headers, so bearer and DPoP connections mint a + * short-lived WebSocket ticket and pass it as `wsTicket`, the same way the + * app's own `/ws` upgrade authenticates. Cookie sessions send the cookie. + * + * A ticket lives five minutes server-side and is bound to the session, not + * to one request, so one ticket covers everything a panel opens at once. + * Callers fetch a fresh one each time they (re)connect a stream. + */ +import * as Effect from "effect/Effect"; +import type { HttpClient } from "effect/unstable/http"; + +import { RemoteEnvironmentAuthorization } from "../authorization/service.ts"; +import type { PreparedConnection } from "../connection/model.ts"; +import { environmentEndpointUrl } from "../environment/endpoint.ts"; +import { ManagedRelayDpopSigner } from "../relay/managedRelay.ts"; +import type { RemoteEnvironmentRequestError } from "../rpc/http.ts"; +import { executeAuthenticatedEnvironmentHttpRequest } from "./environmentHttpAuth.ts"; + +const TICKET_TIMEOUT_MS = 8_000; + +export interface DeviceHubAccess { + /** Absolute origin-relative base, e.g. `https://env.example/api/device-hub`. */ + readonly httpBase: string; + /** Same base with the `ws(s)` scheme. */ + readonly wsBase: string; + /** Query parameters to append to every hub request; empty for cookie sessions. */ + readonly query: Readonly>; + /** Whether requests must include cookies (same-origin session). */ + readonly credentials: boolean; +} + +export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDeviceHubAccess")( + function* (input: { + readonly prepared: PreparedConnection; + readonly hubBasePath: string; + }): Effect.fn.Return { + const httpBase = environmentEndpointUrl(input.prepared.httpBaseUrl, input.hubBasePath); + const wsBase = httpBase.replace(/^http/, "ws"); + if (input.prepared.httpAuthorization === null) { + return { httpBase, wsBase, query: {}, credentials: true }; + } + const signer = yield* Effect.serviceOption(ManagedRelayDpopSigner); + const remoteAuthorization = yield* Effect.serviceOption(RemoteEnvironmentAuthorization); + const ticket = yield* executeAuthenticatedEnvironmentHttpRequest({ + prepared: input.prepared, + signer, + remoteAuthorization, + method: "POST", + url: (httpBaseUrl) => environmentEndpointUrl(httpBaseUrl, "/api/auth/websocket-ticket"), + timeoutMs: TICKET_TIMEOUT_MS, + request: ({ client, headers }) => client.auth.webSocketTicket({ headers }), + }); + return { + httpBase, + wsBase, + query: { wsTicket: ticket.ticket }, + credentials: false, + }; + }, +); + +export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { + const entries = Object.entries(access.query); + if (entries.length === 0) return url; + const separator = url.includes("?") ? "&" : "?"; + return `${url}${separator}${new URLSearchParams(entries).toString()}`; +}; diff --git a/packages/client-runtime/src/work-log/presentation.test.ts b/packages/client-runtime/src/work-log/presentation.test.ts index 5c57b8de3bd6..a0909117ad39 100644 --- a/packages/client-runtime/src/work-log/presentation.test.ts +++ b/packages/client-runtime/src/work-log/presentation.test.ts @@ -221,6 +221,19 @@ describe("resolveWorkEntryToolPresentation", () => { }); }); + it("labels device tools with the device icon", () => { + expect( + resolveWorkEntryToolPresentation({ + label: "mcp__t3-code__device_open", + toolLifecycleStatus: "completed", + }), + ).toEqual({ displayName: "Opened a device in the Device panel", icon: "device" }); + expect(resolveWorkEntryToolPresentation({ label: "t3-code · device_screenshot" })).toEqual({ + displayName: "Taking a screenshot of the device", + icon: "device", + }); + }); + it("uses structured MCP identity when the provider supplies a custom title", () => { expect( resolveWorkEntryToolPresentation({ @@ -647,3 +660,53 @@ describe("pull request tool presentation", () => { ).toBeNull(); }); }); + +describe("device group summaries", () => { + const deviceEntry = (tool: string): WorkLogPresentationEntry => ({ + label: "MCP tool call", + toolData: { server: "t3-code", tool }, + itemType: "mcp_tool_call", + toolLifecycleStatus: "completed", + tone: "tool", + }); + + it.each(["device_list", "device_open", "device_screenshot", "device_close"])( + "recognizes %s as device controls", + (tool) => { + const entry = deviceEntry(tool); + expect(summarizeToolGroup([entry])).toBe("Used device controls 1 time"); + expect(toolGroupSummaryKind([entry])).toBe("device"); + }, + ); + + it("summarizes device calls alongside shell commands", () => { + expect( + summarizeToolGroup([ + { label: "Ran command", itemType: "command_execution", command: "pwd", tone: "tool" }, + deviceEntry("device_list"), + deviceEntry("device_open"), + ]), + ).toBe("Ran 1 command and used device controls 2 times"); + }); + + it("recognizes Claude tool names and preserves screenshot previews", () => { + const entry = { + ...deviceEntry("device_screenshot"), + toolData: { toolName: "mcp__t3_code__device_screenshot" }, + viewedImagePath: "/workspace/device.png", + }; + expect(summarizeToolGroup([entry])).toBe("Used device controls 1 time"); + expect(workEntryViewedImagePath(entry)).toBe("/workspace/device.png"); + }); + + it("does not classify another server's tools as T3 device controls", () => { + expect( + summarizeToolGroup([ + { + ...deviceEntry("device_open"), + toolData: { server: "another-server", tool: "device_open" }, + }, + ]), + ).toBe("Used 1 tool"); + }); +}); diff --git a/packages/client-runtime/src/work-log/presentation.ts b/packages/client-runtime/src/work-log/presentation.ts index 3e62f75d510b..8d62bb0794ca 100644 --- a/packages/client-runtime/src/work-log/presentation.ts +++ b/packages/client-runtime/src/work-log/presentation.ts @@ -44,6 +44,7 @@ export type ToolGroupAction = | "edit" | "command" | "browser" + | "device" | "code-search" | "search" | "other" @@ -104,6 +105,15 @@ const T3_MCP_TOOL_LABELS: Record< preview_set_appearance: ["Set", "Setting", "Set", "preview browser appearance"], preview_recording_start: ["Start", "Starting", "Started", "recording the preview browser"], preview_recording_stop: ["Stop", "Stopping", "Stopped", "recording the preview browser"], + device_list: ["List", "Listing", "Listed", "simulators and emulators"], + device_open: ["Open", "Opening", "Opened", "a device in the Device panel"], + device_screenshot: [ + "Take a screenshot of", + "Taking a screenshot of", + "Took a screenshot of", + "the device", + ], + device_close: ["Close", "Closing", "Closed", "a device"], }; const PR_TOOL_ACTIONS: Readonly> = { @@ -159,7 +169,9 @@ function resolveT3McpToolPresentation( ? ("pull-request" as const) : name.startsWith("preview_") ? ("browser" as const) - : ("t3-code" as const), + : name.startsWith("device_") + ? ("device" as const) + : ("t3-code" as const), ...(actionKind === undefined ? {} : { action: actionKind }), }; } @@ -451,6 +463,7 @@ export function toolGroupAction(entry: WorkLogPresentationEntry): ToolGroupActio const presentation = resolveWorkEntryToolPresentation(entry); if (presentation?.action !== undefined) return presentation.action; if (presentation?.icon === "browser") return "browser"; + if (presentation?.icon === "device") return "device"; if ( entry.requestKind === "file-read" || entry.itemType === "image_view" || @@ -558,6 +571,8 @@ function toolGroupActionLabel(action: ToolGroupAction, count: number): string { return `Changed ${count} ${count === 1 ? "file" : "files"}`; case "command": return `Ran ${count} ${count === 1 ? "command" : "commands"}`; + case "device": + return `Used device controls ${count} ${count === 1 ? "time" : "times"}`; case "browser": return `Used browser ${count} ${count === 1 ? "time" : "times"}`; case "search": diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts new file mode 100644 index 000000000000..309d156430fa --- /dev/null +++ b/packages/contracts/src/device.ts @@ -0,0 +1,516 @@ +/** + * Device - Schemas for first-class iOS Simulator and Android Emulator support. + * + * The server owns device discovery, the streaming helper (expo-device-hub), + * and the agent driver (agent-device). Clients render the live screen from the + * server-proxied stream, and agents reach devices through the `device_*` MCP + * tools plus the `agent-device` CLI the server preconfigures for them. + * + * Devices live on a *host*. Only the local host (the machine the server runs + * on) exists today; the host id is carried everywhere so SSH and cloud hosts + * can be added without changing the client contract. + * + * @module Device + */ +import { Schema } from "effect"; + +import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; + +export const DevicePlatform = Schema.Literals(["ios", "android"]); +export type DevicePlatform = typeof DevicePlatform.Type; + +export const DeviceHostId = TrimmedNonEmptyString.check(Schema.isMaxLength(128)); +export type DeviceHostId = typeof DeviceHostId.Type; + +/** The server machine. Always present; other host kinds are future work. */ +export const LOCAL_DEVICE_HOST_ID = "local" as DeviceHostId; + +/** Simulator udid or adb serial (an AVD name while it is not running). */ +export const DeviceId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)); +export type DeviceId = typeof DeviceId.Type; + +export const DeviceSummary = Schema.Struct({ + hostId: DeviceHostId, + id: DeviceId, + platform: DevicePlatform, + name: TrimmedNonEmptyString, + /** OS label such as "iOS 18.0" or "Android 15.0". */ + version: Schema.String, + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +export type DeviceSummary = typeof DeviceSummary.Type; + +/** + * What the host can do right now. Platforms missing their toolchain are + * reported rather than hidden so the picker and the agent can explain why a + * platform is absent instead of showing an empty list. + */ +export const DevicePlatformAvailability = Schema.Struct({ + platform: DevicePlatform, + available: Schema.Boolean, + reason: Schema.optional(Schema.String), +}); +export type DevicePlatformAvailability = typeof DevicePlatformAvailability.Type; + +export const DeviceHostSummary = Schema.Struct({ + id: DeviceHostId, + kind: Schema.Literals(["local"]), + label: TrimmedNonEmptyString, + platforms: Schema.Array(DevicePlatformAvailability), + hubInstalled: Schema.Boolean, + agentDeviceInstalled: Schema.Boolean, +}); +export type DeviceHostSummary = typeof DeviceHostSummary.Type; + +/** + * Lifecycle of the helper processes on a host. Tools are installed on first + * use, so a fresh install spends a while in `installing` before any device can + * stream; the UI shows that instead of an empty picker. + */ +export const DeviceHostStatus = Schema.Literals([ + "disabled", + "idle", + "installing", + "starting", + "ready", + "failed", +]); +export type DeviceHostStatus = typeof DeviceHostStatus.Type; + +/** + * A device a thread is looking at. One session per (thread, device); the same + * device may be open in several threads, since the stream is shared. + */ +export const DeviceSession = Schema.Struct({ + threadId: ThreadId, + hostId: DeviceHostId, + deviceId: DeviceId, + platform: DevicePlatform, + openedAt: Schema.String, +}); +export type DeviceSession = typeof DeviceSession.Type; + +export const DeviceServiceState = Schema.Struct({ + hosts: Schema.Array(DeviceHostSummary), + hostStatus: DeviceHostStatus, + hostStatusDetail: Schema.optional(Schema.String), + devices: Schema.Array(DeviceSummary), + sessions: Schema.Array(DeviceSession), + bootingDevices: Schema.optional( + Schema.Array(Schema.Struct({ ...DeviceSummary.fields, threadId: ThreadId })), + ), + onboardingCompleted: Schema.Boolean, + agentAccessEnabled: Schema.Boolean, + /** Origin-relative path the client prefixes to hub routes. */ + hubBasePath: Schema.String, + revision: Schema.Int, +}); +export type DeviceServiceState = typeof DeviceServiceState.Type; + +export const DeviceListInput = Schema.Struct({}); +export type DeviceListInput = typeof DeviceListInput.Type; + +export const DeviceConfigureInput = Schema.Struct({ + enabled: Schema.optional(Schema.Boolean), + agentAccessEnabled: Schema.optional(Schema.Boolean), + onboardingCompleted: Schema.optional(Schema.Boolean), +}); +export type DeviceConfigureInput = typeof DeviceConfigureInput.Type; + +export const DeviceOpenInput = Schema.Struct({ + threadId: ThreadId, + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, + platform: DevicePlatform, + /** Boot the simulator or emulator when it is not running. Defaults to true. */ + boot: Schema.optional(Schema.Boolean), +}); +export type DeviceOpenInput = typeof DeviceOpenInput.Type; + +export const DeviceCloseInput = Schema.Struct({ + threadId: ThreadId, + /** Omit to close every device session for the thread. */ + deviceId: Schema.optional(DeviceId), + /** Also shut the simulator or emulator down. Defaults to false. */ + shutdown: Schema.optional(Schema.Boolean), +}); +export type DeviceCloseInput = typeof DeviceCloseInput.Type; + +export const DeviceShutdownInput = Schema.Struct({ + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, + platform: DevicePlatform, +}); +export type DeviceShutdownInput = typeof DeviceShutdownInput.Type; + +// Device settings and actions. Each setting names the platforms that support +// it; the panel hides the rest. Values are normalized across platforms where +// both have the concept (appearance, text size) and platform-specific where +// only one does. + +export const DeviceAppearance = Schema.Literals(["light", "dark"]); +export type DeviceAppearance = typeof DeviceAppearance.Type; + +/** + * iOS content-size categories map onto twelve steps; Android `font_scale` + * is continuous. Four shared steps cover what people actually reach for. + */ +export const DeviceTextSize = Schema.Literals(["small", "default", "large", "extra-large"]); +export type DeviceTextSize = typeof DeviceTextSize.Type; + +export const DeviceColorFilter = Schema.Literals([ + "none", + "grayscale", + "red-green", + "green-red", + "blue-yellow", +]); +export type DeviceColorFilter = typeof DeviceColorFilter.Type; + +export const DeviceOrientation = Schema.Literals([ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]); +export type DeviceOrientation = typeof DeviceOrientation.Type; + +/** Current values as read from the device; `undefined` means unsupported or unread. */ +export const DeviceSettings = Schema.Struct({ + appearance: Schema.optional(DeviceAppearance), + textSize: Schema.optional(DeviceTextSize), + reduceMotion: Schema.optional(Schema.Boolean), + increaseContrast: Schema.optional(Schema.Boolean), + reduceTransparency: Schema.optional(Schema.Boolean), + showBorders: Schema.optional(Schema.Boolean), + voiceOver: Schema.optional(Schema.Boolean), + liquidGlass: Schema.optional(Schema.Literals(["clear", "tinted"])), + colorFilter: Schema.optional(DeviceColorFilter), + networkEnabled: Schema.optional(Schema.Boolean), + location: Schema.optional( + Schema.NullOr(Schema.Struct({ latitude: Schema.Number, longitude: Schema.Number })), + ), +}); +export type DeviceSettings = typeof DeviceSettings.Type; + +/** The app in the foreground, when the platform can tell us. */ +export const DeviceForegroundApp = Schema.Struct({ + id: Schema.String, + name: Schema.optional(Schema.String), + version: Schema.optional(Schema.String), +}); +export type DeviceForegroundApp = typeof DeviceForegroundApp.Type; + +export const DeviceDetail = Schema.Struct({ + hostId: DeviceHostId, + deviceId: DeviceId, + settings: DeviceSettings, + foregroundApp: Schema.NullOr(DeviceForegroundApp), + readAt: Schema.String, +}); +export type DeviceDetail = typeof DeviceDetail.Type; + +export const DevicePermission = Schema.Literals([ + "camera", + "microphone", + "photos", + "contacts", + "calendar", + "reminders", + "location", + "notifications", + "motion", + "media-library", + "faceid", +]); +export type DevicePermission = typeof DevicePermission.Type; + +const DeviceTarget = { + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, +}; + +export const DeviceActionInput = Schema.Union([ + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setAppearance"), + value: DeviceAppearance, + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("setTextSize"), value: DeviceTextSize }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setToggle"), + setting: Schema.Literals([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", + "networkEnabled", + ]), + value: Schema.Boolean, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setLiquidGlass"), + value: Schema.Literals(["clear", "tinted"]), + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setColorFilter"), + value: DeviceColorFilter, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setOrientation"), + value: DeviceOrientation, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setLocation"), + latitude: Schema.Number.check(Schema.isBetween({ minimum: -90, maximum: 90 })), + longitude: Schema.Number.check(Schema.isBetween({ minimum: -180, maximum: 180 })), + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("clearLocation") }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setPermission"), + appId: TrimmedNonEmptyString, + permission: DevicePermission, + decision: Schema.Literals(["grant", "revoke", "reset"]), + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("openUrl"), url: TrimmedNonEmptyString }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("launchApp"), + appId: TrimmedNonEmptyString, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("terminateApp"), + appId: TrimmedNonEmptyString, + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("shake") }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("sendPush"), + appId: TrimmedNonEmptyString, + /** APNs-style payload; a bare string becomes the alert body. */ + payload: Schema.Union([Schema.String, Schema.Record(Schema.String, Schema.Unknown)]), + }), +]); +export type DeviceActionInput = typeof DeviceActionInput.Type; +export type DeviceActionType = DeviceActionInput["type"]; + +export const DeviceDetailInput = Schema.Struct(DeviceTarget); +export type DeviceDetailInput = typeof DeviceDetailInput.Type; + +export class DeviceHostUnavailableError extends Schema.TaggedError()( + "DeviceHostUnavailableError", + { + hostId: DeviceHostId, + reason: Schema.String, + }, +) { + override get message(): string { + return `Device host ${this.hostId} is unavailable: ${this.reason}`; + } +} + +export class DevicePlatformUnavailableError extends Schema.TaggedError()( + "DevicePlatformUnavailableError", + { + hostId: DeviceHostId, + platform: DevicePlatform, + reason: Schema.String, + }, +) { + override get message(): string { + return `${this.platform} devices are unavailable on host ${this.hostId}: ${this.reason}`; + } +} + +export class DeviceNotFoundError extends Schema.TaggedError()( + "DeviceNotFoundError", + { + hostId: DeviceHostId, + deviceId: DeviceId, + }, +) { + override get message(): string { + return `Device ${this.deviceId} was not found on host ${this.hostId}.`; + } +} + +export class DeviceBootError extends Schema.TaggedError()("DeviceBootError", { + hostId: DeviceHostId, + deviceId: DeviceId, + reason: Schema.Literals(["disk_space", "timeout", "launch_failed"]), + cause: Schema.Defect(), +}) { + override get message(): string { + const explanation = { + disk_space: "There is not enough free disk space on the environment server.", + timeout: "The device did not become ready in time.", + launch_failed: + "The simulator or emulator could not start. Check its configuration on the environment server.", + }[this.reason]; + return `Device ${this.deviceId} failed to boot: ${explanation}`; + } +} + +export class DeviceOperationError extends Schema.TaggedError()( + "DeviceOperationError", + { + operation: Schema.String, + reason: Schema.Literals([ + "command_failed", + "request_failed", + "invalid_payload", + "settings_failed", + "hub_rejected", + ]), + exitCode: Schema.optional(Schema.Number), + cause: Schema.Defect(), + }, +) { + override get message(): string { + const explanation = { + command_failed: `The device command failed${this.exitCode === undefined ? "" : ` (exit code ${this.exitCode})`}.`, + request_failed: "Could not communicate with device support. Try refreshing devices.", + invalid_payload: "The device request could not be encoded.", + settings_failed: "Could not read or save device settings.", + hub_rejected: "The device hub could not complete the request.", + }[this.reason]; + return `Device ${this.operation} failed: ${explanation}`; + } +} + +export class DeviceActionUnavailableError extends Schema.TaggedError()( + "DeviceActionUnavailableError", + { + operation: Schema.String, + platform: DevicePlatform, + reason: Schema.Literals(["unsupported", "helper_missing"]), + }, +) { + override get message(): string { + return this.reason === "helper_missing" + ? `Device ${this.operation} requires a helper missing from this install. Set up device support again.` + : `Device ${this.operation} is not supported on ${this.platform}.`; + } +} + +export const DeviceError = Schema.Union([ + DeviceHostUnavailableError, + DevicePlatformUnavailableError, + DeviceNotFoundError, + DeviceBootError, + DeviceOperationError, + DeviceActionUnavailableError, +]); +export type DeviceError = typeof DeviceError.Type; + +// MCP tool shapes. Kept next to the RPC shapes so the tool surface and the +// panel describe devices the same way. + +export const DeviceToolListResult = Schema.Struct({ + hosts: Schema.Array(DeviceHostSummary), + devices: Schema.Array(DeviceSummary), + /** Devices already open in this thread's Device panel. */ + open: Schema.Array(Schema.Struct({ hostId: DeviceHostId, deviceId: DeviceId })), +}); +export type DeviceToolListResult = typeof DeviceToolListResult.Type; + +export const DeviceToolOpenInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: + "Simulator udid or emulator serial from device_list. Omit to use the booted device for the platform, or the most recently used one.", + }), + ), + platform: Schema.optional( + DevicePlatform.annotate({ + description: "Required when deviceId is omitted and both platforms are available.", + }), + ), + hostId: Schema.optional( + DeviceHostId.annotate({ description: "Device host from device_list. Defaults to local." }), + ), +}).annotate({ + description: + "Boots the device if needed, starts its live stream, and opens the Device panel so the user can watch. Returns how to drive it with the agent-device CLI.", +}); +export type DeviceToolOpenInput = typeof DeviceToolOpenInput.Type; + +export const DeviceToolOpenResult = Schema.Struct({ + device: DeviceSummary, + /** Ready-to-run agent-device invocation pinned to this device. */ + agentDevice: Schema.Struct({ + command: Schema.String, + /** Flags that pin every command to this device, e.g. `--udid `. */ + targetArgs: Schema.Array(Schema.String), + }), + quickStart: Schema.String, +}); +export type DeviceToolOpenResult = typeof DeviceToolOpenResult.Type; + +export const DeviceToolTargetInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: + "Device from device_list. Omit to use the device most recently opened in this thread.", + }), + ), + hostId: Schema.optional(DeviceHostId), +}); +export type DeviceToolTargetInput = typeof DeviceToolTargetInput.Type; + +export const DeviceToolScreenshotResult = Schema.Struct({ + device: DeviceSummary, + screenshot: Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String, + width: Schema.Int, + height: Schema.Int, + }), +}); +export type DeviceToolScreenshotResult = typeof DeviceToolScreenshotResult.Type; + +export const DeviceToolCloseInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: "Device to close. Omit to close every device in this thread.", + }), + ), + hostId: Schema.optional(DeviceHostId), + shutdown: Schema.optional( + Schema.Boolean.annotate({ + description: "Also power the simulator or emulator off. Defaults to false.", + }), + ), +}); +export type DeviceToolCloseInput = typeof DeviceToolCloseInput.Type; + +export class DeviceToolUnavailableError extends Schema.TaggedError()( + "DeviceToolUnavailableError", + { + reason: Schema.String, + }, +) { + override get message(): string { + return this.reason; + } +} + +export const DeviceToolError = Schema.Union([ + DeviceToolUnavailableError, + DeviceHostUnavailableError, + DevicePlatformUnavailableError, + DeviceNotFoundError, + DeviceBootError, + DeviceOperationError, + DeviceActionUnavailableError, +]); +export type DeviceToolError = typeof DeviceToolError.Type; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 74a1b4939f1a..32ac53dae9ba 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -34,6 +34,7 @@ export * from "./assets.ts"; export * from "./review.ts"; export * from "./browserImport.ts"; export * from "./browserProfile.ts"; +export * from "./device.ts"; export * from "./preview.ts"; export * from "./previewAutomation.ts"; export * from "./resourceTelemetry.ts"; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 76cca2562afc..a5d35bf2361d 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -181,6 +181,19 @@ import { PreviewResizeInput, PreviewSessionSnapshot, } from "./preview.ts"; +import { + DeviceActionInput, + DeviceCloseInput, + DeviceConfigureInput, + DeviceDetail, + DeviceDetailInput, + DeviceError, + DeviceListInput, + DeviceOpenInput, + DeviceServiceState, + DeviceSession, + DeviceShutdownInput, +} from "./device.ts"; import { PreviewAutomationError, PreviewAutomationHost, @@ -312,6 +325,15 @@ export const WS_METHODS = { previewAutomationRespond: "previewAutomation.respond", previewAutomationFocusHost: "previewAutomation.focusHost", + // Device methods + deviceConfigure: "device.configure", + deviceList: "device.list", + deviceOpen: "device.open", + deviceClose: "device.close", + deviceShutdown: "device.shutdown", + deviceDetail: "device.detail", + deviceAction: "device.action", + // Server meta serverProbe: "server.probe", serverGetConfig: "server.getConfig", @@ -378,6 +400,7 @@ export const WS_METHODS = { subscribeTerminalMetadata: "subscribeTerminalMetadata", subscribePreviewEvents: "subscribePreviewEvents", subscribeDiscoveredLocalServers: "subscribeDiscoveredLocalServers", + subscribeDeviceState: "subscribeDeviceState", subscribeServerConfig: "subscribeServerConfig", subscribeServerLifecycle: "subscribeServerLifecycle", subscribeAuthAccess: "subscribeAuthAccess", @@ -1078,6 +1101,53 @@ const WsSubscribeDiscoveredLocalServersRpc = Rpc.make(WS_METHODS.subscribeDiscov stream: true, }); +const WsDeviceListRpc = Rpc.make(WS_METHODS.deviceList, { + payload: DeviceListInput, + success: DeviceServiceState, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceConfigureRpc = Rpc.make(WS_METHODS.deviceConfigure, { + payload: DeviceConfigureInput, + success: DeviceServiceState, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceOpenRpc = Rpc.make(WS_METHODS.deviceOpen, { + payload: DeviceOpenInput, + success: DeviceSession, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceCloseRpc = Rpc.make(WS_METHODS.deviceClose, { + payload: DeviceCloseInput, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceShutdownRpc = Rpc.make(WS_METHODS.deviceShutdown, { + payload: DeviceShutdownInput, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceDetailRpc = Rpc.make(WS_METHODS.deviceDetail, { + payload: DeviceDetailInput, + success: DeviceDetail, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceActionRpc = Rpc.make(WS_METHODS.deviceAction, { + payload: DeviceActionInput, + success: DeviceDetail, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsSubscribeDeviceStateRpc = Rpc.make(WS_METHODS.subscribeDeviceState, { + payload: Schema.Struct({}), + success: DeviceServiceState, + error: EnvironmentAuthorizationError, + stream: true, +}); + const WsOrchestrationDispatchCommandRpc = Rpc.make(ORCHESTRATION_WS_METHODS.dispatchCommand, { payload: ClientOrchestrationCommand, success: OrchestrationRpcSchemas.dispatchCommand.output, @@ -1308,6 +1378,14 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewAutomationFocusHostRpc, WsSubscribePreviewEventsRpc, WsSubscribeDiscoveredLocalServersRpc, + WsDeviceConfigureRpc, + WsDeviceListRpc, + WsDeviceOpenRpc, + WsDeviceCloseRpc, + WsDeviceShutdownRpc, + WsDeviceDetailRpc, + WsDeviceActionRpc, + WsSubscribeDeviceStateRpc, WsSubscribeServerConfigRpc, WsSubscribeServerLifecycleRpc, WsSubscribeAuthAccessRpc, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 9fd445d0899e..96516367612c 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -967,6 +967,22 @@ export const ServerSettings = Schema.Struct({ defaultModelSelection: Schema.NullOr(ModelSelection).pipe( Schema.withDecodingDefault(Effect.succeed(null)), ), + /** + * Whether agents may drive simulators and emulators. Gates the `device_*` + * MCP tools and the preconfigured `agent-device` CLI the same way + * `enableAgentBrowserAccess` gates the browser: server-authoritative, applied + * when the provider session is prepared. The user's own Device panel is + * unaffected. + */ + enableAgentDeviceAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** + * Whether this server may install and run T3's device helper processes. + * Kept separate from agent access so enabling the user's Device panel does + * not also grant providers control of simulators and emulators. + */ + enableDeviceSupport: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** Whether the server-local Device panel setup flow has been completed. */ + deviceOnboardingCompleted: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), sidebarAutoSettleAfterDays: Schema.NullOr(SidebarAutoSettleAfterDays).pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_SIDEBAR_AUTO_SETTLE_AFTER_DAYS)), ), @@ -1239,6 +1255,9 @@ export const ServerSettingsPatch = Schema.Struct({ Schema.Record(ProjectId, Schema.NullOr(Schema.Boolean)), ), defaultModelSelection: Schema.optionalKey(Schema.NullOr(ModelSelection)), + enableAgentDeviceAccess: Schema.optionalKey(Schema.Boolean), + enableDeviceSupport: Schema.optionalKey(Schema.Boolean), + deviceOnboardingCompleted: Schema.optionalKey(Schema.Boolean), sidebarAutoSettleAfterDays: Schema.optionalKey(Schema.NullOr(SidebarAutoSettleAfterDays)), sidebarAutoSettleOnMerge: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( From e022fa430edbf8d448efe0f34303ff290ebb3dc0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 10 Sep 2026 11:59:33 -0700 Subject: [PATCH 13/15] feat(devices): scope targets and sessions to their hosts (#10854) --- apps/server/src/device/DeviceHubProxy.ts | 3 +- .../server/src/device/DeviceMultiHost.test.ts | 86 ++++++ apps/server/src/device/DeviceService.test.ts | 1 + apps/server/src/device/DeviceService.ts | 109 ++++---- apps/server/src/mcp/McpDeviceToolkit.test.ts | 1 + .../src/mcp/toolkits/device/handlers.ts | 5 +- apps/server/src/server.test.ts | 3 +- apps/web/src/components/ChatView.tsx | 78 ++++-- apps/web/src/components/Icons.tsx | 26 ++ apps/web/src/components/RightPanelTabs.tsx | 73 ++++-- .../components/device/DeviceLoadingView.tsx | 47 ++++ .../web/src/components/device/DevicePanel.tsx | 248 +++++++----------- .../device/DeviceStreamView.test.tsx | 1 + .../components/device/DeviceStreamView.tsx | 21 +- .../settings/IntegrationsSettings.test.tsx | 1 + apps/web/src/rightPanelStore.test.ts | 81 ++++++ apps/web/src/rightPanelStore.ts | 97 ++++++- apps/web/src/state/device.ts | 17 +- packages/contracts/src/device.ts | 8 + 19 files changed, 649 insertions(+), 257 deletions(-) create mode 100644 apps/server/src/device/DeviceMultiHost.test.ts create mode 100644 apps/web/src/components/device/DeviceLoadingView.tsx diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts index f8685727c882..d2cb618fc488 100644 --- a/apps/server/src/device/DeviceHubProxy.ts +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -196,7 +196,7 @@ const handler = Effect.gen(function* () { (!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath)); yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope); const devices = yield* DeviceService.DeviceService; - const ready = yield* devices.currentReadiness(); + const ready = yield* devices.currentReadiness(url.value.searchParams.get("hostId") ?? undefined); if (!ready) { return HttpServerResponse.text("Device hub is not running", { status: 503 }); } @@ -206,6 +206,7 @@ const handler = Effect.gen(function* () { // The ticket authenticates here and must not travel on to the hub. const upstreamSearch = new URLSearchParams(url.value.search); upstreamSearch.delete("wsTicket"); + upstreamSearch.delete("hostId"); const search = upstreamSearch.size > 0 ? `?${upstreamSearch.toString()}` : ""; const upstreamPath = `${hubPath}${search}`; if (upgrade) { diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts new file mode 100644 index 000000000000..d586dffa1567 --- /dev/null +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -0,0 +1,86 @@ +import { expect, it } from "@effect/vitest"; +import { ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; +import { ServerSettingsService } from "../serverSettings.ts"; +import { DeviceHostError, DeviceHost } from "./DeviceHost.ts"; +import { makeWithHosts } from "./DeviceService.ts"; + +it.effect("keeps hosts independent when serials collide and another host fails", () => + Effect.gen(function* () { + const host = (id: string, failed = false): DeviceHost["Service"] => { + const ready = { + hub: { origin: `http://${id}` }, + agentDevice: { baseUrl: `http://${id}`, token: "test", entryPath: "/agent-device" }, + run: () => Effect.succeed({ stdout: "", stderr: "", code: 0 }), + helpers: { serveSimAxSettings: null, serveSimCli: null }, + }; + return { + id, + summary: Effect.succeed({ + id, + label: id, + kind: "local", + hubInstalled: true, + agentDeviceInstalled: true, + platforms: [{ platform: "android", available: true }], + }), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: () => + failed + ? Effect.fail( + new DeviceHostError({ hostId: id, step: "connect", cause: new Error("offline") }), + ) + : Effect.succeed(ready), + ensureAgentReady: () => Effect.succeed(ready), + current: Effect.succeed(ready), + stopAgent: Effect.void, + stop: Effect.void, + }; + }; + const http = HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json({ + simulators: [], + emulators: [ + { + id: "emulator-5554", + name: "Pixel", + version: "36", + platform: "android", + booted: true, + physical: false, + }, + ], + }), + ), + ), + ); + const hosts = new Map(["a", "b", "offline"].map((id) => [id, host(id, id === "offline")])); + const service = yield* makeWithHosts(hosts).pipe( + Effect.provideService(HttpClient.HttpClient, http), + ); + const listed = yield* service.list; + expect(listed.devices.map((device) => device.hostId).sort()).toEqual(["a", "b"]); + expect(listed.hostStatuses.offline?.status).toBe("failed"); + const threadId = ThreadId.make("thread"); + for (const hostId of ["a", "b"]) + yield* service.open({ threadId, hostId, deviceId: "emulator-5554", platform: "android" }); + yield* service.close({ threadId, hostId: "a", deviceId: "emulator-5554" }); + const state = yield* service.state; + expect(state.devices).toHaveLength(2); + expect(state.sessions.map((session) => session.hostId)).toEqual(["b"]); + expect(state.hostStatuses.a?.status).toBe("ready"); + expect(state.hostStatuses.offline?.status).toBe("failed"); + yield* service.agentReadinessIfSupported("b"); + expect((yield* service.state).hostStatuses.b?.status).toBe("ready"); + yield* service.configure({ enabled: false }); + expect((yield* service.state).hostStatuses).toEqual({}); + }).pipe( + Effect.provide( + ServerSettingsService.layerTest({ enableDeviceSupport: true, enableAgentDeviceAccess: true }), + ), + ), +); diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index ed42be64eb02..49a8545bfd6d 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -21,6 +21,7 @@ import { type DeviceService, make, stateStream } from "./DeviceService.ts"; const baseState: DeviceServiceState = { hosts: [], hostStatus: "idle", + hostStatuses: {}, devices: [], sessions: [], onboardingCompleted: false, diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 6cca6631bd7e..597e1a5d2b10 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -136,8 +136,9 @@ interface ServiceState { const vendorPrefix = (platform: DevicePlatform) => platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; -export const make = Effect.gen(function* () { - const localHost = yield* DeviceHost.DeviceHost; +export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ( + hosts: ReadonlyMap, +) { const settings = yield* ServerSettings.ServerSettingsService; const lifecycleLock = yield* Semaphore.make(1); const readDeviceSettings = settings.getSettings.pipe( @@ -152,9 +153,7 @@ export const make = Effect.gen(function* () { ), ); const initialSettings = yield* readDeviceSettings; - const hosts: ReadonlyMap = new Map([ - [localHost.id, localHost], - ]); + const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); const statePubSub = yield* PubSub.unbounded(); const initialHosts = yield* Effect.forEach(hosts.values(), (host) => host.summary); @@ -162,6 +161,7 @@ export const make = Effect.gen(function* () { state: { hosts: initialHosts, hostStatus: initialSettings.enabled ? "idle" : "disabled", + hostStatuses: {}, devices: [], sessions: [], onboardingCompleted: initialSettings.onboardingCompleted, @@ -187,6 +187,18 @@ export const make = Effect.gen(function* () { return host; }); + const setHostStatus = ( + hostId: DeviceHostId, + status: DeviceServiceState["hostStatuses"][string], + ) => + publish((state) => ({ + ...state, + ...(hostId === LOCAL_DEVICE_HOST_ID + ? { hostStatus: status.status, hostStatusDetail: status.detail } + : {}), + hostStatuses: { ...state.hostStatuses, [hostId]: status }, + })); + const readiness: DeviceService["Service"]["readiness"] = Effect.fn("DeviceService.readiness")( function* (hostId) { const host = yield* resolveHost(hostId); @@ -198,34 +210,19 @@ export const make = Effect.gen(function* () { }); } const ready = yield* host - .ensureReady((phase) => - publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( - Effect.asVoid, - ), - ) + .ensureReady((status) => setHostStatus(host.id, { status }).pipe(Effect.asVoid)) .pipe( Effect.tapError((error) => - publish((state) => ({ - ...state, - hostStatus: "failed", - hostStatusDetail: error.message, - })), + setHostStatus(host.id, { status: "failed", detail: error.message }), ), Effect.mapError( (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), ), ); - yield* SynchronizedRef.get(stateRef).pipe( - Effect.flatMap(({ state }) => - state.hostStatus === "ready" - ? Effect.void - : publish((current) => ({ - ...current, - hostStatus: "ready", - hostStatusDetail: undefined, - })), - ), - ); + const { state } = yield* SynchronizedRef.get(stateRef); + if (state.hostStatuses[host.id]?.status !== "ready") { + yield* setHostStatus(host.id, { status: "ready" }); + } return { hostId: host.id, ...ready }; }, lifecycleLock.withPermit, @@ -249,25 +246,18 @@ export const make = Effect.gen(function* () { const summary = yield* host.summary; if (!summary.platforms.some((platform) => platform.available)) return null; const ready = yield* host - .ensureAgentReady((phase) => - publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( - Effect.asVoid, - ), - ) + .ensureAgentReady((phase) => setHostStatus(host.id, { status: phase }).pipe(Effect.asVoid)) .pipe( Effect.tapError((error) => - publish((state) => ({ - ...state, - hostStatus: "failed", - hostStatusDetail: error.message, - })), + setHostStatus(host.id, { status: "failed", detail: error.message }), ), Effect.mapError( (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), ), ); const hostSummaries = yield* Effect.forEach(hosts.values(), (candidate) => candidate.summary); - yield* publish((state) => ({ ...state, hosts: hostSummaries, hostStatus: "ready" })); + yield* publish((state) => ({ ...state, hosts: hostSummaries })); + yield* setHostStatus(host.id, { status: "ready" }); return { hostId: host.id, ...ready }; }, lifecycleLock.withPermit); @@ -357,8 +347,15 @@ export const make = Effect.gen(function* () { return yield* publish((state) => ({ ...state, hosts: hostSummaries, - devices, - hostStatusDetail: detail, + ...(ready.hostId === LOCAL_DEVICE_HOST_ID ? { hostStatusDetail: detail } : {}), + devices: [ + ...state.devices.filter((device) => device.hostId !== ready.hostId), + ...devices, + ], + hostStatuses: { + ...state.hostStatuses, + [ready.hostId]: { status: "ready", ...(detail ? { detail } : {}) }, + }, })); }), ); @@ -366,18 +363,21 @@ export const make = Effect.gen(function* () { const list: DeviceService["Service"]["list"] = Effect.gen(function* () { if (!(yield* readDeviceSettings).enabled) return (yield* SynchronizedRef.get(stateRef)).state; - const ready = yield* readiness(); - return yield* refresh(ready); - }).pipe( - Effect.tapError((error) => - publish((state) => - state.hostStatus === "disabled" - ? state - : { ...state, hostStatus: "failed", hostStatusDetail: error.message }, - ), - ), - Effect.withSpan("DeviceService.list"), - ); + yield* Effect.forEach( + hosts.values(), + (host) => + Effect.gen(function* () { + const ready = yield* readinessIfSupported(host.id); + if (ready) yield* refresh(ready); + }).pipe( + Effect.catch((error) => + setHostStatus(host.id, { status: "failed", detail: error.message }), + ), + ), + { concurrency: 4 }, + ); + return (yield* SynchronizedRef.get(stateRef)).state; + }).pipe(Effect.withSpan("DeviceService.list")); const configure: DeviceService["Service"]["configure"] = Effect.fn("DeviceService.configure")( function* (input) { @@ -415,6 +415,7 @@ export const make = Effect.gen(function* () { ...state, hostStatus: nextEnabled ? "idle" : "disabled", hostStatusDetail: undefined, + hostStatuses: {}, devices: nextEnabled ? state.devices : [], sessions: nextEnabled ? state.sessions : [], bootingDevices: nextEnabled ? state.bootingDevices : [], @@ -636,6 +637,7 @@ export const make = Effect.gen(function* () { const closing = state.sessions.filter( (session) => session.threadId === input.threadId && + (input.hostId === undefined || session.hostId === input.hostId) && (input.deviceId === undefined || session.deviceId === input.deviceId), ); if (closing.length === 0) return; @@ -749,6 +751,11 @@ export const make = Effect.gen(function* () { }); }); +export const make = Effect.gen(function* () { + const host = yield* DeviceHost.DeviceHost; + return yield* makeWithHosts(new Map([[host.id, host]])); +}); + export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); /** State stream for WS subscribers: current snapshot first, then every change. */ diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts index 083ac0c034fe..32f919cbb682 100644 --- a/apps/server/src/mcp/McpDeviceToolkit.test.ts +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -56,6 +56,7 @@ const state = { }, ], hostStatus: "ready" as const, + hostStatuses: { local: { status: "ready" as const } }, devices: [device], sessions: [], onboardingCompleted: true, diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index bab7391af24d..5ad57031306e 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -161,7 +161,9 @@ const handlers = { const target = input.deviceId !== undefined ? { hostId: input.hostId ?? LOCAL_DEVICE_HOST_ID, deviceId: input.deviceId } - : sessions.at(-1); + : sessions + .filter((session) => input.hostId === undefined || session.hostId === input.hostId) + .at(-1); if (!target) { return yield* new DeviceToolUnavailableError({ reason: "No device is open in this thread. Call device_open first.", @@ -183,6 +185,7 @@ const handlers = { const devices = yield* DeviceService.DeviceService; yield* devices.close({ threadId: scope.threadId, + ...(input.hostId === undefined ? {} : { hostId: input.hostId }), ...(input.deviceId === undefined ? {} : { deviceId: input.deviceId }), ...(input.shutdown === undefined ? {} : { shutdown: input.shutdown }), }); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 62412ac25853..afb4adfef79c 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -1669,7 +1669,8 @@ const NodeHttpServerTestWithWsDeflate = HttpServer.layerTestClient.pipe( const EMPTY_DEVICE_STATE: DeviceServiceState = { hosts: [], - hostStatus: "idle", + hostStatus: "disabled", + hostStatuses: {}, devices: [], sessions: [], onboardingCompleted: false, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 225d91310014..283e85365667 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -4143,7 +4143,9 @@ export default function ChatView(props: ChatViewProps) { if (!activeThreadRef || !supportsThreadPullRequests) return; useRightPanelStore.getState().open(activeThreadRef, "pull-requests"); }, [activeThreadRef, supportsThreadPullRequests]); - const { state: deviceState } = useDeviceState(activeThreadRef?.environmentId ?? null); + const { state: deviceState, loaded: deviceStateLoaded } = useDeviceState( + activeThreadRef?.environmentId ?? null, + ); const [deviceSetupThread, setDeviceSetupThread] = useState(null); const addDeviceSurface = useCallback(() => { if (!activeThreadRef) return; @@ -4153,24 +4155,53 @@ export default function ChatView(props: ChatViewProps) { } useRightPanelStore.getState().open(activeThreadRef, "device"); }, [activeThreadRef, deviceState.onboardingCompleted, deviceState.hostStatus]); - // An agent's `device_open` surfaces in every client the same way a - // `preview_open` does: the thread starts a device or gains a session and the panel - // opens on it. Closing the last session leaves the tab in place so the - // user keeps their picker; only new sessions raise the panel. - const threadDeviceSessionCount = activeThreadRef - ? deviceState.sessions.filter((session) => session.threadId === activeThreadRef.threadId) - .length + - (deviceState.bootingDevices?.filter((device) => device.threadId === activeThreadRef.threadId) - .length ?? 0) - : 0; - const previousDeviceSessionCount = useRef(threadDeviceSessionCount); + // Reconcile new server sessions into separate tabs, including sessions opened + // by an agent or another client. The first snapshot is a baseline: persisted + // tabs restore themselves, and existing sessions must not resurrect closed tabs. + const previousDeviceSessions = useRef(new Map>()); useEffect(() => { - const previous = previousDeviceSessionCount.current; - previousDeviceSessionCount.current = threadDeviceSessionCount; - if (!activeThreadRef || threadDeviceSessionCount <= previous) return; - if (shouldUseRightPanelSheet) return; - useRightPanelStore.getState().open(activeThreadRef, "device"); - }, [activeThreadRef, shouldUseRightPanelSheet, threadDeviceSessionCount]); + if (!activeThreadRef || !deviceStateLoaded) return; + const threadKey = `${activeThreadRef.environmentId}:${activeThreadRef.threadId}`; + const sessions = deviceState.sessions.filter( + (session) => session.threadId === activeThreadRef.threadId, + ); + const key = (session: (typeof sessions)[number]) => `${session.hostId}:${session.deviceId}`; + const previous = previousDeviceSessions.current.get(threadKey); + previousDeviceSessions.current.set(threadKey, new Set(sessions.map(key))); + if (!previous || shouldUseRightPanelSheet) return; + for (const session of sessions) { + if (previous?.has(key(session))) continue; + const existing = useRightPanelStore + .getState() + .byThreadKey[scopedThreadKey(activeThreadRef)]?.surfaces.some( + (surface) => + surface.kind === "device" && + surface.target?.hostId === session.hostId && + surface.target.deviceId === session.deviceId, + ); + if (existing) continue; + const device = deviceState.devices.find( + (entry) => entry.hostId === session.hostId && entry.id === session.deviceId, + ); + if (!device) continue; + useRightPanelStore.getState().openDevice( + activeThreadRef, + { + hostId: session.hostId, + deviceId: session.deviceId, + platform: device.platform, + name: device.name, + }, + true, + ); + } + }, [ + activeThreadRef, + deviceStateLoaded, + shouldUseRightPanelSheet, + deviceState.sessions, + deviceState.devices, + ]); const openFileSurface = useCallback( (relativePath: string) => { if (!activeThreadRef || !activeProject) return; @@ -8153,7 +8184,8 @@ export default function ChatView(props: ChatViewProps) { { closeRightPanelSurface(renderedRightPanelSurface); @@ -8725,6 +8757,10 @@ export default function ChatView(props: ChatViewProps) { terminalLabelsById={activeTerminalLabelsById} onActivate={activateRightPanelSurface} onCloseSurface={closeRightPanelSurface} + onRenameDevice={(surfaceId, title) => { + if (activeThreadRef) + useRightPanelStore.getState().renameDevice(activeThreadRef, surfaceId, title); + }} onCloseOtherSurfaces={closeOtherRightPanelSurfaces} onCloseSurfacesToRight={closeRightPanelSurfacesToRight} onCloseAllSurfaces={closeAllRightPanelSurfaces} @@ -8779,6 +8815,10 @@ export default function ChatView(props: ChatViewProps) { terminalLabelsById={activeTerminalLabelsById} onActivate={activateRightPanelSurface} onCloseSurface={closeRightPanelSurface} + onRenameDevice={(surfaceId, title) => { + if (activeThreadRef) + useRightPanelStore.getState().renameDevice(activeThreadRef, surfaceId, title); + }} onCloseOtherSurfaces={closeOtherRightPanelSurfaces} onCloseSurfacesToRight={closeRightPanelSurfacesToRight} onCloseAllSurfaces={closeAllRightPanelSurfaces} diff --git a/apps/web/src/components/Icons.tsx b/apps/web/src/components/Icons.tsx index 199d0ba834d0..b13040152e18 100644 --- a/apps/web/src/components/Icons.tsx +++ b/apps/web/src/components/Icons.tsx @@ -2,6 +2,32 @@ import React, { type SVGProps, useId } from "react"; import { cn } from "~/lib/utils"; export type Icon = React.FC>; +// Apple brand mark from Simple Icons (CC0). +export const AppleIcon: Icon = (props) => ( + +); + +export const AndroidIcon: Icon = (props) => ( + +); + export const LinuxIcon: Icon = ({ className, ...props }) => ( diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index 546c7c5ad45a..79caadf90e06 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -46,6 +46,7 @@ import type { RightPanelSurface } from "~/rightPanelStore"; import { cn } from "~/lib/utils"; import { readLocalApi } from "~/localApi"; import { Button } from "~/components/ui/button"; +import { AndroidIcon, AppleIcon } from "~/components/Icons"; import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; import { Kbd } from "~/components/ui/kbd"; import { @@ -97,6 +98,7 @@ interface RightPanelTabsProps { previewRuntimeTabId?: ((tabId: string) => string) | undefined; terminalLabelsById: ReadonlyMap; onActivate: (surface: RightPanelSurface) => void; + onRenameDevice?: (surfaceId: string, title: string) => void; onCloseSurface: (surface: RightPanelSurface) => void; onCloseOtherSurfaces: (surface: RightPanelSurface) => void; onCloseSurfacesToRight: (surface: RightPanelSurface) => void; @@ -182,6 +184,7 @@ const SURFACE_UNAVAILABLE_HINTS = { } as const; type TabContextMenuAction = + | "rename" | "copy-path" | "toggle-mute" | "close" @@ -648,7 +651,7 @@ function surfaceTitle( case "agents": return "Agents"; case "device": - return "Device"; + return surface.title ?? surface.target?.name ?? "Device"; case "preview": { const snapshot = surface.resourceId ? sessions[surface.resourceId] : null; if (!snapshot || snapshot.navStatus._tag === "Idle") return "Browser"; @@ -733,7 +736,13 @@ function SurfaceIcon({ case "agents": return ; case "device": - return ; + return surface.target?.platform === "ios" ? ( + + ) : surface.target?.platform === "android" ? ( + + ) : ( + + ); } } @@ -832,6 +841,7 @@ export function RightPanelTabs(props: RightPanelTabsProps) { const browserProfiles = useBrowserDefaults().profiles; const { resolvedTheme } = useTheme(); const tabListRef = useRef(null); + const [renamingDevice, setRenamingDevice] = useState(null); const [addSurfaceMenuOpen, setAddSurfaceMenuOpen] = useState(false); const [tabScrollState, setTabScrollState] = useState({ hasOverflow: false, @@ -958,6 +968,8 @@ export function RightPanelTabs(props: RightPanelTabsProps) { if (surfaceIndex < 0) return; const items: ContextMenuItem[] = []; + if (surface.kind === "device" && props.onRenameDevice) + items.push({ id: "rename", label: "Rename" }); if (surface.kind === "file" && surface.attachment === undefined) { items.push({ id: "copy-path", label: "Copy path" }); } @@ -1001,6 +1013,9 @@ export function RightPanelTabs(props: RightPanelTabsProps) { const action = await api.contextMenu.show(items, { x: event.clientX, y: event.clientY }); switch (action) { + case "rename": + setRenamingDevice(surface.id); + break; case "copy-path": if (surface.kind === "file" && surface.attachment === undefined) { props.onCopyFilePath(surface.relativePath); @@ -1202,20 +1217,48 @@ export function RightPanelTabs(props: RightPanelTabsProps) { {audio === "muted" ? "Unmute tab" : "Mute tab"} )} - - props.onActivate(surface)} - > - {title} - - } + {renamingDevice === surface.id ? ( + { + element?.focus(); + element?.select(); + }} + onBlur={(event) => { + props.onRenameDevice?.(surface.id, event.currentTarget.value); + setRenamingDevice(null); + }} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === "Enter") event.currentTarget.blur(); + if (event.key === "Escape") { + event.currentTarget.value = title; + event.currentTarget.blur(); + } + }} /> - {title} - + ) : ( + + { + if (surface.kind === "device" && props.onRenameDevice) + setRenamingDevice(surface.id); + }} + className="cursor-pointer flex min-w-0 items-center" + onClick={() => props.onActivate(surface)} + > + {title} + + } + /> + {title} + + )}
); })} diff --git a/apps/web/src/components/device/DeviceLoadingView.tsx b/apps/web/src/components/device/DeviceLoadingView.tsx new file mode 100644 index 000000000000..70e848c3ce7e --- /dev/null +++ b/apps/web/src/components/device/DeviceLoadingView.tsx @@ -0,0 +1,47 @@ +import { Smartphone } from "lucide-react"; + +import { Spinner } from "~/components/ui/spinner"; + +export function DeviceLoadingView(props: { + readonly name: string; + readonly description?: string; + readonly stage: "opening" | "stream"; + readonly message: string; + readonly error?: boolean; +}) { + return ( +
+
+
+ +
+
+

{props.name}

+ {props.description ? ( +

{props.description}

+ ) : null} +
+
+ {!props.error ? : null} + {props.message} +
+ {!props.error ? ( +
+ + +
+ ) : null} +
+
+ ); +} diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx index d94913b03dff..5b312f7fc4f4 100644 --- a/apps/web/src/components/device/DevicePanel.tsx +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -6,7 +6,6 @@ import type { } from "@t3tools/contracts"; import { ChevronLeft, - Circle, Home, Power, RotateCcw, @@ -15,21 +14,13 @@ import { Square, X, } from "lucide-react"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; +import { useRightPanelStore, type RightPanelSurface } from "~/rightPanelStore"; import { Button } from "~/components/ui/button"; import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list"; import { Dialog } from "~/components/ui/dialog"; import { WizardPopup } from "~/components/ui/wizard"; -import { - Select, - SelectGroup, - SelectGroupLabel, - SelectItem, - SelectPopup, - SelectTrigger, - SelectValue, -} from "~/components/ui/select"; import { Spinner } from "~/components/ui/spinner"; import { Toggle } from "~/components/ui/toggle"; import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; @@ -38,28 +29,22 @@ import { deviceEnvironment, useDeviceHubAccess, useDeviceState } from "~/state/d import { formatEnvironmentQueryError } from "~/state/query"; import { useAtomCommand } from "~/state/use-atom-command"; import { DeviceStreamView, type DeviceStreamHandle } from "./DeviceStreamView"; +import { DeviceLoadingView } from "./DeviceLoadingView"; import { DeviceSetup } from "./DeviceSetup"; import { DeviceToolsPanel } from "./DeviceToolsPanel"; import { PreviewPanelShell, type PreviewPanelMode } from "../preview/PreviewPanelShell"; -const NEW_DEVICE_VALUE = "__new__"; - const platformLabel = (platform: DevicePlatform) => platform === "ios" ? "iOS Simulators" : "Android Emulators"; const deviceKey = (device: Pick) => `${device.hostId}\u0000${device.id}`; -/** - * The Device right-panel surface: one open device (from the thread's device - * sessions) with a picker to switch or boot another. Booting and streaming are - * server-owned; this panel only asks and renders. - */ +/** Each surface owns one host/device; only the visible surface streams. */ export function DevicePanel(props: { readonly mode: PreviewPanelMode; readonly threadRef: ScopedThreadRef; - /** `null` renders the picker with nothing open. */ - readonly deviceId: string | null; + readonly surface: Extract; readonly visible: boolean; readonly onDismissSetup: () => void; }) { @@ -69,7 +54,8 @@ export function DevicePanel(props: { const open = useAtomCommand(deviceEnvironment.open); const close = useAtomCommand(deviceEnvironment.close); const [operationError, setOperationError] = useState(null); - const [pendingDeviceKey, setPendingDeviceKey] = useState(null); + const [pendingDevice, setPendingDevice] = useState(null); + const pendingDeviceKey = pendingDevice ? deviceKey(pendingDevice) : null; const [handle, setHandle] = useState(null); const [toolsOpen, setToolsOpen] = useState(false); const [axOverlay, setAxOverlay] = useState(false); @@ -87,10 +73,13 @@ export function DevicePanel(props: { () => state.sessions.filter((session) => session.threadId === threadId), [state.sessions, threadId], ); - const activeSession = - (props.deviceId - ? sessions.find((session) => session.deviceId === props.deviceId) - : undefined) ?? sessions.at(-1); + const activeSession = props.surface.target + ? sessions.find( + (session) => + session.deviceId === props.surface.target?.deviceId && + session.hostId === props.surface.target.hostId, + ) + : undefined; const activeDevice = activeSession ? state.devices.find( (device) => device.hostId === activeSession.hostId && device.id === activeSession.deviceId, @@ -99,51 +88,67 @@ export function DevicePanel(props: { const grouped = useMemo(() => groupDevices(state), [state]); - const selectDevice = useCallback( - async (value: string) => { - if (value === NEW_DEVICE_VALUE) return; - const device = state.devices.find((candidate) => deviceKey(candidate) === value); - if (!device) return; - setOperationError(null); - setPendingDeviceKey(value); - try { - const result = await open({ - environmentId, - input: { - threadId, - hostId: device.hostId, - deviceId: device.id, - platform: device.platform, - }, - }); - if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); - } finally { - setPendingDeviceKey(null); - } - }, - [environmentId, open, state.devices, threadId], - ); - - const closeActive = useCallback( - (powerOff: boolean) => { - if (!activeSession) return; - setOperationError(null); - void close({ + const selectDevice = async (value: string) => { + const device = state.devices.find((candidate) => deviceKey(candidate) === value); + if (!device) return; + setOperationError(null); + setPendingDevice(device); + try { + const result = await open({ environmentId, - input: { threadId, deviceId: activeSession.deviceId, shutdown: powerOff }, - }).then((result) => { - if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); + input: { + threadId, + hostId: device.hostId, + deviceId: device.id, + platform: device.platform, + }, }); - }, - [activeSession, close, environmentId, threadId], - ); + if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); + else + useRightPanelStore.getState().openDevice(props.threadRef, { + hostId: result.value.hostId, + deviceId: result.value.deviceId, + platform: device.platform, + name: device.name, + }); + } finally { + setPendingDevice(null); + } + }; + + const closeActive = (powerOff: boolean) => { + if (!powerOff) { + useRightPanelStore.getState().closeSurface(props.threadRef, props.surface.id); + return; + } + if (!activeSession) return; + setOperationError(null); + void close({ + environmentId, + input: { + threadId, + hostId: activeSession.hostId, + deviceId: activeSession.deviceId, + shutdown: powerOff, + }, + }).then((result) => { + if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); + else useRightPanelStore.getState().closeSurface(props.threadRef, props.surface.id); + }); + }; const bootingDevices = state.bootingDevices?.filter((device) => device.threadId === threadId) ?? []; - const hostReady = state.hostStatus === "ready"; - const hostBusy = state.hostStatus === "installing" || state.hostStatus === "starting"; + const hostReady = Object.values(state.hostStatuses).some((host) => host.status === "ready"); + const hostBusy = + !hostReady && + Object.values(state.hostStatuses).some( + (host) => host.status === "installing" || host.status === "starting", + ); const unavailablePlatforms = state.hosts.flatMap((host) => - host.platforms.filter((platform) => !platform.available), + host.platforms + .filter((platform) => !platform.available) + .map((platform) => ({ ...platform, hostId: host.id, hostLabel: host.label })), ); if (loaded && (!state.onboardingCompleted || hostDisabled)) { @@ -164,63 +169,11 @@ export function DevicePanel(props: { return (
- + + {props.surface.target + ? `${state.hosts.find((host) => host.id === props.surface.target?.hostId)?.label ?? "Device host"} · ${activeDevice?.version ?? props.surface.target.platform}` + : (pendingDevice?.name ?? "Choose a device")} + {activeDevice ? ( <> host.id === activeDevice.hostId)?.label ?? "Device host"} · ${activeDevice.version}`} deviceId={activeDevice.id} + hostId={activeDevice.hostId} visible={props.visible} axOverlay={axOverlay} onHandle={setHandle} @@ -330,6 +286,25 @@ export function DevicePanel(props: { /> ) : null} + ) : pendingDevice || hostBusy || !loaded ? ( + host.id === pendingDevice.hostId)?.label ?? "Device host"} · ${pendingDevice.version}` + : "" + } + stage="opening" + message={ + pendingDevice + ? pendingDevice.booted + ? "Opening device…" + : "Starting device…" + : state.hostStatus === "installing" + ? "Installing device support…" + : "Finding devices…" + } + /> ) : (
- {grouped.length === 0 || hostBusy || pendingDeviceKey ? ( + {grouped.length === 0 ? ( <> - {hostBusy || pendingDeviceKey ? ( - - ) : ( - - )} +

{state.hostStatus === "failed" ? (state.hostStatusDetail ?? "The device hub failed to start.") - : pendingDeviceKey - ? "Booting device… this can take a minute." - : hostBusy - ? state.hostStatus === "installing" - ? "Installing device tools…" - : "Starting the device hub…" - : !loaded - ? "Connecting…" - : grouped.length === 0 - ? "No simulators or emulators were found on this environment." - : "Choose a device to open."} + : "No simulators or emulators were found on this environment."}

) : null} @@ -380,7 +341,7 @@ export function DevicePanel(props: { } title={device.name} - description={`${device.version} · ${device.booted ? "Running" : "Stopped"}`} + description={`${state.hosts.find((host) => host.id === device.hostId)?.label} · ${device.version} · ${device.booted ? "Running" : "Stopped"}`} disabled={pendingDeviceKey !== null} aria-label={`${device.booted ? "Open" : "Start"} ${device.name}`} onClick={() => void selectDevice(deviceKey(device))} @@ -418,15 +379,6 @@ export function DevicePanel(props: { Refresh devices ) : null} - {unavailablePlatforms.length > 0 && hostReady ? ( -
    - {unavailablePlatforms.map((platform) => ( -
  • - {platform.platform === "ios" ? "iOS" : "Android"}: {platform.reason} -
    • - ))} -
- ) : null}
)} diff --git a/apps/web/src/components/device/DeviceStreamView.test.tsx b/apps/web/src/components/device/DeviceStreamView.test.tsx index a46e5ed7b7b8..9c8a1197926a 100644 --- a/apps/web/src/components/device/DeviceStreamView.test.tsx +++ b/apps/web/src/components/device/DeviceStreamView.test.tsx @@ -36,6 +36,7 @@ it("removes MJPEG requests while hidden and reconnects when shown", async () => ); const view = (visible: boolean) => ( void; readonly onScreen?: (screen: DeviceScreenSize | null) => void; }) { - const access = useDeviceHubAccess(props.environmentId); + const access = useDeviceHubAccess(props.environmentId, props.hostId); const canvasRef = useRef(null); const clientRef = useRef(null); const [status, setStatus] = useState("connecting"); @@ -314,12 +317,14 @@ export function DeviceStreamView(props: {
) : null} {status !== "streaming" ? ( -
- {status === "connecting" ? : null} - {status === "error" ? (detail ?? "Stream failed.") : "Connecting to device…"} - {status === "connecting" && detail ? ( - {detail} - ) : null} +
+
) : null}
diff --git a/apps/web/src/components/settings/IntegrationsSettings.test.tsx b/apps/web/src/components/settings/IntegrationsSettings.test.tsx index 9ce3b2126c0e..4ef218430ae1 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.test.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.test.tsx @@ -105,6 +105,7 @@ const deviceState = (overrides: Partial = {}): DeviceService }, ], hostStatus: "ready", + hostStatuses: {}, devices: [], sessions: [], onboardingCompleted: false, diff --git a/apps/web/src/rightPanelStore.test.ts b/apps/web/src/rightPanelStore.test.ts index ad1788f4cfe9..e2299511b87c 100644 --- a/apps/web/src/rightPanelStore.test.ts +++ b/apps/web/src/rightPanelStore.test.ts @@ -21,6 +21,87 @@ beforeEach(() => { }); describe("rightPanelStore", () => { + it("gives each host/device its own tab and preserves renamed tabs", () => { + const store = useRightPanelStore.getState(); + const android = { + hostId: "nucbox", + deviceId: "emulator-5580", + name: "Pixel", + platform: "android", + } as const; + const ios = { hostId: "macmini", deviceId: "ios-1", name: "iPhone", platform: "ios" } as const; + store.open(refA, "device"); + store.openDevice(refA, android); + store.open(refA, "device"); + expect( + selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, refA).surfaces, + ).toHaveLength(2); + store.openDevice(refA, ios); + let state = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, refA); + expect(state.surfaces.map((surface) => surface.id)).toEqual([ + "device:nucbox:emulator-5580", + "device:macmini:ios-1", + ]); + store.renameDevice(refA, "device:nucbox:emulator-5580", "Android test"); + store.openDevice(refA, android); + state = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, refA); + expect(state.surfaces).toHaveLength(2); + expect(state.surfaces[0]).toMatchObject({ title: "Android test", target: android }); + expect(state.activeSurfaceId).toBe("device:nucbox:emulator-5580"); + store.closeSurface(refA, state.activeSurfaceId!); + expect( + selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, refA).surfaces, + ).toEqual([expect.objectContaining({ target: ios })]); + }); + + it("does not collide when two hosts expose the same device id", () => { + const store = useRightPanelStore.getState(); + const device = { deviceId: "emulator-5554", name: "Pixel", platform: "android" } as const; + store.openDevice(refA, { ...device, hostId: "a:b" }); + store.openDevice(refA, { ...device, hostId: "a" }); + expect( + selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, refA).surfaces, + ).toHaveLength(2); + expect( + selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, refB).surfaces, + ).toHaveLength(0); + }); + + it.each(["one", "all", "others", "right"])( + "keeps device tabs dismissed across reload after closing %s", + (mode) => { + const store = useRightPanelStore.getState(); + const target = { + hostId: "nucbox", + deviceId: "emulator-5580", + name: "Pixel", + platform: "android", + } as const; + store.open(refA, "files"); + store.openDevice(refA, target); + if (mode === "one") store.closeSurface(refA, "device:nucbox:emulator-5580"); + if (mode === "all") store.closeAllSurfaces(refA); + if (mode === "others") store.closeOtherSurfaces(refA, "files"); + if (mode === "right") store.closeSurfacesToRight(refA, "files"); + const persisted = JSON.parse( + JSON.stringify({ byThreadKey: useRightPanelStore.getState().byThreadKey }), + ); + useRightPanelStore.setState(migratePersistedRightPanelState(persisted)); + store.openDevice(refA, target, true); + expect( + selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, refA).surfaces.some( + (surface) => surface.kind === "device", + ), + ).toBe(false); + store.openDevice(refA, target); + expect( + selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, refA).surfaces.some( + (surface) => surface.kind === "device", + ), + ).toBe(true); + }, + ); + const completedDiff = { id: "diff", kind: "diff" } as const; const linkedPullRequest = pullRequestSurface({ projectId: "project-a", diff --git a/apps/web/src/rightPanelStore.ts b/apps/web/src/rightPanelStore.ts index 293733b5b3d9..c44c106c68c8 100644 --- a/apps/web/src/rightPanelStore.ts +++ b/apps/web/src/rightPanelStore.ts @@ -32,15 +32,17 @@ const RIGHT_PANEL_KINDS = [ ] as const; export type RightPanelKind = (typeof RIGHT_PANEL_KINDS)[number]; +export interface DeviceTabTarget { + hostId: string; + deviceId: string; + platform: "ios" | "android"; + name: string; +} + export type RightPanelSurface = | { id: `browser:${string}`; kind: "preview"; resourceId: string } | { id: "browser:new"; kind: "preview"; resourceId: null } - /** - * One Device tab per thread. The tab is the surface; which device it shows - * comes from the thread's server-side device sessions, so an agent opening a - * device from another client lands in the same tab. - */ - | { id: "device"; kind: "device" } + | { id: "device" | `device:${string}`; kind: "device"; target?: DeviceTabTarget; title?: string } | { id: `terminal:${string}`; kind: "terminal"; @@ -90,7 +92,7 @@ const RIGHT_PANEL_STORAGE_KEY = "t3code:right-panel-state:v2"; // v10 keys pull-request surfaces by reference instead of a singleton tab. // v11 stops persisting the pull-request list's shared panel, so a restart opens the page fresh. // v12 adds the device surface. -const RIGHT_PANEL_STORAGE_VERSION = 12; +const RIGHT_PANEL_STORAGE_VERSION = 13; /** A fixed workspace-level ref: each PR surface carries its own real environment. */ export const PULL_REQUESTS_PANEL_REF = scopeThreadRef( @@ -108,6 +110,7 @@ export interface ThreadRightPanelState { isOpen: boolean; activeSurfaceId: string | null; surfaces: RightPanelSurface[]; + dismissedDeviceSurfaceIds?: string[]; } interface RightPanelStoreState { @@ -128,6 +131,8 @@ interface RightPanelStoreState { ref: ScopedThreadRef, kind: Exclude, ) => void; + openDevice: (ref: ScopedThreadRef, target: DeviceTabTarget, automatic?: boolean) => void; + renameDevice: (ref: ScopedThreadRef, surfaceId: string, title: string) => void; openBrowser: (ref: ScopedThreadRef, tabId: string | null) => void; openFile: (ref: ScopedThreadRef, relativePath: string, line?: number) => void; openAttachment: (ref: ScopedThreadRef, attachment: ChatFileAttachment) => void; @@ -281,7 +286,12 @@ const updateThread = ( ): Record => { const current = byThreadKey[threadKey] ?? EMPTY_THREAD_STATE; const next = updater(current); - if (!next.isOpen && next.activeSurfaceId === null && next.surfaces.length === 0) { + if ( + !next.isOpen && + next.activeSurfaceId === null && + next.surfaces.length === 0 && + !next.dismissedDeviceSurfaceIds?.length + ) { if (!(threadKey in byThreadKey)) return byThreadKey; const { [threadKey]: _removed, ...rest } = byThreadKey; return rest; @@ -306,7 +316,25 @@ const userAction = ( threadKey: string, updater: (current: ThreadRightPanelState) => ThreadRightPanelState, ): Partial => ({ - byThreadKey: updateThread(state.byThreadKey, threadKey, updater), + byThreadKey: updateThread(state.byThreadKey, threadKey, (current) => { + const next = updater(current); + const removed = current.surfaces.filter( + (surface) => + surface.kind === "device" && + surface.target && + !next.surfaces.some((entry) => entry.id === surface.id), + ); + if (removed.length === 0) return next; + return { + ...next, + dismissedDeviceSurfaceIds: [ + ...new Set([ + ...(next.dismissedDeviceSurfaceIds ?? []), + ...removed.map((surface) => surface.id), + ]), + ], + }; + }), userActionRevisionByThreadKey: { ...state.userActionRevisionByThreadKey, [threadKey]: (state.userActionRevisionByThreadKey[threadKey] ?? 0) + 1, @@ -426,7 +454,22 @@ export function migratePersistedRightPanelState(persistedState: unknown): { // first survivor instead of rendering an open empty panel. const activeSurfaceId = persistedActiveSurfaceId ?? (isOpen ? (surfaces[0]?.id ?? null) : null); - return [threadKey, { isOpen, surfaces, activeSurfaceId }]; + return [ + threadKey, + { + isOpen, + surfaces, + activeSurfaceId, + ...(Array.isArray(validThreadState?.dismissedDeviceSurfaceIds) + ? { + dismissedDeviceSurfaceIds: + validThreadState.dismissedDeviceSurfaceIds.filter( + (id): id is string => typeof id === "string", + ), + } + : {}), + }, + ]; }), ) : {}; @@ -472,6 +515,40 @@ export const useRightPanelStore = create()( return upsertSurface(current, singletonSurface(kind)); }), ), + openDevice: (ref, target, automatic = false) => + set((state) => + (automatic ? automaticUpdate : userAction)(state, scopedThreadKey(ref), (current) => { + const id = + `device:${encodeURIComponent(target.hostId)}:${encodeURIComponent(target.deviceId)}` as const; + if (automatic && current.dismissedDeviceSurfaceIds?.includes(id)) return current; + const surface: RightPanelSurface = { id, kind: "device", target }; + const existing = current.surfaces.find((entry) => entry.id === id); + const surfaces = existing + ? current.surfaces.filter((entry) => entry.id !== "device") + : current.surfaces.map((entry) => (entry.id === "device" ? surface : entry)); + return upsertSurface( + { + ...current, + surfaces, + dismissedDeviceSurfaceIds: (current.dismissedDeviceSurfaceIds ?? []).filter( + (entry) => entry !== id, + ), + }, + existing ?? surface, + ); + }), + ), + renameDevice: (ref, surfaceId, title) => + set((state) => + userAction(state, scopedThreadKey(ref), (current) => ({ + ...current, + surfaces: current.surfaces.map((surface) => + surface.id === surfaceId && surface.kind === "device" + ? { ...surface, title: title.trim() || surface.target?.name || "Device" } + : surface, + ), + })), + ), openBrowser: (ref, tabId) => set((state) => userAction(state, scopedThreadKey(ref), (current) => { diff --git a/apps/web/src/state/device.ts b/apps/web/src/state/device.ts index b9c1adf86cdd..9a42cb0b1305 100644 --- a/apps/web/src/state/device.ts +++ b/apps/web/src/state/device.ts @@ -1,3 +1,4 @@ +import { useMemo } from "react"; import { useAtomValue } from "@effect/atom-react"; import { createDeviceEnvironmentAtoms } from "@t3tools/client-runtime/state/device"; import { @@ -18,7 +19,8 @@ export const deviceEnvironment = createDeviceEnvironmentAtoms(connectionAtomRunt const EMPTY_DEVICE_STATE: DeviceServiceState = { hosts: [], - hostStatus: "idle", + hostStatus: "disabled", + hostStatuses: {}, devices: [], sessions: [], onboardingCompleted: false, @@ -54,11 +56,20 @@ const deviceHubAccessAtom = Atom.family((environmentId: EnvironmentId) => .pipe(Atom.setIdleTTL(60_000), Atom.withLabel(`device-hub-access:${environmentId}`)), ); -export function useDeviceHubAccess(environmentId: EnvironmentId | null): DeviceHubAccess | null { +export function useDeviceHubAccess( + environmentId: EnvironmentId | null, + hostId = "local", +): DeviceHubAccess | null { const result = useAtomValue( environmentId === null ? EMPTY_ACCESS_ATOM : deviceHubAccessAtom(environmentId), ); - return AsyncResult.isSuccess(result) ? result.value : null; + return useMemo( + () => + AsyncResult.isSuccess(result) + ? { ...result.value, query: { ...result.value.query, hostId } } + : null, + [result, hostId], + ); } const EMPTY_ACCESS_ATOM = Atom.make(AsyncResult.initial()).pipe( diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts index 309d156430fa..895a954e2e40 100644 --- a/packages/contracts/src/device.ts +++ b/packages/contracts/src/device.ts @@ -95,6 +95,13 @@ export const DeviceServiceState = Schema.Struct({ hosts: Schema.Array(DeviceHostSummary), hostStatus: DeviceHostStatus, hostStatusDetail: Schema.optional(Schema.String), + hostStatuses: Schema.Record( + DeviceHostId, + Schema.Struct({ + status: DeviceHostStatus, + detail: Schema.optional(Schema.String), + }), + ), devices: Schema.Array(DeviceSummary), sessions: Schema.Array(DeviceSession), bootingDevices: Schema.optional( @@ -129,6 +136,7 @@ export const DeviceOpenInput = Schema.Struct({ export type DeviceOpenInput = typeof DeviceOpenInput.Type; export const DeviceCloseInput = Schema.Struct({ + hostId: Schema.optional(DeviceHostId), threadId: ThreadId, /** Omit to close every device session for the thread. */ deviceId: Schema.optional(DeviceId), From 7734c6d71b6dd5df9c9b4856cff8e419b8837fe8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 10 Sep 2026 11:59:33 -0700 Subject: [PATCH 14/15] feat(devices): target concurrent agent sessions across hosts (#10855) --- apps/server/src/device/AgentDeviceShim.ts | 28 +++++- .../src/device/AgentDeviceTarget.test.ts | 89 ++++++++++++++++++ apps/server/src/device/AgentDeviceTarget.ts | 43 +++++++++ apps/server/src/device/DeviceService.test.ts | 4 +- apps/server/src/device/DeviceService.ts | 90 ++++++++++++++++++- apps/server/src/mcp/McpDeviceToolkit.test.ts | 49 +++++++++- .../src/mcp/toolkits/device/handlers.test.ts | 13 +++ .../src/mcp/toolkits/device/handlers.ts | 66 +++++++++++--- apps/server/src/mcp/toolkits/device/tools.ts | 5 +- .../provider/CodexDeveloperInstructions.ts | 2 +- .../src/provider/Layers/ProviderService.ts | 35 ++------ 11 files changed, 376 insertions(+), 48 deletions(-) create mode 100644 apps/server/src/device/AgentDeviceTarget.test.ts create mode 100644 apps/server/src/device/AgentDeviceTarget.ts diff --git a/apps/server/src/device/AgentDeviceShim.ts b/apps/server/src/device/AgentDeviceShim.ts index c2a28127e31d..e3f92de492f3 100644 --- a/apps/server/src/device/AgentDeviceShim.ts +++ b/apps/server/src/device/AgentDeviceShim.ts @@ -1,3 +1,4 @@ +// @effect-diagnostics preferSchemaOverJson:off - JSON string literals embed paths safely into generated JavaScript. /** * A directory holding an `agent-device` launcher that runs the pinned install * with the server's Node. Prepended to provider subprocess PATHs so the agent @@ -22,11 +23,34 @@ export const ensureAgentDeviceShim = Effect.fn("AgentDeviceShim.ensure")(functio const shimDir = path.join(input.stateDir, SHIM_DIR); yield* fs.makeDirectory(shimDir, { recursive: true }); const node = process.execPath; + const launcherPath = path.join(shimDir, "agent-device-launcher.mjs"); + yield* fs.writeFileString( + launcherPath, + `import { spawn } from "node:child_process"; +const args = process.argv.slice(2); +const informational = args.length === 1 && ["help", "--help", "-h", "--version", "version"].includes(args[0]); +const hasValue = flag => { const index = args.indexOf(flag); return index >= 0 && !!args[index + 1] && !args[index + 1].startsWith("--"); }; +if (!informational && !(hasValue("--config") && hasValue("--session"))) { + console.error("Call device_open first and include its --config and --session flags."); + process.exit(1); +} +const env = { ...process.env }; +delete env.AGENT_DEVICE_DAEMON_BASE_URL; +delete env.AGENT_DEVICE_DAEMON_AUTH_TOKEN; +delete env.AGENT_DEVICE_CONFIG; +const child = spawn(${JSON.stringify(node)}, [${JSON.stringify(entryPath)}, ...args], { stdio: "inherit", env }); +child.on("error", error => { console.error(error.message); process.exitCode = 1; }); +child.on("exit", code => { process.exitCode = code ?? 1; }); +`, + ); if (platform === "win32") { - const script = `@echo off\r\n"${node}" "${entryPath}" %*\r\n`; + const script = `@echo off\r\n"${node}" "${launcherPath}" %*\r\n`; yield* fs.writeFileString(path.join(shimDir, "agent-device.cmd"), script); } else { - const script = `#!/bin/sh\nexec "${node}" "${entryPath}" "$@"\n`; + const command = [node, launcherPath] + .map((value) => "'" + value.replaceAll("'", "'\"'\"'") + "'") + .join(" "); + const script = `#!/bin/sh\nexec ${command} "$@"\n`; const shimPath = path.join(shimDir, "agent-device"); yield* fs.writeFileString(shimPath, script); yield* fs.chmod(shimPath, 0o755); diff --git a/apps/server/src/device/AgentDeviceTarget.test.ts b/apps/server/src/device/AgentDeviceTarget.test.ts new file mode 100644 index 000000000000..3552beaf3cde --- /dev/null +++ b/apps/server/src/device/AgentDeviceTarget.test.ts @@ -0,0 +1,89 @@ +// @effect-diagnostics nodeBuiltinImport:off - exercises concurrent real CLI subprocesses. +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { describe, expect, it } from "@effect/vitest"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeUtil from "node:util"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import { ensureAgentDeviceShim } from "./AgentDeviceShim.ts"; +import { + agentDeviceConfigPath, + agentDeviceSession, + writeAgentDeviceConfig, +} from "./AgentDeviceTarget.ts"; + +const exec = NodeUtil.promisify(NodeChildProcess.execFile); + +describe("host-bound agent commands", () => { + it.effect("runs two hosts concurrently and only updates the reconnected host", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const temp = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-target-" }); + const platform = yield* HostProcessPlatform; + const dir = path.join( + temp, + platform === "win32" ? "paths with spaces" : "quotes '\" $HOME `literal`", + ); + yield* fs.makeDirectory(dir); + const entryPath = path.join(dir, "cli.mjs"); + yield* fs.writeFileString( + entryPath, + `import { readFileSync } from 'node:fs'; +const args = process.argv.slice(2); +console.log(readFileSync(args[args.indexOf('--config') + 1], 'utf8')); +if (process.env.AGENT_DEVICE_DAEMON_BASE_URL) process.exit(2);`, + ); + const shim = yield* ensureAgentDeviceShim({ entryPath, stateDir: dir }); + const files = ["mini", "android"].map((host) => agentDeviceConfigPath(dir, host, path)); + for (const [index, file] of files.entries()) + yield* writeAgentDeviceConfig(file, { + baseUrl: `http://127.0.0.1:${1000 + index}`, + token: `token-${index}`, + entryPath, + }); + const invoke = (file: string) => + exec( + platform === "win32" ? process.execPath : path.join(shim, "agent-device"), + [ + ...(platform === "win32" ? [path.join(shim, "agent-device-launcher.mjs")] : []), + "snapshot", + "--config", + file, + "--session", + "test-session", + ], + { env: { ...process.env, AGENT_DEVICE_DAEMON_BASE_URL: "http://wrong-host" } }, + ).then((result) => JSON.parse(result.stdout)); + expect(yield* Effect.promise(() => Promise.all(files.map(invoke)))).toEqual([ + { daemonBaseUrl: "http://127.0.0.1:1000", daemonAuthToken: "token-0" }, + { daemonBaseUrl: "http://127.0.0.1:1001", daemonAuthToken: "token-1" }, + ]); + const second = yield* fs.readFileString(files[1]!); + yield* writeAgentDeviceConfig(files[0]!, { + baseUrl: "http://127.0.0.1:2000", + token: "new", + entryPath, + }); + expect((yield* Effect.promise(() => invoke(files[0]!))).daemonAuthToken).toBe("new"); + expect(yield* fs.readFileString(files[1]!)).toBe(second); + expect(agentDeviceSession("thread", "mini", "same-id")).not.toBe( + agentDeviceSession("thread", "android", "same-id"), + ); + for (const args of [ + ["snapshot"], + ["snapshot", "--config", files[0]!], + ["snapshot", "--config", "help"], + ["snapshot", "--config", files[0]!, "--session"], + ]) { + yield* Effect.promise(() => + expect( + exec(process.execPath, [path.join(shim, "agent-device-launcher.mjs"), ...args]), + ).rejects.toThrow("Call device_open first"), + ); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/device/AgentDeviceTarget.ts b/apps/server/src/device/AgentDeviceTarget.ts new file mode 100644 index 000000000000..379649b1257c --- /dev/null +++ b/apps/server/src/device/AgentDeviceTarget.ts @@ -0,0 +1,43 @@ +import * as NodeCrypto from "node:crypto"; +import * as Schema from "effect/Schema"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import type { AgentDeviceEndpoint } from "./DeviceHost.ts"; + +const encodeEndpoint = Schema.encodeEffect( + Schema.fromJsonString( + Schema.Struct({ daemonBaseUrl: Schema.String, daemonAuthToken: Schema.String }), + ), +); + +const key = (value: string) => + NodeCrypto.createHash("sha256").update(value).digest("hex").slice(0, 24); + +/** A stable file per host lets forwarded endpoints change without retargeting other commands. */ +export const agentDeviceConfigPath = (stateDir: string, hostId: string, path: Path.Path) => + path.join(stateDir, "device", "hosts", `${key(hostId)}.json`); + +export const agentDeviceSession = (threadId: string, hostId: string, deviceId: string) => + `t3-${key(JSON.stringify([threadId, hostId, deviceId]))}`; + +export const writeAgentDeviceConfig = Effect.fn("AgentDeviceTarget.writeConfig")(function* ( + file: string, + endpoint: AgentDeviceEndpoint, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + const content = yield* encodeEndpoint({ + daemonBaseUrl: endpoint.baseUrl, + daemonAuthToken: endpoint.token, + }); + if ((yield* fs.readFileString(file).pipe(Effect.orElseSucceed(() => ""))) === content) return; + const temporary = yield* fs.makeTempFile({ directory: path.dirname(file), prefix: ".endpoint-" }); + yield* Effect.gen(function* () { + yield* fs.chmod(temporary, 0o600); + yield* fs.writeFileString(temporary, content); + yield* fs.rename(temporary, file); + }).pipe(Effect.ensuring(fs.remove(temporary, { force: true }).pipe(Effect.ignore))); +}); diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index 49a8545bfd6d..b45db236fab6 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -16,7 +16,7 @@ import { HttpClient, HttpClientResponse } from "effect/unstable/http"; import { ServerSettingsService } from "../serverSettings.ts"; import * as DeviceHost from "./DeviceHost.ts"; -import { type DeviceService, make, stateStream } from "./DeviceService.ts"; +import { type DeviceService, makeWithHosts, stateStream } from "./DeviceService.ts"; const baseState: DeviceServiceState = { hosts: [], @@ -108,7 +108,7 @@ const fixture = Effect.fn("fixture")(function* ( starts.push("stop"); }), }; - const service = yield* make.pipe( + const service = yield* makeWithHosts(new Map([[host.id, host]])).pipe( Effect.provideService(DeviceHost.DeviceHost, host), Effect.provideService( ServerSettingsService, diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 597e1a5d2b10..866663dbf21c 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -33,6 +33,15 @@ import { LOCAL_DEVICE_HOST_ID, type ThreadId, } from "@t3tools/contracts"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import { ensureAgentDevice } from "./DeviceToolchain.ts"; +import * as ServerConfig from "../config.ts"; +import { + agentDeviceConfigPath, + agentDeviceSession, + writeAgentDeviceConfig, +} from "./AgentDeviceTarget.ts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; @@ -49,6 +58,7 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstab import * as ServerSettings from "../serverSettings.ts"; import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; +import * as ProcessRunner from "../processRunner.ts"; import * as DeviceHost from "./DeviceHost.ts"; import * as LocalDeviceHost from "./LocalDeviceHost.ts"; @@ -94,6 +104,12 @@ export interface DeviceAgentReadiness extends DeviceReadiness { export class DeviceService extends Context.Service< DeviceService, { + readonly agentCli: Effect.Effect; + readonly agentTarget: (input: { + threadId: ThreadId; + hostId: DeviceHostId; + deviceId: DeviceId; + }) => Effect.Effect, DeviceError>; readonly state: Effect.Effect; readonly subscribe: Effect.Effect, never, Scope.Scope>; readonly configure: ( @@ -138,6 +154,16 @@ const vendorPrefix = (platform: DevicePlatform) => export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ( hosts: ReadonlyMap, + configureAgent: ( + hostId: DeviceHostId, + ready: DeviceHost.DeviceHostAgentReady, + ) => Effect.Effect = (hostId) => + Effect.fail( + new DeviceHostUnavailableError({ + hostId, + reason: "Agent configuration is unavailable in this device service.", + }), + ), ) { const settings = yield* ServerSettings.ServerSettingsService; const lifecycleLock = yield* Semaphore.make(1); @@ -733,6 +759,29 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ); return DeviceService.of({ + agentCli: Effect.fail( + new DeviceHostUnavailableError({ + hostId: LOCAL_DEVICE_HOST_ID, + reason: "Agent CLI installation is unavailable in this device service.", + }), + ), + agentTarget: (input) => + Effect.gen(function* () { + const ready = yield* agentReadinessIfSupported(input.hostId); + if (!ready) + return yield* new DeviceHostUnavailableError({ + hostId: input.hostId, + reason: + "Agent device access requires enabled device support, agent access, and an available simulator platform on this host.", + }); + const configPath = yield* configureAgent(input.hostId, ready); + return [ + "--config", + configPath, + "--session", + agentDeviceSession(input.threadId, input.hostId, input.deviceId), + ]; + }), state: SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)), subscribe: PubSub.subscribe(statePubSub), configure, @@ -751,9 +800,46 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* }); }); +/** @public Service construction is part of the canonical Effect module API. */ export const make = Effect.gen(function* () { - const host = yield* DeviceHost.DeviceHost; - return yield* makeWithHosts(new Map([[host.id, host]])); + const localHost = yield* DeviceHost.DeviceHost; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const runner = yield* ProcessRunner.ProcessRunner; + const service = yield* makeWithHosts(new Map([[localHost.id, localHost]]), (hostId, ready) => { + const file = agentDeviceConfigPath(config.stateDir, hostId, path); + return writeAgentDeviceConfig(file, ready.agentDevice).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "configure agent", + reason: "settings_failed", + cause, + }), + ), + Effect.as(file), + ); + }); + return { + ...service, + agentCli: ensureAgentDevice(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.map((tool) => tool.entryPath), + Effect.mapError( + (error) => + new DeviceOperationError({ + operation: "install agent CLI", + reason: "command_failed", + cause: error, + }), + ), + ), + }; }); export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts index 32f919cbb682..24097a92faac 100644 --- a/apps/server/src/mcp/McpDeviceToolkit.test.ts +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -1,10 +1,16 @@ import { expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import { + DeviceHostUnavailableError, + EnvironmentId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import { McpSchema, McpServer } from "effect/unstable/ai"; +import * as ServerConfig from "../config.ts"; import * as DeviceService from "../device/DeviceService.ts"; import * as McpHttpServer from "./McpHttpServer.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; @@ -85,11 +91,14 @@ const DeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ sessionsForThread: () => Effect.succeed([]), screenshot: () => Effect.succeed({ device, png }), close: () => Effect.void, + agentCli: Effect.succeed("/cli"), + agentTarget: () => Effect.succeed(["--config", "/host.json", "--session", "thread-device"]), }); const TestLayer = McpHttpServer.DeviceToolkitRegistrationLive.pipe( Layer.provideMerge(McpServer.McpServer.layer), Layer.provideMerge(DeviceServiceMock), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-device-toolkit-test-" })), Layer.provide(NodeServices.layer), ); @@ -126,3 +135,41 @@ it.effect("registers the device tools and returns the screenshot as image conten }), ).pipe(Effect.provide(TestLayer)), ); + +it.effect("rejects unavailable agent access before booting or opening a device", () => { + const unavailable = Layer.mock(DeviceService.DeviceService)({ + list: Effect.succeed(state), + agentTarget: () => + Effect.fail( + new DeviceHostUnavailableError({ hostId: "local", reason: "Agent access is disabled." }), + ), + open: () => Effect.die("Must not boot or register a device when agent access fails"), + }); + return Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const result = yield* server + .callTool({ name: "device_open", arguments: { platform: "ios" } }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation(["device"])), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(result.isError).toBe(true); + expect(result.content).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: "text", + text: expect.stringContaining("Agent access is disabled."), + }), + ]), + ); + }).pipe( + Effect.scoped, + Effect.provide( + McpHttpServer.DeviceToolkitRegistrationLive.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provide(unavailable), + Layer.provide(NodeServices.layer), + ), + ), + ); +}); diff --git a/apps/server/src/mcp/toolkits/device/handlers.test.ts b/apps/server/src/mcp/toolkits/device/handlers.test.ts index e37b2a8b22c2..110d4cbba72a 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.test.ts @@ -30,6 +30,19 @@ describe("device tool helpers", () => { expect(text).toContain("XCTest runner"); }); + it("uses the absolute launcher in every quick-start command", () => { + const text = agentDeviceQuickStart( + device, + ["--session", "thread-1", "--config", "/tmp/host.json"], + "/tmp/t3 tools/agent-device", + ); + expect(text).toContain( + "'/tmp/t3 tools/agent-device' snapshot -i --session thread-1 --config /tmp/host.json", + ); + expect(text).not.toContain(" agent-device "); + expect(text).not.toContain("is on PATH"); + }); + it("reads PNG dimensions from the IHDR chunk", () => { const png = new Uint8Array(24); new DataView(png.buffer).setUint32(0, 0x89504e47); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index 5ad57031306e..3bc89d22bc60 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -8,6 +8,10 @@ import { LOCAL_DEVICE_HOST_ID, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Path from "effect/Path"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { ServerConfig } from "../../../config.ts"; +import { ensureAgentDeviceShim } from "../../../device/AgentDeviceShim.ts"; import * as DeviceService from "../../../device/DeviceService.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; @@ -26,24 +30,37 @@ export function agentDeviceTargetArgs(device: DeviceSummary): ReadonlyArray + /^[a-zA-Z0-9_./:-]+$/.test(arg) ? arg : "'" + arg.replaceAll("'", "'\"'\"'") + "'", + ) + .join(" "); const platformNotes = device.platform === "ios" ? "First use builds an XCTest runner and can take a couple of minutes; later commands are fast." : "The Android snapshot helper installs itself on first use."; return [ `The user is watching ${device.name} (${device.version}) in the Device panel.`, - `Drive it with the agent-device CLI, which is on PATH and already connected to this environment. Always pass ${target}.`, + `Drive it with ${executable}. Use this exact executable path; login shells may reset PATH. Always pass ${target}.`, "Typical loop:", - ` agent-device open ${target} # or: open `, - ` agent-device snapshot -i ${target} # accessibility tree with @eN refs`, - ` agent-device click @e3 ${target}`, - ` agent-device fill @e5 "text" ${target}`, - ` agent-device screenshot /tmp/shot.png ${target} # or call device_screenshot`, - ` agent-device install ${target}`, - "Prefer snapshot refs over coordinates. Run `agent-device help` for workflow guides and `agent-device --help` for flags.", + ` ${executable} open ${target} # or: open `, + ` ${executable} snapshot -i ${target} # accessibility tree with @eN refs`, + ` ${executable} click @e3 ${target}`, + ` ${executable} fill @e5 "text" ${target}`, + ` ${executable} screenshot /tmp/shot.png ${target} # or call device_screenshot`, + ` ${executable} install ${target}`, + `Prefer snapshot refs over coordinates. Run ${executable} help for workflow guides and ${executable} --help for flags.`, "Do not call simctl, adb, xcrun, or serve-sim directly while these tools are attached; use agent-device.", + "For remote hosts, arrange builds, app installation, and any Metro reverse forwarding yourself. T3 provides discovery, streaming, and control only.", + "Keep the returned --config and --session flags on every command. Other hosts can be used concurrently; opening one does not switch these commands.", platformNotes, ].join("\n"); } @@ -136,6 +153,12 @@ const handlers = { }); } const target = yield* pickDevice(state.devices, input); + // Resolve consent and agent connectivity before booting or registering a session. + const agentArgs = yield* devices.agentTarget({ + threadId: scope.threadId, + hostId: target.hostId, + deviceId: target.id, + }); const session = yield* devices.open({ threadId: scope.threadId, hostId: target.hostId, @@ -147,10 +170,29 @@ const handlers = { after.devices.find( (candidate) => candidate.hostId === session.hostId && candidate.id === session.deviceId, ) ?? target; + const targetArgs = [...agentDeviceTargetArgs(device), ...agentArgs]; + const config = yield* ServerConfig; + const path = yield* Path.Path; + const platform = yield* HostProcessPlatform; + const shimDir = yield* ensureAgentDeviceShim({ + entryPath: yield* devices.agentCli, + stateDir: config.stateDir, + }).pipe( + Effect.mapError( + () => + new DeviceToolUnavailableError({ + reason: "Could not prepare the agent-device launcher.", + }), + ), + ); + const command = path.join( + shimDir, + platform === "win32" ? "agent-device.cmd" : "agent-device", + ); return { device, - agentDevice: { command: "agent-device", targetArgs: agentDeviceTargetArgs(device) }, - quickStart: agentDeviceQuickStart(device), + agentDevice: { command, targetArgs }, + quickStart: agentDeviceQuickStart(device, targetArgs, command), }; }).pipe(Effect.mapError(toolError)), device_screenshot: (input) => diff --git a/apps/server/src/mcp/toolkits/device/tools.ts b/apps/server/src/mcp/toolkits/device/tools.ts index e98d6ab64829..58bfed41d940 100644 --- a/apps/server/src/mcp/toolkits/device/tools.ts +++ b/apps/server/src/mcp/toolkits/device/tools.ts @@ -8,6 +8,9 @@ import { DeviceToolTargetInput, } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import { ServerConfig } from "../../../config.ts"; import { Tool, Toolkit } from "effect/unstable/ai"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; @@ -48,7 +51,7 @@ const DeviceOpenTool = Tool.make("device_open", { parameters: DeviceToolOpenInput, success: DeviceToolOpenResult, failure: DeviceToolError, - dependencies, + dependencies: [...dependencies, FileSystem.FileSystem, Path.Path, ServerConfig], }) .annotate(Tool.Title, "Open device") .annotate(Tool.Readonly, false) diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index 85784d21ca4b..6a7fee351bce 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -16,7 +16,7 @@ const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = ` ## T3 Code devices -The \`t3-code\` MCP server also exposes \`device_*\` tools for iOS Simulators and Android Emulators on this environment. For mobile verification, call \`device_list\`, then \`device_open\` so the user can watch the device in their Device panel; its result explains how to drive the device. Driving happens through the \`agent-device\` CLI, which is on PATH and already connected: prefer \`agent-device snapshot -i\` refs over coordinates, and use \`device_screenshot\` when you need to see the screen. Do not call simctl, adb, xcrun, or serve-sim directly while these tools are present. If \`device_list\` reports a platform as unavailable, say so instead of trying another route. +The \`t3-code\` MCP server also exposes \`device_*\` tools for iOS Simulators and Android Emulators on this environment. For mobile verification, call \`device_list\`, then \`device_open\` so the user can watch the device in their Device panel; its result explains how to drive the device. Driving happens through the \`agent-device\` CLI, which is on PATH. Keep the host config and session flags returned by \`device_open\` on every command so concurrent devices stay independent: prefer \`agent-device snapshot -i\` refs over coordinates, and use \`device_screenshot\` when you need to see the screen. Do not call simctl, adb, xcrun, or serve-sim directly while these tools are present. If \`device_list\` reports a platform as unavailable, say so instead of trying another route. `; export interface T3CodeToolAvailability { diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index f2d94e735b0d..d04dcae7f126 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -54,8 +54,8 @@ import * as Stream from "effect/Stream"; import { appendUserInputAttachmentPaths } from "../userInputAttachments.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import * as ServerConfig from "../../config.ts"; -import { ensureAgentDeviceShim } from "../../device/AgentDeviceShim.ts"; import * as DeviceService from "../../device/DeviceService.ts"; +import { ensureAgentDeviceShim } from "../../device/AgentDeviceShim.ts"; import type * as McpInvocationContext from "../../mcp/McpInvocationContext.ts"; import { increment, @@ -254,8 +254,6 @@ export interface ProviderServiceLiveOptions { * test see whether a credential was requested at all. */ readonly issueMcpCredential?: typeof McpSessionRegistry.issueActiveMcpCredential; - /** Overrides the device host lookup used to build the agent-device environment. */ - readonly deviceReadiness?: DeviceService.DeviceService["Service"]["agentReadinessIfSupported"]; } interface TurnAnalyticsMetadata { @@ -484,14 +482,6 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ); const issueMcpCredential = options?.issueMcpCredential ?? McpSessionRegistry.issueActiveMcpCredential; - const deviceReadiness = - options?.deviceReadiness ?? - (() => - Effect.serviceOption(DeviceService.DeviceService).pipe( - Effect.flatMap((service) => - Option.isSome(service) ? service.value.agentReadinessIfSupported() : Effect.succeed(null), - ), - )); const fileSystem = yield* FileSystem.FileSystem; const pathService = yield* Path.Path; const runtimeEventPubSub = yield* PubSub.unbounded(); @@ -914,26 +904,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( return capabilities; }); - /** - * Starting a session with device access also brings the device host up, so - * the `agent-device` CLI is on the provider's PATH from its first turn. The - * environment is fixed at spawn time, so a host started later by - * `device_open` could not reach an already-running agent. Tools install once - * and the host is idempotent, so this is cheap after the first session; - * a host that fails to start withholds only the CLI, not the MCP tools. - */ + /** Install only the local CLI here. device_open supplies a separate config for each host. */ const hostPlatform = yield* HostProcessPlatform; const agentDeviceEnvironment = Effect.gen(function* () { - const readiness = yield* deviceReadiness().pipe( + const devices = yield* Effect.serviceOption(DeviceService.DeviceService); + if (Option.isNone(devices)) return undefined; + const entryPath = yield* devices.value.agentCli.pipe( Effect.catch((cause) => - Effect.logWarning("Device host unavailable; starting session without agent-device", { - cause, - }).pipe(Effect.as(null)), + Effect.logWarning("Agent device CLI unavailable", { cause }).pipe(Effect.as(null)), ), ); - if (!readiness) return undefined; + if (!entryPath) return undefined; const shimDir = yield* ensureAgentDeviceShim({ - entryPath: readiness.agentDevice.entryPath, + entryPath, stateDir: serverConfig.stateDir, }).pipe( Effect.provideService(FileSystem.FileSystem, fileSystem), @@ -944,8 +927,6 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( return { PATH: shimDir, PATH_SEPARATOR: hostPlatform === "win32" ? ";" : ":", - AGENT_DEVICE_DAEMON_BASE_URL: readiness.agentDevice.baseUrl, - AGENT_DEVICE_DAEMON_AUTH_TOKEN: readiness.agentDevice.token, AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", } satisfies Record; }); From d2eeacd8ccde4d8763ca315b4cddf2964db93329 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 10 Sep 2026 11:59:34 -0700 Subject: [PATCH 15/15] feat(devices): connect simulator hosts over SSH (#10856) --- apps/server/package.json | 1 + apps/server/src/auth/RpcAuthorization.ts | 1 + apps/server/src/device/DeviceActions.test.ts | 1 + apps/server/src/device/DeviceActions.ts | 2 +- apps/server/src/device/DeviceHost.ts | 3 +- .../server/src/device/DeviceMultiHost.test.ts | 54 ++- apps/server/src/device/DeviceService.test.ts | 1 + apps/server/src/device/DeviceService.ts | 272 ++++++++--- apps/server/src/device/DeviceToolchain.ts | 4 +- apps/server/src/device/LocalDeviceHost.ts | 1 + apps/server/src/device/SshDeviceHost.test.ts | 131 ++++++ apps/server/src/device/SshDeviceHost.ts | 434 ++++++++++++++++++ .../server/src/device/sshDeviceScript.test.ts | 220 +++++++++ apps/server/src/device/sshDeviceScript.ts | 192 ++++++++ apps/server/src/mcp/McpDeviceToolkit.test.ts | 1 + .../src/mcp/toolkits/device/handlers.ts | 3 + apps/server/src/ws.ts | 4 + .../device/DeviceHostAvailability.tsx | 27 ++ .../settings/DeviceHostsSettings.tsx | 334 ++++++++++++++ .../settings/IntegrationsSettings.tsx | 72 ++- .../src/components/settings/settingsSearch.ts | 6 + docs/internals/devices.md | 12 +- docs/user/devices.md | 29 +- packages/client-runtime/src/state/device.ts | 4 + packages/contracts/src/device.ts | 24 +- packages/contracts/src/rpc.ts | 10 + packages/contracts/src/settings.test.ts | 13 + packages/contracts/src/settings.ts | 3 + packages/shared/src/serverSettings.test.ts | 10 + pnpm-lock.yaml | 3 + 30 files changed, 1794 insertions(+), 78 deletions(-) create mode 100644 apps/server/src/device/SshDeviceHost.test.ts create mode 100644 apps/server/src/device/SshDeviceHost.ts create mode 100644 apps/server/src/device/sshDeviceScript.test.ts create mode 100644 apps/server/src/device/sshDeviceScript.ts create mode 100644 apps/web/src/components/device/DeviceHostAvailability.tsx create mode 100644 apps/web/src/components/settings/DeviceHostsSettings.tsx diff --git a/apps/server/package.json b/apps/server/package.json index 7d27d39d2eb3..84a8c6a5988b 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -40,6 +40,7 @@ "@effect/vitest": "catalog:", "@t3tools/contracts": "workspace:*", "@t3tools/shared": "workspace:*", + "@t3tools/ssh": "workspace:*", "@t3tools/tailscale": "workspace:*", "@t3tools/web": "workspace:*", "@types/bun": "1.3.14", diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 97afa4f40775..d7d1be455fa1 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -145,6 +145,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.subscribePreviewEvents]: AuthOrchestrationReadScope, [WS_METHODS.subscribeDiscoveredLocalServers]: AuthOrchestrationReadScope, [WS_METHODS.deviceConfigure]: AuthOrchestrationOperateScope, + [WS_METHODS.deviceTestHost]: AuthOrchestrationOperateScope, [WS_METHODS.deviceList]: AuthOrchestrationReadScope, [WS_METHODS.deviceOpen]: AuthOrchestrationOperateScope, [WS_METHODS.deviceClose]: AuthOrchestrationOperateScope, diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts index ed6b5a87335d..c22dcfe86610 100644 --- a/apps/server/src/device/DeviceActions.test.ts +++ b/apps/server/src/device/DeviceActions.test.ts @@ -16,6 +16,7 @@ const makeReady = ( ) => { const calls: Call[] = []; const ready: DeviceHostReady = { + nodePath: process.execPath, hub: { origin: "http://127.0.0.1:1" }, helpers, run: (command, args, options) => { diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts index 12781b31b4b5..509dac526b93 100644 --- a/apps/server/src/device/DeviceActions.ts +++ b/apps/server/src/device/DeviceActions.ts @@ -327,7 +327,7 @@ const serveSimPermissions = ( reason: "helper_missing", }); yield* ready - .run(process.execPath, [ + .run(ready.nodePath, [ cli, "permissions", input.decision, diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts index 8d1bafbc5dfc..34ff2769362e 100644 --- a/apps/server/src/device/DeviceHost.ts +++ b/apps/server/src/device/DeviceHost.ts @@ -46,11 +46,12 @@ export interface DeviceHubEndpoint { export interface AgentDeviceEndpoint { readonly baseUrl: string; readonly token: string; - /** Absolute path of the agent-device entry script for the provider PATH shim. */ + /** Host-local path of the agent-device entry script. The provider uses a separate local CLI install. */ readonly entryPath: string; } export interface DeviceHostReady { + readonly nodePath: string; readonly hub: DeviceHubEndpoint; /** * Runs a host command (`xcrun`, `adb`, or a helper bundled with the hub) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index d586dffa1567..7a40e1e53c72 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -1,5 +1,7 @@ import { expect, it } from "@effect/vitest"; import { ThreadId } from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; import * as Effect from "effect/Effect"; import { HttpClient, HttpClientResponse } from "effect/unstable/http"; import { ServerSettingsService } from "../serverSettings.ts"; @@ -10,6 +12,7 @@ it.effect("keeps hosts independent when serials collide and another host fails", Effect.gen(function* () { const host = (id: string, failed = false): DeviceHost["Service"] => { const ready = { + nodePath: process.execPath, hub: { origin: `http://${id}` }, agentDevice: { baseUrl: `http://${id}`, token: "test", entryPath: "/agent-device" }, run: () => Effect.succeed({ stdout: "", stderr: "", code: 0 }), @@ -20,10 +23,10 @@ it.effect("keeps hosts independent when serials collide and another host fails", summary: Effect.succeed({ id, label: id, - kind: "local", + kind: id === "b" ? "ssh" : "local", hubInstalled: true, agentDeviceInstalled: true, - platforms: [{ platform: "android", available: true }], + platforms: id === "b" ? [] : [{ platform: "android", available: true }], }), platformAvailability: (platform) => Effect.succeed({ platform, available: true }), ensureReady: () => @@ -59,9 +62,19 @@ it.effect("keeps hosts independent when serials collide and another host fails", ), ); const hosts = new Map(["a", "b", "offline"].map((id) => [id, host(id, id === "offline")])); - const service = yield* makeWithHosts(hosts).pipe( - Effect.provideService(HttpClient.HttpClient, http), - ); + const writeStarted = yield* Deferred.make(); + const finishWrite = yield* Deferred.make(); + const order: string[] = []; + const service = yield* makeWithHosts(hosts, undefined, () => + Effect.gen(function* () { + order.push("write started"); + yield* Deferred.succeed(writeStarted, undefined); + yield* Deferred.await(finishWrite); + order.push("write finished"); + return "/host-config.json"; + }), + ).pipe(Effect.provideService(HttpClient.HttpClient, http)); + expect(yield* service.agentReadinessIfSupported("b")).not.toBeNull(); const listed = yield* service.list; expect(listed.devices.map((device) => device.hostId).sort()).toEqual(["a", "b"]); expect(listed.hostStatuses.offline?.status).toBe("failed"); @@ -74,8 +87,35 @@ it.effect("keeps hosts independent when serials collide and another host fails", expect(state.sessions.map((session) => session.hostId)).toEqual(["b"]); expect(state.hostStatuses.a?.status).toBe("ready"); expect(state.hostStatuses.offline?.status).toBe("failed"); - yield* service.agentReadinessIfSupported("b"); - expect((yield* service.state).hostStatuses.b?.status).toBe("ready"); + const targeting = yield* service + .agentTarget({ threadId, hostId: "b", deviceId: "emulator-5554" }) + .pipe(Effect.forkChild); + yield* Deferred.await(writeStarted); + const replacing = yield* service + .withLifecycleLock( + Effect.gen(function* () { + order.push("replace"); + hosts.set("b", host("b")); + yield* service.refreshHosts; + }), + ) + .pipe(Effect.forkChild); + yield* Deferred.succeed(finishWrite, undefined); + yield* Fiber.join(targeting); + yield* Fiber.join(replacing); + expect(order).toEqual(["write started", "write finished", "replace"]); + const replaced = yield* service.state; + expect(replaced.sessions).toEqual([]); + expect(replaced.devices.map((device) => device.hostId)).toEqual(["a"]); + expect(replaced.hostStatuses.b).toBeUndefined(); + yield* service.open({ threadId, hostId: "b", deviceId: "emulator-5554", platform: "android" }); + hosts.delete("b"); + yield* service.refreshHosts; + yield* service.setHostStatus("b", { status: "ready" }); + expect((yield* service.state).hostStatuses.b).toBeUndefined(); + expect((yield* service.state).sessions).toEqual([]); + yield* service.agentReadinessIfSupported("a"); + expect((yield* service.state).hostStatuses.a?.status).toBe("ready"); yield* service.configure({ enabled: false }); expect((yield* service.state).hostStatuses).toEqual({}); }).pipe( diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index b45db236fab6..46e159dbfc33 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -70,6 +70,7 @@ const fixture = Effect.fn("fixture")(function* ( let booted = false; let shutDown = false; const ready: DeviceHost.DeviceHostReady = { + nodePath: process.execPath, hub: { origin: "http://device.test" }, helpers: { serveSimAxSettings: null, serveSimCli: null }, run: () => Effect.succeed({ code: 0, stdout: "Pixel_API_35\n", stderr: "" }), diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 866663dbf21c..a60cda76c648 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -30,6 +30,8 @@ import { type DeviceSession, type DeviceShutdownInput, type DeviceSummary, + type SshDeviceHostConfig, + type DeviceHostSummary, LOCAL_DEVICE_HOST_ID, type ThreadId, } from "@t3tools/contracts"; @@ -60,6 +62,8 @@ import * as ServerSettings from "../serverSettings.ts"; import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as DeviceHost from "./DeviceHost.ts"; +import * as SshDeviceHost from "./SshDeviceHost.ts"; +import * as Exit from "effect/Exit"; import * as LocalDeviceHost from "./LocalDeviceHost.ts"; /** Origin-relative prefix the hub is proxied under. See DeviceHubProxy. */ @@ -105,6 +109,9 @@ export class DeviceService extends Context.Service< DeviceService, { readonly agentCli: Effect.Effect; + readonly testHost: ( + config: SshDeviceHostConfig, + ) => Effect.Effect; readonly agentTarget: (input: { threadId: ThreadId; hostId: DeviceHostId; @@ -154,6 +161,13 @@ const vendorPrefix = (platform: DevicePlatform) => export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ( hosts: ReadonlyMap, + testHost: DeviceService["Service"]["testHost"] = (host) => + Effect.fail( + new DeviceHostUnavailableError({ + hostId: host.id, + reason: "SSH probing is unavailable in this device service.", + }), + ), configureAgent: ( hostId: DeviceHostId, ready: DeviceHost.DeviceHostAgentReady, @@ -183,6 +197,7 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); const statePubSub = yield* PubSub.unbounded(); const initialHosts = yield* Effect.forEach(hosts.values(), (host) => host.summary); + let publishedHosts = new Map(hosts); const stateRef = yield* SynchronizedRef.make({ state: { hosts: initialHosts, @@ -217,13 +232,17 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* hostId: DeviceHostId, status: DeviceServiceState["hostStatuses"][string], ) => - publish((state) => ({ - ...state, - ...(hostId === LOCAL_DEVICE_HOST_ID - ? { hostStatus: status.status, hostStatusDetail: status.detail } - : {}), - hostStatuses: { ...state.hostStatuses, [hostId]: status }, - })); + Effect.suspend(() => + !hosts.has(hostId) + ? SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)) + : publish((state) => ({ + ...state, + ...(hostId === LOCAL_DEVICE_HOST_ID + ? { hostStatus: status.status, hostStatusDetail: status.detail } + : {}), + hostStatuses: { ...state.hostStatuses, [hostId]: status }, + })), + ); const readiness: DeviceService["Service"]["readiness"] = Effect.fn("DeviceService.readiness")( function* (hostId) { @@ -245,6 +264,11 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), ), ); + if (hosts.get(host.id) !== host) + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: "Host configuration changed. Retry the operation.", + }); const { state } = yield* SynchronizedRef.get(stateRef); if (state.hostStatuses[host.id]?.status !== "ready") { yield* setHostStatus(host.id, { status: "ready" }); @@ -260,7 +284,8 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* if (!(yield* readDeviceSettings).enabled) return null; const host = yield* resolveHost(hostId); const summary = yield* host.summary; - if (!summary.platforms.some((platform) => platform.available)) return null; + if (summary.kind === "local" && !summary.platforms.some((platform) => platform.available)) + return null; return yield* readiness(host.id); }); @@ -270,7 +295,8 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* if (!deviceSettings.enabled || !deviceSettings.agentAccessEnabled) return null; const host = yield* resolveHost(hostId); const summary = yield* host.summary; - if (!summary.platforms.some((platform) => platform.available)) return null; + if (summary.kind === "local" && !summary.platforms.some((platform) => platform.available)) + return null; const ready = yield* host .ensureAgentReady((phase) => setHostStatus(host.id, { status: phase }).pipe(Effect.asVoid)) .pipe( @@ -364,11 +390,12 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* }); const refresh = Effect.fn("DeviceService.refresh")(function* (ready: DeviceReadiness) { + const host = hosts.get(ready.hostId); const { devices, detail } = yield* fetchDevices(ready); const hostSummaries = yield* Effect.forEach(hosts.values(), (host) => host.summary); return yield* lifecycleLock.withPermit( Effect.gen(function* () { - if (!(yield* readDeviceSettings).enabled) + if (!(yield* readDeviceSettings).enabled || !host || hosts.get(ready.hostId) !== host) return (yield* SynchronizedRef.get(stateRef)).state; return yield* publish((state) => ({ ...state, @@ -578,6 +605,11 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ), ); } + if (hosts.get(host.id) !== host) + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: "Host configuration changed. Retry the operation.", + }); const openedAt = DateTime.formatIso(yield* DateTime.now); const session: DeviceSession = { threadId: input.threadId, @@ -758,46 +790,76 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* Effect.map(({ state }) => state.sessions.filter((session) => session.threadId === threadId)), ); - return DeviceService.of({ - agentCli: Effect.fail( - new DeviceHostUnavailableError({ - hostId: LOCAL_DEVICE_HOST_ID, - reason: "Agent CLI installation is unavailable in this device service.", - }), - ), - agentTarget: (input) => - Effect.gen(function* () { - const ready = yield* agentReadinessIfSupported(input.hostId); - if (!ready) - return yield* new DeviceHostUnavailableError({ - hostId: input.hostId, - reason: - "Agent device access requires enabled device support, agent access, and an available simulator platform on this host.", - }); - const configPath = yield* configureAgent(input.hostId, ready); - return [ - "--config", - configPath, - "--session", - agentDeviceSession(input.threadId, input.hostId, input.deviceId), - ]; - }), - state: SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)), - subscribe: PubSub.subscribe(statePubSub), - configure, - list, - open, - close, - shutdown, - detail, - action, - screenshot, - readiness, - readinessIfSupported, - agentReadinessIfSupported, - currentReadiness, - sessionsForThread, - }); + return { + ...DeviceService.of({ + testHost, + agentCli: Effect.fail( + new DeviceHostUnavailableError({ + hostId: LOCAL_DEVICE_HOST_ID, + reason: "Agent CLI installation is unavailable in this device service.", + }), + ), + agentTarget: (input) => + Effect.gen(function* () { + const host = yield* resolveHost(input.hostId); + const ready = yield* agentReadinessIfSupported(input.hostId); + if (!ready) + return yield* new DeviceHostUnavailableError({ + hostId: input.hostId, + reason: + "Agent device access requires enabled device support, agent access, and an available simulator platform on this host.", + }); + const configPath = yield* lifecycleLock.withPermit( + Effect.gen(function* () { + if (hosts.get(host.id) !== host) + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: "Host configuration changed. Retry the operation.", + }); + return yield* configureAgent(input.hostId, ready); + }), + ); + return [ + "--config", + configPath, + "--session", + agentDeviceSession(input.threadId, input.hostId, input.deviceId), + ]; + }), + state: SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)), + subscribe: PubSub.subscribe(statePubSub), + configure, + list, + open, + close, + shutdown, + detail, + action, + screenshot, + readiness, + readinessIfSupported, + agentReadinessIfSupported, + currentReadiness, + sessionsForThread, + }), + setHostStatus, + withLifecycleLock: lifecycleLock.withPermit, + refreshHosts: Effect.gen(function* () { + const summaries = yield* Effect.forEach(hosts.values(), (host) => host.summary); + const unchanged = (id: DeviceHostId) => + hosts.has(id) && hosts.get(id) === publishedHosts.get(id); + yield* publish((state) => ({ + ...state, + hosts: summaries, + hostStatuses: Object.fromEntries( + Object.entries(state.hostStatuses).filter(([id]) => unchanged(id)), + ), + devices: state.devices.filter((device) => unchanged(device.hostId)), + sessions: state.sessions.filter((session) => unchanged(session.hostId)), + })); + publishedHosts = new Map(hosts); + }), + }; }); /** @public Service construction is part of the canonical Effect module API. */ @@ -807,7 +869,12 @@ export const make = Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const runner = yield* ProcessRunner.ProcessRunner; - const service = yield* makeWithHosts(new Map([[localHost.id, localHost]]), (hostId, ready) => { + const settings = yield* ServerSettings.ServerSettingsService; + const scope = yield* Scope.Scope; + const hosts = new Map([ + [localHost.id, localHost], + ]); + const configureAgent = (hostId: DeviceHostId, ready: DeviceHost.DeviceHostAgentReady) => { const file = agentDeviceConfigPath(config.stateDir, hostId, path); return writeAgentDeviceConfig(file, ready.agentDevice).pipe( Effect.provideService(FileSystem.FileSystem, fs), @@ -822,7 +889,108 @@ export const make = Effect.gen(function* () { ), Effect.as(file), ); - }); + }; + const probeContext = + yield* Effect.context>>(); + const service = yield* makeWithHosts( + hosts, + (host) => + SshDeviceHost.probe(host).pipe( + Effect.provide(probeContext), + Effect.mapError( + (error) => + new DeviceOperationError({ + operation: "probe host", + reason: "request_failed", + cause: error, + }), + ), + ), + configureAgent, + ); + const hostContext = + yield* Effect.context>>(); + const configured = new Map(); + const reconcile = (next: ReadonlyArray) => + Effect.gen(function* () { + const removed = yield* service.withLifecycleLock( + Effect.gen(function* () { + const removed: Array<{ id: string; scope: Scope.Closeable }> = []; + for (const [id, previous] of configured) { + if ( + next.some( + (host) => + host.id === id && + host.label === previous.config.label && + host.target === previous.config.target && + host.port === previous.config.port && + host.identityFile === previous.config.identityFile, + ) + ) + continue; + hosts.delete(id); + configured.delete(id); + removed.push({ id, scope: previous.scope }); + } + yield* service.refreshHosts; + return removed; + }), + ); + // Stop old writers before deleting config files or publishing replacements, without blocking healthy hosts. + yield* Effect.forEach( + removed, + ({ id, scope }) => + Effect.gen(function* () { + yield* Scope.close(scope, Exit.void); + yield* fs + .remove(agentDeviceConfigPath(config.stateDir, id, path), { force: true }) + .pipe(Effect.ignore); + }), + { concurrency: 4, discard: true }, + ); + yield* service.withLifecycleLock( + Effect.gen(function* () { + for (const host of next) { + if (configured.has(host.id)) continue; + const hostScope = yield* Scope.fork(scope); + const instance = yield* SshDeviceHost.make( + host, + (ready) => + configureAgent(host.id, ready).pipe( + Effect.asVoid, + Effect.mapError( + (error) => + new DeviceHost.DeviceHostError({ + hostId: host.id, + step: "configuring agent access", + cause: error, + }), + ), + ), + (status, detail) => + service + .setHostStatus(host.id, { status, ...(detail ? { detail } : {}) }) + .pipe(Effect.asVoid), + ).pipe(Effect.provideService(Scope.Scope, hostScope), Effect.provide(hostContext)); + hosts.set(host.id, instance); + configured.set(host.id, { config: host, scope: hostScope }); + } + yield* service.refreshHosts; + }), + ); + }); + const changes = yield* settings.subscribeChanges; + yield* reconcile((yield* settings.getSettings).deviceHosts); + yield* changes.pipe( + Stream.runForEach((value) => reconcile(value.deviceHosts)), + Effect.forkIn(scope), + ); + yield* Effect.addFinalizer(() => + Effect.forEach(configured.values(), (value) => Scope.close(value.scope, Exit.void), { + discard: true, + concurrency: 4, + }), + ); return { ...service, agentCli: ensureAgentDevice(config.baseDir).pipe( diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts index e6b43cd81ee8..fa8d8cd11d17 100644 --- a/apps/server/src/device/DeviceToolchain.ts +++ b/apps/server/src/device/DeviceToolchain.ts @@ -25,9 +25,9 @@ import * as Semaphore from "effect/Semaphore"; import * as ProcessRunner from "../processRunner.ts"; const DEVICE_HUB_PACKAGE = "expo-device-hub"; -const DEVICE_HUB_VERSION = "0.9.0"; +export const DEVICE_HUB_VERSION = "0.9.0"; const AGENT_DEVICE_PACKAGE = "agent-device"; -const AGENT_DEVICE_VERSION = "0.20.10"; +export const AGENT_DEVICE_VERSION = "0.20.10"; const INSTALL_TIMEOUT = Duration.minutes(10); const installLock = Semaphore.makeUnsafe(1); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index b24ebfbd98b6..5a18d6e826a6 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -644,6 +644,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const toReady = (running: RunningHost): DeviceHost.DeviceHostReady => ({ hub: { origin: running.hub.origin } satisfies DeviceHost.DeviceHubEndpoint, + nodePath: process.execPath, run, helpers: running.helpers, }); diff --git a/apps/server/src/device/SshDeviceHost.test.ts b/apps/server/src/device/SshDeviceHost.test.ts new file mode 100644 index 000000000000..d968d3383fae --- /dev/null +++ b/apps/server/src/device/SshDeviceHost.test.ts @@ -0,0 +1,131 @@ +// @effect-diagnostics preferSchemaOverJson:off - the external process fixture emits raw JSON over SSH stdout. +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Net from "@t3tools/shared/Net"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as PlatformError from "effect/PlatformError"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; +import * as ServerConfig from "../config.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import * as SshDeviceHost from "./SshDeviceHost.ts"; + +it.effect("preserves installed status after probes and cleans failed agent activation", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped(); + const modes: string[] = []; + let forwards = 0; + let failForward = true; + let rejectConfig = true; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (command._tag !== "StandardCommand") return yield* Effect.die("Unexpected command"); + const forwarding = command.args.includes("-N"); + let output = ""; + if (forwarding) { + if (failForward) { + failForward = false; + return yield* PlatformError.systemError({ + _tag: "AlreadyExists", + module: "ChildProcess", + method: "spawn", + description: "Port already bound", + }); + } + forwards++; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + forwards--; + }), + ); + } else { + const stdin = command.options.stdin; + if ( + !stdin || + typeof stdin !== "object" || + !("stream" in stdin) || + !Stream.isStream(stdin.stream) + ) + return yield* Effect.die("Missing script"); + const script = yield* stdin.stream.pipe( + Stream.decodeText(), + Stream.runFold( + () => "", + (a, b) => a + b, + ), + ); + const mode = /const mode = "([^"]+)"/.exec(script)?.[1] ?? ""; + modes.push(mode); + output = JSON.stringify({ + nodePath: "/node", + platforms: [{ platform: "ios", available: true }], + hubPort: 1234, + helpers: { serveSimAxSettings: null, serveSimCli: null }, + ...(mode === "agent-start" + ? { daemonPort: 1235, token: "fixture", entryPath: "/agent.mjs" } + : {}), + }); + } + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(123), + stdout: Stream.make(new TextEncoder().encode(output)), + stderr: Stream.empty, + all: Stream.empty, + exitCode: forwarding ? Effect.never : Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(forwarding), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ); + const host = yield* SshDeviceHost.make( + { id: "test", label: "Test", target: "test.example" }, + () => + rejectConfig + ? Effect.fail( + new DeviceHost.DeviceHostError({ + hostId: "test", + step: "configuring agent access", + cause: new Error("fixture failure"), + }), + ) + : Effect.void, + ).pipe( + Effect.provide(Layer.mergeAll(ServerConfig.layerTest(home, home), Net.layer)), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, new Response("ok"))), + ), + ), + ); + yield* host.ensureReady(() => Effect.void); + expect(forwards).toBe(1); + expect(modes.filter((mode) => mode === "start")).toHaveLength(2); + yield* host.platformAvailability("ios"); + expect((yield* host.summary).hubInstalled).toBe(true); + const failed = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.result); + expect(failed._tag).toBe("Failure"); + expect(forwards).toBe(0); + expect(modes.at(-1)).toBe("stop-agent"); + expect(yield* host.current).toBeNull(); + rejectConfig = false; + yield* host.ensureAgentReady(() => Effect.void); + yield* host.platformAvailability("ios"); + expect((yield* host.summary).agentDeviceInstalled).toBe(true); + yield* host.stopAgent; + expect(forwards).toBe(1); + yield* host.stop; + expect(forwards).toBe(0); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/device/SshDeviceHost.ts b/apps/server/src/device/SshDeviceHost.ts new file mode 100644 index 000000000000..4ccda0fdefe9 --- /dev/null +++ b/apps/server/src/device/SshDeviceHost.ts @@ -0,0 +1,434 @@ +import * as NodeCrypto from "node:crypto"; +import { + type DeviceHostSummary, + DevicePlatformAvailability, + type SshDeviceHostConfig, +} from "@t3tools/contracts"; +import { runSshCommand, baseSshArgs, resolveSshCommand } from "@t3tools/ssh/command"; +import * as NetService from "@t3tools/shared/Net"; +import { waitForHttpReady } from "@t3tools/shared/httpReadiness"; +import * as Exit from "effect/Exit"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import * as ChildProcess from "effect/unstable/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; +import * as ServerConfig from "../config.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import { quoteRemoteArg, remoteDeviceEnvironment, remoteDeviceScript } from "./sshDeviceScript.ts"; + +const Probe = Schema.Struct({ + nodePath: Schema.String, + platforms: Schema.Array(DevicePlatformAvailability), +}); +const Started = Schema.Struct({ + ...Probe.fields, + hubPort: Schema.Int, + daemonPort: Schema.optionalKey(Schema.Int), + token: Schema.optionalKey(Schema.String), + entryPath: Schema.optionalKey(Schema.String), + helpers: Schema.Struct({ + serveSimAxSettings: Schema.NullOr(Schema.String), + serveSimCli: Schema.NullOr(Schema.String), + }), +}); +const decodeProbe = Schema.decodeUnknownEffect(Schema.fromJsonString(Probe)); +const decodeStarted = Schema.decodeUnknownEffect(Schema.fromJsonString(Started)); +const targetFor = (config: SshDeviceHostConfig) => ({ + alias: config.target, + hostname: config.target, + username: null, + port: config.port ?? null, +}); +const identityArgs = (config: SshDeviceHostConfig) => + config.identityFile ? ["-i", config.identityFile] : []; +const commandArgs = (script: string) => [ + "sh", + "-c", + quoteRemoteArg(remoteDeviceEnvironment + script), +]; +const bootstrap = ( + config: SshDeviceHostConfig, + owner: string, + mode: "probe" | "start" | "agent-start" | "stop-agent" | "stop", +) => + runSshCommand(targetFor(config), { + preHostArgs: identityArgs(config), + remoteCommandArgs: commandArgs( + 'command -v node >/dev/null 2>&1 || { echo "Node is missing from the non-interactive SSH PATH" >&2; exit 1; }; exec node', + ), + stdin: remoteDeviceScript(owner, mode), + timeoutMs: mode === "start" || mode === "agent-start" ? 1_300_000 : 45_000, + }).pipe( + Effect.mapError( + (cause) => new DeviceHost.DeviceHostError({ hostId: config.id, step: mode, cause }), + ), + ); + +export const probe = Effect.fn("SshDeviceHost.probe")(function* (config: SshDeviceHostConfig) { + const result = yield* bootstrap(config, "probe", "probe"); + const value = yield* decodeProbe(result.stdout.trim()).pipe( + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ hostId: config.id, step: "reading probe result", cause }), + ), + ); + return { + id: config.id, + label: config.label, + kind: "ssh", + hubInstalled: false, + agentDeviceInstalled: false, + platforms: value.platforms, + } satisfies DeviceHostSummary; +}); + +export const make = Effect.fn("SshDeviceHost.make")(function* ( + config: SshDeviceHostConfig, + onReady: ( + ready: DeviceHost.DeviceHostAgentReady, + ) => Effect.Effect = () => Effect.void, + onStatus: ( + status: "starting" | "ready" | "failed", + detail?: string, + ) => Effect.Effect = () => Effect.void, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const server = yield* ServerConfig.ServerConfig; + const net = yield* NetService.NetService; + const http = yield* HttpClient.HttpClient; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const parentScope = yield* Scope.Scope; + const ssh = yield* resolveSshCommand; + const environmentId = yield* fs + .readFileString(server.environmentIdPath) + .pipe(Effect.orElseSucceed(() => server.stateDir)); + const owner = NodeCrypto.createHash("sha256") + .update(`${environmentId}\0${server.stateDir}\0${config.id}`) + .digest("hex") + .slice(0, 24); + const provide = ( + effect: Effect.Effect< + A, + E, + FileSystem.FileSystem | Path.Path | ChildProcessSpawner.ChildProcessSpawner + >, + ) => + effect.pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const lock = yield* Semaphore.make(1); + let stopped = false; + let activated = false; + let wantsAgent = false; + let ready: + | (DeviceHost.DeviceHostReady & { + agentDevice?: DeviceHost.DeviceHostAgentReady["agentDevice"]; + }) + | null = null; + let connectionScope: Scope.Closeable | null = null; + let summary: DeviceHostSummary = { + id: config.id, + label: config.label, + kind: "ssh", + hubInstalled: false, + agentDeviceInstalled: false, + platforms: [], + }; + + const run: DeviceHost.DeviceHostReady["run"] = (command, args, options) => + provide( + runSshCommand(targetFor(config), { + preHostArgs: identityArgs(config), + remoteCommandArgs: commandArgs(`exec ${[command, ...args].map(quoteRemoteArg).join(" ")}`), + ...(options?.stdin === undefined ? {} : { stdin: options.stdin }), + ...(options?.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), + }), + ).pipe( + Effect.map((result) => ({ ...result, code: 0 })), + Effect.catch((error) => + Effect.succeed({ + stdout: "stdout" in error ? (error.stdout ?? "") : "", + stderr: error.message, + code: "exitCode" in error ? (error.exitCode ?? 127) : 127, + }), + ), + ); + + const connectOnce = Effect.fn("SshDeviceHost.connectOnce")(function* (): Effect.fn.Return< + DeviceHost.DeviceHostReady & { agentDevice?: DeviceHost.DeviceHostAgentReady["agentDevice"] }, + DeviceHost.DeviceHostError + > { + activated = true; + const result = yield* provide(bootstrap(config, owner, wantsAgent ? "agent-start" : "start")); + yield* onStatus("starting"); + const remote = yield* decodeStarted(result.stdout.trim()).pipe( + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId: config.id, + step: "reading host endpoints", + cause, + }), + ), + ); + summary = { + ...summary, + platforms: remote.platforms, + hubInstalled: true, + agentDeviceInstalled: wantsAgent || summary.agentDeviceInstalled, + }; + const hubPort = yield* net.reserveLoopbackPort("127.0.0.1").pipe( + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId: config.id, + step: "reserving hub port", + cause, + }), + ), + ); + const daemonPort = yield* net.reserveLoopbackPort("127.0.0.1").pipe( + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId: config.id, + step: "reserving daemon port", + cause, + }), + ), + ); + const scope = yield* Scope.make(); + connectionScope = scope; + const child = yield* spawner + .spawn( + ChildProcess.make( + ssh, + [ + ...baseSshArgs(targetFor(config), { batchMode: "yes" }), + ...identityArgs(config), + "-o", + "ExitOnForwardFailure=yes", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-N", + "-L", + `127.0.0.1:${hubPort}:127.0.0.1:${remote.hubPort}`, + ...(remote.daemonPort === undefined + ? [] + : ["-L", `127.0.0.1:${daemonPort}:127.0.0.1:${remote.daemonPort}`]), + config.target, + ], + { stdin: "ignore", stdout: "ignore", stderr: "pipe" }, + ), + ) + .pipe( + Effect.provideService(Scope.Scope, scope), + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ hostId: config.id, step: "forwarding ports", cause }), + ), + ); + let stderr = ""; + yield* child.stderr.pipe( + Stream.decodeText(), + Stream.runForEach((chunk) => + Effect.sync(() => { + stderr = (stderr + chunk).slice(-2000); + }), + ), + Effect.forkIn(scope), + ); + const next = { + nodePath: remote.nodePath, + hub: { origin: `http://127.0.0.1:${hubPort}` }, + ...(remote.daemonPort !== undefined && + remote.token !== undefined && + remote.entryPath !== undefined + ? { + agentDevice: { + baseUrl: `http://127.0.0.1:${daemonPort}`, + token: remote.token, + entryPath: remote.entryPath, + }, + } + : {}), + helpers: remote.helpers, + run, + }; + for (const [baseUrl, route] of [ + [next.hub.origin, "/readyz"], + ...(next.agentDevice ? [[next.agentDevice.baseUrl, "/health"]] : []), + ]) { + yield* waitForHttpReady({ + baseUrl: baseUrl!, + path: route!, + timeoutMs: 15000, + makeError: () => + new DeviceHost.DeviceHostError({ + hostId: config.id, + step: "waiting for SSH forward", + cause: new Error(stderr || "Forwarded endpoint did not answer."), + }), + }).pipe(Effect.provideService(HttpClient.HttpClient, http)); + } + if (next.agentDevice) yield* onReady({ ...next, agentDevice: next.agentDevice }); + ready = next; + yield* onStatus("ready"); + // Reconnect also repairs helpers that died while SSH itself stayed connected. + const unhealthy = Effect.gen(function* () { + while (true) { + yield* Effect.sleep("10 seconds"); + const alive = yield* http.get(`${next.hub.origin}/readyz`).pipe( + Effect.timeout("5 seconds"), + Effect.map((r) => r.status === 200), + Effect.orElseSucceed(() => false), + ); + const daemonAlive = next.agentDevice + ? yield* http.get(`${next.agentDevice!.baseUrl}/health`).pipe( + Effect.timeout("5 seconds"), + Effect.map((r) => r.status === 200), + Effect.orElseSucceed(() => false), + ) + : true; + if (!alive || !daemonAlive) return; + } + }); + yield* Effect.gen(function* () { + yield* Effect.raceFirst(child.exitCode.pipe(Effect.ignore), unhealthy); + if (stopped || connectionScope !== scope) return; + ready = null; + yield* onStatus("starting", "Reconnecting to device host…"); + yield* Scope.close(scope, Exit.void); + let delay = 1000; + while (true) { + if (stopped || connectionScope !== scope) return; + yield* Effect.sleep(delay); + const result = yield* lock + .withPermit( + Effect.suspend(() => (stopped || ready ? Effect.void : connect().pipe(Effect.asVoid))), + ) + .pipe(Effect.result); + if (result._tag === "Success") return; + yield* onStatus("failed", result.failure.message); + if (connectionScope && connectionScope !== scope) + yield* Scope.close(connectionScope, Exit.void); + connectionScope = scope; + delay = Math.min(delay * 2, 30000); + } + }).pipe(Effect.forkIn(parentScope)); + return next; + }); + + const connect = Effect.fn("SshDeviceHost.connect")(function* () { + for (let attempt = 0; ; attempt++) { + const result = yield* connectOnce().pipe(Effect.result); + if (result._tag === "Success") return result.success; + const failedScope = connectionScope; + connectionScope = null; + if (failedScope) yield* Scope.close(failedScope, Exit.void); + // SSH binds after the reservation is released, so a competing bind needs fresh ports. + if ( + attempt >= 2 || + !["forwarding ports", "waiting for SSH forward"].includes(result.failure.step) + ) + return yield* result.failure; + } + }); + + const ensureReady: DeviceHost.DeviceHost["Service"]["ensureReady"] = (onPhase) => + lock.withPermit( + Effect.gen(function* () { + stopped = false; + if (ready) return ready; + summary = yield* provide(probe(config)); + yield* onPhase("installing"); + return yield* connect().pipe( + Effect.tapError(() => + connectionScope ? Scope.close(connectionScope, Exit.void) : Effect.void, + ), + ); + }), + ); + const stop = lock.withPermit( + Effect.gen(function* () { + stopped = true; + ready = null; + if (connectionScope) yield* Scope.close(connectionScope, Exit.void); + connectionScope = null; + if (activated) yield* provide(bootstrap(config, owner, "stop")).pipe(Effect.ignore); + activated = false; + wantsAgent = false; + }), + ); + const changeAgent = (enabled: boolean) => + lock.withPermit( + Effect.gen(function* () { + wantsAgent = enabled; + if (enabled && ready?.agentDevice) return { ...ready, agentDevice: ready.agentDevice }; + if (!enabled && !ready?.agentDevice) return null; + ready = null; + const previousScope = connectionScope; + connectionScope = null; + if (previousScope) yield* Scope.close(previousScope, Exit.void); + if (!enabled) yield* provide(bootstrap(config, owner, "stop-agent")); + return yield* connect().pipe( + Effect.onError(() => + Effect.gen(function* () { + const failedScope = connectionScope; + connectionScope = null; + if (failedScope) yield* Scope.close(failedScope, Exit.void); + if (enabled) + yield* provide(bootstrap(config, owner, "stop-agent")).pipe(Effect.ignore); + }), + ), + ); + }), + ); + yield* Effect.addFinalizer(() => stop); + return { + id: config.id, + summary: Effect.sync(() => summary), + current: Effect.sync(() => ready), + ensureReady, + ensureAgentReady: (onPhase) => + onPhase("installing").pipe( + Effect.flatMap(() => changeAgent(true)), + Effect.flatMap((value) => + value?.agentDevice + ? Effect.succeed({ ...value, agentDevice: value.agentDevice }) + : Effect.fail( + new DeviceHost.DeviceHostError({ + hostId: config.id, + step: "starting agent tools", + cause: new Error("Daemon endpoint missing"), + }), + ), + ), + ), + stopAgent: changeAgent(false).pipe(Effect.asVoid, Effect.ignore), + stop, + platformAvailability: (platform) => + provide(probe(config)).pipe( + Effect.map((value) => { + summary = { ...summary, platforms: value.platforms }; + return value.platforms.find((p) => p.platform === platform)!; + }), + Effect.orElseSucceed(() => ({ + platform, + available: false, + reason: "Cannot reach device host. Test its SSH connection in Settings.", + })), + ), + } satisfies DeviceHost.DeviceHost["Service"]; +}); diff --git a/apps/server/src/device/sshDeviceScript.test.ts b/apps/server/src/device/sshDeviceScript.test.ts new file mode 100644 index 000000000000..eadd85fd047d --- /dev/null +++ b/apps/server/src/device/sshDeviceScript.test.ts @@ -0,0 +1,220 @@ +// @effect-diagnostics nodeBuiltinImport:off globalFetchInEffect:off preferSchemaOverJson:off - verifies generated remote scripts using real shell and Node processes. +import * as Effect from "effect/Effect"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { describe, expect, it } from "@effect/vitest"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeUtil from "node:util"; +import { quoteRemoteArg, remoteDeviceEnvironment, remoteDeviceScript } from "./sshDeviceScript.ts"; +import { AGENT_DEVICE_VERSION, DEVICE_HUB_VERSION } from "./DeviceToolchain.ts"; + +const exec = NodeUtil.promisify(NodeChildProcess.execFile); + +it.effect("finds Android Studio Java for a non-interactive SSH session", () => + Effect.gen(function* () { + if ((yield* HostProcessPlatform) === "win32") return; + yield* Effect.promise(async () => { + const home = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-ssh-java-")); + try { + const javaHome = NodePath.join(home, ".local/opt/android-studio/jbr"); + await NodeFSP.mkdir(NodePath.join(javaHome, "bin"), { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(javaHome, "bin/java"), + "#!/bin/sh\necho test-java\n", + { mode: 0o755 }, + ); + const result = await exec("/bin/sh", ["-c", `${remoteDeviceEnvironment}\njava`], { + env: { HOME: home, PATH: "/nonexistent", JAVA_HOME: "" }, + }); + expect(result.stdout.trim()).toBe("test-java"); + } finally { + await NodeFSP.rm(home, { recursive: true, force: true }); + } + }); + }), +); + +it.effect("preserves shell metacharacters and newlines in remote arguments", () => + Effect.gen(function* () { + if ((yield* HostProcessPlatform) === "win32") return; + yield* Effect.promise(async () => { + const value = "quotes ' \" ; $(echo expanded) $HOME\nnext line"; + const result = await exec("sh", ["-c", `printf %s ${quoteRemoteArg(value)}`]); + expect(result.stdout).toBe(value); + }); + }), +); + +describe("remote helper lifecycle", () => { + it.effect("reuses its own healthy helpers and stops only its own runtime", () => + Effect.gen(function* () { + if ((yield* HostProcessPlatform) === "win32") return; + yield* Effect.promise(async () => { + const home = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-remote-script-")); + const bin = NodePath.join(home, "bin"); + await NodeFSP.mkdir(bin); + await NodeFSP.writeFile(NodePath.join(bin, "adb"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + const root = NodePath.join(home, ".t3/device"); + const hubDir = NodePath.join(root, `tools/expo-device-hub@${DEVICE_HUB_VERSION}`); + const agentDir = NodePath.join(root, `tools/agent-device@${AGENT_DEVICE_VERSION}`); + const hub = NodePath.join(hubDir, "node_modules/expo-device-hub/dist/server/cli.mjs"); + const agent = NodePath.join(agentDir, "node_modules/agent-device/bin/agent-device.mjs"); + await NodeFSP.mkdir(NodePath.join(hubDir, "node_modules/expo-device-hub/dist/server"), { + recursive: true, + }); + await NodeFSP.mkdir(NodePath.join(agentDir, "node_modules/agent-device/bin"), { + recursive: true, + }); + await NodeFSP.writeFile(NodePath.join(hubDir, ".install-complete"), DEVICE_HUB_VERSION); + await NodeFSP.writeFile(NodePath.join(agentDir, ".install-complete"), AGENT_DEVICE_VERSION); + await NodeFSP.writeFile( + hub, + `import http from 'node:http'; import fs from 'node:fs'; +if(fs.existsSync('fail-start-once')) {fs.unlinkSync('fail-start-once');process.exit(1);} +const args=process.argv.slice(2); http.createServer((req,res)=>{res.statusCode=fs.existsSync('unhealthy-'+process.pid)?503:200;res.end('ok');}).listen(Number(args[args.indexOf('--port')+1]),'127.0.0.1');`, + ); + await NodeFSP.writeFile( + agent, + `import fs from 'node:fs'; import path from 'node:path'; import http from 'node:http'; import {spawn} from 'node:child_process'; +const args=process.argv.slice(2); +const state=process.env.AGENT_DEVICE_STATE_DIR || args[args.indexOf('--state-dir')+1]; +const file=path.join(state,'daemon.json'); +if(args[0]==='daemon') { const data=JSON.parse(fs.readFileSync(file,'utf8')); fs.writeFileSync(path.join(state,'stopped-agent'),String(data.pid)); try {process.kill(data.pid,'SIGTERM')} catch {} } +else if(args[0]==='serve') { const server=http.createServer((req,res)=>{res.statusCode=fs.existsSync(path.join(state,'unhealthy-agent-'+process.pid))?503:200;res.end('ok');}); server.listen(0,'127.0.0.1',()=>{fs.writeFileSync(file,JSON.stringify({httpPort:server.address().port,pid:process.pid,token:'test'}));process.send?.('ready');process.disconnect?.();}); } +else { const child=spawn(process.execPath,[process.argv[1],'serve'],{detached:true,stdio:['ignore','ignore','ignore','ipc'],env:process.env});await new Promise((resolve,reject)=>{child.once('message',resolve);child.once('error',reject);});child.unref(); } +`, + ); + const nextHubVersion = DEVICE_HUB_VERSION + "-upgrade"; + const nextAgentVersion = AGENT_DEVICE_VERSION + "-upgrade"; + let invocation = 0; + const invoke = async ( + owner: string, + mode: "start" | "agent-start" | "stop-agent" | "stop", + upgraded = false, + ) => { + const file = NodePath.join(home, `${owner}-${mode}-${invocation++}.cjs`); + await NodeFSP.writeFile( + file, + `const originalKill = process.kill; process.kill = (pid, signal) => { if (signal === 'SIGTERM') require('node:fs').appendFileSync(${JSON.stringify(NodePath.join(home, "stops"))}, pid+'\\n'); return originalKill(pid, signal); };\n` + + remoteDeviceScript(owner, mode) + .replace(DEVICE_HUB_VERSION, upgraded ? nextHubVersion : DEVICE_HUB_VERSION) + .replace(AGENT_DEVICE_VERSION, upgraded ? nextAgentVersion : AGENT_DEVICE_VERSION), + ); + const result = await exec(process.execPath, [file], { + env: { ...process.env, HOME: home, PATH: `${bin}:${process.env.PATH}` }, + }); + return result.stdout ? JSON.parse(result.stdout) : null; + }; + const template = NodePath.join(home, "hub-template"); + await NodeFSP.cp(hubDir, template, { recursive: true }); + await NodeFSP.rm(NodePath.join(hubDir, ".install-complete")); + const installLock = hubDir + ".lock"; + await NodeFSP.symlink("2147483647:exited-installer", installLock); + await NodeFSP.writeFile( + NodePath.join(bin, "npm"), + `#!${process.execPath}\nconst fs=require('node:fs');const args=process.argv.slice(2);fs.cpSync(${JSON.stringify(template)},args[args.indexOf('--prefix')+1],{recursive:true});`, + { mode: 0o755 }, + ); + await NodeFSP.mkdir(NodePath.join(root, "hosts/one"), { recursive: true }); + await NodeFSP.writeFile(NodePath.join(root, "hosts/one/fail-start-once"), ""); + try { + const [manual, concurrent] = await Promise.all([ + invoke("one", "start"), + invoke("one", "start"), + ]); + expect(concurrent.hubPort).toBe(manual.hubPort); + expect(manual.daemonPort).toBeUndefined(); + await expect( + NodeFSP.stat(NodePath.join(root, "hosts/one/daemon.json")), + ).rejects.toThrow(); + const [first, concurrentAgent] = await Promise.all([ + invoke("one", "agent-start"), + invoke("one", "agent-start"), + ]); + expect(concurrentAgent.hubPort).toBe(first.hubPort); + expect(concurrentAgent.daemonPort).toBe(first.daemonPort); + const second = await invoke("two", "agent-start"); + const reused = await invoke("one", "agent-start"); + expect(reused.hubPort).toBe(first.hubPort); + expect(reused.daemonPort).toBe(first.daemonPort); + expect(second.hubPort).not.toBe(first.hubPort); + expect(second.daemonPort).not.toBe(first.daemonPort); + const firstHub = JSON.parse( + await NodeFSP.readFile(NodePath.join(root, "hosts/one/hub.json"), "utf8"), + ); + const secondHub = JSON.parse( + await NodeFSP.readFile(NodePath.join(root, "hosts/two/hub.json"), "utf8"), + ); + await NodeFSP.writeFile(NodePath.join(root, `hosts/one/unhealthy-${firstHub.pid}`), ""); + let repaired = await invoke("one", "agent-start"); + expect(repaired.hubPort).not.toBe(first.hubPort); + const stopped = (await NodeFSP.readFile(NodePath.join(home, "stops"), "utf8")) + .trim() + .split("\n"); + expect(stopped).toContain(String(firstHub.pid)); + expect(stopped).not.toContain(String(secondHub.pid)); + const previousDaemon = JSON.parse( + await NodeFSP.readFile(NodePath.join(root, "hosts/one/daemon.json"), "utf8"), + ); + for (const [source, name, version] of [ + [hubDir, "expo-device-hub", nextHubVersion], + [agentDir, "agent-device", nextAgentVersion], + ]) { + const destination = NodePath.join(root, `tools/${name}@${version}`); + await NodeFSP.cp(source!, destination, { recursive: true }); + await NodeFSP.writeFile(NodePath.join(destination, ".install-complete"), version!); + } + const upgraded = await invoke("one", "agent-start", true); + expect(upgraded.entryPath).toContain(nextAgentVersion); + const upgradedHub = JSON.parse( + await NodeFSP.readFile(NodePath.join(root, "hosts/one/hub.json"), "utf8"), + ); + expect(upgradedHub.entryPath).toContain(nextHubVersion); + const upgradedDaemon = JSON.parse( + await NodeFSP.readFile(NodePath.join(root, "hosts/one/daemon.json"), "utf8"), + ); + expect(upgradedDaemon.pid).not.toBe(previousDaemon.pid); + expect(await invoke("one", "agent-start", true)).toEqual(upgraded); + await NodeFSP.writeFile( + NodePath.join(root, `hosts/one/unhealthy-agent-${upgradedDaemon.pid}`), + "", + ); + repaired = await invoke("one", "agent-start", true); + expect( + await NodeFSP.readFile(NodePath.join(root, "hosts/one/stopped-agent"), "utf8"), + ).toBe(String(upgradedDaemon.pid)); + expect(repaired.daemonPort).not.toBe(upgraded.daemonPort); + // Stop still uses the recorded entry when a future pinned package is not installed yet. + const originalScript = remoteDeviceScript("one", "stop-agent"); + const upgradedStop = NodePath.join(home, "upgraded-stop.cjs"); + await NodeFSP.writeFile( + upgradedStop, + originalScript.replace(AGENT_DEVICE_VERSION, "999.0.0"), + ); + await exec(process.execPath, [upgradedStop], { + env: { ...process.env, HOME: home, PATH: `${bin}:${process.env.PATH}` }, + }); + const daemon = JSON.parse( + await NodeFSP.readFile(NodePath.join(root, "hosts/one/daemon.json"), "utf8"), + ); + expect( + await NodeFSP.readFile(NodePath.join(root, "hosts/one/stopped-agent"), "utf8"), + ).toBe(String(daemon.pid)); + expect((await fetch(`http://127.0.0.1:${repaired.hubPort}/readyz`)).ok).toBe(true); + await invoke("one", "stop"); + expect((await fetch(`http://127.0.0.1:${second.hubPort}/readyz`)).ok).toBe(true); + expect( + JSON.parse(await NodeFSP.readFile(NodePath.join(root, "hosts/two/hub.json"), "utf8")) + .owner, + ).toBe("two"); + } finally { + await invoke("one", "stop").catch(() => {}); + await invoke("two", "stop").catch(() => {}); + await NodeFSP.rm(home, { recursive: true, force: true }); + } + }); + }), + ); +}); diff --git a/apps/server/src/device/sshDeviceScript.ts b/apps/server/src/device/sshDeviceScript.ts new file mode 100644 index 000000000000..bbdb828c1a74 --- /dev/null +++ b/apps/server/src/device/sshDeviceScript.ts @@ -0,0 +1,192 @@ +import { AGENT_DEVICE_VERSION, DEVICE_HUB_VERSION } from "./DeviceToolchain.ts"; + +export const quoteRemoteArg = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`; + +/** Resolve common non-interactive SDK and Node locations without sourcing user shell scripts. */ +export const remoteDeviceEnvironment = `export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" +if [ -z "$ANDROID_HOME" ]; then + if [ -d "$HOME/Library/Android/sdk" ]; then export ANDROID_HOME="$HOME/Library/Android/sdk"; + elif [ -d "$HOME/Android/Sdk" ]; then export ANDROID_HOME="$HOME/Android/Sdk"; fi +fi +if [ -n "$ANDROID_HOME" ]; then export PATH="$ANDROID_HOME/platform-tools:$ANDROID_HOME/emulator:$PATH"; fi +if [ -z "$JAVA_HOME" ] && ! command -v java >/dev/null 2>&1; then + for device_java_home in "$HOME/.local/opt/android-studio/jbr" /opt/android-studio/jbr /Applications/Android\\ Studio.app/Contents/jbr "$HOME/Applications/Android Studio.app/Contents/jbr"; do + if [ -x "$device_java_home/bin/java" ]; then export JAVA_HOME="$device_java_home"; break; fi + done +fi +if [ -n "$JAVA_HOME" ]; then export PATH="$JAVA_HOME/bin:$PATH"; fi +`; + +/** Node runs this on the host. All paths it returns belong to that host. */ +export const remoteDeviceScript = ( + owner: string, + mode: "probe" | "start" | "agent-start" | "stop-agent" | "stop", +) => + ` +const owner = ${JSON.stringify(owner)}; +const mode = ${JSON.stringify(mode)}; +const hubVersion = ${JSON.stringify(DEVICE_HUB_VERSION)}; +const agentVersion = ${JSON.stringify(AGENT_DEVICE_VERSION)}; +` + + String.raw` +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const net = require('node:net'); +const { spawn, spawnSync } = require('node:child_process'); +const root = path.join(os.homedir(), '.t3', 'device'); +const state = path.join(root, 'hosts', owner); +const run = (command, args, options = {}) => spawnSync(command, args, { encoding: 'utf8', timeout: 30000, ...options }); +const read = (file) => { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } }; +const write = (file, value) => { const tmp = file + '.' + process.pid; fs.writeFileSync(tmp, JSON.stringify(value), { mode: 0o600 }); fs.renameSync(tmp, file); }; +const stopHub = hub => { + if (!hub || hub.owner !== owner) return; + const command = run('ps', ['-p', String(hub.pid), '-o', 'command=']).stdout || ''; + if (command.includes(hub.entryPath) && command.includes(String(hub.port))) { + try { process.kill(hub.pid, 'SIGTERM'); } catch {} + } +}; +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const healthy = async (port, route) => { try { return (await fetch('http://127.0.0.1:' + port + route, { signal: AbortSignal.timeout(2000) })).ok; } catch { return false; } }; +const port = () => new Promise((resolve, reject) => { const server = net.createServer(); server.once('error', reject); server.listen(0, '127.0.0.1', () => { const value = server.address().port; server.close(() => resolve(value)); }); }); +async function acquireLock(lock, complete = () => false) { + const deadline = Date.now() + 600000; + const token = process.pid + ':' + require('node:crypto').randomUUID(); + const owner = () => { try { return fs.readlinkSync(lock); } catch (error) { if (error.code === 'ENOENT') return null; throw error; } }; + while (true) { + try { + // Publishing the PID and token is atomic; suspension cannot leave an incomplete owner. + fs.symlinkSync(token, lock); + return () => { if (owner() === token) fs.unlinkSync(lock); }; + } catch (error) { + if (error.code !== 'EEXIST') throw error; + if (complete()) return null; + const previous = owner(); + if (previous === null) continue; + const pid = Number(previous.split(':')[0]); + if (!Number.isSafeInteger(pid) || pid <= 0) throw Error('Invalid device lock at ' + lock); + try { process.kill(pid, 0); } catch (error) { + if (error.code === 'ESRCH' && owner() === previous) { + try { fs.unlinkSync(lock); } catch (error) { if (error.code !== 'ENOENT') throw error; } + continue; + } + } + if (Date.now() > deadline) throw Error('Device operation is locked at ' + lock + '. Check the other installer before removing the lock.'); + await sleep(500); + } + } +} +async function install(name, version, entry) { + const dir = path.join(root, 'tools', name + '@' + version); + const file = path.join(dir, 'node_modules', name, entry); + const complete = () => fs.existsSync(file) && fs.existsSync(path.join(dir, '.install-complete')) && fs.readFileSync(path.join(dir, '.install-complete'), 'utf8').trim() === version; + if (complete()) return file; + fs.mkdirSync(path.dirname(dir), { recursive: true }); + const lock = dir + '.lock'; + const release = await acquireLock(lock, complete); + if (!release) return file; + let staging; + try { + if (complete()) return file; + staging = fs.mkdtempSync(path.join(path.dirname(dir), '.install-')); + const result = run('npm', ['install', '--prefix', staging, '--no-fund', '--no-audit', name + '@' + version], { timeout: 600000, maxBuffer: 8 * 1024 * 1024 }); + if (result.status !== 0) throw Error('Installing ' + name + ': ' + (result.error?.message || result.stderr?.slice(-2000))); + if (!fs.existsSync(path.join(staging, 'node_modules', name, entry))) throw Error('Missing installed entry for ' + name); + fs.writeFileSync(path.join(staging, '.install-complete'), version); + fs.rmSync(dir, { recursive: true, force: true }); + fs.renameSync(staging, dir); + return file; + } finally { + if (staging) fs.rmSync(staging, { recursive: true, force: true }); + release(); + } +} +(async () => { + const ios = process.platform === 'darwin' && run('xcrun', ['simctl', 'help']).status === 0; + const android = run('adb', ['version']).status === 0; + const platforms = [ + { platform: 'ios', available: ios, ...(!ios ? { reason: 'iOS needs macOS with Xcode and working xcrun simctl.' } : {}) }, + { platform: 'android', available: android, ...(!android ? { reason: 'Android SDK missing. Set ANDROID_HOME or put adb on the SSH PATH.' } : {}) }, + ]; + if (mode === 'probe') { + if (Number(process.versions.node.split('.')[0]) < 22) throw Error('Node 22 or newer is required on the device host.'); + if (run('npm', ['--version']).status !== 0) throw Error('npm is missing from the non-interactive SSH PATH.'); + console.log(JSON.stringify({ nodePath: process.execPath, platforms })); return; + } + fs.mkdirSync(state, { recursive: true, mode: 0o700 }); + // Serialize starts and stops for this environment/host owner, including agent startup. + const hostLock = path.join(state, 'runtime.lock'); + const releaseHost = await acquireLock(hostLock); + try { + const hubFile = path.join(state, 'hub.json'); + const daemonFile = path.join(state, 'daemon.json'); + const agentFile = path.join(state, 'agent.json'); + if (mode === 'stop' || mode === 'stop-agent') { + const hub = read(hubFile); + if (mode === 'stop' && hub && hub.owner === owner) { + stopHub(hub); + fs.rmSync(hubFile, { force: true }); + } + const entry = read(agentFile)?.entryPath || path.join(root, 'tools', 'agent-device@' + agentVersion, 'node_modules', 'agent-device', 'bin', 'agent-device.mjs'); + if (fs.existsSync(entry)) run(process.execPath, [entry, 'daemon', 'stop', '--state-dir', state]); + return; + } + if (!ios && !android) throw Error(platforms.map(p => p.reason).join(' ')); + fs.mkdirSync(state, { recursive: true, mode: 0o700 }); + const hubEntry = await install('expo-device-hub', hubVersion, 'dist/server/cli.mjs'); + let hub = read(hubFile); + if (!hub || hub.owner !== owner || hub.entryPath !== hubEntry || !await healthy(hub.port, '/readyz')) { + stopHub(hub); + for (let attempt = 0; attempt < 5; attempt++) { + const hubPort = await port(); + const log = fs.openSync(path.join(state, 'hub.log'), 'a'); + const child = spawn(process.execPath, [hubEntry, '--port', String(hubPort), '--host', '127.0.0.1', '--hide-sidebar', '--hide-boot-device'], { + cwd: state, detached: true, stdio: ['ignore', log, log], env: { ...process.env, FORCE_COLOR: '0', NO_COLOR: '1' }, + }); + try { await new Promise((resolve, reject) => { child.once('spawn', resolve); child.once('error', reject); }); } + finally { fs.closeSync(log); } + child.unref(); + hub = { owner, pid: child.pid, port: hubPort, entryPath: hubEntry }; + write(hubFile, hub); + const deadline = Date.now() + 30000; + let listening = false; + while (child.exitCode === null && child.signalCode === null) { + if (await healthy(hub.port, '/readyz')) { listening = true; break; } + if (Date.now() > deadline) { stopHub(hub); throw Error('Device hub did not become ready. See ' + path.join(state, 'hub.log')); } + await sleep(200); + } + if (listening) break; + // Port reservation and binding happen in different processes. Retry an early exit with a fresh port. + fs.rmSync(hubFile, { force: true }); + if (attempt === 4) throw Error('Device hub exited before becoming ready. See ' + path.join(state, 'hub.log')); + } + } + let agentResult = {}; + if (mode === 'agent-start') { + const agentEntry = await install('agent-device', agentVersion, 'bin/agent-device.mjs'); + const previousAgent = read(agentFile)?.entryPath; + let daemon = read(daemonFile); + if (daemon && (previousAgent !== agentEntry || !await healthy(daemon.httpPort, '/health'))) { + const stopped = run(process.execPath, [previousAgent || agentEntry, 'daemon', 'stop', '--state-dir', state]); + if (stopped.status !== 0) throw Error('Could not stop the previous agent-device version.'); + fs.rmSync(daemonFile, { force: true }); + daemon = null; + } + if (!daemon) { + fs.rmSync(daemonFile, { force: true }); + const env = { ...process.env, AGENT_DEVICE_STATE_DIR: state, AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', AGENT_DEVICE_NO_UPDATE_NOTIFIER: '1' }; + delete env.AGENT_DEVICE_DAEMON_BASE_URL; delete env.AGENT_DEVICE_DAEMON_AUTH_TOKEN; delete env.AGENT_DEVICE_CONFIG; + run(process.execPath, [agentEntry, 'devices', '--json'], { env }); + daemon = read(daemonFile); + } + if (!daemon || !await healthy(daemon.httpPort, '/health')) throw Error('agent-device daemon did not become ready in ' + state); + write(agentFile, { entryPath: agentEntry }); + agentResult = { daemonPort: daemon.httpPort, token: daemon.token, entryPath: agentEntry }; + } + const vendor = path.resolve(path.dirname(hubEntry), '../../vendor/serve-sim/dist'); + const optional = file => fs.existsSync(file) ? file : null; + console.log(JSON.stringify({ nodePath: process.execPath, platforms, hubPort: hub.port, ...agentResult, + helpers: { serveSimAxSettings: optional(path.join(vendor, 'simax/serve-sim-ax-settings')), serveSimCli: optional(path.join(vendor, 'serve-sim.js')) } })); + } finally { releaseHost(); } +})().catch(error => { console.error(error.message); process.exitCode = 1; }); +`; diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts index 24097a92faac..3a9307314c39 100644 --- a/apps/server/src/mcp/McpDeviceToolkit.test.ts +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -92,6 +92,7 @@ const DeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ screenshot: () => Effect.succeed({ device, png }), close: () => Effect.void, agentCli: Effect.succeed("/cli"), + testHost: () => Effect.die("not used"), agentTarget: () => Effect.succeed(["--config", "/host.json", "--session", "thread-device"]), }); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index 3bc89d22bc60..feb69ec630d6 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -134,6 +134,9 @@ const handlers = { .filter((session) => session.threadId === scope.threadId) .map((session) => ({ hostId: session.hostId, deviceId: session.deviceId })); return { + hostStatuses: Object.fromEntries( + Object.entries(state.hostStatuses).filter(([id]) => !hostId || id === hostId), + ), hosts: hostId ? state.hosts.filter((host) => host.id === hostId) : state.hosts, devices: hostId ? state.devices.filter((device) => device.hostId === hostId) diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 71a8ed19d905..f59a753d9c72 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2770,6 +2770,10 @@ const makeWsRpcLayer = ( observeRpcEffect(WS_METHODS.deviceConfigure, deviceService.configure(input), { "rpc.aggregate": "device", }), + [WS_METHODS.deviceTestHost]: (input) => + observeRpcEffect(WS_METHODS.deviceTestHost, deviceService.testHost(input), { + "rpc.aggregate": "device", + }), [WS_METHODS.deviceList]: (_input) => observeRpcEffect(WS_METHODS.deviceList, deviceService.list, { "rpc.aggregate": "device", diff --git a/apps/web/src/components/device/DeviceHostAvailability.tsx b/apps/web/src/components/device/DeviceHostAvailability.tsx new file mode 100644 index 000000000000..03a0b175dae8 --- /dev/null +++ b/apps/web/src/components/device/DeviceHostAvailability.tsx @@ -0,0 +1,27 @@ +import type { DevicePlatformAvailability } from "@t3tools/contracts"; +import { Check, Minus } from "lucide-react"; +import { Tooltip, TooltipTrigger, TooltipPopup } from "../ui/tooltip"; + +export function DeviceHostAvailability({ + platforms, +}: { + platforms: ReadonlyArray; +}) { + return ( +
+ {platforms.map((platform) => ( + + }> + {platform.available ? : } + {platform.platform === "ios" ? "iOS" : "Android"}{" "} + {platform.available ? "available" : "unavailable"} + + + {platform.reason ?? + (platform.platform === "ios" ? "iOS available" : "Android available")} + + + ))} +
+ ); +} diff --git a/apps/web/src/components/settings/DeviceHostsSettings.tsx b/apps/web/src/components/settings/DeviceHostsSettings.tsx new file mode 100644 index 000000000000..e61cf742dfd5 --- /dev/null +++ b/apps/web/src/components/settings/DeviceHostsSettings.tsx @@ -0,0 +1,334 @@ +import { Tooltip, TooltipTrigger, TooltipPopup } from "../ui/tooltip"; +import { AppleIcon, AndroidIcon } from "../Icons"; +import { DeviceHostAvailability } from "../device/DeviceHostAvailability"; +import { Spinner } from "../ui/spinner"; +import type { + DevicePlatformAvailability, + EnvironmentId, + SshDeviceHostConfig, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import { randomUUID } from "../../lib/utils"; +import { useState } from "react"; +import { deviceEnvironment, useDeviceState } from "../../state/device"; +import { serverEnvironment } from "../../state/server"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { Button } from "../ui/button"; +import { Input } from "../ui/input"; +import { MoreVertical, PlusIcon } from "lucide-react"; +import { Menu, MenuTrigger, MenuPopup, MenuItem } from "../ui/menu"; +import { SettingsRow } from "./settingsLayout"; + +/** Host names and identity paths belong to the selected environment, never all environments. */ +export function DeviceHostsSettings(props: { + environmentId: EnvironmentId | null; + hosts: ReadonlyArray; +}) { + const update = useAtomCommand(serverEnvironment.updateSettings); + const test = useAtomCommand(deviceEnvironment.testHost, { reportFailure: false }); + const { state } = useDeviceState(props.environmentId); + const [editing, setEditing] = useState(null); + const [busy, setBusy] = useState(false); + const validPort = (port: number | undefined) => + port === undefined || (Number.isInteger(port) && port >= 1 && port <= 65535); + const [checks, setChecks] = useState< + Record< + string, + { pending?: boolean; platforms?: ReadonlyArray; error?: string } + > + >({}); + const setCheck = (id: string, value: (typeof checks)[string]) => + setChecks((current) => ({ ...current, [id]: value })); + const save = async (hosts: ReadonlyArray) => { + if (!props.environmentId) return; + setBusy(true); + try { + const saved = await update({ + environmentId: props.environmentId, + input: { patch: { deviceHosts: hosts } }, + }); + if (saved._tag === "Success") { + setEditing(null); + } + } finally { + setBusy(false); + } + }; + const testConnection = async (host: SshDeviceHostConfig) => { + if (!props.environmentId || checks[host.id]?.pending) return; + setCheck(host.id, { pending: true }); + try { + const summary = await test({ environmentId: props.environmentId, input: host }); + setCheck( + host.id, + summary._tag === "Failure" + ? { error: Cause.pretty(summary.cause) } + : { platforms: summary.value.platforms }, + ); + } catch (error) { + setCheck(host.id, { error: error instanceof Error ? error.message : String(error) }); + } + }; + return ( + { + setEditing({ id: randomUUID(), label: "", target: "" }); + }} + > + Add host + + } + > +
+ {!props.environmentId ? ( +

+ Select one connected environment to manage its device hosts. +

+ ) : ( + <> + {props.hosts.map((host) => { + const status = state.hostStatuses[host.id]; + const check = checks[host.id]; + const platforms = + check?.platforms ?? + state.hosts.find((value) => value.id === host.id)?.platforms ?? + []; + const progress = check?.pending + ? "Checking connection…" + : status?.status === "installing" + ? "Installing device support…" + : status?.status === "starting" + ? "Connecting…" + : null; + const error = + check?.error ?? (status?.status === "failed" ? status.detail : undefined); + return ( +
+
+
+

{host.label}

+ {platforms + .filter((platform) => platform.available) + .map((platform) => ( + + + } + > + {platform.platform === "ios" ? ( + + ) : ( + + )} + + + {platform.platform === "ios" ? "iOS available" : "Android available"} + + + ))} +
+

{host.target}

+ {error ? ( +
+
+ Connection failed +

{error}

+
+
+ ) : null} +
+ {progress ? ( + + + {progress} + + ) : null} + + + } + > + + + + { + setEditing(host); + }} + > + Edit + + + void save(props.hosts.filter((value) => value.id !== host.id)) + } + > + Remove + + + + +
+ ); + })} + {editing ? ( +
{ + event.preventDefault(); + void save([...props.hosts.filter((host) => host.id !== editing.id), editing]); + }} + > + + + + +
+ + + +
+ {checks[editing.id]?.pending ? ( + + + Checking connection… + + ) : null} + {checks[editing.id]?.platforms ? ( + + ) : null} + {checks[editing.id]?.error ? ( +

+ {checks[editing.id]?.error} +

+ ) : null} + + ) : null} + + )} +
+ + ); +} diff --git a/apps/web/src/components/settings/IntegrationsSettings.tsx b/apps/web/src/components/settings/IntegrationsSettings.tsx index a3f76f49cd5f..8cb21532a426 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.tsx @@ -1,3 +1,4 @@ +import { DeviceHostsSettings } from "./DeviceHostsSettings"; /** * Integrations settings - preferences for surfaces T3 Code embeds rather than * owns. Browser is the first section: the defaults a preview tab opens at, @@ -12,6 +13,7 @@ import { type BrowserLinkTarget, type BrowserProfile, type EnvironmentId, + type SshDeviceHostConfig, BROWSER_PROFILE_NAME_MAX_LENGTH, BROWSER_RECORDING_FRAME_RATES, DEFAULT_BROWSER_AUTO_SHOW_FLOATING_PREVIEW, @@ -584,17 +586,74 @@ function AgentBrowserAccessSetting() { function DeviceIntegrationSettings() { const primaryEnvironment = usePrimaryEnvironment(); - const environmentId = primaryEnvironment?.environmentId ?? null; + const { environments } = useEnvironments(); + const [selectedId, setSelectedId] = useState(null); + const selected = + environments.find((environment) => environment.environmentId === selectedId) ?? + environments.find( + (environment) => environment.environmentId === primaryEnvironment?.environmentId, + ) ?? + environments[0]; + const connected = selected?.connection.phase === "connected" && selected.serverConfig !== null; + const environmentId = connected ? selected.environmentId : null; + + return ( + + {environments.length > 1 ? ( + setSelectedId(value)} + > + + {selected?.label ?? "Select environment"} + + + {environments.map((environment) => ( + + {environment.label} + {environment.connection.phase === "connected" ? "" : " · Offline"} + + ))} + + + } + /> + ) : null} + + + ); +} + +function DeviceIntegrationControls({ + environmentId, + hosts, + enabled, + agentAccessEnabled, +}: { + environmentId: EnvironmentId | null; + hosts: ReadonlyArray; + enabled: boolean; + agentAccessEnabled: boolean; +}) { const { state, loaded } = useDeviceState(environmentId); const configure = useAtomCommand(deviceEnvironment.configure); const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); const [pending, setPending] = useState<"hub" | "check" | "agent" | null>(null); - const enabled = state.hostStatus !== "disabled"; const busy = state.hostStatus === "installing" || state.hostStatus === "starting"; const [platformsRevealed, setPlatformsRevealed] = useState(false); // Keep diagnostics visible through subsequent agent setup and refresh phases. if (platformsRevealed && !enabled) setPlatformsRevealed(false); - if (!platformsRevealed && state.hostStatus === "ready" && pending !== "hub") { + if (enabled && !platformsRevealed && state.hostStatus === "ready" && pending !== "hub") { setPlatformsRevealed(true); } @@ -615,7 +674,7 @@ function DeviceIntegrationSettings() { }; return ( - + <> {pending === "agent" ? : null} @@ -689,7 +748,8 @@ function DeviceIntegrationSettings() { {state.hostStatusDetail}

) : null} - + + ); } diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 24892b0e1e3b..f56579990714 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -402,6 +402,12 @@ export const SETTINGS_SEARCH_ITEMS = [ to: "/settings/projects", searchTerms: ["allow open drive preview tools sessions"], }, + { + id: "device-hosts", + title: "Device hosts", + to: "/settings/integrations", + searchTerms: ["ssh remote simulator emulator ios android mac mini identity key connection"], + }, { id: "agent-device-access", title: "Agent device access", diff --git a/docs/internals/devices.md b/docs/internals/devices.md index 49027022fdbc..9b34851439ca 100644 --- a/docs/internals/devices.md +++ b/docs/internals/devices.md @@ -3,8 +3,7 @@ The environment server owns simulators and emulators the way it owns terminals: discovery, streaming, and agent access all run there, and every client reaches them through the environment connection. This is what makes the -Device panel work over Tailscale and T3 Connect, and what will let a device -host on another machine slot in later. +Device panel work over Tailscale and T3 Connect, including when an SSH host runs the devices. ## Two external tools, one seam @@ -21,8 +20,8 @@ native addon, and a crash there must not take the server down. Everything platform-specific sits behind [`DeviceHost`](../../apps/server/src/device/DeviceHost.ts). The service, the proxy, and the MCP tools only see a hub origin and an agent-device endpoint. -An SSH or cloud host would forward those two things to the server and change -nothing above it. +SSH hosts forward both endpoints to server loopback. Every proxied request +also carries the host id; device ids alone are not unique across hosts. ## The hub is never exposed @@ -57,9 +56,8 @@ screenshot capture and stream tuning. The `device_*` toolkit is deliberately four tools: list, open, screenshot, and close. Driving happens through the `agent-device` CLI, which has the semantic snapshot model agents need and stays current with its own releases. T3 prepends -a shim directory to the provider's PATH and sets -`AGENT_DEVICE_DAEMON_BASE_URL` and `AGENT_DEVICE_DAEMON_AUTH_TOKEN` so the -agent never handles the endpoint or token. +a shim directory to the provider's PATH. The CLI installs on the environment +server even when that server cannot run simulators. Hosts start on demand. That environment is fixed when the provider subprocess spawns, so [`prepareMcpSession`](../../apps/server/src/provider/Layers/ProviderService.ts) diff --git a/docs/user/devices.md b/docs/user/devices.md index 51a0a578b607..7d1b91792ae3 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -15,6 +15,9 @@ installed, the setup screen says so and reuses it. Choose a running device to watch it, or choose **Start** next to a stopped device to boot it. The panel shows when you or an agent starts a device. +Each device opens in its own tab. Use **+ → Device** to open another, and +double-click a tab name or choose **Rename** from its context menu to rename it. +Only the visible tab streams video; switching tabs keeps both devices running. Turn off the device hub in **Settings → Integrations → Devices** to stop the helper processes; simulators and emulators keep running until you power them off. @@ -29,7 +32,8 @@ After installing them, restart the environment server and refresh devices. The screen is interactive: click and drag to touch, type while the screen is focused, and use the toolbar for Home, Back, and Recents on Android, rotate on iOS, and power off. Close the tab to stop watching; the device keeps running -unless you power it off. +unless you power it off. Closed tabs stay closed after a reload. To watch the +device again, choose it from **+ → Device**. ## Tools @@ -61,3 +65,26 @@ The device stream goes through the environment server, so it works over the local network, Tailscale, and T3 Connect. Live video needs a secure page (HTTPS or localhost); on a plain-HTTP remote origin iOS falls back to a slower still-image stream and Android cannot show video. + +## SSH device hosts + +In Settings → Integrations → Devices, select one connected environment +and add a host under **Device hosts**. Enter an SSH alias or `user@host`, with +an optional identity file and port. These resolve on the environment server, +so use the SSH configuration and keys available there. Password prompts are +not supported. + +**Test connection** checks SSH, Node, npm, and platform tools without installing +anything. The first device listing installs pinned device tools on the host. +Node 22 or newer and npm must be available to non-interactive SSH commands. +T3 checks common Homebrew and Android SDK locations; custom installations need +the appropriate PATH and ANDROID_HOME on the host. + +The picker identifies devices by host when several hosts are configured. +Connections recover after interruptions. Removing a host closes its device +sessions and stops its T3 helpers when reachable; simulators keep running. + +T3 provides discovery, streaming, and control. Arrange app builds, +installation, and connectivity to development servers such as Metro separately. +A simulator on another machine cannot reach Metro through your environment's +localhost without forwarding or another reachable address. diff --git a/packages/client-runtime/src/state/device.ts b/packages/client-runtime/src/state/device.ts index df27f793720c..1e6523497ef3 100644 --- a/packages/client-runtime/src/state/device.ts +++ b/packages/client-runtime/src/state/device.ts @@ -28,6 +28,10 @@ export function createDeviceEnvironmentAtoms( scheduler, concurrency, }), + testHost: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:test-host", + tag: WS_METHODS.deviceTestHost, + }), list: createEnvironmentRpcCommand(runtime, { label: "environment-data:device:list", tag: WS_METHODS.deviceList, diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts index 895a954e2e40..03102dd62b44 100644 --- a/packages/contracts/src/device.ts +++ b/packages/contracts/src/device.ts @@ -25,6 +25,27 @@ export type DeviceHostId = typeof DeviceHostId.Type; /** The server machine. Always present; other host kinds are future work. */ export const LOCAL_DEVICE_HOST_ID = "local" as DeviceHostId; +/** SSH aliases and key paths are resolved on the environment server. */ +export const SshDeviceHostConfig = Schema.Struct({ + id: DeviceHostId.check( + Schema.isPattern(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + Schema.makeFilter((id) => id !== "local" || "The local host id is reserved."), + ), + label: TrimmedNonEmptyString, + target: TrimmedNonEmptyString.check(Schema.isPattern(/^[^\s-][^\s]*$/)), + identityFile: Schema.optional(TrimmedNonEmptyString), + port: Schema.optional(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 }))), +}); +export type SshDeviceHostConfig = typeof SshDeviceHostConfig.Type; + +export const SshDeviceHostConfigs = Schema.Array(SshDeviceHostConfig).check( + Schema.makeFilter( + (hosts) => + new Set(hosts.map((host) => host.id)).size === hosts.length || + "Device host ids must be unique.", + ), +); + /** Simulator udid or adb serial (an AVD name while it is not running). */ export const DeviceId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)); export type DeviceId = typeof DeviceId.Type; @@ -55,7 +76,7 @@ export type DevicePlatformAvailability = typeof DevicePlatformAvailability.Type; export const DeviceHostSummary = Schema.Struct({ id: DeviceHostId, - kind: Schema.Literals(["local"]), + kind: Schema.Literals(["local", "ssh"]), label: TrimmedNonEmptyString, platforms: Schema.Array(DevicePlatformAvailability), hubInstalled: Schema.Boolean, @@ -424,6 +445,7 @@ export type DeviceError = typeof DeviceError.Type; // panel describe devices the same way. export const DeviceToolListResult = Schema.Struct({ + hostStatuses: DeviceServiceState.fields.hostStatuses, hosts: Schema.Array(DeviceHostSummary), devices: Schema.Array(DeviceSummary), /** Devices already open in this thread's Device panel. */ diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index a5d35bf2361d..5e11aeb28fa3 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -189,6 +189,8 @@ import { DeviceDetailInput, DeviceError, DeviceListInput, + SshDeviceHostConfig, + DeviceHostSummary, DeviceOpenInput, DeviceServiceState, DeviceSession, @@ -328,6 +330,7 @@ export const WS_METHODS = { // Device methods deviceConfigure: "device.configure", deviceList: "device.list", + deviceTestHost: "device.testHost", deviceOpen: "device.open", deviceClose: "device.close", deviceShutdown: "device.shutdown", @@ -1101,6 +1104,12 @@ const WsSubscribeDiscoveredLocalServersRpc = Rpc.make(WS_METHODS.subscribeDiscov stream: true, }); +const WsDeviceTestHostRpc = Rpc.make(WS_METHODS.deviceTestHost, { + payload: SshDeviceHostConfig, + success: DeviceHostSummary, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + const WsDeviceListRpc = Rpc.make(WS_METHODS.deviceList, { payload: DeviceListInput, success: DeviceServiceState, @@ -1380,6 +1389,7 @@ export const WsRpcGroup = RpcGroup.make( WsSubscribeDiscoveredLocalServersRpc, WsDeviceConfigureRpc, WsDeviceListRpc, + WsDeviceTestHostRpc, WsDeviceOpenRpc, WsDeviceCloseRpc, WsDeviceShutdownRpc, diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index 8cbf0e55b9df..7322307c1f89 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -759,3 +759,16 @@ describe("ServerSettings environment icon", () => { expect(encodeServerSettings(linuxSettings).environmentIcon).toBe("linux"); }); }); + +const decodeDeviceHostSettings = Schema.decodeSync(ServerSettings); + +it("validates remote device hosts and rejects ambiguous host ids", () => { + const host = { id: "mini", label: "Mac mini", target: "user@mini", port: 2222 }; + expect(decodeDeviceHostSettings({ deviceHosts: [host] }).deviceHosts).toEqual([host]); + expect(() => decodeDeviceHostSettings({ deviceHosts: [host, host] })).toThrow(); + expect(() => decodeDeviceHostSettings({ deviceHosts: [{ ...host, id: "local" }] })).toThrow(); + expect(() => + decodeDeviceHostSettings({ deviceHosts: [{ ...host, target: "-oProxyCommand=bad" }] }), + ).toThrow(); + expect(() => decodeDeviceHostSettings({ deviceHosts: [{ ...host, port: 0 }] })).toThrow(); +}); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 96516367612c..dd6136461fc1 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1,3 +1,4 @@ +import { SshDeviceHostConfigs } from "./device.ts"; import * as Effect from "effect/Effect"; import * as Duration from "effect/Duration"; import * as Schema from "effect/Schema"; @@ -983,6 +984,7 @@ export const ServerSettings = Schema.Struct({ enableDeviceSupport: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), /** Whether the server-local Device panel setup flow has been completed. */ deviceOnboardingCompleted: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + deviceHosts: SshDeviceHostConfigs.pipe(Schema.withDecodingDefault(Effect.succeed([]))), sidebarAutoSettleAfterDays: Schema.NullOr(SidebarAutoSettleAfterDays).pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_SIDEBAR_AUTO_SETTLE_AFTER_DAYS)), ), @@ -1258,6 +1260,7 @@ export const ServerSettingsPatch = Schema.Struct({ enableAgentDeviceAccess: Schema.optionalKey(Schema.Boolean), enableDeviceSupport: Schema.optionalKey(Schema.Boolean), deviceOnboardingCompleted: Schema.optionalKey(Schema.Boolean), + deviceHosts: Schema.optionalKey(SshDeviceHostConfigs), sidebarAutoSettleAfterDays: Schema.optionalKey(Schema.NullOr(SidebarAutoSettleAfterDays)), sidebarAutoSettleOnMerge: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( diff --git a/packages/shared/src/serverSettings.test.ts b/packages/shared/src/serverSettings.test.ts index a5e428fcdaac..cc783fe64bf3 100644 --- a/packages/shared/src/serverSettings.test.ts +++ b/packages/shared/src/serverSettings.test.ts @@ -21,6 +21,16 @@ import { } from "./serverSettings.ts"; describe("serverSettings helpers", () => { + it("replaces SSH host lists when saving, editing, and removing hosts", () => { + const host = { id: "mini", label: "Mac mini", target: "mini" }; + const saved = applyServerSettingsPatch(DEFAULT_SERVER_SETTINGS, { deviceHosts: [host] }); + expect(saved.deviceHosts).toEqual([host]); + const replacement = { ...host, target: "other-mini" }; + const edited = applyServerSettingsPatch(saved, { deviceHosts: [replacement] }); + expect(edited.deviceHosts).toEqual([replacement]); + expect(applyServerSettingsPatch(edited, { deviceHosts: [] }).deviceHosts).toEqual([]); + }); + it("inherits actions, preserves existing actions, and supports empty overrides and reset", () => { const project = { id: ProjectId.make("project-actions"), scripts: [] }; const action = { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6c58c455b465..bebd36cec2e8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -540,6 +540,9 @@ importers: '@t3tools/shared': specifier: workspace:* version: link:../../packages/shared + '@t3tools/ssh': + specifier: workspace:* + version: link:../../packages/ssh '@t3tools/tailscale': specifier: workspace:* version: link:../../packages/tailscale