Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .agents/skills/headless-computer-use/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ Chromium capabilities; macOS WebKit reports them as unsupported.
4. Start with `inspect --context summary --task "..."`, then use an outline,
scoped text, or scoped actions only when the task needs them.
5. Prefer role/name targeting; otherwise use a ref from the latest inspection.
For native dropdowns, use `select` with exactly one exact `--label` or
`--value`; do not treat custom ARIA widgets as native selects.
6. After navigation or a substantial rerender, wait for the expected URL, text,
settled state, or Chromium network idle and inspect again before the next
interaction.
Expand Down
9 changes: 6 additions & 3 deletions .agents/skills/headless-computer-use/references/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ headless --session NAME click REF
headless --session NAME click --role ROLE --name NAME
headless --session NAME fill REF "TEXT"
headless --session NAME fill REF -- "--json stays literal"
headless --session NAME select REF --label LABEL
headless --session NAME select --role combobox --name NAME --value VALUE
headless --session NAME upload REF --artifact FILE
headless --session NAME upload --role textbox --name NAME --artifact FILE
headless --session NAME press KEY
Expand All @@ -46,9 +48,10 @@ large pages, request `outline`, select a returned `@rN` region, then use
bound the result; check `omitted` before assuming it describes the whole page.
Use `click --role ... --name ...` for unique accessible controls. Use an `@eN`
ref from the latest inspection when role/name is ambiguous. Inspect again after
navigation or a large rerender. File inputs advertise `upload` for an existing
private artifact-store basename. Upload never accepts or imports a filesystem
path. Ask before uploading, as in [safety.md](safety.md).
navigation or a large rerender. Native single-select controls advertise
`select`; use exactly one exact `--label` or `--value`. File inputs advertise
`upload` for an existing private artifact-store basename. Upload never accepts
or imports a filesystem path. Ask before uploading, as in [safety.md](safety.md).

Pass fill text as one quoted shell argument so whitespace is preserved. Put
`--` before a value that contains a literal global flag such as `--json` or
Expand Down
12 changes: 12 additions & 0 deletions apps/headless/Host/AgentBridge.swift
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,15 @@ extension BrowserWindowController {
return try callAgent("return globalThis.__headlessAgent.fill(args);", arguments: ["args": args])
}

func agentSelect(parameters: [String: JSONValue]) throws -> JSONValue {
var args = try browserTargetArguments(parameters)
if let label = parameters["label"]?.stringValue { args["label"] = label }
if let value = parameters["value"]?.stringValue { args["value"] = value }
return try callAgent(
"return globalThis.__headlessAgent.select(args);", arguments: ["args": args]
)
}

func agentAuthenticationState() throws -> JSONValue {
try callAgent("return globalThis.__headlessAgent.authentication();")
}
Expand Down Expand Up @@ -550,6 +559,9 @@ extension BrowserWindowController: BrowserEngineSession {
func hostFill(parameters: [String: JSONValue]) throws -> JSONValue {
try agentFill(parameters: parameters)
}
func hostSelect(parameters: [String: JSONValue]) throws -> JSONValue {
try agentSelect(parameters: parameters)
}
func hostPress(parameters: [String: JSONValue]) throws -> JSONValue {
try agentPress(parameters: parameters)
}
Expand Down
9 changes: 9 additions & 0 deletions apps/headless/LinuxHost/BrowserProcess.swift
Original file line number Diff line number Diff line change
Expand Up @@ -805,6 +805,15 @@ final class LinuxBrowserSession: @unchecked Sendable {
])
}

func select(parameters: [String: JSONValue]) throws -> JSONValue {
var args = try browserTargetArguments(parameters)
if let label = parameters["label"]?.stringValue { args["label"] = label }
if let value = parameters["value"]?.stringValue { args["value"] = value }
return try evaluate(
"return globalThis.__headlessAgent.select(args);", input: ["args": args]
)
}

func authenticationState() throws -> JSONValue {
try evaluate("return globalThis.__headlessAgent.authentication();")
}
Expand Down
3 changes: 3 additions & 0 deletions apps/headless/LinuxHost/main.swift
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ final class ChromiumBrowserEngineSession: BrowserEngineSession {
func hostFill(parameters: [String: JSONValue]) throws -> JSONValue {
try browserSession.fill(parameters: parameters)
}
func hostSelect(parameters: [String: JSONValue]) throws -> JSONValue {
try browserSession.select(parameters: parameters)
}
func hostUpload(parameters: [String: JSONValue], artifactURL: URL) throws -> JSONValue {
try browserSession.upload(parameters: parameters, artifactURL: artifactURL)
}
Expand Down
22 changes: 22 additions & 0 deletions apps/headless/Sources/HeadlessProtocol/CLI.swift
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,8 @@ public struct CLIParser {
return remote(.fill, session: session, parameters: [
"target": .string(arguments[0]), "value": .string(arguments[1]),
], jsonOutput: jsonOutput)
case "select":
return try parseSelect(arguments, session: session, jsonOutput: jsonOutput)
case "press":
guard arguments.count == 1 else { throw CLIParseError.missingArgument("KEY") }
return remote(.press, session: session, parameters: ["key": .string(arguments[0])], jsonOutput: jsonOutput)
Expand Down Expand Up @@ -401,6 +403,24 @@ public struct CLIParser {
return remote(.upload, session: session, parameters: parameters, jsonOutput: jsonOutput)
}

private func parseSelect(
_ arguments: [String], session: String?, jsonOutput: Bool
) throws -> CLIInvocation {
var args = arguments
let label = try removeOption("--label", from: &args)
let value = try removeOption("--value", from: &args)
guard (label == nil) != (value == nil) else {
throw CLIParseError.missingArgument("exactly one of --label or --value")
}
let invocation = try parseTargeted(
.select, arguments: args, session: session, jsonOutput: jsonOutput
)
var parameters = invocation.request?.parameters ?? [:]
if let label { parameters["label"] = .string(label) }
if let value { parameters["value"] = .string(value) }
return remote(.select, session: session, parameters: parameters, jsonOutput: jsonOutput)
}

private func parseTargeted(
_ command: CommandName,
arguments: [String],
Expand Down Expand Up @@ -856,6 +876,8 @@ Commands:
[--within @rN] [--limit N] [--budget TOKENS] [--depth N] [--text]
click REF | click --role ROLE [--name NAME]
fill REF TEXT | fill REF -- TEXT_WITH_LITERAL_FLAGS | press KEY
select REF --label LABEL | select REF --value VALUE
select --role ROLE [--name NAME] (--label LABEL | --value VALUE)
upload REF --artifact FILE | upload --role ROLE [--name NAME] --artifact FILE
scroll [up|down|top|bottom] [--amount PX]
back | reload
Expand Down
4 changes: 4 additions & 0 deletions apps/headless/Sources/HeadlessProtocol/Capabilities.swift
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public struct BrowserEngineCapabilities: Sendable {
public let qaDiagnosticSynchronization: String
public let screenshotClipboard: Bool
public let inputDispatch: String
public let selectDispatch: String
public let networkIdleWait: Bool
public let normalProfileStorage: String
public let fileUpload: Bool
Expand Down Expand Up @@ -69,6 +70,7 @@ public struct BrowserEngineCapabilities: Sendable {
"screenshotClipboard": .bool(screenshotClipboard),
"tourTimeoutMs": .number(65_000),
"inputDispatch": .string(inputDispatch),
"selectDispatch": .string(selectDispatch),
"networkIdleWait": .bool(networkIdleWait),
"fileUpload": .bool(fileUpload),
"normalProfile": .object([
Expand Down Expand Up @@ -115,6 +117,7 @@ public struct BrowserEngineCapabilities: Sendable {
qaDiagnosticSynchronization: "best-effort-page-world-observer",
screenshotClipboard: true,
inputDispatch: "synthetic-dom",
selectDispatch: "synthetic-dom",
networkIdleWait: false,
normalProfileStorage: "persistent-wkwebsite-data-store",
fileUpload: false
Expand All @@ -139,6 +142,7 @@ public struct BrowserEngineCapabilities: Sendable {
qaDiagnosticSynchronization: "runtime-round-trip-flush",
screenshotClipboard: false,
inputDispatch: "trusted-cdp",
selectDispatch: "synthetic-dom",
networkIdleWait: true,
normalProfileStorage: "private-xdg-data-directory",
fileUpload: true
Expand Down
2 changes: 2 additions & 0 deletions apps/headless/Sources/HeadlessProtocol/HostCore.swift
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public protocol BrowserEngineSession: AnyObject {
func hostInspect(parameters: [String: JSONValue]) throws -> JSONValue
func hostClick(parameters: [String: JSONValue]) throws -> JSONValue
func hostFill(parameters: [String: JSONValue]) throws -> JSONValue
func hostSelect(parameters: [String: JSONValue]) throws -> JSONValue
func hostPress(parameters: [String: JSONValue]) throws -> JSONValue
func hostScroll(parameters: [String: JSONValue]) throws -> JSONValue
func hostWait(parameters: [String: JSONValue]) throws -> JSONValue
Expand Down Expand Up @@ -554,6 +555,7 @@ public final class HostCore<Engine: BrowserEngine>: @unchecked Sendable {
case .inspect: return try session.hostInspect(parameters: request.parameters)
case .click: return try session.hostClick(parameters: request.parameters)
case .fill: return try session.hostFill(parameters: request.parameters)
case .select: return try session.hostSelect(parameters: request.parameters)
case .upload:
guard let artifact = request.parameters["artifact"]?.stringValue else {
throw HostError(code: .missingParameter, message: "Artifact name is required.")
Expand Down
10 changes: 10 additions & 0 deletions apps/headless/Sources/HeadlessProtocol/Protocol.swift
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ public enum CommandName: String, Codable, CaseIterable, Sendable {
case inspect
case click
case fill
case select
case upload
case press
case scroll
Expand Down Expand Up @@ -265,6 +266,15 @@ public struct CommandRequest: Codable, Equatable, Sendable {
try target(allowValue: false)
case .fill:
try target(allowValue: true)
case .select:
try target(allowValue: false)
let label = try string("label", maximumBytes: 1_000)
let value = try string("value", maximumBytes: 1_000)
guard (label == nil) != (value == nil) else {
throw ProtocolValidationError.invalidParameter(
"Choose exactly one option label or value"
)
}
case .upload:
try target(allowValue: false)
if let artifact = try string("artifact", required: true, maximumBytes: 128) {
Expand Down
18 changes: 18 additions & 0 deletions apps/headless/Sources/HeadlessProtocol/ProtocolSchema.swift
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,11 @@ public func protocolResultDefinition(for command: CommandName) -> ProtocolResult
])
case .fill:
return result("Fill", [resultField("filled", .string), resultField("valueLength", .number)])
case .select:
return result("Select", [
resultField("selected", .string), resultField("role", .string),
resultField("name", .string), resultField("optionIndex", .number),
])
case .upload:
return result("Upload", [
resultField("uploaded", .string), resultField("role", .string),
Expand Down Expand Up @@ -701,6 +706,19 @@ public let protocolCommandDefinitions: [CommandName: ProtocolCommandDefinition]
untrusted: true,
constraints: ["exactly one target reference or semantic role/name target"]
),
command(
.select,
targetParameters + [
string("label", maximumBytes: 1_000, sensitive: true),
string("value", maximumBytes: 1_000, sensitive: true),
],
untrusted: true,
constraints: [
"exactly one target reference or semantic role/name target",
"exactly one option label or value",
"native single-selection HTML select controls only",
]
),
command(
.upload,
targetParameters + [string("artifact", required: true, maximumBytes: 128)],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ if (!globalThis.__headlessAgent) {
const elementRole = role(element);
// Only advertise verbs implemented by the public Headless protocol.
// Unsupported controls can still appear for context, but must not route
// an agent toward nonexistent select/slide commands. File inputs advertise
// an agent toward nonexistent verbs. File inputs advertise
// upload only when the host injected __headlessFileUpload, never fill or
// click as the primary verb.
if (element instanceof HTMLInputElement) {
Expand All @@ -161,6 +161,8 @@ if (!globalThis.__headlessAgent) {
if (tag === 'a' && element.hasAttribute('href')) hints.push('click');
if (tag === 'button' || elementRole === 'button') hints.push('click');
if (tag === 'summary' || elementRole === 'tab' || elementRole === 'menuitem') hints.push('click');
if (element instanceof HTMLSelectElement && !element.multiple && !element.disabled
&& element.getAttribute('aria-disabled') !== 'true') hints.push('select');
if (element instanceof HTMLTextAreaElement || element.isContentEditable) hints.push('fill');
if (element.tabIndex >= 0 && hints.length === 0) hints.push('click');
return Array.from(new Set(hints));
Expand Down Expand Up @@ -743,6 +745,57 @@ if (!globalThis.__headlessAgent) {
element.dispatchEvent(new Event('change', {bubbles: true}));
return {filled: refFor(element), valueLength: String(args.value).length};
};
const select = args => {
const element = target(args);
if (!(element instanceof HTMLSelectElement)) {
fail('INVALID_INPUT', 'INVALID_INPUT: target is not a native select');
}
if (element.multiple) {
fail('INVALID_INPUT', 'INVALID_INPUT: multi-select controls are not supported');
}
if (element.disabled || element.getAttribute('aria-disabled') === 'true') {
fail('INVALID_INPUT', 'INVALID_INPUT: select control is disabled');
}
const hasLabel = typeof args.label === 'string';
const hasValue = typeof args.value === 'string';
if (hasLabel === hasValue) {
fail('INVALID_INPUT', 'INVALID_INPUT: choose exactly one option label or value');
}
const wanted = hasLabel ? normalize(args.label) : args.value;
const matches = Array.from(element.options).filter(option =>
hasLabel ? normalize(option.label || option.textContent) === wanted : option.value === wanted
);
if (matches.length === 0) {
fail('INVALID_INPUT', 'INVALID_INPUT: matching option was not found');
}
if (matches.length > 1) {
fail('INVALID_INPUT', `INVALID_INPUT: option matcher is ambiguous (${matches.length} matches)`);
}
const option = matches[0];
const parentDisabled = option.parentElement instanceof HTMLOptGroupElement
&& option.parentElement.disabled;
if (option.disabled || parentDisabled) {
fail('INVALID_INPUT', 'INVALID_INPUT: matching option is disabled');
}
const optionIndex = Array.from(element.options).indexOf(option);
if (optionIndex < 0) fail('INVALID_INPUT', 'INVALID_INPUT: matching option is detached');
if (element.selectedIndex !== optionIndex) {
const setter = Object.getOwnPropertyDescriptor(
HTMLSelectElement.prototype, 'selectedIndex'
)?.set;
if (!setter) fail('OPERATION_FAILED', 'OPERATION_FAILED: native select setter unavailable');
element.focus({preventScroll: false});
setter.call(element, optionIndex);
if (element.selectedIndex !== optionIndex) {
fail('OPERATION_FAILED', 'OPERATION_FAILED: native select did not accept the option');
}
element.dispatchEvent(new Event('input', {bubbles: true}));
element.dispatchEvent(new Event('change', {bubbles: true}));
}
return {
selected: refFor(element), role: role(element), name: name(element), optionIndex
};
};
const credentialFill = args => {
const initialOrigin = String(location.origin || '');
if (initialOrigin !== args.origin) throw new Error('AUTH_ORIGIN_CHANGED');
Expand Down Expand Up @@ -1036,7 +1089,7 @@ if (!globalThis.__headlessAgent) {
return {count: document.getAnimations().length, animations: all, truncated: document.getAnimations().length > all.length};
};
return {
snapshot, click, fill, credentialFill, finishCredentialFill, press, inputTarget, fileInput, fileInputMetadata,
snapshot, click, fill, select, credentialFill, finishCredentialFill, press, inputTarget, fileInput, fileInputMetadata,
authentication, scroll, state, tour, screenshotPlan, scrollToCapturePoint, rectangle, styles, storage,
performance: performanceSummary, animations
};
Expand Down
42 changes: 42 additions & 0 deletions apps/headless/Tests/Fixtures/select.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Native select fixture</title>
</head>
<body>
<main>
<h1>Native select fixture</h1>
<label for="country">Country</label>
<select id="country">
<option value="">Choose a country</option>
<option value="CA"> Canada </option>
<option value="US">United States</option>
</select>

<select aria-label="Duplicate country">
<option value="first">Canada</option>
<option value="second"> Canada </option>
</select>
<select aria-label="Disabled country" disabled><option>Canada</option></select>
<select aria-label="Disabled option"><option>Choose</option><option disabled>Canada</option></select>
<select aria-label="Disabled group">
<option>Choose</option>
<optgroup label="Unavailable" disabled><option>Canada</option></optgroup>
</select>
<select aria-label="Many countries" multiple><option>Canada</option></select>
<div role="combobox" aria-label="Custom country">Canada</div>
<output aria-label="Selection state">country=none events=none</output>
</main>
<script>
const country = document.querySelector('#country');
const output = document.querySelector('output');
const events = [];
const update = () => {
output.textContent = `country=${country.value || 'none'} events=${events.join(',') || 'none'}`;
};
country.addEventListener('input', () => { events.push('input'); update(); });
country.addEventListener('change', () => { events.push('change'); update(); });
</script>
</body>
</html>
Loading
Loading