From f666e2bfd4550a851c8867e402853208c31609a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Kondratiuk?= Date: Sat, 17 Oct 2026 15:34:51 -0300 Subject: [PATCH 1/2] Hold modifier keys on tap and double tap The agent's tap and double_tap tools take a modifiers list on an engine that declares TapModifiers, as upstream's tools do on an engine that declares tapModifiers. The web engine holds the keys through Playwright. The held keys are recorded in the trace and replayed from the cache. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01KWUdrjfarTEM4dmBbK6GaV --- COMPATIBILITY.md | 5 +- skills/e2e-dotnet/references/agent.md | 10 +- skills/e2e-dotnet/references/mcp.md | 2 +- src/E2E.Cli/Mcp/SessionCatalog.cs | 12 +- src/E2E/Agent/ActContext.cs | 2 +- src/E2E/Agent/ActReplayHost.cs | 7 +- src/E2E/Agent/ActTools.cs | 32 ++- src/E2E/Agent/ActionDispatcher.cs | 71 +++++-- src/E2E/Agent/AgentTools.cs | 23 ++- src/E2E/Agent/DefaultAgent.cs | 5 + src/E2E/Agent/Executor.cs | 20 +- src/E2E/Agent/Replay.cs | 8 +- src/E2E/Agent/StepCache.cs | 2 +- src/E2E/Cache/StepCache.cs | 15 ++ src/E2E/Engine/IEngine.cs | 3 + src/E2E/Engine/KeyModifier.cs | 33 ++++ src/E2E/Engine/LocatorAction.cs | 6 +- src/E2E/Engine/WebEngine.cs | 22 ++- tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs | 2 +- .../AgentTapModifiers/TapModifiersEngine.cs | 54 ++++++ .../TapModifiersInTheGrammarToolsTests.cs | 183 ++++++++++++++++++ tests/E2E.Tests/StepCache/StepCacheHarness.cs | 4 +- .../CreateToolLoopExecutorTests.cs | 6 +- tests/E2E.Tests/TraceReplay/TestHost.cs | 13 +- tests/E2E.Tests/WebEngineTests.cs | 30 +++ 25 files changed, 523 insertions(+), 47 deletions(-) create mode 100644 src/E2E/Engine/KeyModifier.cs create mode 100644 tests/E2E.Tests/AgentTapModifiers/TapModifiersEngine.cs create mode 100644 tests/E2E.Tests/AgentTapModifiers/TapModifiersInTheGrammarToolsTests.cs diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 75df131b..17d6b5a7 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -114,7 +114,7 @@ Names are C# versions of the JavaScript API: `agent.act` is `ActAsync`, `screen. - Parallel workers and serial suites - The full locator action set (`dragTo`, `swipe`, `hover`, `setInputFiles`, `secondaryTap`, `longPress`, pointer points). `secondaryTap` and `longPress` have no counterpart in the document engine - The screen-level `screen.tapAt`, `screen.swipe`, and `screen.scrollUntilVisible` -- Per-action options other than `timeout` (and `delay` for `pressSequentially`): click `modifiers`, tap `position`, and the `{ label, value, index }` form of `selectOption` +- Per-action options other than `timeout` (and `delay` for `pressSequentially`): click `modifiers` on a locator (the agent's `tap` and `double_tap` hold them, above), tap `position`, and the `{ label, value, index }` form of `selectOption` - Route patterns that ignore a record id. This port compares the URL path exactly and ignores the query and fragment - Diff-only observations. Each model turn receives a full text snapshot - Telemetry @@ -275,6 +275,7 @@ Not ported: - As upstream, the judge prompt says a value shown in more than one place holds only when every place agrees, unless the statement names the place or is about some item among several. The wording follows upstream, with "instruction" as "statement" and the `"fails"` verdict as "false", because the port's judge answers through `done` - `ExtractAsync`: the type argument is the schema, in place of a Standard Schema. The judge gets the JSON schema of `T` (`JsonSchemaExporter`), and the answer must deserialize into `T` with required members and nullable annotations respected. A failure gets one repair round with the validation error, then `MODEL_OUTPUT_INVALID` (`extracted data failed schema validation: ...`). A judge that says the data is not shown ends `ASSERTION_INCONCLUSIVE` (`nothing to extract: ...`) - `tools` (`AgentOptions.Tools`, `AgentTools` on the default agent) are project tools the model is offered beside the agent's own. As upstream's `defineTool`, an `AgentTool` declares `mutates` and `platforms`: a tool is offered only on the platforms it names, and a mutating tool takes an action slot. Its output is redacted and cut at 16384 characters. A name the agent already uses (`observe`, `done`, `navigate`, `back`, `scroll`, `scroll_to`, or a verb) is refused with `INVALID_CONFIG`. Differences: the tool is an `AgentTool` built with its schema as a `JsonObject` and a delegate, since the port has no `defineTool`, AI SDK tool, or `toModelOutput`; a tool that wants upstream's `getToolContext` takes the second constructor, whose delegate gets an `AgentToolContext` (`ObserveAsync`, `AttachScreenshotAsync`), and the first constructor keeps working; and upstream's reserved names for tools it adds elsewhere (`complete_step`, `locate`, the recording tools, `report_finding`) have no counterpart. Upstream records a mutating tool as a gap in the trace, and so does the port: a recording holds an action of kind `tool` with the tool name, a replay runs the actions before it and then hands the step to the model, and the step is never replayed whole. Older versions of this port cannot read that action and run the step live. As upstream, a recording that stops at such a gap (reason `gap`) is not stale: a strict cache run replays what it can and runs the rest of the step live. The recording keeps the tool's name only, never its arguments +- `tap` and `double_tap` take an optional `modifiers` list (`Shift`, `Control`, `Alt`, `Meta`, `ControlOrMeta`) on an engine that declares `EngineCapabilities.TapModifiers`, as upstream's `tap`, `double_tap`, and `right_click` do on an engine that declares `tapModifiers`. The web engine declares it. The device engines and the document engine do not, and refuse the field with `UNSUPPORTED_CAPABILITY`. The held keys are recorded as `modifiers` of the `tap` or `doubleTap` action and replayed from the cache. An entry whose `modifiers` is empty, repeats a key, or sits on another action is a miss (`invalid-entry`). The tools do not close their schemas, so a tool called with a field its engine does not declare is refused by the action, not by the schema. There is no `right_click`, because no engine here has a secondary tap. - `act` also offers a `double_tap` tool, which upstream does not have. Without it, the agent cannot open a control that reacts only to a double-click, such as a TodoMVC label that opens its editor. It runs the locator `doubleTap`, and the replay cache records it as `doubleTap`. Upstream and older versions of this port cannot replay that entry, so they run the step live - `act` params are checked as upstream: JSON-safe values, at most 32 levels deep, no cycle, and at most 64 KiB once a `Secret` is projected to its name and purpose and a `Values.Unique` to its value. The instruction is at most 8192 UTF-8 bytes. Each limit throws `INVALID_ARGUMENT` with the upstream message before any model call. A value reached twice through different paths is not a cycle - `Vision` on `AssertAsync`, `WaitForAsync`, and `ExtractAsync` is a `VisionMode` (`Off`, `On`, `Only`) in place of upstream's `false`, `true`, and `'only'`, and an undefined value throws `INVALID_ARGUMENT`. `On` sends a masked screenshot beside the tree, and `Only` sends the screenshot alone, with a note that no tree is attached on purpose. The screenshot is sized for the model (long side at most 768 pixels) and its note names the regions masked for security. A secret filled in the attempt (by a locator or by the agent), an engine that captures no pixels, or masking the engine cannot prove (fewer masked regions than secure nodes in the tree, or no count at all) takes the image away and the judgment goes on with the tree; `Only` has no tree to go on with, so it fails with `POLICY_DENIED` that names `PIXEL_TAINTED`, `MASKING_UNPROVEN`, or `UNSUPPORTED_CAPABILITY`, before any model call. The judge's system text says an attached screenshot is data, as upstream's policy does. `WaitForAsync` takes a screenshot with each look and counts a changed picture as a changed screen. An inconclusive judgment made from the tree alone ends with the hint to pass `Vision = VisionMode.On`, firm once a judgment of the attempt has received pixels and conditional before ("if the engine captures pixels"), and none when pixels cannot help: a secret was filled, the engine declares no `Screenshot` capability, or an earlier request for pixels was degraded. The hint's words name the C# option @@ -282,7 +283,7 @@ Not ported: - `executor` (`AgentOptions.Executor`, `AgentExecutor` on `E2ESessionOptions` and the fixtures) replaces the built-in agent for `act` and `assert`. The socket is `IStepExecutor`: `Name`, and `Version`, `Cache`, `Model`, and `Judge` with defaults, so a hand-rolled executor writes `Name` and `RunStepAsync(context, cancellationToken)`. The token is upstream's `signal`; the context's `Attempt.CancellationToken` is upstream's `attempt.signal`. `StepExecutorContext` is a class a test can build and override: `Step`, `Attempt`, `Target`, `ReplayedPrefix`, `Model`, `ProviderOptions`, `Ledger`, `AgentContext`, `Budgets`, `Actions`, `PixelsTainted`, `ObserveAsync`, `AttachTranscript`, and `AttachScreenshotAsync`. An executor is code, so `agents..executor` in `e2e.config.json` stays `INVALID_CONFIG` and names `AgentOptions.Executor`. As upstream, `System` and `Tools` are refused beside an executor, and the `Model` and `Judge` an executor brought are the agent's, for the judgments too. An entry that names another is `INVALID_CONFIG` (`agents.default.model (gpt-5.4) and the executor's own model (gpt-5.4-mini) differ; configure the model in one place`). Two models are the same when they are one instance or share a `Name`, where upstream compares the provider and the id - `ToolLoop.CreateToolLoopExecutor` and `DefaultAgent.CreateBuiltInAgent` are public. Differences: `ToolLoopExecutorOptions.Tools` returns a list of `ToolLoopTool` (a `ModelTool` and a delegate that answers a `ToolLoopResult`), where upstream's returns an AI SDK `ToolSet`; `BuildPrompt` answers a string, where upstream's can answer a whole message history; the chassis adds `done`, where upstream adds `complete_step`; there is no `maxTurns` to remove. `ToolLoopHelpers` has `GuardAsync`, `Concluding`, and `ReportHardStop`, and the loop guards every tool call itself. The built-in agent drives the context the runtime builds, so it runs as a step's executor or inside a custom executor that passes its context on, and not on a context built by hand -- The verbs are the ones the port's engines have: `TapAsync`, `DoubleTapAsync`, `TypeAsync`, `TypeSecretAsync`, `ClearAsync` (upstream clears with `type`), `PressAsync`, `SelectAsync`, `CheckAsync`, `ScrollToAsync`, `ScrollUntilAsync`, `ScrollAsync`, `NavigateAsync`, and `BackAsync`. Not ported, because no engine here has them: long press, secondary tap, hover, drag, upload, the point verbs (`tapAt`, `hoverAt`, `hitTest`), `typeText`, `pressKey`, and `dismissKeyboard`. `Target.Verbs` follows the engine's capabilities: the scroll verbs need `Scroll`, `Back` needs `History`. A node is named by its ref (`new ExecutorTarget("e12")`, the `[ref=e12]` of a line), where upstream names `n42`. Actions and looks run in call order, mutating project tools included, and a mutating `RunToolAsync` body that calls an action or a look waits behind itself, as upstream's does. A verb checks its argument before it takes a slot (`INVALID_ARGUMENT`), but the port has no key grammar, so `PressAsync` takes any non-blank key +- The verbs are the ones the port's engines have: `TapAsync` and `DoubleTapAsync` (both take `ClickOptions`, whose `Modifiers` need `Target.TapModifiers`, as upstream's `tap(target, { modifiers })` needs `target.tapModifiers`), `TypeAsync`, `TypeSecretAsync`, `ClearAsync` (upstream clears with `type`), `PressAsync`, `SelectAsync`, `CheckAsync`, `ScrollToAsync`, `ScrollUntilAsync`, `ScrollAsync`, `NavigateAsync`, and `BackAsync`. Not ported, because no engine here has them: long press, secondary tap, hover, drag, upload, the point verbs (`tapAt`, `hoverAt`, `hitTest`), `typeText`, `pressKey`, and `dismissKeyboard`. `Target.Verbs` follows the engine's capabilities: the scroll verbs need `Scroll`, `Back` needs `History`. A node is named by its ref (`new ExecutorTarget("e12")`, the `[ref=e12]` of a line), where upstream names `n42`. Actions and looks run in call order, mutating project tools included, and a mutating `RunToolAsync` body that calls an action or a look waits behind itself, as upstream's does. A verb checks its argument before it takes a slot (`INVALID_ARGUMENT`), but the port has no key grammar, so `PressAsync` takes any non-blank key - `ObserveAsync` answers `Text`, `Truncated`, and `Path`, and with `ExecutorObserveOptions` `Tree` (a list of root `ExecutorNode`s, redacted, with no value or selection on a secure field, a link target absolute and cut as a line cuts it) and `Pixels` (the masked screenshot sized for the model, or `PixelsWithheld`, a `VisionDegradation`). It has no `Revision`, no `Viewport`, and no `framePath`, since the port has none. `ReplayedPrefix` carries the actions as the text the hand-off notice lists, `ReplayHandOffReason` adds the port's `ActionBudget` and `InvalidEntry`, and has no `viewport-changed`. A replay that meets `ACTION_MAY_HAVE_COMMITTED` stops as `action-uncertain` and names the action, which the built-in agent's notice repeats - The runtime owns the budgets and the clock. `RecordModelCall` throws `STEP_BUDGET_EXHAUSTED` past `MaxModelCalls`, and a verdict never outranks a hard stop: a passed one over a spent budget, a timeout, or a code-less failure after one ends with the runtime's code. An executor that ignores its token gets one second to unwind after the step is stopped, and then the step settles without it. The closed verdict grammar is upstream's: an empty summary, an unknown code, a passed verdict with a code, or a blocked one without a blockable code is `MODEL_OUTPUT_INVALID`, and so is a runtime code the runtime did not record. A thrown error that is no agent error becomes `ACTION_FAILED`, as upstream's `toAgentError` does, and a cancel by the caller stays an `OperationCanceledException`. The message of a failed or blocked act is the executor's summary alone, where upstream's reads `agent.act failed: ` - `AssertAsync` on an agent with an executor goes to it as a step of kind `Assert`, with the agent's budgets and `JudgmentTimeout`. An assert is never cached, and a passing one verifies the acts before it. `Vision` is refused with `UNSUPPORTED_CAPABILITY`, since the executor decides what its model sees, and `Screenshot` keeps its evidence after the verdict, bounded by the action timeout: a capture that outlives it is dropped and the verdict stands. `WaitForAsync` and `ExtractAsync` keep the judge, which is the executor's `Judge` or `Model`. `ExecutorCache.Off` never replays or records, and takes no look before the step. A step with the cache on that cannot read its first screen (no app opened) still runs, and records nothing diff --git a/skills/e2e-dotnet/references/agent.md b/skills/e2e-dotnet/references/agent.md index 717ba754..87529d77 100644 --- a/skills/e2e-dotnet/references/agent.md +++ b/skills/e2e-dotnet/references/agent.md @@ -99,6 +99,10 @@ The tools the model can call: `observe`, `tap`, `double_tap`, `fill`, `back` with one that has history. There is no `hover`, `drag`, `upload`, `long_press`, `right_click`, `screenshot`, or point tool: a step that needs one is `AUTOMATION_UNSUPPORTED`. Do that part with `Screen` actions instead. +On an engine that declares `EngineCapabilities.TapModifiers` (the web engine), +`tap` and `double_tap` take an optional `modifiers` list (`Shift`, `Control`, +`Alt`, `Meta`, `ControlOrMeta`) to hold keys for the click, as a Shift-click +extends a selection. The held keys are recorded and replayed from the cache. A project tool adds its own name to this list (see "Project tools"). `fill_secret` fills any editable field, a rich-text editor included, with a generic secret; a password secret needs a password field. @@ -423,7 +427,7 @@ public sealed class TapOnlyTests : E2ETest new ModelTool { Name = "tap", Description = "Tap a node by its ref from the latest read_screen", Parameters = Schema("""{ "type": "object", "properties": { "ref": { "type": "string" } }, "required": ["ref"] }""") }, async (call, token) => { - await context.Actions.TapAsync(new ExecutorTarget(call.Arguments.GetProperty("ref").GetString()!), token); + await context.Actions.TapAsync(new ExecutorTarget(call.Arguments.GetProperty("ref").GetString()!), cancellationToken: token); return new ToolLoopResult("Tapped."); }), ], @@ -472,8 +476,8 @@ What the context holds: | --- | --- | | `Step` | The `Kind`, `Instruction`, `Params`, `Index`, and `Secrets`. A secret value the attempt knows reads ``, and a `Secret` in the params is its name and purpose, never its value | | `ObserveAsync(options)` | Redacted screen text, and with `Tree` the node tree (`ExecutorNode`, whose `Id` is the ref) and with `Pixels` a masked screenshot, or `PixelsWithheld` with the reason | -| `Actions` | Checked and recorded verbs: `TapAsync`, `DoubleTapAsync`, `TypeAsync`, `TypeSecretAsync`, `ClearAsync`, `PressAsync`, `SelectAsync`, `CheckAsync`, `ScrollToAsync`, `ScrollUntilAsync`, `ScrollAsync`, `NavigateAsync`, `BackAsync`. A node is `new ExecutorTarget(ref)` | -| `Target` | The target's `Name`, `Platform`, and `Verbs`, the verbs its engine supports. Offer a model only these | +| `Actions` | Checked and recorded verbs: `TapAsync` and `DoubleTapAsync` (both take `new ClickOptions { Modifiers = [KeyModifier.Shift] }` when `Target.TapModifiers`), `TypeAsync`, `TypeSecretAsync`, `ClearAsync`, `PressAsync`, `SelectAsync`, `CheckAsync`, `ScrollToAsync`, `ScrollUntilAsync`, `ScrollAsync`, `NavigateAsync`, `BackAsync`. A node is `new ExecutorTarget(ref)` | +| `Target` | The target's `Name`, `Platform`, and `Verbs`, the verbs its engine supports, and `TapModifiers`, true when `TapAsync` and `DoubleTapAsync` take `ClickOptions.Modifiers`. Offer a model only these | | `PixelsTainted` | True after a secret fill. Then no screenshots are available | | `ReplayedPrefix` | Set when a cache replay ran part of the step first. The actions and why it stopped. Continue from the current screen without repeating them | | `Model`, `ProviderOptions`, `AgentContext`, `Ledger` | The agent's model and provider options, its project context, and the steps completed before this one | diff --git a/skills/e2e-dotnet/references/mcp.md b/skills/e2e-dotnet/references/mcp.md index e6d8fff9..38710646 100644 --- a/skills/e2e-dotnet/references/mcp.md +++ b/skills/e2e-dotnet/references/mcp.md @@ -84,7 +84,7 @@ config declares a `secrets` entry. | Tool | Does | | --- | --- | | `observe` | The whole current screen | -| `tap`, `double_tap`, `check`, `uncheck`, `clear` `{target}` | Act on one node | +| `tap`, `double_tap`, `check`, `uncheck`, `clear` `{target}` | Act on one node. On an engine that declares `TapModifiers` (the web engine), `tap` and `double_tap` also take an optional `modifiers` list (`Shift`, `Control`, `Alt`, `Meta`, `ControlOrMeta`) to hold keys for the click, as a Shift-click extends a selection | | `type {target, text}` | Replace the text in a field | | `type_secret {target, secret}` | Fill a configured secret into an editable field by name; you never see the value | | `press {key, target?}` | Press a key on a node, or on whatever has focus | diff --git a/src/E2E.Cli/Mcp/SessionCatalog.cs b/src/E2E.Cli/Mcp/SessionCatalog.cs index 9f4802ad..4c8e18e9 100644 --- a/src/E2E.Cli/Mcp/SessionCatalog.cs +++ b/src/E2E.Cli/Mcp/SessionCatalog.cs @@ -61,8 +61,9 @@ public static SessionCatalog Create(ISessionContext context, EngineCapabilities void Add(string name, CatalogTool tool) => tools.Add(new(name, tool)); Add("observe", Observe(context)); - Add("tap", Verb(context, "tap", "tap", "Tap or click one node: the gesture for a button, link, menu item, tab, checkbox, row, or field.", Schema(Target))); - Add("double_tap", Verb(context, "double_tap", "double_tap", "Double-click one node. Use it only when a tap does nothing, such as a label that opens an editor on a double-click.", Schema(Target))); + var holdable = capabilities.HasFlag(EngineCapabilities.TapModifiers) ? new[] { ("modifiers", Modifiers, false) } : []; + Add("tap", Verb(context, "tap", "tap", "Tap or click one node: the gesture for a button, link, menu item, tab, checkbox, row, or field.", Schema(Target, holdable))); + Add("double_tap", Verb(context, "double_tap", "double_tap", "Double-click one node. Use it only when a tap does nothing, such as a label that opens an editor on a double-click.", Schema(Target, holdable))); Add("type", Verb(context, "type", "fill", "Replace the text in a field with text.", Schema(Target, ("text", Property("string", "The text the field should hold"), true)), ("text", "value"))); Add("press", Verb(context, "press", "press", "Press a key such as Enter, Escape, or Tab. With a target, the key goes to that node; without one, to whatever has focus.", Schema(OptionalTarget, ("key", Property("string", "The key, such as Enter"), true)))); Add("select", Verb(context, "select", "select", "Choose an option of a combobox or select, by its value or label.", Schema(Target, ("value", Property("string", "The option's value or label"), true)))); @@ -200,6 +201,13 @@ private static readonly (string Name, JsonNode Schema, bool Required)[] Optional private static JsonNode Direction => new JsonObject { ["type"] = "string", ["enum"] = new JsonArray("up", "down", "left", "right") }; + private static JsonNode Modifiers => new JsonObject + { + ["type"] = "array", + ["items"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray([.. KeyModifiers.Names.Select(name => (JsonNode)name)]) }, + ["description"] = "Keys held while the pointer clicks. A Shift-click extends a selection, a Control-click (Meta on macOS) toggles one item", + }; + private static JsonNode TimesProperty => new JsonObject { ["type"] = "integer", diff --git a/src/E2E/Agent/ActContext.cs b/src/E2E/Agent/ActContext.cs index 0201cd6d..aac476d1 100644 --- a/src/E2E/Agent/ActContext.cs +++ b/src/E2E/Agent/ActContext.cs @@ -66,7 +66,7 @@ public ActContext(AttemptScope scope, ObservationFeed feed, StepAccounting budge CancellationToken = scope.Ended.Token, Memory = scope.ExecutorMemory, }; - Target = new ExecutorTargetInfo { Name = scope.TargetName, Platform = scope.EnginePlatform, Verbs = VerbsOf(scope.EngineCapabilities) }; + Target = new ExecutorTargetInfo { Name = scope.TargetName, Platform = scope.EnginePlatform, Verbs = VerbsOf(scope.EngineCapabilities), TapModifiers = scope.EngineCapabilities.HasFlag(EngineCapabilities.TapModifiers) }; ReplayedPrefix = prefix; Model = agent.Model; ProviderOptions = agent.ProviderOptions; diff --git a/src/E2E/Agent/ActReplayHost.cs b/src/E2E/Agent/ActReplayHost.cs index b4163d8d..d8bff3b9 100644 --- a/src/E2E/Agent/ActReplayHost.cs +++ b/src/E2E/Agent/ActReplayHost.cs @@ -43,9 +43,12 @@ private sealed class ActReplayHost( public void Commit(RecordedAction action) => budget.Recorder.Record(action); - public Task TapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => PerformAsync(target, new LocatorAction.Tap(), "tap", cancellationToken); + public Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => ClickAsync("tap", target, options, cancellationToken); - public Task DoubleTapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => PerformAsync(target, new LocatorAction.DoubleTap(), "doubleTap", cancellationToken); + public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => ClickAsync("doubleTap", target, options, cancellationToken); + + private Task ClickAsync(string kind, ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken) => + PerformAsync(target, ActionDispatcher.ClickAction(kind, ActionDispatcher.RequireClickModifiers(options, kind, scope.EngineCapabilities)), kind, cancellationToken); // A value is expanded with this call's params before it is read: one that expands to a secret's marker fills that secret. public Task TypeAsync(ExecutorTarget target, string value, CancellationToken cancellationToken = default) diff --git a/src/E2E/Agent/ActTools.cs b/src/E2E/Agent/ActTools.cs index 1020fcb2..d89a5b19 100644 --- a/src/E2E/Agent/ActTools.cs +++ b/src/E2E/Agent/ActTools.cs @@ -132,10 +132,10 @@ private async Task RunActionAsync(ModelToolCall call, IExecutorActi switch (call.Name) { case "tap": - await actions.TapAsync(target, token).ConfigureAwait(false); + await actions.TapAsync(target, ClickOptionsOf(call.Arguments), token).ConfigureAwait(false); return ToolOutcome.Ok(await DescribeAsync("tapped " + Label(node), token).ConfigureAwait(false)); case "double_tap": - await actions.DoubleTapAsync(target, token).ConfigureAwait(false); + await actions.DoubleTapAsync(target, ClickOptionsOf(call.Arguments), token).ConfigureAwait(false); return ToolOutcome.Ok(await DescribeAsync("double-tapped " + Label(node), token).ConfigureAwait(false)); case "fill": var value = Args.String(call.Arguments, "value") ?? ""; @@ -300,6 +300,34 @@ private async Task FillSecretAsync(ExecutorTarget target, JsonEleme return ToolOutcome.Ok(await DescribeAsync("filled secret into " + Label(target.Node!), token).ConfigureAwait(false)); } + // The keys a tap call holds: absent for none. A value that is no list of key modifier names is the model's error. + private static ClickOptions? ClickOptionsOf(JsonElement arguments) + { + if (arguments.ValueKind != JsonValueKind.Object || !arguments.TryGetProperty("modifiers", out var held) || held.ValueKind == JsonValueKind.Null) + { + return null; + } + + var invalid = new TestException("INVALID_ARGUMENT", "modifiers must be an array of " + string.Join(", ", KeyModifiers.Names)); + if (held.ValueKind != JsonValueKind.Array) + { + throw invalid; + } + + var modifiers = new List(); + foreach (var item in held.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String || !KeyModifiers.TryParse(item.GetString(), out var modifier)) + { + throw invalid; + } + + modifiers.Add(modifier); + } + + return new ClickOptions { Modifiers = modifiers }; + } + private async Task ResolveAsync(JsonElement arguments, CancellationToken token) { var observation = await _feed.ObserveAsync(SettleMode.Raw, token).ConfigureAwait(false); diff --git a/src/E2E/Agent/ActionDispatcher.cs b/src/E2E/Agent/ActionDispatcher.cs index 4f321696..40e8e187 100644 --- a/src/E2E/Agent/ActionDispatcher.cs +++ b/src/E2E/Agent/ActionDispatcher.cs @@ -43,25 +43,72 @@ public ActionDispatcher(AttemptScope scope, ObservationFeed feed, StepAccounting _secrets = secrets; } - public Task TapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => - RunAsync( + public Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => + ClickAsync("tap", target, options, cancellationToken); + + public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => + ClickAsync("doubleTap", target, options, cancellationToken); + + // Upstream's nodeVerb for the tap family: the modifiers are checked before a slot is taken, and recorded with the action. + private Task ClickAsync(string kind, ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken) + { + var modifiers = RequireClickModifiers(options, kind, _scope.EngineCapabilities); + return RunAsync( async token => { var node = Resolve(target); - await TargetedAsync(node, new LocatorAction.Tap(), token).ConfigureAwait(false); - Commit("tap", node); + await TargetedAsync(node, ClickAction(kind, modifiers), token).ConfigureAwait(false); + var recorded = Target(node, kind); + recorded.Modifiers = modifiers?.Select(modifier => modifier.ToString()).ToList(); + Commit(recorded); }, cancellationToken); + } - public Task DoubleTapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => - RunAsync( - async token => + /// + /// The modifiers of a tap-family action, validated: distinct key modifiers, on an engine that declares + /// . An empty list holds nothing and is dropped, so the recorded action stays + /// clean. Upstream's requireClickModifiers; the capability check is its session's perform. + /// + internal static IReadOnlyList? RequireClickModifiers(ClickOptions? options, string verb, EngineCapabilities capabilities) + { + if (options?.Modifiers is not { } given) + { + return null; + } + + var modifiers = new List(); + foreach (var modifier in given) + { + if (!Enum.IsDefined(modifier)) { - var node = Resolve(target); - await TargetedAsync(node, new LocatorAction.DoubleTap(), token).ConfigureAwait(false); - Commit("doubleTap", node); - }, - cancellationToken); + throw new TestException("INVALID_ARGUMENT", verb + "() modifiers must be an array of " + string.Join(", ", KeyModifiers.Names)); + } + + if (modifiers.Contains(modifier)) + { + throw new TestException("INVALID_ARGUMENT", verb + "() names a modifier twice"); + } + + modifiers.Add(modifier); + } + + if (modifiers.Count == 0) + { + return null; + } + + if (!capabilities.HasFlag(EngineCapabilities.TapModifiers)) + { + throw new EngineException(EngineErrorCodes.UnsupportedCapability, "The engine cannot hold modifiers on " + verb + "."); + } + + return modifiers; + } + + /// The engine action of a tap or doubleTap holding . + internal static LocatorAction ClickAction(string kind, IReadOnlyList? modifiers) => + string.Equals(kind, "doubleTap", StringComparison.Ordinal) ? new LocatorAction.DoubleTap(modifiers) : new LocatorAction.Tap(modifiers); public Task TypeAsync(ExecutorTarget target, string value, CancellationToken cancellationToken = default) { diff --git a/src/E2E/Agent/AgentTools.cs b/src/E2E/Agent/AgentTools.cs index bd0f91ed..12c465bb 100644 --- a/src/E2E/Agent/AgentTools.cs +++ b/src/E2E/Agent/AgentTools.cs @@ -119,6 +119,27 @@ internal static class AgentTools Tool("clear", "Clear a field.", Target), ]; + // The tap family names the keys it holds for the click, on an engine that declares them. + private static ModelTool HoldingModifiers(ModelTool verb) => + verb.Name is "tap" or "double_tap" + ? new ModelTool + { + Name = verb.Name, + Description = verb.Description + " Pass modifiers to hold keys for the click.", + Parameters = Extend( + Target, + $$""" + { + "modifiers": { + "type": "array", + "items": { "type": "string", "enum": {{JsonSerializer.Serialize(KeyModifiers.Names)}} }, + "description": "Keys held while the pointer clicks: Shift, Control, Alt, Meta, or ControlOrMeta. A Shift-click extends a selection, a Control-click (Meta on macOS) toggles one item." + } + } + """), + } + : verb; + private static readonly ModelTool Navigate = Tool("navigate", "Open a path or URL.", Parse("""{ "type": "object", "properties": { "url": { "type": "string" } }, "required": ["url"] }""")); /// The verdict tool every loop offers, whatever its vocabulary, with the policy that accepts or refuses a verdict. @@ -129,7 +150,7 @@ internal static VerdictTool CreateVerdictTool() => public static IReadOnlyList CreateGrammarTools(EngineCapabilities capabilities, IReadOnlyList? project = null) { var tools = new List { Observe }; - tools.AddRange(Verbs); + tools.AddRange(capabilities.HasFlag(EngineCapabilities.TapModifiers) ? Verbs.Select(HoldingModifiers) : Verbs); if (capabilities.HasFlag(EngineCapabilities.Scroll)) { tools.Add(ScrollTo); diff --git a/src/E2E/Agent/DefaultAgent.cs b/src/E2E/Agent/DefaultAgent.cs index 22fabe64..70c47964 100644 --- a/src/E2E/Agent/DefaultAgent.cs +++ b/src/E2E/Agent/DefaultAgent.cs @@ -153,6 +153,11 @@ internal static string Describe(RecordedAction action) } } + if (action.Modifiers is { Count: > 0 }) + { + builder.Append(" with ").Append(string.Join('+', action.Modifiers)).Append(" held"); + } + if (action.Url is not null) { builder.Append(' ').Append(action.Url); diff --git a/src/E2E/Agent/Executor.cs b/src/E2E/Agent/Executor.cs index fde29f9f..55e8888e 100644 --- a/src/E2E/Agent/Executor.cs +++ b/src/E2E/Agent/Executor.cs @@ -204,6 +204,9 @@ public sealed class ExecutorTargetInfo public required string Platform { get; init; } public IReadOnlySet Verbs { get; init; } = new HashSet(); + + /// True when and take . + public bool TapModifiers { get; init; } } /// One verb of . The port has the verbs its engines can perform. @@ -299,6 +302,13 @@ public sealed record ExecutorObservation(string Text) public VisionDegradation? PixelsWithheld { get; init; } } +/// What a tap-family action takes. Upstream's ClickOptions, of which the port has the modifiers. +public sealed class ClickOptions +{ + /// Keys held for the click, distinct. An empty list holds nothing. + public IReadOnlyList? Modifiers { get; init; } +} + /// /// A node named by its id from the newest observation (upstream's { id }). The id is the node's ref in the text and in /// . A ref the screen no longer lists fails the action instead of acting on the wrong node. @@ -339,10 +349,14 @@ internal ExecutorTarget(SemanticNode node) /// public interface IExecutorActions { - Task TapAsync(ExecutorTarget target, CancellationToken cancellationToken = default); + /// + /// Taps one node. holds keys for the click, as a Shift-click extends a selection; it + /// needs , UNSUPPORTED_CAPABILITY otherwise. + /// + Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default); - /// Double-taps one node. - Task DoubleTapAsync(ExecutorTarget target, CancellationToken cancellationToken = default); + /// Double-taps one node. Takes the tap's . + Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default); Task TypeAsync(ExecutorTarget target, string value, CancellationToken cancellationToken = default); diff --git a/src/E2E/Agent/Replay.cs b/src/E2E/Agent/Replay.cs index d69bf33a..d7b1149f 100644 --- a/src/E2E/Agent/Replay.cs +++ b/src/E2E/Agent/Replay.cs @@ -283,9 +283,9 @@ private static Task PerformAsync(IExecutorActions actions, ExecutorTarget target switch (action.Kind) { case "tap": - return actions.TapAsync(target, token); + return actions.TapAsync(target, ClickOptionsOf(action), token); case "doubleTap": - return actions.DoubleTapAsync(target, token); + return actions.DoubleTapAsync(target, ClickOptionsOf(action), token); case "fill" when action.Value is { } value && value.StartsWith("'): return actions.TypeSecretAsync(target, value[" + action.Modifiers is { } names ? new ClickOptions { Modifiers = [.. names.Select(name => Enum.Parse(name))] } : null; + /// /// Re-finds a recorded target. The first look is the one in hand, or a capture settled as far as the previous action /// requires; the looks after it read the screen as it is. Exactly one match is the target: several are ambiguity, which diff --git a/src/E2E/Agent/StepCache.cs b/src/E2E/Agent/StepCache.cs index 6ffb857d..433603e9 100644 --- a/src/E2E/Agent/StepCache.cs +++ b/src/E2E/Agent/StepCache.cs @@ -225,7 +225,7 @@ private EntryRead ReadEntry() Pending.ReadEntry = true; // Any store can return a hit, so its shape is checked here, at the one read site. - return entry.Schema != FileStepCache.SchemaVersion || entry.Actions is not { Count: > 0 } || entry.Appeared is null || entry.Gone is null + return entry.Schema != FileStepCache.SchemaVersion || entry.Actions is not { Count: > 0 } || entry.Appeared is null || entry.Gone is null || !entry.Actions.All(action => action.HasValidModifiers()) ? new EntryRead(null, "invalid-entry") : new EntryRead(entry, null); } diff --git a/src/E2E/Cache/StepCache.cs b/src/E2E/Cache/StepCache.cs index 050b917b..a49305af 100644 --- a/src/E2E/Cache/StepCache.cs +++ b/src/E2E/Cache/StepCache.cs @@ -8,6 +8,7 @@ using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; +using E2E.Engine; using E2E.Internal; namespace E2E; @@ -87,6 +88,20 @@ public sealed class RecordedAction /// The text a scrollUntil paged toward. public string? Text { get; set; } + + /// Keys a tap or doubleTap held for the click, replayed with it. Null holds none. + public List? Modifiers { get; set; } + + /// + /// Whether is well formed: absent, or on a tap or doubleTap as a non-empty list of + /// distinct key modifier names. Upstream's readModifiers, which makes a malformed entry a miss. + /// + internal bool HasValidModifiers() => + Modifiers is null + || (Kind is "tap" or "doubleTap" + && Modifiers.Count > 0 + && Modifiers.Distinct(StringComparer.Ordinal).Count() == Modifiers.Count + && Modifiers.All(name => KeyModifiers.TryParse(name, out _))); } public sealed class RecordedTarget diff --git a/src/E2E/Engine/IEngine.cs b/src/E2E/Engine/IEngine.cs index d73b524a..37159ea1 100644 --- a/src/E2E/Engine/IEngine.cs +++ b/src/E2E/Engine/IEngine.cs @@ -22,6 +22,9 @@ public enum EngineCapabilities /// , which toHaveScreenshot compares. Screenshot = 64, + + /// The Modifiers of and : keys held for the click. + TapModifiers = 128, } /// Passed to once per test attempt. diff --git a/src/E2E/Engine/KeyModifier.cs b/src/E2E/Engine/KeyModifier.cs new file mode 100644 index 00000000..a7e4b4fd --- /dev/null +++ b/src/E2E/Engine/KeyModifier.cs @@ -0,0 +1,33 @@ +// Copyright 2026 TesterArmy. +// Modified by Dario Kondratiuk. +// SPDX-License-Identifier: Apache-2.0 + +namespace E2E.Engine; + +/// +/// A key held for a click, as a Shift-click extends a selection. Upstream's KeyModifier (engine/contract.ts). +/// is Control, or Meta on macOS. +/// +public enum KeyModifier +{ + Shift, + Control, + Alt, + Meta, + ControlOrMeta, +} + +/// The names of , which a recorded action and an MCP argument spell. Upstream's KEY_MODIFIERS and isKeyModifier. +public static class KeyModifiers +{ + /// The names in declaration order: Shift, Control, Alt, Meta, ControlOrMeta. + public static IReadOnlyList Names { get; } = [.. Enum.GetNames()]; + + /// Reads a name exactly as spelled: no other case, no number. + public static bool TryParse(string? name, out KeyModifier modifier) + { + var index = name is null ? -1 : Array.IndexOf(Enum.GetNames(), name); + modifier = index < 0 ? default : Enum.GetValues()[index]; + return index >= 0; + } +} diff --git a/src/E2E/Engine/LocatorAction.cs b/src/E2E/Engine/LocatorAction.cs index 3686322a..c0187fde 100644 --- a/src/E2E/Engine/LocatorAction.cs +++ b/src/E2E/Engine/LocatorAction.cs @@ -11,9 +11,11 @@ private LocatorAction() { } - public sealed record Tap : LocatorAction; + /// A tap. are held for the click, and only an engine that declares receives them. + public sealed record Tap(IReadOnlyList? Modifiers = null) : LocatorAction; - public sealed record DoubleTap : LocatorAction; + /// A double tap. are held for both clicks, as for . + public sealed record DoubleTap(IReadOnlyList? Modifiers = null) : LocatorAction; public sealed record Fill(string Value, bool Sensitive = false) : LocatorAction; diff --git a/src/E2E/Engine/WebEngine.cs b/src/E2E/Engine/WebEngine.cs index e3923ba2..e9cf77de 100644 --- a/src/E2E/Engine/WebEngine.cs +++ b/src/E2E/Engine/WebEngine.cs @@ -302,7 +302,8 @@ public WebEngine(WebEngineOptions options) | EngineCapabilities.Keyboard | EngineCapabilities.Scroll | EngineCapabilities.History - | EngineCapabilities.Screenshot; + | EngineCapabilities.Screenshot + | EngineCapabilities.TapModifiers; public async Task StartAsync(EngineStartOptions options, CancellationToken cancellationToken) { @@ -1633,11 +1634,11 @@ private async Task DispatchAsync(IElementHandle element, LocatorAction action, O { switch (action) { - case LocatorAction.Tap: - await element.ClickAsync(new ElementHandleClickOptions { Timeout = budget.PlaywrightTimeout }).ConfigureAwait(false); + case LocatorAction.Tap tap: + await element.ClickAsync(new ElementHandleClickOptions { Timeout = budget.PlaywrightTimeout, Modifiers = Held(tap.Modifiers) }).ConfigureAwait(false); break; - case LocatorAction.DoubleTap: - await element.DblClickAsync(new ElementHandleDblClickOptions { Timeout = budget.PlaywrightTimeout }).ConfigureAwait(false); + case LocatorAction.DoubleTap doubleTap: + await element.DblClickAsync(new ElementHandleDblClickOptions { Timeout = budget.PlaywrightTimeout, Modifiers = Held(doubleTap.Modifiers) }).ConfigureAwait(false); break; case LocatorAction.Fill fill: await element.FillAsync(fill.Value, new ElementHandleFillOptions { Timeout = budget.PlaywrightTimeout }).ConfigureAwait(false); @@ -2333,6 +2334,17 @@ EngineException Unchanged(string detail = "") => } } + // The keys Playwright holds for a click. Null holds none. + private static IEnumerable? Held(IReadOnlyList? modifiers) => + modifiers?.Select(modifier => modifier switch + { + KeyModifier.Shift => KeyboardModifier.Shift, + KeyModifier.Control => KeyboardModifier.Control, + KeyModifier.Alt => KeyboardModifier.Alt, + KeyModifier.Meta => KeyboardModifier.Meta, + _ => KeyboardModifier.ControlOrMeta, + }); + private static string Direction(ScrollDirection direction) { return direction switch diff --git a/tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs b/tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs index 231a52e3..b914b452 100644 --- a/tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs +++ b/tests/E2E.Cli.Tests/McpServer/E2EMcpTests.cs @@ -93,7 +93,7 @@ public async Task Opens_a_session_then_locates_acts_fills_a_secret_and_withholds "observe", "tap", "double_tap", "type", "press", "select", "check", "uncheck", "clear", "scroll_to", "scroll", "navigate", "back", "screenshot", "type_secret", "locate", ]; Assert.Equal(names, ServedCli.CatalogNames(opened.Text)); - Assert.Contains("- tap {target}: Tap or click one node: the gesture for a button, link, menu item, tab, checkbox, row, or field.\n", opened.Text, StringComparison.Ordinal); + Assert.Contains("- tap {target, modifiers?}: Tap or click one node: the gesture for a button, link, menu item, tab, checkbox, row, or field.\n", opened.Text, StringComparison.Ordinal); Assert.Matches(new Regex(@"^- locate \{role\?, name\?, text\?, label\?, placeholder\?, testId\?, exact\?\}: .* \[read-only\]$", RegexOptions.Multiline, TimeSpan.FromSeconds(5)), opened.Text); Assert.Contains("Screen (/):", opened.Text, StringComparison.Ordinal); Assert.Contains("button \"Increment\"", opened.Text, StringComparison.Ordinal); diff --git a/tests/E2E.Tests/AgentTapModifiers/TapModifiersEngine.cs b/tests/E2E.Tests/AgentTapModifiers/TapModifiersEngine.cs new file mode 100644 index 00000000..3490ba30 --- /dev/null +++ b/tests/E2E.Tests/AgentTapModifiers/TapModifiersEngine.cs @@ -0,0 +1,54 @@ +// Copyright 2026 TesterArmy. +// Modified by Dario Kondratiuk. +// SPDX-License-Identifier: Apache-2.0 + +using E2E.Engine; + +namespace E2E.Tests.AgentTapModifiers; + +/// An engine over another that declares and keeps every tap and double tap it is sent. +internal sealed class TapModifiersEngine(IEngine inner) : IEngine +{ + /// The taps and double taps the engine received, in order. + public List Clicks { get; } = []; + + public string Platform => inner.Platform; + + public string Version => inner.Version; + + public EngineCapabilities Capabilities => inner.Capabilities | EngineCapabilities.TapModifiers; + + public async Task StartAsync(EngineStartOptions options, CancellationToken cancellationToken) => + new Session(this, await inner.StartAsync(options, cancellationToken).ConfigureAwait(false)); + + private sealed class Session(TapModifiersEngine engine, IEngineSession session) : IEngineSession + { + public string Route => session.Route; + + public Task OpenAsync(string url, CancellationToken cancellationToken) => session.OpenAsync(url, cancellationToken); + + public Task ObserveAsync(CancellationToken cancellationToken) => session.ObserveAsync(cancellationToken); + + public Task PerformAsync(SemanticNode node, LocatorAction action, CancellationToken cancellationToken) + { + if (action is LocatorAction.Tap or LocatorAction.DoubleTap) + { + engine.Clicks.Add(action); + } + + return session.PerformAsync(node, action, cancellationToken); + } + + public Task PressAsync(string key, CancellationToken cancellationToken) => session.PressAsync(key, cancellationToken); + + public Task SwipeAsync(ScrollDirection direction, CancellationToken cancellationToken) => session.SwipeAsync(direction, cancellationToken); + + public Task BackAsync(CancellationToken cancellationToken) => session.BackAsync(cancellationToken); + + public Task RestartAsync(CancellationToken cancellationToken) => session.RestartAsync(cancellationToken); + + public Task ClearStateAsync(CancellationToken cancellationToken) => session.ClearStateAsync(cancellationToken); + + public ValueTask DisposeAsync() => session.DisposeAsync(); + } +} diff --git a/tests/E2E.Tests/AgentTapModifiers/TapModifiersInTheGrammarToolsTests.cs b/tests/E2E.Tests/AgentTapModifiers/TapModifiersInTheGrammarToolsTests.cs new file mode 100644 index 00000000..f9b5ae49 --- /dev/null +++ b/tests/E2E.Tests/AgentTapModifiers/TapModifiersInTheGrammarToolsTests.cs @@ -0,0 +1,183 @@ +// Copyright 2026 TesterArmy. +// Modified by Dario Kondratiuk. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text.Json; +using E2E.Cache; +using E2E.Engine; +using E2E.Internal; +using E2E.Tests.StepCache; +using E2E.Tests.TraceReplay; +using Recorder = E2E.Cache.TraceRecorder; + +namespace E2E.Tests.AgentTapModifiers; + +/// +/// Upstream's agent-tap-modifiers.test.ts. The port has no right_click (it has no secondary tap), so the tap family +/// here is tap and double_tap. Its tools do not close their schemas, so a field an engine does not declare is refused +/// when the tool runs, not by the schema. The tools take a role and a name, not a node id, and the prose of a recorded action is +/// DefaultAgent.Describe, which the hand-off uses. +/// +public sealed class TapModifiersInTheGrammarToolsTests +{ + private static readonly string[] TapFamily = ["tap", "double_tap"]; + + // Each click changes the screen, so the step does not wait out the change window after every action. + private static DocumentWorld World() + { + var clicks = 0; + return new DocumentWorld().Map("/", page => + { + var count = page.Status("Clicked 0 times"); + void Click() => count.Name = count.Text = $"Clicked {++clicks} times"; + page.Button("Save", Click); + page.Button("Open", Click); + page.Button("Close", Click); + }); + } + + private static JsonElement PropertiesOf(IReadOnlyList tools, string name) => tools.Single(tool => tool.Name == name).Parameters.GetProperty("properties"); + + /// One act over in which the model calls in turn and then ends the step. + private static async Task<(IReadOnlyList Results, TapModifiersEngine? Engine)> ActAsync(bool declared, params ModelResponse[] calls) + { + TapModifiersEngine? engine = null; + var next = 0; + var model = new ScriptedModel(_ => next < calls.Length ? calls[next++] : ModelResponses.Done("passed", "Done.")); + var step = await StepCachePipelineTests.StepAsync(World(), new FileStepCache(CoreTests.TempCache()), "/", null, model, wrap: inner => declared ? engine = new TapModifiersEngine(inner) : inner); + Assert.Null(step.Error); + var results = model.Requests[^1].Messages.Where(message => message.Role == "tool").Select(message => message.Content ?? "").Where(content => !content.StartsWith("done", StringComparison.Ordinal)).ToList(); + return (results, engine); + } + + private static ModelResponse Call(string tool, string name, params string[]? modifiers) => + ModelResponses.Call(tool, modifiers is null ? new { role = "button", name } : (object)new { role = "button", name, modifiers }); + + /// Partial: the port has no right_click, and its schemas are not closed, so what the schema accepts is checked by the property it declares, and what it refuses by running the tool. + [Fact] + public async Task Partial_offers_modifiers_on_the_tap_family_only_when_the_engine_declares_tapModifiers() + { + var holding = AgentTools.CreateGrammarTools(EngineCapabilities.TapModifiers); + foreach (var name in TapFamily) + { + var modifiers = PropertiesOf(holding, name).GetProperty("modifiers"); + Assert.Equal("array", modifiers.GetProperty("type").GetString()); + Assert.Equal(["Shift", "Control", "Alt", "Meta", "ControlOrMeta"], modifiers.GetProperty("items").GetProperty("enum").EnumerateArray().Select(item => item.GetString())); + } + + // The rest of the grammar never takes modifiers, even on a holding engine. + foreach (var tool in holding.Where(tool => !TapFamily.Contains(tool.Name))) + { + Assert.False(tool.Parameters.GetProperty("properties").TryGetProperty("modifiers", out _), tool.Name + " refuses modifiers"); + } + + // A modifier that is unknown, in another case, repeated, or not a list is refused before the engine is touched. + var (results, engine) = await ActAsync( + declared: true, + Call("tap", "Save", "shift"), + Call("tap", "Save", "Bogus"), + Call("tap", "Save", "Shift", "Shift"), + ModelResponses.Call("tap", new { role = "button", name = "Save", modifiers = "Shift" })); + Assert.Empty(engine!.Clicks); + Assert.Equal(4, results.Count); + Assert.All(results, result => Assert.Matches("^failed: .*modifier", result)); + } + + /// Partial: the port has no right_click, and its schemas are not closed, so the engine that does not declare tapModifiers is shown to refuse the field instead. + [Fact] + public async Task Partial_keeps_modifiers_out_of_the_vocabulary_when_the_engine_does_not_declare_tapModifiers() + { + var plain = AgentTools.CreateGrammarTools(EngineCapabilities.None); + foreach (var name in TapFamily) + { + var properties = PropertiesOf(plain, name); + Assert.True(properties.TryGetProperty("role", out _), name + " still takes a bare tap"); + Assert.False(properties.TryGetProperty("modifiers", out _), name + " refuses the undeclared field"); + } + + var (results, _) = await ActAsync(declared: false, Call("tap", "Save", "Shift")); + Assert.Equal("failed: The engine cannot hold modifiers on tap.", Assert.Single(results)); + } + + /// Partial: the port has no right_click, so the third call upstream makes without modifiers is a double tap here. + [Fact] + public async Task Partial_threads_modifiers_through_to_the_tap_family_actions() + { + var (_, engine) = await ActAsync( + declared: true, + Call("tap", "Save", "Shift"), + Call("double_tap", "Open", "Control"), + Call("double_tap", "Close"), + Call("tap", "Close", "Alt", "Meta")); + + Assert.Equal( + [ + (typeof(LocatorAction.Tap), (KeyModifier[]?)[KeyModifier.Shift]), + (typeof(LocatorAction.DoubleTap), [KeyModifier.Control]), + (typeof(LocatorAction.DoubleTap), null), + (typeof(LocatorAction.Tap), [KeyModifier.Alt, KeyModifier.Meta]), + ], + engine!.Clicks.Select(click => (click.GetType(), click is LocatorAction.Tap tap ? tap.Modifiers?.ToArray() : ((LocatorAction.DoubleTap)click).Modifiers?.ToArray()))); + } + + [Fact] + public void Says_what_keys_a_tap_held_in_its_prose() + { + var held = DefaultAgent.Describe(new RecordedAction { Kind = "tap", Role = "button", Name = "Save", Modifiers = ["Shift", "Control"] }); + Assert.Contains("with Shift+Control held", held); + + var plain = DefaultAgent.Describe(new RecordedAction { Kind = "tap", Role = "button", Name = "Save" }); + Assert.DoesNotContain("held", plain); + } + + [Fact] + public void Keeps_modifiers_through_the_trace_round_trip() + { + var recorder = new Recorder(() => Redactor.None); + var save = new SemanticNode { Ref = "n1", Role = "button", Name = "Save" }; + recorder.Record(new RecordedAction { Kind = "tap", Role = save.Role, Name = save.Name, Modifiers = ["Shift"] }); + recorder.Record("tap", save); + var store = new FileStepCache(CoreTests.TempCache()); + store.Write("abc123", recorder.Finalize(StepCacheHarness.ExampleStep, "/", "/", [], [])!); + + var entry = Assert.IsType(store.Read("abc123").Entry); + + Assert.Equal(2, entry.Actions.Count); + Assert.Equal(["Shift"], entry.Actions[0].Modifiers); + Assert.Null(entry.Actions[1].Modifiers); + } + + /// Partial: a stored null reads as no modifiers, since the port's reader does not tell a null from an absent field. + [Fact] + public async Task Partial_rejects_a_trace_whose_modifiers_are_malformed_or_on_a_verb_that_cannot_hold_them() + { + static async Task ReasonAsync(RecordedAction action) + { + var entry = new CacheEntry { Schema = FileStepCache.SchemaVersion, Actions = [action], Appeared = [], Gone = [] }; + var session = StepCacheHarness.MakeSession(StepCacheHarness.FakeContext(() => new CacheLookup { Entry = entry }), StepCacheHarness.MakeHost(["/"], [StepCacheHarness.Node("n1", "button", "Save")])); + await session.BeginAsync(); + return session.CacheInfo?.Reason; + } + + Assert.NotEqual("invalid-entry", await ReasonAsync(new RecordedAction { Kind = "tap", Role = "button", Name = "Save", Modifiers = ["Shift"] })); + foreach (var modifiers in new[] { Array.Empty(), ["shift"], ["Shift", "Shift"] }) + { + Assert.Equal("invalid-entry", await ReasonAsync(new RecordedAction { Kind = "tap", Role = "button", Name = "Save", Modifiers = [.. modifiers] })); + } + + Assert.Equal("invalid-entry", await ReasonAsync(new RecordedAction { Kind = "scrollTo", Role = "button", Name = "Save", Modifiers = ["Shift"] })); + } + + [Fact] + public async Task Replays_a_recorded_tap_with_the_keys_it_held() + { + var host = TestHost.MakeHost(); + var action = new RecordedAction { Kind = "tap", Role = "button", Name = "Upgrade", Modifiers = ["Shift"] }; + + var outcome = await Replay.ReplayTrace(host, [action]); + + Assert.True(outcome.Completed); + Assert.Equal(1, outcome.Executed); + Assert.Equal([KeyModifier.Shift], Assert.Single(host.Clicks)!.Modifiers); + } +} diff --git a/tests/E2E.Tests/StepCache/StepCacheHarness.cs b/tests/E2E.Tests/StepCache/StepCacheHarness.cs index e4b16df5..0d2bb30b 100644 --- a/tests/E2E.Tests/StepCache/StepCacheHarness.cs +++ b/tests/E2E.Tests/StepCache/StepCacheHarness.cs @@ -200,9 +200,9 @@ public Task ObserveAsync(SettleMode mode) private sealed class Grammar : IExecutorActions { - public Task TapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => Task.CompletedTask; - public Task DoubleTapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => Task.CompletedTask; public Task TypeAsync(ExecutorTarget target, string value, CancellationToken cancellationToken = default) => Task.CompletedTask; diff --git a/tests/E2E.Tests/ToolLoopExecutor/CreateToolLoopExecutorTests.cs b/tests/E2E.Tests/ToolLoopExecutor/CreateToolLoopExecutorTests.cs index 05f79e39..31ad4f3a 100644 --- a/tests/E2E.Tests/ToolLoopExecutor/CreateToolLoopExecutorTests.cs +++ b/tests/E2E.Tests/ToolLoopExecutor/CreateToolLoopExecutorTests.cs @@ -31,7 +31,7 @@ public sealed partial class CreateToolLoopExecutorTests new ModelTool { Name = "tap", Description = "Tap a node by its ref", Parameters = Schema("""{ "type": "object", "properties": { "ref": { "type": "string" } }, "required": ["ref"] }""") }, async (call, token) => { - await context.Actions.TapAsync(new ExecutorTarget(call.Arguments.GetProperty("ref").GetString()!), token); + await context.Actions.TapAsync(new ExecutorTarget(call.Arguments.GetProperty("ref").GetString()!), cancellationToken: token); return new ToolLoopResult("Tapped."); }); @@ -140,8 +140,8 @@ public async Task Ends_the_loop_when_a_tool_raises_a_hard_stop_of_the_step() { var observation = await context.ObserveAsync(cancellationToken: token); var increment = TargetFor(observation.Text, "button \"Increment\""); - await context.Actions.TapAsync(increment, token); - await context.Actions.TapAsync(increment, token); + await context.Actions.TapAsync(increment, cancellationToken: token); + await context.Actions.TapAsync(increment, cancellationToken: token); return new ToolLoopResult("Tapped twice."); }), ]); diff --git a/tests/E2E.Tests/TraceReplay/TestHost.cs b/tests/E2E.Tests/TraceReplay/TestHost.cs index e6c6cdb4..defde282 100644 --- a/tests/E2E.Tests/TraceReplay/TestHost.cs +++ b/tests/E2E.Tests/TraceReplay/TestHost.cs @@ -74,6 +74,15 @@ public void MarkActing() public void Commit(RecordedAction action) => Committed.Add(action); + /// The options of each tap and double tap, in call order. + public List Clicks { get; } = []; + + private Task Click(string name, ExecutorTarget target, ClickOptions? options) + { + Clicks.Add(options); + return Act(name, target.Id); + } + private Task Act(string name, object? detail = null) { Calls.Add(name); @@ -83,9 +92,9 @@ private Task Act(string name, object? detail = null) private sealed class Grammar(TestHost host) : IExecutorActions { - public Task TapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => host.Act("tap", target.Id); + public Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => host.Click("tap", target, options); - public Task DoubleTapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => host.Act("doubleTap", target.Id); + public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => host.Click("doubleTap", target, options); public Task TypeAsync(ExecutorTarget target, string value, CancellationToken cancellationToken = default) => host.Act("type", (target.Id, value)); diff --git a/tests/E2E.Tests/WebEngineTests.cs b/tests/E2E.Tests/WebEngineTests.cs index 78c20668..5eaeed10 100644 --- a/tests/E2E.Tests/WebEngineTests.cs +++ b/tests/E2E.Tests/WebEngineTests.cs @@ -43,6 +43,36 @@ public sealed class WebEngineTests """; + private const string SelectionPage = """ + + + +
+ + + """; + + /// .NET-only: the web engine adapter holds the keys a tap names. Upstream's agent-tap-modifiers.test.ts has no browser. + [Fact] + public async Task Chromium_holds_the_modifiers_of_a_tap_the_agent_sends() + { + using var site = await TinySite.StartAsync(SelectionPage); + var model = new ScriptedModel(request => request.Messages.Any(message => message.Role == "tool") + ? ModelResponses.Done("passed", "clicked") + : ModelResponses.Call("tap", new { role = "button", name = "Row", modifiers = new[] { "Shift", "Alt" } })); + var session = await StartAsync(site, model, cache: null); + await RunAsync(session, async () => + { + await session.App.OpenAsync("/"); + await session.Agent.ActAsync("select the row with Shift and Alt held"); + await Expect.That(session.Screen.GetByRole("status")).ToContainTextAsync("Clicked with Shift with Alt"); + }); + } + [Fact] public async Task Chromium_acts_on_the_element_a_ref_names_when_two_share_a_role_and_name() { From 3ac665372ec38c08ae9c15cb1cd758beda145ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Kondratiuk?= Date: Sat, 17 Oct 2026 15:51:26 -0300 Subject: [PATCH 2/2] Keep the tap overloads that take no options Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01KWUdrjfarTEM4dmBbK6GaV --- skills/e2e-dotnet/references/agent.md | 2 +- src/E2E.Cli/Mcp/SessionCatalog.cs | 9 +-------- src/E2E/Agent/ActReplayHost.cs | 4 ++-- src/E2E/Agent/ActionDispatcher.cs | 4 ++-- src/E2E/Agent/AgentTools.cs | 20 +++++++++---------- src/E2E/Agent/Executor.cs | 12 ++++++++--- src/E2E/Agent/Replay.cs | 2 +- src/E2E/Engine/KeyModifier.cs | 7 ++++--- tests/E2E.Tests/StepCache/StepCacheHarness.cs | 4 ++-- .../CreateToolLoopExecutorTests.cs | 6 +++--- tests/E2E.Tests/TraceReplay/TestHost.cs | 4 ++-- 11 files changed, 36 insertions(+), 38 deletions(-) diff --git a/skills/e2e-dotnet/references/agent.md b/skills/e2e-dotnet/references/agent.md index 87529d77..f1a20ce6 100644 --- a/skills/e2e-dotnet/references/agent.md +++ b/skills/e2e-dotnet/references/agent.md @@ -427,7 +427,7 @@ public sealed class TapOnlyTests : E2ETest new ModelTool { Name = "tap", Description = "Tap a node by its ref from the latest read_screen", Parameters = Schema("""{ "type": "object", "properties": { "ref": { "type": "string" } }, "required": ["ref"] }""") }, async (call, token) => { - await context.Actions.TapAsync(new ExecutorTarget(call.Arguments.GetProperty("ref").GetString()!), cancellationToken: token); + await context.Actions.TapAsync(new ExecutorTarget(call.Arguments.GetProperty("ref").GetString()!), token); return new ToolLoopResult("Tapped."); }), ], diff --git a/src/E2E.Cli/Mcp/SessionCatalog.cs b/src/E2E.Cli/Mcp/SessionCatalog.cs index 4c8e18e9..d2769996 100644 --- a/src/E2E.Cli/Mcp/SessionCatalog.cs +++ b/src/E2E.Cli/Mcp/SessionCatalog.cs @@ -61,7 +61,7 @@ public static SessionCatalog Create(ISessionContext context, EngineCapabilities void Add(string name, CatalogTool tool) => tools.Add(new(name, tool)); Add("observe", Observe(context)); - var holdable = capabilities.HasFlag(EngineCapabilities.TapModifiers) ? new[] { ("modifiers", Modifiers, false) } : []; + var holdable = capabilities.HasFlag(EngineCapabilities.TapModifiers) ? new (string Name, JsonNode Schema, bool Required)[] { ("modifiers", AgentTools.ModifiersSchema(), false) } : []; Add("tap", Verb(context, "tap", "tap", "Tap or click one node: the gesture for a button, link, menu item, tab, checkbox, row, or field.", Schema(Target, holdable))); Add("double_tap", Verb(context, "double_tap", "double_tap", "Double-click one node. Use it only when a tap does nothing, such as a label that opens an editor on a double-click.", Schema(Target, holdable))); Add("type", Verb(context, "type", "fill", "Replace the text in a field with text.", Schema(Target, ("text", Property("string", "The text the field should hold"), true)), ("text", "value"))); @@ -201,13 +201,6 @@ private static readonly (string Name, JsonNode Schema, bool Required)[] Optional private static JsonNode Direction => new JsonObject { ["type"] = "string", ["enum"] = new JsonArray("up", "down", "left", "right") }; - private static JsonNode Modifiers => new JsonObject - { - ["type"] = "array", - ["items"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray([.. KeyModifiers.Names.Select(name => (JsonNode)name)]) }, - ["description"] = "Keys held while the pointer clicks. A Shift-click extends a selection, a Control-click (Meta on macOS) toggles one item", - }; - private static JsonNode TimesProperty => new JsonObject { ["type"] = "integer", diff --git a/src/E2E/Agent/ActReplayHost.cs b/src/E2E/Agent/ActReplayHost.cs index d8bff3b9..71f4a07e 100644 --- a/src/E2E/Agent/ActReplayHost.cs +++ b/src/E2E/Agent/ActReplayHost.cs @@ -43,9 +43,9 @@ private sealed class ActReplayHost( public void Commit(RecordedAction action) => budget.Recorder.Record(action); - public Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => ClickAsync("tap", target, options, cancellationToken); + public Task TapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default) => ClickAsync("tap", target, options, cancellationToken); - public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => ClickAsync("doubleTap", target, options, cancellationToken); + public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default) => ClickAsync("doubleTap", target, options, cancellationToken); private Task ClickAsync(string kind, ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken) => PerformAsync(target, ActionDispatcher.ClickAction(kind, ActionDispatcher.RequireClickModifiers(options, kind, scope.EngineCapabilities)), kind, cancellationToken); diff --git a/src/E2E/Agent/ActionDispatcher.cs b/src/E2E/Agent/ActionDispatcher.cs index 40e8e187..4d3e84e9 100644 --- a/src/E2E/Agent/ActionDispatcher.cs +++ b/src/E2E/Agent/ActionDispatcher.cs @@ -43,10 +43,10 @@ public ActionDispatcher(AttemptScope scope, ObservationFeed feed, StepAccounting _secrets = secrets; } - public Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => + public Task TapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default) => ClickAsync("tap", target, options, cancellationToken); - public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => + public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default) => ClickAsync("doubleTap", target, options, cancellationToken); // Upstream's nodeVerb for the tap family: the modifiers are checked before a slot is taken, and recorded with the action. diff --git a/src/E2E/Agent/AgentTools.cs b/src/E2E/Agent/AgentTools.cs index 12c465bb..5230c54d 100644 --- a/src/E2E/Agent/AgentTools.cs +++ b/src/E2E/Agent/AgentTools.cs @@ -119,6 +119,14 @@ internal static class AgentTools Tool("clear", "Clear a field.", Target), ]; + /// The modifiers property of a tap-family tool: the keys held while the pointer clicks. + internal static JsonObject ModifiersSchema() => new() + { + ["type"] = "array", + ["items"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray([.. KeyModifiers.Names.Select(name => (JsonNode)name)]) }, + ["description"] = "Keys held while the pointer clicks. A Shift-click extends a selection, a Control-click (Meta on macOS) toggles one item", + }; + // The tap family names the keys it holds for the click, on an engine that declares them. private static ModelTool HoldingModifiers(ModelTool verb) => verb.Name is "tap" or "double_tap" @@ -126,17 +134,7 @@ verb.Name is "tap" or "double_tap" { Name = verb.Name, Description = verb.Description + " Pass modifiers to hold keys for the click.", - Parameters = Extend( - Target, - $$""" - { - "modifiers": { - "type": "array", - "items": { "type": "string", "enum": {{JsonSerializer.Serialize(KeyModifiers.Names)}} }, - "description": "Keys held while the pointer clicks: Shift, Control, Alt, Meta, or ControlOrMeta. A Shift-click extends a selection, a Control-click (Meta on macOS) toggles one item." - } - } - """), + Parameters = Extend(Target, new JsonObject { ["modifiers"] = ModifiersSchema() }.ToJsonString()), } : verb; diff --git a/src/E2E/Agent/Executor.cs b/src/E2E/Agent/Executor.cs index 55e8888e..e3c7fe02 100644 --- a/src/E2E/Agent/Executor.cs +++ b/src/E2E/Agent/Executor.cs @@ -205,7 +205,7 @@ public sealed class ExecutorTargetInfo public IReadOnlySet Verbs { get; init; } = new HashSet(); - /// True when and take . + /// True when and take . public bool TapModifiers { get; init; } } @@ -353,10 +353,16 @@ public interface IExecutorActions /// Taps one node. holds keys for the click, as a Shift-click extends a selection; it /// needs , UNSUPPORTED_CAPABILITY otherwise. /// - Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default); + Task TapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default); + + /// Taps one node, holding no keys. + Task TapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => TapAsync(target, null, cancellationToken); /// Double-taps one node. Takes the tap's . - Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default); + Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default); + + /// Double-taps one node, holding no keys. + Task DoubleTapAsync(ExecutorTarget target, CancellationToken cancellationToken = default) => DoubleTapAsync(target, null, cancellationToken); Task TypeAsync(ExecutorTarget target, string value, CancellationToken cancellationToken = default); diff --git a/src/E2E/Agent/Replay.cs b/src/E2E/Agent/Replay.cs index d7b1149f..2a080f31 100644 --- a/src/E2E/Agent/Replay.cs +++ b/src/E2E/Agent/Replay.cs @@ -309,7 +309,7 @@ private static Task PerformAsync(IExecutorActions actions, ExecutorTarget target // The keys a recorded tap held, which it replays with. Null for a tap that held none. private static ClickOptions? ClickOptionsOf(RecordedAction action) => - action.Modifiers is { } names ? new ClickOptions { Modifiers = [.. names.Select(name => Enum.Parse(name))] } : null; + action.Modifiers is { } names ? new ClickOptions { Modifiers = [.. names.Select(name => KeyModifiers.TryParse(name, out var modifier) ? modifier : throw new AgentException("AUTOMATION_UNSUPPORTED", "Cannot replay modifier " + name + "."))] } : null; /// /// Re-finds a recorded target. The first look is the one in hand, or a capture settled as far as the previous action diff --git a/src/E2E/Engine/KeyModifier.cs b/src/E2E/Engine/KeyModifier.cs index a7e4b4fd..5c651fe2 100644 --- a/src/E2E/Engine/KeyModifier.cs +++ b/src/E2E/Engine/KeyModifier.cs @@ -23,11 +23,12 @@ public static class KeyModifiers /// The names in declaration order: Shift, Control, Alt, Meta, ControlOrMeta. public static IReadOnlyList Names { get; } = [.. Enum.GetNames()]; + private static readonly Dictionary ByName = Enum.GetValues().ToDictionary(modifier => modifier.ToString(), StringComparer.Ordinal); + /// Reads a name exactly as spelled: no other case, no number. public static bool TryParse(string? name, out KeyModifier modifier) { - var index = name is null ? -1 : Array.IndexOf(Enum.GetNames(), name); - modifier = index < 0 ? default : Enum.GetValues()[index]; - return index >= 0; + modifier = default; + return name is not null && ByName.TryGetValue(name, out modifier); } } diff --git a/tests/E2E.Tests/StepCache/StepCacheHarness.cs b/tests/E2E.Tests/StepCache/StepCacheHarness.cs index 0d2bb30b..8b816d23 100644 --- a/tests/E2E.Tests/StepCache/StepCacheHarness.cs +++ b/tests/E2E.Tests/StepCache/StepCacheHarness.cs @@ -200,9 +200,9 @@ public Task ObserveAsync(SettleMode mode) private sealed class Grammar : IExecutorActions { - public Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task TapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default) => Task.CompletedTask; - public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default) => Task.CompletedTask; public Task TypeAsync(ExecutorTarget target, string value, CancellationToken cancellationToken = default) => Task.CompletedTask; diff --git a/tests/E2E.Tests/ToolLoopExecutor/CreateToolLoopExecutorTests.cs b/tests/E2E.Tests/ToolLoopExecutor/CreateToolLoopExecutorTests.cs index 31ad4f3a..05f79e39 100644 --- a/tests/E2E.Tests/ToolLoopExecutor/CreateToolLoopExecutorTests.cs +++ b/tests/E2E.Tests/ToolLoopExecutor/CreateToolLoopExecutorTests.cs @@ -31,7 +31,7 @@ public sealed partial class CreateToolLoopExecutorTests new ModelTool { Name = "tap", Description = "Tap a node by its ref", Parameters = Schema("""{ "type": "object", "properties": { "ref": { "type": "string" } }, "required": ["ref"] }""") }, async (call, token) => { - await context.Actions.TapAsync(new ExecutorTarget(call.Arguments.GetProperty("ref").GetString()!), cancellationToken: token); + await context.Actions.TapAsync(new ExecutorTarget(call.Arguments.GetProperty("ref").GetString()!), token); return new ToolLoopResult("Tapped."); }); @@ -140,8 +140,8 @@ public async Task Ends_the_loop_when_a_tool_raises_a_hard_stop_of_the_step() { var observation = await context.ObserveAsync(cancellationToken: token); var increment = TargetFor(observation.Text, "button \"Increment\""); - await context.Actions.TapAsync(increment, cancellationToken: token); - await context.Actions.TapAsync(increment, cancellationToken: token); + await context.Actions.TapAsync(increment, token); + await context.Actions.TapAsync(increment, token); return new ToolLoopResult("Tapped twice."); }), ]); diff --git a/tests/E2E.Tests/TraceReplay/TestHost.cs b/tests/E2E.Tests/TraceReplay/TestHost.cs index defde282..22c092b4 100644 --- a/tests/E2E.Tests/TraceReplay/TestHost.cs +++ b/tests/E2E.Tests/TraceReplay/TestHost.cs @@ -92,9 +92,9 @@ private Task Act(string name, object? detail = null) private sealed class Grammar(TestHost host) : IExecutorActions { - public Task TapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => host.Click("tap", target, options); + public Task TapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default) => host.Click("tap", target, options); - public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options = null, CancellationToken cancellationToken = default) => host.Click("doubleTap", target, options); + public Task DoubleTapAsync(ExecutorTarget target, ClickOptions? options, CancellationToken cancellationToken = default) => host.Click("doubleTap", target, options); public Task TypeAsync(ExecutorTarget target, string value, CancellationToken cancellationToken = default) => host.Act("type", (target.Id, value));