feat(runner): tap, swipe and fill in runner actions sequences (AGE-5427)

.changeset/runner-actions-mobile-actions.md

@@ -0,0 +1,5 @@
+---
+"@qawolf/cli": minor
+---
+
+`qawolf runner actions` takes the same actions as `qawolf runner act`, including the mobile `tap`, `swipe` and `fill`. A browser runner refuses a mobile action in a sequence with `action-not-supported-on-browser`; with `--continue-on-failure` the rest of the sequence still runs. Pins `@qawolf/api-contracts` 0.81.0.

package.json

@@ -71,7 +71,7 @@
     "@clack/prompts": "1.5.1",
     "@napi-rs/keyring": "1.3.0",
     "@oxc-node/core": "0.1.0",
-    "@qawolf/api-contracts": "0.77.0",
+    "@qawolf/api-contracts": "0.81.0",
     "@qawolf/emails": "1.1.1",
     "@qawolf/flow-targets": "1.0.0",
     "@qawolf/flows": "0.1.4",

skills/qawolf-cli/references/run-results.md

@@ -201,6 +201,7 @@ Every documented field of the `run.get` response. `[]` marks an array, so
 - `flows[].attempts[].kind` — One of: automated, manual
 - `flows[].attempts[].startedAt` — Absent when the attempt failed before it could start.
 - `flows[].attempts[].status` — One of: passed, failed, canceled
+- `flows[].attempts[].canceledReason` — Why the attempt was canceled. userCanceled means a person stopped it; do not immediately retry that activity.
 - `flows[].failure.diagnosis` — QA Wolf's investigation verdict for the failure: `bug` means the application is broken, `maintenance` means the test needed an update and the failure does not indicate an application problem. Absent until the investigation reaches a verdict, and absent when the flow was opted out of investigation. Pass issueId to issue.get for details.
 - `flows[].failure.diagnosis.issueId` — The id of the issue.
 - `flows[].failure.diagnosis.type` — One of: bug, maintenance

src/commands/publicApi/index.environment.test.ts

@@ -10,6 +10,7 @@ import {
 import { createSignalRegistry } from "~/shell/signals/createSignalRegistry.js";
 
 import { registerPublicApiCommands } from "./index.js";
+import { standInAnnotationJustifications } from "~/domains/publicApi/contract.fixtures.js";
 
 const originalAiTaskId = process.env["QAWOLF_AI_TASK_ID"];
 const originalChatSessionId = process.env["QAWOLF_CHAT_SESSION_ID"];
@@ -45,6 +46,7 @@ function makeNotificationContract() {
       openWorldHint: true,
       readOnlyHint: false,
     },
+    annotationJustifications: standInAnnotationJustifications,
     description: "Create a run.",
     input: z.object({
       aiTaskId: z.string().optional(),
@@ -86,6 +88,7 @@ it("defaults public API options from their environment variables", async () => {
       openWorldHint: true,
       readOnlyHint: false,
     },
+    annotationJustifications: standInAnnotationJustifications,
     description: "Create a run.",
     input: z.object({
       aiTaskId: z.string().optional(),

src/commands/publicApi/index.nested.test.ts

@@ -9,6 +9,7 @@ import {
 import { createSignalRegistry } from "~/shell/signals/createSignalRegistry.js";
 
 import { registerPublicApiCommands } from "./index.js";
+import { standInAnnotationJustifications } from "~/domains/publicApi/contract.fixtures.js";
 
 // `process.exitCode` is global state and persists across tests; reset to 0
 // (not undefined — bun ignores an undefined assignment) so leftover state
@@ -33,6 +34,7 @@ const contract = {
     openWorldHint: false,
     readOnlyHint: false,
   },
+  annotationJustifications: standInAnnotationJustifications,
   name: "fake.reportStatus",
   kind: "write" as const,
   description: "Synthetic nested contract.",

src/commands/publicApi/index.test.ts

@@ -10,6 +10,7 @@ import {
 import { createSignalRegistry } from "~/shell/signals/createSignalRegistry.js";
 
 import { registerPublicApiCommands } from "./index.js";
+import { standInAnnotationJustifications } from "~/domains/publicApi/contract.fixtures.js";
 
 afterEach(() => {
   mock.restore();
@@ -162,6 +163,7 @@ describe("registerPublicApiCommands", () => {
         openWorldHint: false,
         readOnlyHint: true,
       },
+      annotationJustifications: standInAnnotationJustifications,
       description: "List the flows of an environment.",
       input: z.object({ environmentId: z.string() }),
       kind: "read",
@@ -174,6 +176,7 @@ describe("registerPublicApiCommands", () => {
         openWorldHint: false,
         readOnlyHint: true,
       },
+      annotationJustifications: standInAnnotationJustifications,
       description: "Look up a run.",
       input: z.object({ runId: z.string() }),
       kind: "read",
@@ -215,6 +218,7 @@ describe("registerPublicApiCommands", () => {
         openWorldHint: false,
         readOnlyHint: true,
       },
+      annotationJustifications: standInAnnotationJustifications,
       description: "Look up a run attempt.",
       input: z.object({ runId: z.string() }),
       kind: "read",

src/commands/runner/actions.register.ts

@@ -13,7 +13,11 @@ Examples:
   $ qawolf runner actions '[{"type":"click","button":"left","x":480,"y":260},{"type":"type","text":"hello@example.com"},{"type":"keypress","keys":["Enter"]}]' --screenshot after-login.jpg
   $ echo '[{"type":"click","button":"left","x":1,"y":2},{"type":"type","text":"hi"}]' | qawolf runner actions -
   $ qawolf runner actions '[{"type":"click","button":"left","x":480,"y":260},{"type":"type","text":"hello"}]' --screenshot-mode each --screenshot step.jpg
-  $ qawolf runner actions '[{"type":"scroll","x":480,"y":260,"scroll_x":0,"scroll_y":600},{"type":"click","button":"left","x":120,"y":700}]' --continue-on-failure`;
+  $ qawolf runner actions '[{"type":"scroll","x":480,"y":260,"scroll_x":0,"scroll_y":600},{"type":"click","button":"left","x":120,"y":700}]' --continue-on-failure
+
+Mobile:
+  $ qawolf runner actions '[{"type":"tap","selector":"//*[@content-desc=\\"Email\\"]"},{"type":"type","text":"hello@example.com"},{"type":"tap","x":540,"y":1650}]' --screenshot after-login.jpg
+  $ qawolf runner actions '[{"type":"fill","selector":"name == \\"Postal code\\"","strategy":"ios-predicate","text":"94107"},{"type":"swipe","from":{"x":540,"y":1600},"to":{"x":540,"y":600}}]'`;
 
 type ActionsFlags = {
   continueOnFailure?: boolean;

src/commands/skillRunResults.test.ts

@@ -7,6 +7,7 @@ import {
   renderResponseFields,
   spliceResponseFields,
 } from "./skillRunResults.js";
+import { standInAnnotationJustifications } from "~/domains/publicApi/contract.fixtures.js";
 
 const runResultsMdPath = join(
   import.meta.dirname,
@@ -25,6 +26,7 @@ describe("renderResponseFields", () => {
         openWorldHint: false,
         readOnlyHint: true,
       },
+      annotationJustifications: standInAnnotationJustifications,
       description: "Look up a run.",
       input: z.object({ runId: z.string() }),
       kind: "read",
@@ -47,6 +49,7 @@ describe("renderResponseFields", () => {
         openWorldHint: false,
         readOnlyHint: true,
       },
+      annotationJustifications: standInAnnotationJustifications,
       description: "Look up a run.",
       input: z.object({ runId: z.string() }),
       kind: "read",

src/core/interactiveRunner/browserAction.test.ts

@@ -3,7 +3,6 @@ import { describe, expect, it } from "bun:test";
 import {
   type ActionFlags,
   buildRunnerAction,
-  parseBrowserAction,
   parseRunnerAction,
 } from "./browserAction.js";
 
@@ -219,31 +218,3 @@ describe("parseRunnerAction", () => {
     ).toBe(true);
   });
 });
-
-describe("parseBrowserAction", () => {
-  it("refuses a touchscreen action, which a sequence does not take", () => {
-    expect(parseBrowserAction({ type: "tap", x: 1, y: 2 }).ok).toBe(false);
-  });
-
-  it("takes a complete action as a model emitted it", () => {
-    const parsed = parseBrowserAction({
-      button: "left",
-      type: "click",
-      x: 1,
-      y: 2,
-    });
-
-    expect(parsed.ok).toBe(true);
-    if (!parsed.ok) return;
-    expect(parsed.action).toEqual({
-      button: "left",
-      type: "click",
-      x: 1,
-      y: 2,
-    });
-  });
-
-  it("refuses one the published schema does not admit", () => {
-    expect(parseBrowserAction({ type: "click" }).ok).toBe(false);
-  });
-});

src/core/interactiveRunner/browserAction.ts

@@ -1,7 +1,5 @@
 import {
-  type BrowserAction,
   type RunnerAction,
-  browserActionSchema,
   runnerActionSchema,
 } from "@qawolf/api-contracts/v1";
 import { z } from "zod";
@@ -104,27 +102,13 @@ export function buildRunnerAction(
   return parseRunnerAction(candidate);
 }
 
-function parseWith<Action>(
-  schema: z.ZodType<Action>,
+/** Puts one complete action, as a caller's model emitted it, to the schema `runner act` and `runner actions` send. */
+export function parseRunnerAction(
   candidate: unknown,
-): BuiltAction<Action> {
-  const parsed = schema.safeParse(candidate);
+): BuiltAction<RunnerAction> {
+  const parsed = runnerActionSchema.safeParse(candidate);
   if (!parsed.success) {
     return { error: z.prettifyError(parsed.error), ok: false };
   }
   return { action: parsed.data, ok: true };
 }
-
-/** Puts one complete action, as a caller's model emitted it, to the schema `runner act` sends. */
-export function parseRunnerAction(
-  candidate: unknown,
-): BuiltAction<RunnerAction> {
-  return parseWith(runnerActionSchema, candidate);
-}
-
-/** Puts one action of a `runner actions` sequence, which takes browser actions only, to its schema. */
-export function parseBrowserAction(
-  candidate: unknown,
-): BuiltAction<BrowserAction> {
-  return parseWith(browserActionSchema, candidate);
-}

src/core/publicApi/flagSpecs.test.ts

@@ -181,7 +181,7 @@ describe("buildFlagSpecs", () => {
       "One of: unprioritized, low, medium, high, urgent",
     );
     expect(described("status")).toBe(
-      "One of: pending, inProgress, paused, resolved, canceled, archived",
+      "One of: backlog, pending, inProgress, paused, resolved, canceled, archived",
     );
   });
 

src/domains/interactiveRunner/performActions.touchscreen.test.ts

@@ -0,0 +1,86 @@
+import { publicContractsV1 } from "@qawolf/api-contracts/v1";
+import { describe, expect, it } from "bun:test";
+
+import { makeAuthCtx, makeTestDeps } from "./deps.testUtils.js";
+import { handleRunnerActions } from "./performActions.js";
+import {
+  actionsOptions,
+  performed,
+  sequenceAnswer,
+  someTyping,
+} from "./performActions.fixtures.js";
+
+const sequenceTimeoutMs = 180_000;
+
+describe("handleRunnerActions on a touchscreen", () => {
+  it("sends a touchscreen sequence, as runner act takes it", async () => {
+    const { callPublicApi, ctx } = makeAuthCtx();
+    callPublicApi.mockResolvedValue({
+      ok: true,
+      value: sequenceAnswer({
+        lastCompletedIndex: 2,
+        outcome: "success",
+        results: [performed(0), performed(1), performed(2)],
+      }),
+    });
+    const touchscreenSequence = [
+      { selector: "//*[@content-desc='Email']", type: "tap" },
+      { selector: "~Postal code", text: "94107", type: "fill" },
+      { from: { x: 540, y: 1600 }, to: { x: 540, y: 600 }, type: "swipe" },
+    ];
+
+    const result = await handleRunnerActions(
+      ctx,
+      actionsOptions({ actions: JSON.stringify(touchscreenSequence) }),
+      makeTestDeps(),
+    );
+
+    expect(result).toBeUndefined();
+    expect(callPublicApi).toHaveBeenCalledWith(
+      publicContractsV1.runner.performActions,
+      {
+        actions: touchscreenSequence,
+        id: "ci",
+        screenshotMode: "none",
+        stopOnFailure: true,
+      },
+      { timeoutMs: sequenceTimeoutMs },
+    );
+  });
+
+  it("names a touchscreen action a browser runner refused, as a bad argument", async () => {
+    const { callPublicApi, ctx } = makeAuthCtx();
+    callPublicApi.mockResolvedValue({
+      ok: true,
+      value: sequenceAnswer({
+        failedIndex: 0,
+        failureReason: "action-not-supported-on-browser",
+        lastCompletedIndex: 1,
+        outcome: "failure",
+        results: [
+          {
+            effect: "not-performed",
+            failureReason: "action-not-supported-on-browser",
+            index: 0,
+            outcome: "failure",
+          },
+          performed(1),
+        ],
+        stoppedEarly: false,
+      }),
+    });
+
+    const result = await handleRunnerActions(
+      ctx,
+      actionsOptions({
+        actions: JSON.stringify([{ type: "tap", x: 1, y: 2 }, someTyping]),
+        continueOnFailure: true,
+      }),
+      makeTestDeps(),
+    );
+
+    expect(result?.exitCode).toBe(2);
+    expect(result?.error).toContain("tap");
+    expect(result?.error).toContain("only a mobile runner performs");
+  });
+});

src/domains/interactiveRunner/performActionsFailure.ts

@@ -1,4 +1,4 @@
-import type { BrowserAction } from "@qawolf/api-contracts/v1";
+import type { RunnerAction } from "@qawolf/api-contracts/v1";
 
 import { interactiveRunnerMessages } from "~/core/messages/index.js";
 import { appendSentence } from "~/core/sentences.js";
@@ -13,7 +13,7 @@ import {
 
 /** Every action that did not succeed, why, and what the caller should do about it. */
 export function describeSequenceFailure(options: {
-  actions: BrowserAction[];
+  actions: RunnerAction[];
   answer: SequenceFailure;
 }): Exclude<CommandResult, void> {
   const { actions, answer } = options;

src/domains/interactiveRunner/performActionsReasons.ts

@@ -80,6 +80,11 @@ export function describeReason(step: FailedStep): Described {
         exitCode: exitCodes.invalidArgs,
         why: "has no touchscreen equivalent on a mobile runner.",
       };
+    case "action-not-supported-on-browser":
+      return {
+        exitCode: exitCodes.invalidArgs,
+        why: "is a touchscreen action, which only a mobile runner performs.",
+      };
     case "screen-needs-a-run":
       return {
         exitCode: exitCodes.invalidArgs,

src/domains/interactiveRunner/readActions.ts

@@ -1,7 +1,7 @@
-import type { BrowserAction, ScreenshotMode } from "@qawolf/api-contracts/v1";
+import type { RunnerAction, ScreenshotMode } from "@qawolf/api-contracts/v1";
 import { maxActionsPerRequest } from "@qawolf/api-contracts/v1";
 
-import { parseBrowserAction } from "~/core/interactiveRunner/browserAction.js";
+import { parseRunnerAction } from "~/core/interactiveRunner/browserAction.js";
 import { interactiveRunnerMessages } from "~/core/messages/index.js";
 import type {
   AuthCommandContext,
@@ -16,7 +16,7 @@ import type { InteractiveRunnerDeps } from "./deps.js";
 const stdinArgument = "-";
 
 export type ReadActions =
-  | { actions: BrowserAction[]; ok: true }
+  | { actions: RunnerAction[]; ok: true }
   | { error: string; ok: false };
 
 /**
@@ -48,7 +48,7 @@ export async function readActions(
     };
   }
 
-  const built = parsed.items.map(parseBrowserAction);
+  const built = parsed.items.map(parseRunnerAction);
   const firstRefused = built.findIndex((action) => !action.ok);
   const refused = built[firstRefused];
   if (refused !== undefined && !refused.ok) {