feat(runner): key a runner run to its flow with --flow-id

.changeset/runner-run-flow-id.md

@@ -0,0 +1,5 @@
+---
+"@qawolf/cli": minor
+---
+
+`qawolf runner run --flow-id` names the flow a run is for when the platform cannot tell from the entry point. The run receives it as `QAWOLF_WORKFLOW_ID`, like a platform run of that flow does, so fixtures keyed by that id match across both. The runner SDK's `run` verb takes the same `flowId`.

skills/qawolf-cli/SKILL.md

@@ -40,7 +40,11 @@ exported variables, so pass the selected environment explicitly instead of
 rediscovering it. Check the command's help for its environment flag: variable
 commands use `--environment-id`; `qawolf flows run` uses `--env`; and
 `qawolf runner run` uses `--env-id`, which falls back to `QAWOLF_ENVIRONMENT`
-when neither it nor `--env-file` is passed.
+when neither it nor `--env-file` is passed. The run receives the flow it is for
+as `process.env.QAWOLF_WORKFLOW_ID`, found from the entry point on the
+environment's flow-code branch; `--flow-id` names the flow when that lookup
+cannot. A run with neither has that variable `undefined`, which is expected, not
+a missing id.
 
 If multiple environments are returned and the task context does not identify
 the target, ask instead of guessing. Do not default to the newest environment.

skills/qawolf-cli/references/runner.md

@@ -477,6 +477,27 @@ The lines-file has to be one of the files that travel, so it lives under the
 directory you run from. A range whose file is not collected is refused before a
 runner is addressed, naming the path.
 
+### Naming the flow the run is for
+
+The run receives the id of the flow it is for as `QAWOLF_WORKFLOW_ID`, like a
+platform run of that flow does, so fixtures keyed by that id, such as
+`entity-${process.env.QAWOLF_WORKFLOW_ID}`, and cleaned up by it match across the
+runner and the platform. When `--env-id` names an environment with a flow-code
+branch, the platform finds the flow itself: the entry point is looked up on that
+branch. `--flow-id` names the flow when that lookup cannot: the run uses
+`--env-file`, the environment has no branch, or the entry point is a file the
+branch does not know yet.
+
+```sh
+qawolf runner run flows/checkout.flow.ts --flow-id <id>
+```
+
+A run whose flow neither the lookup nor the flag names has no
+`QAWOLF_WORKFLOW_ID`, and the name above becomes `entity-undefined`. That is
+expected, not a missing id. Your shell's own `QAWOLF_WORKFLOW_ID` is never read:
+only an AI Job's pod sets one, and it names the pod's flow, not every flow run
+from there.
+
 ### Giving the run environment variables
 
 There are two ways, and a run takes one of them. `--env-id` names a QA Wolf

src/commands/__snapshots__/help.test.ts.snap

@@ -369,6 +369,9 @@ Options:
                        server and no size limit applies to them
   --env-file <path>    Dotenv file on this machine whose variables the run is
                        given. Pass this or --env-id, not both
+  --flow-id <id>       The QA Wolf flow this run is for, when the platform
+                       cannot tell from the entry point. The run receives it as
+                       QAWOLF_WORKFLOW_ID
   --lines <start-end>  Run only these 1-indexed lines against the browser as it
                        stands, instead of the whole flow from a fresh one
   --lines-file <path>  File the --lines range lives in. Defaults to <flowFile>;
@@ -388,6 +391,7 @@ Examples:
   $ qawolf runner run flows/checkout.flow.ts --lines 12-40
   $ qawolf runner run flows/checkout.flow.ts --lines 4-9 --lines-file pages/login.ts
   $ qawolf runner run flows/checkout.flow.ts --env-id staging
+  $ qawolf runner run flows/checkout.flow.ts --flow-id <id>
 "
 `;
 

src/commands/qawolfCliSkill.template.md

@@ -40,7 +40,11 @@ exported variables, so pass the selected environment explicitly instead of
 rediscovering it. Check the command's help for its environment flag: variable
 commands use `--environment-id`; `qawolf flows run` uses `--env`; and
 `qawolf runner run` uses `--env-id`, which falls back to `QAWOLF_ENVIRONMENT`
-when neither it nor `--env-file` is passed.
+when neither it nor `--env-file` is passed. The run receives the flow it is for
+as `process.env.QAWOLF_WORKFLOW_ID`, found from the entry point on the
+environment's flow-code branch; `--flow-id` names the flow when that lookup
+cannot. A run with neither has that variable `undefined`, which is expected, not
+a missing id.
 
 If multiple environments are returned and the task context does not identify
 the target, ask instead of guessing. Do not default to the newest environment.

src/commands/runner/run.register.ts

@@ -15,11 +15,13 @@ Examples:
   $ qawolf runner run flows/checkout.flow.ts --follow --logs
   $ qawolf runner run flows/checkout.flow.ts --lines 12-40
   $ qawolf runner run flows/checkout.flow.ts --lines 4-9 --lines-file pages/login.ts
-  $ qawolf runner run flows/checkout.flow.ts --env-id staging`;
+  $ qawolf runner run flows/checkout.flow.ts --env-id staging
+  $ qawolf runner run flows/checkout.flow.ts --flow-id <id>`;
 
 type RunFlags = {
   envFile?: string;
   envId?: string;
+  flowId?: string;
   follow: boolean;
   lines?: string;
   linesFile?: string;
@@ -68,6 +70,10 @@ export function registerRunCommand(
       "--env-file <path>",
       "Dotenv file on this machine whose variables the run is given. Pass this or --env-id, not both",
     )
+    .option(
+      "--flow-id <id>",
+      "The QA Wolf flow this run is for, when the platform cannot tell from the entry point. The run receives it as QAWOLF_WORKFLOW_ID",
+    )
     .option(
       "--lines <start-end>",
       "Run only these 1-indexed lines against the browser as it stands, instead of the whole flow from a fresh one",
@@ -91,6 +97,7 @@ export function registerRunCommand(
             entryPoint: flowFile,
             envFile: opts.envFile,
             envId: opts.envId,
+            flowId: opts.flowId,
             follow: opts.follow,
             lines: opts.lines,
             linesFile: opts.linesFile,

src/core/interactiveRunner/runFlowId.ts

@@ -0,0 +1,17 @@
+import { interactiveRunnerMessages } from "~/core/messages/index.js";
+
+/**
+ * The flow a run is for, from the flag alone. A blank flag is refused, like a
+ * blank `--env-id`. The process's own `QAWOLF_WORKFLOW_ID` is not read: only
+ * an AI Job's pod has one, and it names the pod's flow, not every flow run
+ * from there.
+ */
+export function resolveRunFlowId(
+  flag: string | undefined,
+): { ok: true; flowId: string | undefined } | { ok: false; error: string } {
+  const flowId = flag?.trim();
+  if (flowId === "") {
+    return { error: interactiveRunnerMessages.flowIdBlank, ok: false };
+  }
+  return { flowId, ok: true };
+}

src/core/messages/interactiveRunner/run.ts

@@ -10,6 +10,8 @@ export const runMessages = {
     "--env-id was given nothing. Pass the id or alias of a QA Wolf environment, or drop the flag to send no environment.",
   envIdWithEnvFile:
     "--env-id and --env-file both give the run its environment, so only one may be passed. Use --env-id for a QA Wolf environment, which QA Wolf reads itself, or --env-file for a dotenv file on this machine.",
+  flowIdBlank:
+    "--flow-id was given nothing. Pass the id of the flow this run is for, or drop the flag.",
   fileNotCollected: (path: string) =>
     `"${path}" is not one of the files that travel to a runner. It must be inside the current directory and end in .ts, .tsx, .js, .mjs, .cjs or .json.`,
   fileUnreadable: (path: string, reason: string) =>

src/domains/interactiveRunner/prepareRun.ts

@@ -6,6 +6,7 @@ import {
   type BuiltRunEnvironment,
   buildRunEnvironment,
 } from "~/core/interactiveRunner/runEnvironment.js";
+import { resolveRunFlowId } from "~/core/interactiveRunner/runFlowId.js";
 import {
   checkRunFiles,
   describeRunFilesCheck,
@@ -23,6 +24,7 @@ export type PreparedRun =
       environment: Record<string, string> | undefined;
       environmentId: string | undefined;
       files: RunFiles;
+      flowId: string | undefined;
       selection: RunSelection | undefined;
     }
   | { ok: false; error: string; exitCode: number };
@@ -44,6 +46,7 @@ export async function prepareRun(
     entryPointPath: string;
     envFile: string | undefined;
     envId: string | undefined;
+    flowId: string | undefined;
     lines: string | undefined;
     linesFile: string | undefined;
   },
@@ -65,6 +68,9 @@ export async function prepareRun(
     explicitEnvId ??
     (options.envFile === undefined && fromEnvVar ? fromEnvVar : undefined);
 
+  const flow = resolveRunFlowId(options.flowId);
+  if (!flow.ok) return refused(flow.error, exitCodes.invalidArgs);
+
   const linesFilePath =
     options.linesFile === undefined
       ? undefined
@@ -95,6 +101,7 @@ export async function prepareRun(
           environment: given,
           environmentId,
           files,
+          flowId: flow.flowId,
           ok: true,
           selection: undefined,
         }
@@ -118,6 +125,7 @@ export async function prepareRun(
         environment: given,
         environmentId,
         files,
+        flowId: flow.flowId,
         ok: true,
         selection: built.selection,
       }

src/domains/interactiveRunner/runFlow.delta.test.ts

@@ -35,6 +35,7 @@ async function run(
       entryPoint: "flow.ts",
       envFile: undefined,
       envId: undefined,
+      flowId: undefined,
       follow: false,
       lines: undefined,
       linesFile: undefined,
@@ -173,6 +174,7 @@ describe("handleRunnerRun file delta", () => {
         entryPoint: "flow.ts",
         envFile: undefined,
         envId: undefined,
+        flowId: undefined,
         follow: false,
         lines: undefined,
         linesFile: undefined,

src/domains/interactiveRunner/runFlow.environment.test.ts

@@ -42,6 +42,7 @@ describe("handleRunnerRun with --env-file and --env-id", () => {
         entryPoint: "flow.ts",
         envFile,
         envId,
+        flowId: undefined,
         follow: false,
         lines: undefined,
         linesFile: undefined,

src/domains/interactiveRunner/runFlow.files.test.ts

@@ -14,6 +14,7 @@ describe("handleRunnerRun file refusals", () => {
         follow: false,
         envFile: undefined,
         envId: undefined,
+        flowId: undefined,
         lines: undefined,
         linesFile: undefined,
         runner: "ci",
@@ -40,6 +41,7 @@ describe("handleRunnerRun file refusals", () => {
         follow: false,
         envFile: undefined,
         envId: undefined,
+        flowId: undefined,
         lines: undefined,
         linesFile: undefined,
         runner: "ci",
@@ -71,6 +73,7 @@ describe("handleRunnerRun file refusals", () => {
         follow: false,
         envFile: undefined,
         envId: undefined,
+        flowId: undefined,
         lines: undefined,
         linesFile: undefined,
         runner: undefined,

src/domains/interactiveRunner/runFlow.flowId.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from "bun:test";
+
+import { makeAuthCtx, makeTestDeps } from "./deps.testUtils.js";
+import { handleRunnerRun } from "./runFlow.js";
+
+const submitted = { outcome: "success" as const, runId: "run-a" };
+
+describe("handleRunnerRun with --flow-id", () => {
+  const run = async ({
+    env = {},
+    flowId,
+  }: {
+    env?: Record<string, string>;
+    flowId?: string;
+  }) => {
+    const { callPublicApi, ctx } = makeAuthCtx();
+    callPublicApi.mockResolvedValue({ ok: true, value: submitted });
+    const result = await handleRunnerRun(
+      ctx,
+      {
+        entryPoint: "flow.ts",
+        envFile: undefined,
+        envId: undefined,
+        flowId,
+        follow: false,
+        lines: undefined,
+        linesFile: undefined,
+        logs: false,
+        recorderEvents: false,
+        runEvents: false,
+        runner: "ci",
+        timeout: undefined,
+      },
+      makeTestDeps({
+        env,
+        collectRunFiles: async () => ({
+          files: {
+            "flow.ts": "export default {};",
+            "package.json": "{}",
+          },
+          unresolvedImports: [],
+        }),
+      }),
+    );
+    return { callPublicApi, result };
+  };
+
+  const sentRequest = (callPublicApi: { mock: { calls: unknown[][] } }) =>
+    callPublicApi.mock.calls[0]?.[1] ?? {};
+
+  it("names the flow the run is for", async () => {
+    const { callPublicApi, result } = await run({ flowId: "flow-7c2e" });
+
+    expect(result).toBeUndefined();
+    expect(sentRequest(callPublicApi)).toMatchObject({ flowId: "flow-7c2e" });
+  });
+
+  it("trims the id it was given", async () => {
+    const { callPublicApi } = await run({ flowId: "  flow-7c2e  " });
+
+    expect(sentRequest(callPublicApi)).toMatchObject({ flowId: "flow-7c2e" });
+  });
+
+  it("names no flow when the flag is absent", async () => {
+    const { callPublicApi } = await run({});
+
+    expect(Object.hasOwn(sentRequest(callPublicApi), "flowId")).toBe(false);
+  });
+
+  // An AI Job's pod exports the id of its own flow, which is not the flow of
+  // every run started from that pod. Reading it here would key a run of one
+  // flow to another, so the flag is the only source.
+  it("ignores a QAWOLF_WORKFLOW_ID in its own environment", async () => {
+    const { callPublicApi } = await run({
+      env: { QAWOLF_WORKFLOW_ID: "flow-of-this-pod" },
+    });
+
+    expect(Object.hasOwn(sentRequest(callPublicApi), "flowId")).toBe(false);
+  });
+
+  it("refuses a blank flag before addressing a runner", async () => {
+    const { callPublicApi, result } = await run({ flowId: "  " });
+
+    expect(result?.error).toContain("--flow-id was given nothing");
+    expect(result?.exitCode).toBe(2);
+    expect(callPublicApi).not.toHaveBeenCalled();
+  });
+});

src/domains/interactiveRunner/runFlow.follow.test.ts

@@ -24,6 +24,7 @@ const runFollowing = (
       follow: options.follow ?? true,
       envFile: undefined,
       envId: undefined,
+      flowId: undefined,
       lines: undefined,
       linesFile: undefined,
       logs: options.logs ?? false,

src/domains/interactiveRunner/runFlow.selection.test.ts

@@ -42,6 +42,7 @@ async function runWith(options: {
       follow: false,
       envFile: undefined,
       envId: undefined,
+      flowId: undefined,
       lines: options.lines,
       linesFile: options.linesFile,
       logs: false,

src/domains/interactiveRunner/runFlow.test.ts

@@ -24,6 +24,7 @@ async function runWith(
       follow: false,
       envFile: undefined,
       envId: undefined,
+      flowId: undefined,
       lines: options.lines,
       linesFile: options.linesFile,
       runner: "ci",
@@ -120,6 +121,7 @@ describe("handleRunnerRun", () => {
         follow: false,
         envFile: undefined,
         envId: undefined,
+        flowId: undefined,
         lines: undefined,
         linesFile: undefined,
         runner: undefined,
@@ -158,6 +160,7 @@ describe("handleRunnerRun", () => {
         follow: false,
         envFile: undefined,
         envId: undefined,
+        flowId: undefined,
         lines: undefined,
         linesFile: undefined,
         runner: "ci",

src/domains/interactiveRunner/runFlow.ts

@@ -25,6 +25,7 @@ export async function handleRunnerRun(
     entryPoint: string;
     envFile: string | undefined;
     envId: string | undefined;
+    flowId: string | undefined;
     follow: boolean;
     lines: string | undefined;
     linesFile: string | undefined;
@@ -53,6 +54,7 @@ export async function handleRunnerRun(
       entryPointPath,
       envFile: options.envFile,
       envId: options.envId,
+      flowId: options.flowId,
       lines: options.lines,
       linesFile: options.linesFile,
     },
@@ -90,6 +92,7 @@ export async function handleRunnerRun(
       environment: prepared.environment,
       environmentId: prepared.environmentId,
       files: prepared.files,
+      flowId: prepared.flowId,
       resolved,
       selection: prepared.selection,
     },