diff --git a/.changeset/runner-run-flow-id.md b/.changeset/runner-run-flow-id.md new file mode 100644 index 000000000..5f807dfeb --- /dev/null +++ b/.changeset/runner-run-flow-id.md @@ -0,0 +1,5 @@ +--- +"@qawolf/cli": minor +--- + +`qawolf runner run --flow-id` names the flow a run is for when the platform cannot tell from the entry point. The run receives it as `QAWOLF_WORKFLOW_ID`, like a platform run of that flow does, so fixtures keyed by that id match across both. The runner SDK's `run` verb takes the same `flowId`. diff --git a/skills/qawolf-cli/SKILL.md b/skills/qawolf-cli/SKILL.md index da6032fce..9964b1138 100644 --- a/skills/qawolf-cli/SKILL.md +++ b/skills/qawolf-cli/SKILL.md @@ -40,7 +40,11 @@ exported variables, so pass the selected environment explicitly instead of rediscovering it. Check the command's help for its environment flag: variable commands use `--environment-id`; `qawolf flows run` uses `--env`; and `qawolf runner run` uses `--env-id`, which falls back to `QAWOLF_ENVIRONMENT` -when neither it nor `--env-file` is passed. +when neither it nor `--env-file` is passed. The run receives the flow it is for +as `process.env.QAWOLF_WORKFLOW_ID`, found from the entry point on the +environment's flow-code branch; `--flow-id` names the flow when that lookup +cannot. A run with neither has that variable `undefined`, which is expected, not +a missing id. If multiple environments are returned and the task context does not identify the target, ask instead of guessing. Do not default to the newest environment. diff --git a/skills/qawolf-cli/references/runner.md b/skills/qawolf-cli/references/runner.md index 0c112b002..e0acf133f 100644 --- a/skills/qawolf-cli/references/runner.md +++ b/skills/qawolf-cli/references/runner.md @@ -477,6 +477,27 @@ The lines-file has to be one of the files that travel, so it lives under the directory you run from. A range whose file is not collected is refused before a runner is addressed, naming the path. +### Naming the flow the run is for + +The run receives the id of the flow it is for as `QAWOLF_WORKFLOW_ID`, like a +platform run of that flow does, so fixtures keyed by that id, such as +`entity-${process.env.QAWOLF_WORKFLOW_ID}`, and cleaned up by it match across the +runner and the platform. When `--env-id` names an environment with a flow-code +branch, the platform finds the flow itself: the entry point is looked up on that +branch. `--flow-id` names the flow when that lookup cannot: the run uses +`--env-file`, the environment has no branch, or the entry point is a file the +branch does not know yet. + +```sh +qawolf runner run flows/checkout.flow.ts --flow-id +``` + +A run whose flow neither the lookup nor the flag names has no +`QAWOLF_WORKFLOW_ID`, and the name above becomes `entity-undefined`. That is +expected, not a missing id. Your shell's own `QAWOLF_WORKFLOW_ID` is never read: +only an AI Job's pod sets one, and it names the pod's flow, not every flow run +from there. + ### Giving the run environment variables There are two ways, and a run takes one of them. `--env-id` names a QA Wolf diff --git a/src/commands/__snapshots__/help.test.ts.snap b/src/commands/__snapshots__/help.test.ts.snap index 879d1f4ef..56dd5ceab 100644 --- a/src/commands/__snapshots__/help.test.ts.snap +++ b/src/commands/__snapshots__/help.test.ts.snap @@ -369,6 +369,9 @@ Options: server and no size limit applies to them --env-file Dotenv file on this machine whose variables the run is given. Pass this or --env-id, not both + --flow-id The QA Wolf flow this run is for, when the platform + cannot tell from the entry point. The run receives it as + QAWOLF_WORKFLOW_ID --lines Run only these 1-indexed lines against the browser as it stands, instead of the whole flow from a fresh one --lines-file File the --lines range lives in. Defaults to ; @@ -388,6 +391,7 @@ Examples: $ qawolf runner run flows/checkout.flow.ts --lines 12-40 $ qawolf runner run flows/checkout.flow.ts --lines 4-9 --lines-file pages/login.ts $ qawolf runner run flows/checkout.flow.ts --env-id staging + $ qawolf runner run flows/checkout.flow.ts --flow-id " `; diff --git a/src/commands/qawolfCliSkill.template.md b/src/commands/qawolfCliSkill.template.md index f272e2f03..8ba5c4dd2 100644 --- a/src/commands/qawolfCliSkill.template.md +++ b/src/commands/qawolfCliSkill.template.md @@ -40,7 +40,11 @@ exported variables, so pass the selected environment explicitly instead of rediscovering it. Check the command's help for its environment flag: variable commands use `--environment-id`; `qawolf flows run` uses `--env`; and `qawolf runner run` uses `--env-id`, which falls back to `QAWOLF_ENVIRONMENT` -when neither it nor `--env-file` is passed. +when neither it nor `--env-file` is passed. The run receives the flow it is for +as `process.env.QAWOLF_WORKFLOW_ID`, found from the entry point on the +environment's flow-code branch; `--flow-id` names the flow when that lookup +cannot. A run with neither has that variable `undefined`, which is expected, not +a missing id. If multiple environments are returned and the task context does not identify the target, ask instead of guessing. Do not default to the newest environment. diff --git a/src/commands/runner/run.register.ts b/src/commands/runner/run.register.ts index 3ed6a579b..6c2ff0d3d 100644 --- a/src/commands/runner/run.register.ts +++ b/src/commands/runner/run.register.ts @@ -15,11 +15,13 @@ Examples: $ qawolf runner run flows/checkout.flow.ts --follow --logs $ qawolf runner run flows/checkout.flow.ts --lines 12-40 $ qawolf runner run flows/checkout.flow.ts --lines 4-9 --lines-file pages/login.ts - $ qawolf runner run flows/checkout.flow.ts --env-id staging`; + $ qawolf runner run flows/checkout.flow.ts --env-id staging + $ qawolf runner run flows/checkout.flow.ts --flow-id `; type RunFlags = { envFile?: string; envId?: string; + flowId?: string; follow: boolean; lines?: string; linesFile?: string; @@ -68,6 +70,10 @@ export function registerRunCommand( "--env-file ", "Dotenv file on this machine whose variables the run is given. Pass this or --env-id, not both", ) + .option( + "--flow-id ", + "The QA Wolf flow this run is for, when the platform cannot tell from the entry point. The run receives it as QAWOLF_WORKFLOW_ID", + ) .option( "--lines ", "Run only these 1-indexed lines against the browser as it stands, instead of the whole flow from a fresh one", @@ -91,6 +97,7 @@ export function registerRunCommand( entryPoint: flowFile, envFile: opts.envFile, envId: opts.envId, + flowId: opts.flowId, follow: opts.follow, lines: opts.lines, linesFile: opts.linesFile, diff --git a/src/core/interactiveRunner/runFlowId.ts b/src/core/interactiveRunner/runFlowId.ts new file mode 100644 index 000000000..401b7005c --- /dev/null +++ b/src/core/interactiveRunner/runFlowId.ts @@ -0,0 +1,17 @@ +import { interactiveRunnerMessages } from "~/core/messages/index.js"; + +/** + * The flow a run is for, from the flag alone. A blank flag is refused, like a + * blank `--env-id`. The process's own `QAWOLF_WORKFLOW_ID` is not read: only + * an AI Job's pod has one, and it names the pod's flow, not every flow run + * from there. + */ +export function resolveRunFlowId( + flag: string | undefined, +): { ok: true; flowId: string | undefined } | { ok: false; error: string } { + const flowId = flag?.trim(); + if (flowId === "") { + return { error: interactiveRunnerMessages.flowIdBlank, ok: false }; + } + return { flowId, ok: true }; +} diff --git a/src/core/messages/interactiveRunner/run.ts b/src/core/messages/interactiveRunner/run.ts index 69700dfcc..5f8244a7e 100644 --- a/src/core/messages/interactiveRunner/run.ts +++ b/src/core/messages/interactiveRunner/run.ts @@ -10,6 +10,8 @@ export const runMessages = { "--env-id was given nothing. Pass the id or alias of a QA Wolf environment, or drop the flag to send no environment.", envIdWithEnvFile: "--env-id and --env-file both give the run its environment, so only one may be passed. Use --env-id for a QA Wolf environment, which QA Wolf reads itself, or --env-file for a dotenv file on this machine.", + flowIdBlank: + "--flow-id was given nothing. Pass the id of the flow this run is for, or drop the flag.", fileNotCollected: (path: string) => `"${path}" is not one of the files that travel to a runner. It must be inside the current directory and end in .ts, .tsx, .js, .mjs, .cjs or .json.`, fileUnreadable: (path: string, reason: string) => diff --git a/src/domains/interactiveRunner/prepareRun.ts b/src/domains/interactiveRunner/prepareRun.ts index f50032578..913d49443 100644 --- a/src/domains/interactiveRunner/prepareRun.ts +++ b/src/domains/interactiveRunner/prepareRun.ts @@ -6,6 +6,7 @@ import { type BuiltRunEnvironment, buildRunEnvironment, } from "~/core/interactiveRunner/runEnvironment.js"; +import { resolveRunFlowId } from "~/core/interactiveRunner/runFlowId.js"; import { checkRunFiles, describeRunFilesCheck, @@ -23,6 +24,7 @@ export type PreparedRun = environment: Record | undefined; environmentId: string | undefined; files: RunFiles; + flowId: string | undefined; selection: RunSelection | undefined; } | { ok: false; error: string; exitCode: number }; @@ -44,6 +46,7 @@ export async function prepareRun( entryPointPath: string; envFile: string | undefined; envId: string | undefined; + flowId: string | undefined; lines: string | undefined; linesFile: string | undefined; }, @@ -65,6 +68,9 @@ export async function prepareRun( explicitEnvId ?? (options.envFile === undefined && fromEnvVar ? fromEnvVar : undefined); + const flow = resolveRunFlowId(options.flowId); + if (!flow.ok) return refused(flow.error, exitCodes.invalidArgs); + const linesFilePath = options.linesFile === undefined ? undefined @@ -95,6 +101,7 @@ export async function prepareRun( environment: given, environmentId, files, + flowId: flow.flowId, ok: true, selection: undefined, } @@ -118,6 +125,7 @@ export async function prepareRun( environment: given, environmentId, files, + flowId: flow.flowId, ok: true, selection: built.selection, } diff --git a/src/domains/interactiveRunner/runFlow.delta.test.ts b/src/domains/interactiveRunner/runFlow.delta.test.ts index 81a21fa0e..4ad62ebc1 100644 --- a/src/domains/interactiveRunner/runFlow.delta.test.ts +++ b/src/domains/interactiveRunner/runFlow.delta.test.ts @@ -35,6 +35,7 @@ async function run( entryPoint: "flow.ts", envFile: undefined, envId: undefined, + flowId: undefined, follow: false, lines: undefined, linesFile: undefined, @@ -173,6 +174,7 @@ describe("handleRunnerRun file delta", () => { entryPoint: "flow.ts", envFile: undefined, envId: undefined, + flowId: undefined, follow: false, lines: undefined, linesFile: undefined, diff --git a/src/domains/interactiveRunner/runFlow.environment.test.ts b/src/domains/interactiveRunner/runFlow.environment.test.ts index 25c65eb02..32d7ee2ab 100644 --- a/src/domains/interactiveRunner/runFlow.environment.test.ts +++ b/src/domains/interactiveRunner/runFlow.environment.test.ts @@ -42,6 +42,7 @@ describe("handleRunnerRun with --env-file and --env-id", () => { entryPoint: "flow.ts", envFile, envId, + flowId: undefined, follow: false, lines: undefined, linesFile: undefined, diff --git a/src/domains/interactiveRunner/runFlow.files.test.ts b/src/domains/interactiveRunner/runFlow.files.test.ts index a58ce96f6..c610b2905 100644 --- a/src/domains/interactiveRunner/runFlow.files.test.ts +++ b/src/domains/interactiveRunner/runFlow.files.test.ts @@ -14,6 +14,7 @@ describe("handleRunnerRun file refusals", () => { follow: false, envFile: undefined, envId: undefined, + flowId: undefined, lines: undefined, linesFile: undefined, runner: "ci", @@ -40,6 +41,7 @@ describe("handleRunnerRun file refusals", () => { follow: false, envFile: undefined, envId: undefined, + flowId: undefined, lines: undefined, linesFile: undefined, runner: "ci", @@ -71,6 +73,7 @@ describe("handleRunnerRun file refusals", () => { follow: false, envFile: undefined, envId: undefined, + flowId: undefined, lines: undefined, linesFile: undefined, runner: undefined, diff --git a/src/domains/interactiveRunner/runFlow.flowId.test.ts b/src/domains/interactiveRunner/runFlow.flowId.test.ts new file mode 100644 index 000000000..582ed23d7 --- /dev/null +++ b/src/domains/interactiveRunner/runFlow.flowId.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "bun:test"; + +import { makeAuthCtx, makeTestDeps } from "./deps.testUtils.js"; +import { handleRunnerRun } from "./runFlow.js"; + +const submitted = { outcome: "success" as const, runId: "run-a" }; + +describe("handleRunnerRun with --flow-id", () => { + const run = async ({ + env = {}, + flowId, + }: { + env?: Record; + flowId?: string; + }) => { + const { callPublicApi, ctx } = makeAuthCtx(); + callPublicApi.mockResolvedValue({ ok: true, value: submitted }); + const result = await handleRunnerRun( + ctx, + { + entryPoint: "flow.ts", + envFile: undefined, + envId: undefined, + flowId, + follow: false, + lines: undefined, + linesFile: undefined, + logs: false, + recorderEvents: false, + runEvents: false, + runner: "ci", + timeout: undefined, + }, + makeTestDeps({ + env, + collectRunFiles: async () => ({ + files: { + "flow.ts": "export default {};", + "package.json": "{}", + }, + unresolvedImports: [], + }), + }), + ); + return { callPublicApi, result }; + }; + + const sentRequest = (callPublicApi: { mock: { calls: unknown[][] } }) => + callPublicApi.mock.calls[0]?.[1] ?? {}; + + it("names the flow the run is for", async () => { + const { callPublicApi, result } = await run({ flowId: "flow-7c2e" }); + + expect(result).toBeUndefined(); + expect(sentRequest(callPublicApi)).toMatchObject({ flowId: "flow-7c2e" }); + }); + + it("trims the id it was given", async () => { + const { callPublicApi } = await run({ flowId: " flow-7c2e " }); + + expect(sentRequest(callPublicApi)).toMatchObject({ flowId: "flow-7c2e" }); + }); + + it("names no flow when the flag is absent", async () => { + const { callPublicApi } = await run({}); + + expect(Object.hasOwn(sentRequest(callPublicApi), "flowId")).toBe(false); + }); + + // An AI Job's pod exports the id of its own flow, which is not the flow of + // every run started from that pod. Reading it here would key a run of one + // flow to another, so the flag is the only source. + it("ignores a QAWOLF_WORKFLOW_ID in its own environment", async () => { + const { callPublicApi } = await run({ + env: { QAWOLF_WORKFLOW_ID: "flow-of-this-pod" }, + }); + + expect(Object.hasOwn(sentRequest(callPublicApi), "flowId")).toBe(false); + }); + + it("refuses a blank flag before addressing a runner", async () => { + const { callPublicApi, result } = await run({ flowId: " " }); + + expect(result?.error).toContain("--flow-id was given nothing"); + expect(result?.exitCode).toBe(2); + expect(callPublicApi).not.toHaveBeenCalled(); + }); +}); diff --git a/src/domains/interactiveRunner/runFlow.follow.test.ts b/src/domains/interactiveRunner/runFlow.follow.test.ts index c93253bca..1e68783cb 100644 --- a/src/domains/interactiveRunner/runFlow.follow.test.ts +++ b/src/domains/interactiveRunner/runFlow.follow.test.ts @@ -24,6 +24,7 @@ const runFollowing = ( follow: options.follow ?? true, envFile: undefined, envId: undefined, + flowId: undefined, lines: undefined, linesFile: undefined, logs: options.logs ?? false, diff --git a/src/domains/interactiveRunner/runFlow.selection.test.ts b/src/domains/interactiveRunner/runFlow.selection.test.ts index c5ac772b5..75c980c0b 100644 --- a/src/domains/interactiveRunner/runFlow.selection.test.ts +++ b/src/domains/interactiveRunner/runFlow.selection.test.ts @@ -42,6 +42,7 @@ async function runWith(options: { follow: false, envFile: undefined, envId: undefined, + flowId: undefined, lines: options.lines, linesFile: options.linesFile, logs: false, diff --git a/src/domains/interactiveRunner/runFlow.test.ts b/src/domains/interactiveRunner/runFlow.test.ts index 22c33098b..879d215ad 100644 --- a/src/domains/interactiveRunner/runFlow.test.ts +++ b/src/domains/interactiveRunner/runFlow.test.ts @@ -24,6 +24,7 @@ async function runWith( follow: false, envFile: undefined, envId: undefined, + flowId: undefined, lines: options.lines, linesFile: options.linesFile, runner: "ci", @@ -120,6 +121,7 @@ describe("handleRunnerRun", () => { follow: false, envFile: undefined, envId: undefined, + flowId: undefined, lines: undefined, linesFile: undefined, runner: undefined, @@ -158,6 +160,7 @@ describe("handleRunnerRun", () => { follow: false, envFile: undefined, envId: undefined, + flowId: undefined, lines: undefined, linesFile: undefined, runner: "ci", diff --git a/src/domains/interactiveRunner/runFlow.ts b/src/domains/interactiveRunner/runFlow.ts index 3e61fbcca..b081f9168 100644 --- a/src/domains/interactiveRunner/runFlow.ts +++ b/src/domains/interactiveRunner/runFlow.ts @@ -25,6 +25,7 @@ export async function handleRunnerRun( entryPoint: string; envFile: string | undefined; envId: string | undefined; + flowId: string | undefined; follow: boolean; lines: string | undefined; linesFile: string | undefined; @@ -53,6 +54,7 @@ export async function handleRunnerRun( entryPointPath, envFile: options.envFile, envId: options.envId, + flowId: options.flowId, lines: options.lines, linesFile: options.linesFile, }, @@ -90,6 +92,7 @@ export async function handleRunnerRun( environment: prepared.environment, environmentId: prepared.environmentId, files: prepared.files, + flowId: prepared.flowId, resolved, selection: prepared.selection, }, diff --git a/src/domains/interactiveRunner/sendRunFlowRequest.ts b/src/domains/interactiveRunner/sendRunFlowRequest.ts index b6d98ef1a..244b82f8e 100644 --- a/src/domains/interactiveRunner/sendRunFlowRequest.ts +++ b/src/domains/interactiveRunner/sendRunFlowRequest.ts @@ -29,6 +29,7 @@ export async function sendRunFlowRequest( entryPointPath: string; environment: Record | undefined; environmentId: string | undefined; + flowId: string | undefined; resolved: Runner; selection: RunSelection | undefined; }, @@ -49,6 +50,7 @@ export async function sendRunFlowRequest( ...(options.environmentId === undefined ? {} : { environmentId: options.environmentId }), + ...(options.flowId === undefined ? {} : { flowId: options.flowId }), ...(options.selection === undefined ? {} : { selection: options.selection }), diff --git a/src/domains/interactiveRunner/submitRun.ts b/src/domains/interactiveRunner/submitRun.ts index f503acace..c45f1a543 100644 --- a/src/domains/interactiveRunner/submitRun.ts +++ b/src/domains/interactiveRunner/submitRun.ts @@ -33,6 +33,7 @@ export async function submitRun( environment: Record | undefined; environmentId: string | undefined; files: RunFiles; + flowId: string | undefined; resolved: Runner; selection: RunSelection | undefined; }, diff --git a/src/runnerSdk/runVerbs.ts b/src/runnerSdk/runVerbs.ts index 974dc640b..29db32eb8 100644 --- a/src/runnerSdk/runVerbs.ts +++ b/src/runnerSdk/runVerbs.ts @@ -55,6 +55,7 @@ export function createRunVerbs({ deps, platformClient }: SdkContext) { async run({ entryPointPath, environment, + flowId, runnerId, selection, }: RunRequest): Promise> { @@ -63,6 +64,7 @@ export function createRunVerbs({ deps, platformClient }: SdkContext) { entryPointPath, envFile: undefined, envId: environment === "ambient" ? undefined : environment.id, + flowId, lines: toLineRange(selection), linesFile: toLinesFile(selection), }, @@ -77,6 +79,7 @@ export function createRunVerbs({ deps, platformClient }: SdkContext) { environment: prepared.environment, environmentId: prepared.environmentId, files: prepared.files, + flowId: prepared.flowId, resolved: { ...givenRunner(runnerId), type: "resolved" }, selection: prepared.selection, }, diff --git a/src/runnerSdk/types.ts b/src/runnerSdk/types.ts index dce6b7a86..a93b52313 100644 --- a/src/runnerSdk/types.ts +++ b/src/runnerSdk/types.ts @@ -74,6 +74,7 @@ export type Recordings = PublicApiOutput; export type RunRequest = RunnerRequest & { entryPointPath: string; environment: RunEnvironment; + flowId?: string; // the run receives it as QAWOLF_WORKFLOW_ID selection: RunSelection; };