diff --git a/evals/README.md b/evals/README.md index b0ac7b3e..92b1c152 100644 --- a/evals/README.md +++ b/evals/README.md @@ -34,6 +34,7 @@ npm run eval:agentcontrol-tools # agentcontrol/tools npm run eval:configs-variations # agentcontrol/configs-variations npm run eval:flag-create # feature-flags/launchdarkly-flag-create npm run eval:onboarding # onboarding +npm run eval:experiment-setup # experiments/launchdarkly-experiment-setup npm run eval:experiment-orchestration # aws-devops-agent/experiment-orchestration npm run eval:release-autopilot-onboarding # release-autopilot/launchdarkly-release-autopilot-onboarding @@ -44,6 +45,7 @@ npm run eval:agentcontrol-tools:single npm run eval:configs-variations:single npm run eval:flag-create:single npm run eval:onboarding:single +npm run eval:experiment-setup:single npm run eval:experiment-orchestration:single npm run eval:release-autopilot-onboarding:single diff --git a/evals/experiment-setup/mocks.json b/evals/experiment-setup/mocks.json new file mode 100644 index 00000000..98202bd1 --- /dev/null +++ b/evals/experiment-setup/mocks.json @@ -0,0 +1,67 @@ +{ + "get-flag-status-across-envs": { + "flagKey": "{{flagKey}}", + "environments": [ + { + "key": "test", + "status": "active" + }, + { + "key": "production", + "status": "active" + } + ], + "summary": { + "active": [ + "test", + "production" + ], + "inactive": [], + "new": [], + "launched": [] + } + }, + "list-metric-events": { + "events": [ + { + "eventKey": "checkout-completed", + "eventKind": "custom", + "lastSeen": "1792454400000" + }, + { + "eventKey": "page-load-ms", + "eventKind": "custom", + "lastSeen": "1792454400000" + }, + { + "eventKey": "cta-click", + "eventKind": "custom", + "lastSeen": "1792281600000" + }, + { + "eventKey": "cta-clicked-v2", + "eventKind": "custom", + "lastSeen": "1792281600000" + } + ], + "count": 4, + "_note": "4 event key(s) found with recent activity." + }, + "search-contexts": { + "items": [ + { + "kind": "user", + "key": "user-1", + "lastSeen": "2026-10-20T00:00:00.000Z" + } + ], + "totalCount": 1 + }, + "save-and-start-experiment-iteration": { + "key": "{{experimentKey}}", + "environmentKey": "{{environmentKey}}", + "currentIteration": { + "status": "running" + } + } +} diff --git a/evals/experiment-setup/promptfooconfig.yaml b/evals/experiment-setup/promptfooconfig.yaml new file mode 100644 index 00000000..da6f42e6 --- /dev/null +++ b/evals/experiment-setup/promptfooconfig.yaml @@ -0,0 +1,342 @@ +# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json +# +# Run with shared defaults: +# promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml +# +# Scores the approval gates in launchdarkly-experiment-setup. The user message +# is the whole turn: the agent shows a design or a question and waits. No case +# scripts an ask-question answer. Each case has one weight-3 tool-call check +# (lookups that should happen, and writes that must not) and a weight-1 rubric +# graded from the assistant's messages. The tool check decides pass or fail. +description: "Evaluation of the experiment-setup skill: look up, design, and wait" + +prompts: + - file://../../skills/experiments/launchdarkly-experiment-setup/SKILL.md + +providers: + - id: file://../providers/claude-skill-agent-sdk.js + label: claude-skill-agent-sdk + config: + skill_slug: launchdarkly-experiment-setup + allow_builtins: false + expose_mcp_tools: true + # No case scripts an answer. An unscripted ask-question call returns the + # first option, which can approve a create, so leave the tool off. + expose_ask_question: false + # Paste SKILL.md into the agent instructions. A slash-command first turn + # arrives before the request and aborts in-flight tool calls, and the + # skills list only supplies the name and description. + force_skill_invocation: false + inline_skill: true + # Schemas and default templates for the tools only this suite needs. + # Sentinel responses still come from the shared mock renderer. + tool_definitions: experiment-setup/tools.json + tool_mocks: experiment-setup/mocks.json + # In the shared catalog, but not a hosted MCP tool and it has no mock. + # A call returns "no mock configured", and the agent can then treat a + # lifecycle status of active as the flag being on. + exclude_mcp_tools: [get-feature-flag] + +tests: + # ------------------------------------------------------------------ + # 1. Every required piece is already named. Look up the flag and the + # metric, show one design, and wait. checkout-flow's get-flag template + # is version 1 and its fallthrough rule id is "fallthrough". + # ------------------------------------------------------------------ + - description: "Complete design: one design, then wait, with nothing created or started" + vars: + user_request: > + Set up an experiment in project "my-project", environment "production". + Flag: checkout-flow. Variations: true (id var-true) and false (id var-false). + Control: the false variation. Hypothesis: the new checkout will lift + completion rate by 3%. Primary metric: the existing checkout-completed + metric. Randomization unit: user. + max_turns: 16 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-flow'); + const listedMetrics = tools.includes('list-metrics'); + const pass = gotFlag && listedMetrics && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' list-metrics=' + listedMetrics + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_flag_and_metric_and_does_not_write + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. Score 1.0 if they show one complete + design and then wait for approval. That design names checkout-flow, + both variations, the false variation as control, the hypothesis, + checkout-completed, and randomization unit user. It includes ruleId + fallthrough and a flagConfigVersion. Statistics are the LaunchDarkly + default (frequentist, sequential testing). Score 0 if a required piece + is missing, statistics are Bayesian or another methodology, or the + message says the experiment was created or started. + metric: one_complete_design_then_waits + weight: 1 + + # ------------------------------------------------------------------ + # 2. Flag is on and active, and user is the only context kind. Suggest + # the control and the randomization unit. Do not create. + # ------------------------------------------------------------------ + - description: "Suggests control and randomization unit when the flag is active and only user contexts exist" + vars: + user_request: > + Set up an experiment in project "only-user", environment "production". + Flag: checkout-active. Compare its variations. Hypothesis: the new + checkout will lift completion rate by 3%. Primary metric: the existing + checkout-completed metric. I have not chosen a control variation or a + randomization unit. Show the design and wait. Do not create anything yet. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-active'); + const searches = traj.filter(t => t.tool === 'search-contexts'); + const revealsOthers = searches.some(t => { + const a = t.arguments || {}; + const filter = String(a.filter || ''); + if (/kind\s+notEquals\s+"?user"?/i.test(filter)) return true; + if (a.kind === 'user' || /kind\s+equals\s+"?user"?/i.test(filter)) return false; + const kindBlank = a.kind == null || String(a.kind).trim() === ''; + const filterBlank = a.filter == null || filter.trim() === ''; + return kindBlank && filterBlank; + }); + const pass = gotFlag && revealsOthers && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' revealsOthers=' + revealsOthers + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_active_flag_and_contexts + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The user did not name a control or a + randomization unit. Score 1.0 if the design labels both the served + variation ("Current checkout") and randomization unit "user" as + suggested, each with a short reason. Offering to confirm or change + those suggestions before approval is fine. Score 0 only if it asks + the user to supply the control or the randomization unit instead of + suggesting them, or omits the suggested label on either. + metric: labels_suggestions + weight: 1 + + # ------------------------------------------------------------------ + # 3. Flag is off and inactive, and another context kind is stored. + # Ask for the control and the randomization unit. Do not suggest them + # and do not create. + # ------------------------------------------------------------------ + - description: "Asks for control and randomization unit when the flag is inactive and contexts are mixed" + vars: + user_request: > + Set up an experiment in project "mixed-contexts", environment "production". + Flag: checkout-inactive. Compare its variations. Hypothesis: the new + checkout will lift completion rate by 3%. Primary metric: the existing + checkout-completed metric. I have not chosen a control variation or a + randomization unit. Ask me for anything you still need. Do not create + anything yet. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-inactive'); + const gotStatus = traj.some(t => t.tool === 'get-flag-status-across-envs' && (t.arguments || {}).flagKey === 'checkout-inactive'); + const searches = traj.filter(t => t.tool === 'search-contexts'); + const revealsOthers = searches.some(t => { + const a = t.arguments || {}; + const filter = String(a.filter || ''); + if (/kind\s+notEquals\s+"?user"?/i.test(filter)) return true; + if (a.kind === 'user' || /kind\s+equals\s+"?user"?/i.test(filter)) return false; + const kindBlank = a.kind == null || String(a.kind).trim() === ''; + const filterBlank = a.filter == null || filter.trim() === ''; + return kindBlank && filterBlank; + }); + const pass = gotFlag && gotStatus && revealsOthers && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' status=' + gotStatus + ' revealsOthers=' + revealsOthers + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_inactive_flag_and_mixed_contexts + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The flag is off and inactive, and more + than one context kind is stored. Score 1.0 if the agent asks for both + the control variation and the randomization unit, leaves both blank, + and does not label either as suggested. Score 0 if it suggests either + value or shows a design that fills either in. + metric: asks_instead_of_suggesting + weight: 1 + + # ------------------------------------------------------------------ + # 4. Flag, hypothesis, and primary metric are all missing. One gather + # message, and nothing is created. No lookup is required. + # ------------------------------------------------------------------ + - description: "Missing pieces: one gather message and no create" + vars: + user_request: > + I want to run an experiment in project "my-project", environment + "production". I don't have a flag, a hypothesis, or a primary metric yet. + max_turns: 8 + assert: + - type: javascript + value: | + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const pass = called.length === 0; + return { pass, score: pass ? 1 : 0, reason: called.length ? 'wrote ' + called.join(', ') : 'no writes' }; + metric: does_not_write + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. Score 1.0 if a single message asks for + the missing flag and its variations, the hypothesis, and the primary + metric. Score 0 if any of those three is left out, they are split + across separate messages, or a design is shown. + metric: gather_names_missing_pieces + weight: 1 + + # ------------------------------------------------------------------ + # 5. The custom event has not arrived. list-metric-events runs, the + # design warns and lists similar keys, and nothing is created. + # ------------------------------------------------------------------ + - description: "Missing event: list-metric-events runs, the design warns, and nothing is created" + vars: + user_request: > + Set up an experiment in project "my-project", environment "production". + Flag: checkout-flow. Variations: true (id var-true) and false (id var-false). + Control: the false variation. Hypothesis: the new checkout will lift + completion rate by 3%. Primary metric: a new custom metric "CTA clicked" + (key cta-clicked) measuring the share of users who sent the cta-clicked + event. Randomization unit: user. + max_turns: 16 + assert: + - type: javascript + value: | + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const listedEvents = tools.includes('list-metric-events'); + const pass = listedEvents && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'list-metric-events=' + listedEvents + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: lists_events_and_does_not_create + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The cta-clicked event is not arriving. + Similar keys cta-click and cta-clicked-v2 are. Score 1.0 if the design + warns that the event has not arrived in the environment and lists those + similar keys. Score 0 if the warning is missing or neither similar key + is named. + metric: warns_and_lists_similar_keys + weight: 1 + + # ------------------------------------------------------------------ + # 6. The user says to start. The flag is off and the event is still + # missing. One message covers both. No flag write and no start. + # ------------------------------------------------------------------ + - description: "Start blocked: flag off and event missing, no flag change and no start" + vars: + user_request: > + Start the existing experiment "checkout-draft" in project "my-project", + environment "production". It uses flag checkout-flow and a custom metric + whose event key is cta-clicked. Start it now. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-flow'); + const listedEvents = tools.includes('list-metric-events'); + const pass = gotFlag && listedEvents && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' list-metric-events=' + listedEvents + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_flag_and_event_and_does_not_start + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The flag is off and the cta-clicked + event is not arriving. Score 1.0 if one message covers both: the flag + must be on before starting, and whether to start while the event is + missing. Score 0 if those are separate messages, or the message says + the flag was turned on or the iteration was started. + metric: one_message_covers_flag_and_event + weight: 1 + + # ------------------------------------------------------------------ + # 7. The named flag does not exist. Look up the key and similar flags. + # Offer three choices. Do not create the flag. The project key is unique + # so only this case's list-flags calls return the similar checkout flags. + # ------------------------------------------------------------------ + - description: "Missing flag: three choices, and the flag is not created" + vars: + user_request: > + Set up an experiment in project "exp-setup-missing-flag", environment "production". + Flag: missing-checkout. Variations: true and false. Control: false. + Hypothesis: the new checkout will lift completion rate by 3%. Primary + metric: the existing checkout-completed metric. Randomization unit: user. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const lookedUp = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'missing-checkout'); + const listed = traj.some(t => t.tool === 'list-flags'); + const pass = lookedUp && listed && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + lookedUp + ' list-flags=' + listed + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_missing_flag_and_does_not_create + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The flag missing-checkout does not exist. + Score 1.0 if one message offers three choices: correct the key + (mentioning a similar flag such as checkout-flow or new-checkout), + create the flag themselves and then continue, or have the agent create + it. For that last choice the message proposes the key, name, type, and + variations and says the flag will be created off. Score 0 if it offers + fewer than three choices. + metric: offers_three_flag_choices + weight: 1 + + # ------------------------------------------------------------------ + # 8. A running experiment's allocation change would start a new + # iteration. Show the change and wait. The user has not said to start. + # ------------------------------------------------------------------ + - description: "Running experiment: show the allocation change and do not start" + vars: + user_request: > + The experiment "checkout-running" in project "my-project", environment + "production" is running. Change the treatment allocation to 30% control + and 70% treatment. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const sawRunning = traj.some(t => t.tool === 'get-experiment' && ((t.arguments || {}).experimentKey === 'checkout-running' || (t.arguments || {}).key === 'checkout-running')); + const pass = sawRunning && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-experiment=' + sawRunning + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: reads_running_experiment_and_does_not_start + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The experiment is running and the user + wants a 30/70 allocation but has not said to start. Score 1.0 if the + agent shows that allocation change and waits. Score 0 if it says a new + iteration is already running. + metric: shows_change_and_waits + weight: 1 diff --git a/evals/experiment-setup/tools.json b/evals/experiment-setup/tools.json new file mode 100644 index 00000000..aa765987 --- /dev/null +++ b/evals/experiment-setup/tools.json @@ -0,0 +1,301 @@ +[ + { + "name": "get-flag-status-across-envs", + "description": "Check a flag's lifecycle status across all environments. Returns per-environment state (active, inactive, new, launched) and a summary of which environments the flag is active, inactive, or launched in. Use before cleanup to verify the flag is inactive everywhere.", + "input_schema": { + "type": "object", + "properties": { + "flagKey": { + "type": "string" + }, + "projectKey": { + "type": "string" + } + }, + "required": [ + "projectKey", + "flagKey" + ] + } + }, + { + "name": "list-metric-events", + "description": "List recent event keys received by a LaunchDarkly project \u2014 used to check which events are actively flowing before creating a metric. Returns up to 50 event keys with last-seen timestamps. If the event key you need isn't here, it may not be instrumented yet. Use this BEFORE create-metric to: 1. Confirm the event key exists and is receiving data 2. Discover event keys when the user isn't sure what to use 3. Check the correct spelling of an event key Supports fuzzy search via `query` to narrow results. Filter by `environmentKey` to check a specific environment. If the desired event key is absent, surface a warning and ask whether the user wants to instrument the event first or proceed and wire it up later.", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + } + }, + "required": [ + "projectKey" + ] + } + }, + { + "name": "search-contexts", + "description": "Search for stored LaunchDarkly contexts in an environment by kind, key, or attribute value. Returns context records including their stored attributes (e.g., merchantCountryCode, disbursementRail) and when the context was last seen. Convenience param: - `kind`: filter to a specific context kind (e.g. `Merchant`, `user`, `organization`). Emitted as the filter clause `kind equals \"\"`. Raw filter syntax (passed in `filter`) uses `field operator value` clauses. Operators: `equals`, `notEquals`, `anyOf`, `startsWith`, `exists`, `before`, `after`; `contains` and `notContains` are not supported. Combine clauses with `,` (AND) or `|` (OR); group with `()`. Supported fields: `kind`, `key`, `kindKey`, `name`, `applicationId`, `q` (fuzzy), and `.` for context attributes. Examples: - `kind equals \"Merchant\"` \u2014 contexts of a specific kind - `Merchant.merchantCountryCode equals \"US\"` \u2014 Merchant contexts whose attribute matches - `key startsWith \"merchant-\"` \u2014 contexts whose key has a prefix - `applicationId equals \"GoSDK/1.2\"` \u2014 contexts reported by a specific SDK/application Example combining both params: set `kind=Merchant` and `filter=Merchant.disbursementRail equals \"PIX\"` to find Merchant contexts on the PIX rail. Note: the contexts API cannot filter by which flag a context evaluated; there is no `flagEvaluated` field. Filter by kind, key, applicationId, or attribute values instead. Ordering is fixed by the API; there is no sort parameter. Pagination: responses include `totalCount` and `continuationToken`. Pass `continuationToken` from the previous response to get the next page.", + "input_schema": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "limit": { + "type": "number" + }, + "filter": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + }, + "continuationToken": { + "type": "string" + } + }, + "required": [ + "projectKey", + "environmentKey" + ] + } + }, + { + "name": "save-and-start-experiment-iteration", + "description": "Stop the current running iteration, create a new draft iteration with the provided field updates applied, and start it \u2014 the API-recommended way to mutate treatments, metrics, methodology, or other fields that are locked while an iteration is running. Provide `changeJustification` to record why a new iteration is needed. Field inputs are the same as `update-experiment`; updates are checked against the new iteration's not_started mutability rules. Fields not allowed in not_started are returned under `skipped` rather than being sent.", + "input_schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "flags": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "required": [ + "ruleId", + "flagConfigVersion" + ], + "properties": { + "ruleId": { + "type": "string" + }, + "flagConfigVersion": { + "type": "number" + }, + "notInExperimentVariationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "key" + ], + "properties": { + "key": { + "type": "string" + }, + "isGroup": { + "type": "boolean" + } + }, + "additionalProperties": false + } + }, + "holdoutId": { + "type": "string" + }, + "attributes": { + "type": "array", + "items": { + "type": "string" + } + }, + "dataSource": { + "type": "string", + "enum": [ + "launchdarkly", + "snowflake", + "databricks" + ] + }, + "hypothesis": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "treatments": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "baseline", + "allocationPercent", + "parameters" + ], + "properties": { + "name": { + "type": "string" + }, + "baseline": { + "type": "boolean" + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "required": [ + "flagKey", + "variationId" + ], + "properties": { + "flagKey": { + "type": "string" + }, + "variationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "allocationPercent": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "covariateId": { + "type": "string" + }, + "description": { + "type": "string" + }, + "methodology": { + "type": "string", + "enum": [ + "bayesian", + "frequentist", + "export_only" + ] + }, + "flagSettings": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "required": [ + "ruleId", + "flagConfigVersion" + ], + "properties": { + "ruleId": { + "type": "string" + }, + "flagConfigVersion": { + "type": "number" + }, + "notInExperimentVariationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "maintainerId": { + "type": "string" + }, + "experimentKey": { + "type": "string" + }, + "analysisConfig": { + "type": "object", + "properties": { + "testDirection": { + "type": "string" + }, + "bayesianThreshold": { + "type": "string" + }, + "significanceThreshold": { + "type": "string" + }, + "sequentialTestingEnabled": { + "type": "boolean" + }, + "multipleComparisonCorrectionScope": { + "type": "string", + "enum": [ + "variations", + "variations-and-metrics", + "metrics" + ] + }, + "multipleComparisonCorrectionMethod": { + "type": "string", + "enum": [ + "bonferroni", + "benjamini-hochberg" + ] + } + }, + "additionalProperties": false + }, + "environmentKey": { + "type": "string" + }, + "primaryFunnelKey": { + "type": "string" + }, + "randomizationUnit": { + "type": "string" + }, + "canReshuffleTraffic": { + "type": "boolean" + }, + "changeJustification": { + "type": "string" + }, + "primarySingleMetricKey": { + "type": "string" + } + }, + "required": [ + "projectKey", + "environmentKey", + "experimentKey" + ] + } + } +] diff --git a/evals/package.json b/evals/package.json index 9c1e1d26..baa65bdb 100644 --- a/evals/package.json +++ b/evals/package.json @@ -29,6 +29,8 @@ "eval:release-autopilot-onboarding:single": "promptfoo eval -c shared/defaults.yaml -c release-autopilot-onboarding/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "eval:experiment-guide": "promptfoo eval -c shared/defaults.yaml -c experiment-guide/promptfooconfig.yaml --env-file .env --no-cache -o experiment-guide/results.json", "eval:experiment-guide:single": "promptfoo eval -c shared/defaults.yaml -c experiment-guide/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", + "eval:experiment-setup": "promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml --env-file .env --no-cache -o experiment-setup/results.json", + "eval:experiment-setup:single": "promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "eval:experiment-orchestration": "promptfoo eval -c shared/defaults.yaml -c experiment-orchestration/promptfooconfig.yaml --env-file .env --no-cache -o experiment-orchestration/results.json", "eval:experiment-orchestration:single": "promptfoo eval -c shared/defaults.yaml -c experiment-orchestration/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "fixture:from-pr": "node scripts/build-pr-fixture.js", diff --git a/evals/providers/_mock.js b/evals/providers/_mock.js index 136c6836..339c516e 100644 --- a/evals/providers/_mock.js +++ b/evals/providers/_mock.js @@ -132,6 +132,149 @@ function createMockState() { }; } +// Lifecycle status for get-flag-status-across-envs. The default matches the +// environments in the get-environments mock and reports both as active, so +// flag-drift sees no cross-environment difference and flag-command gets no +// cleanup signal. `federal` is intentionally absent: the flag-drift mock +// flips that environment's fallthrough, and listing it here would make +// single-environment drift cases look divergent. +function flagStatusResponse(flagKey, productionStatus) { + const environments = [ + { key: "test", status: "active" }, + { key: "production", status: productionStatus }, + ]; + const summary = { active: [], inactive: [], new: [], launched: [] }; + for (const env of environments) { + if (summary[env.status]) summary[env.status].push(env.key); + } + return { flagKey, environments, summary }; +} + +// experiment-setup sentinel. The exact key cta-clicked is not in that suite's +// list-metric-events template; a query for it returns the similar keys from +// the template that was passed in. Every other call falls through and returns +// that template unchanged, so another suite keeps its own event list. +function similarCtaMetricEvents(template, replacements) { + const rendered = walk(template, replacements); + const events = Array.isArray(rendered.events) + ? rendered.events.filter( + (event) => event.eventKey === "cta-click" || event.eventKey === "cta-clicked-v2", + ) + : []; + return { + events, + count: events.length, + _note: `${events.length} event key(s) found with recent activity.`, + }; +} + +function isUserKindQuery(input) { + if (input.kind === "user") return true; + return typeof input.filter === "string" && /kind\s+equals\s+"?user"?/i.test(input.filter); +} + +function isNotEqualsUser(input) { + return ( + typeof input.filter === "string" && + /kind\s+notEquals\s+"?user"?/i.test(input.filter) + ); +} + +function contextItem(kind, key) { + return { kind, key, lastSeen: "2026-10-20T00:00:00.000Z" }; +} + +function isUnfilteredContextSearch(input) { + const kindBlank = input.kind == null || String(input.kind).trim() === ""; + const filterBlank = input.filter == null || String(input.filter).trim() === ""; + return kindBlank && filterBlank; +} + +function searchContextsResponse(input) { + const mixed = input.projectKey === "mixed-contexts"; + if (isNotEqualsUser(input)) { + if (mixed) { + return { items: [contextItem("organization", "org-1")], totalCount: 1 }; + } + return { items: [], totalCount: 0 }; + } + if (isUserKindQuery(input)) { + return { items: [contextItem("user", "user-1")], totalCount: 1 }; + } + // No kind and no filter lists every stored kind: user in both projects, + // and organization as well when other kinds are stored. + if (isUnfilteredContextSearch(input)) { + const items = [contextItem("user", "user-1")]; + if (mixed) items.push(contextItem("organization", "org-1")); + return { items, totalCount: items.length }; + } + return { items: [], totalCount: 0 }; +} + +function checkoutActiveFlag() { + return { + key: "checkout-active", + name: "Checkout active", + kind: "multivariate", + description: "Checkout experiment flag", + temporary: true, + tags: [], + variations: [ + { _id: "var-current", value: "current", name: "Current checkout" }, + { _id: "var-new", value: "new", name: "New checkout" }, + ], + defaults: { onVariation: 0, offVariation: 1 }, + environment: { + on: true, + version: 7, + rules: [], + targets: [], + fallthrough: { variation: 0 }, + offVariation: 1, + }, + }; +} + +function similarCheckoutFlags() { + const flag = (key, name, on) => ({ + key, + name, + kind: "boolean", + description: name, + temporary: true, + tags: ["checkout"], + on, + variations: [ + { value: true, name: "Enabled" }, + { value: false, name: "Disabled" }, + ], + }); + return { + flags: [ + flag("checkout-flow", "Checkout flow", true), + flag("new-checkout", "New checkout", false), + ], + totalCount: 2, + pageInfo: { limit: 20, offset: 0 }, + }; +} + +function runningExperiment(input) { + return { + key: "checkout-running", + name: "Checkout running", + environmentKey: input.environmentKey || input.env || "production", + currentIteration: { + status: "running", + hypothesis: "The new checkout will lift completion.", + treatments: [ + { _id: "treat-control", name: "Control", baseline: true, allocationPercent: 50 }, + { _id: "treat-treatment", name: "Treatment", baseline: false, allocationPercent: 50 }, + ], + }, + }; +} + /** * Render a mock response for `toolName` given `input` (raw tool arguments). * `state` is the per-invocation map from `createMockState()`. @@ -252,6 +395,17 @@ function renderMockResponse(template, input, toolName, state) { const key = input.flagKey || input.key; if (key && state.flags[key]) return state.flags[key]; + // Sentinel keys for the experiment-setup suite. Checked before the + // federal fallthrough hook so they do not inherit that override. + if (key === "missing-checkout") { + return { + error: "not_found", + status: 404, + message: "Flag 'missing-checkout' was not found in this project.", + }; + } + if (key === "checkout-active") return checkoutActiveFlag(); + // Cross-environment divergence hook: a `federal`-style environment serves // the opposite default (variation 0 / `true`) from every other environment // (which resolve to variation 1 / `false` via the static template). This @@ -425,6 +579,33 @@ function renderMockResponse(template, input, toolName, state) { return flag; } + if (toolName === "get-flag-status-across-envs") { + const flagKey = input.flagKey || "unknown-flag"; + const productionStatus = flagKey === "checkout-inactive" ? "inactive" : "active"; + return flagStatusResponse(flagKey, productionStatus); + } + + if (toolName === "search-contexts") return searchContextsResponse(input || {}); + + if (toolName === "list-metric-events") { + const query = typeof (input || {}).query === "string" ? input.query.trim().toLowerCase() : ""; + if (query === "cta-clicked") return similarCtaMetricEvents(template, replacements); + } + + if (toolName === "list-flags") { + // experiment-setup's missing-flag case uses this project. Any search there + // returns the similar keys and omits the missing one. Other projects, and + // list-feature-flags, keep the template they were given. + const projectKey = + typeof (input || {}).projectKey === "string" ? input.projectKey.trim() : ""; + if (projectKey === "exp-setup-missing-flag") return similarCheckoutFlags(); + } + + if (toolName === "get-experiment") { + const experimentKey = input.experimentKey || input.key; + if (experimentKey === "checkout-running") return runningExperiment(input); + } + // Default: stateless template render return walk(template, replacements); } diff --git a/evals/scripts/_manifest.js b/evals/scripts/_manifest.js index a4eb0707..4c9da8a2 100644 --- a/evals/scripts/_manifest.js +++ b/evals/scripts/_manifest.js @@ -91,6 +91,12 @@ const SUITES = [ skillDir: "skills/experiments/launchdarkly-experiment-guide", readme: "skills/experiments/launchdarkly-experiment-guide/README.md", }, + { + suite: "experiment-setup", + skillKey: "experiments/launchdarkly-experiment-setup", + skillDir: "skills/experiments/launchdarkly-experiment-setup", + readme: "skills/experiments/launchdarkly-experiment-setup/README.md", + }, { // No readme: the skill folder holds only the SKILL.md that ldcli embeds. suite: "experiment-orchestration", diff --git a/evals/tests/mock.test.js b/evals/tests/mock.test.js index 852dae5a..103427a2 100644 --- a/evals/tests/mock.test.js +++ b/evals/tests/mock.test.js @@ -4,6 +4,10 @@ const assert = require("node:assert/strict"); const { renderMockResponse, createMockState } = require("../providers/_mock"); const TEMPLATES = require("../mocks/tool-responses.json"); +// These four tools' default templates live with the experiment-setup suite. +// The sentinel branches in _mock.js still override them. +const SUITE_TEMPLATES = require("../experiment-setup/mocks.json"); +const ORCH_TEMPLATES = require("../experiment-orchestration/mocks.json"); // --------------------------------------------------------------------------- // createMockState @@ -423,6 +427,263 @@ test("update-ai-config uses template substitution", () => { assert.equal(result.name, "Bot"); }); +// --------------------------------------------------------------------------- +// experiment-setup sentinels +// --------------------------------------------------------------------------- + +test("experiment-setup suite files own the four tools the shared catalog no longer has", () => { + const sharedDefs = require("../tools/definitions.json").map((def) => def.name); + const suiteDefs = require("../experiment-setup/tools.json").map((def) => def.name); + const owned = [ + "get-flag-status-across-envs", + "list-metric-events", + "search-contexts", + "save-and-start-experiment-iteration", + ]; + for (const name of owned) { + assert.equal(sharedDefs.includes(name), false); + assert.equal(TEMPLATES[name], undefined); + assert.equal(suiteDefs.includes(name), true); + assert.ok(SUITE_TEMPLATES[name]); + } +}); + +test("save-and-start returns the suite template when no sentinel matches", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["save-and-start-experiment-iteration"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-running", + }, "save-and-start-experiment-iteration", state); + assert.equal(result.key, "checkout-running"); + assert.equal(result.currentIteration.status, "running"); +}); + +test("get-flag-status-across-envs defaults to active in test and production", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["get-flag-status-across-envs"], { + projectKey: "payments", + flagKey: "billing-v2", + }, "get-flag-status-across-envs", state); + assert.deepEqual( + result.environments.map((env) => [env.key, env.status]), + [["test", "active"], ["production", "active"]], + ); + assert.deepEqual(result.summary.active, ["test", "production"]); + assert.deepEqual(result.summary.inactive, []); + assert.equal( + result.environments.some((env) => /federal/i.test(env.key)), + false, + ); +}); + +test("get-flag-status-across-envs reports checkout-inactive as inactive in production", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["get-flag-status-across-envs"], { + projectKey: "mixed-contexts", + flagKey: "checkout-inactive", + }, "get-flag-status-across-envs", state); + const production = result.environments.find((env) => env.key === "production"); + assert.equal(production.status, "inactive"); + assert.deepEqual(result.summary.inactive, ["production"]); +}); + +test("get-flag returns checkout-active on, serving one variation", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["get-flag"], { + projectKey: "only-user", + flagKey: "checkout-active", + environmentKey: "production", + }, "get-flag", state); + assert.equal(result.environment.on, true); + assert.equal(result.environment.version, 7); + assert.equal(result.environment.fallthrough.variation, 0); + assert.equal(result.environment.fallthrough.rollout, undefined); + assert.equal(result.variations[0].name, "Current checkout"); +}); + +test("get-flag returns not-found for missing-checkout", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["get-flag"], { + projectKey: "my-project", + flagKey: "missing-checkout", + }, "get-flag", state); + assert.equal(result.status, 404); +}); + +test("get-flag returns a stored flag before the missing-checkout not-found branch", () => { + const state = createMockState(); + renderMockResponse(TEMPLATES["create-flag"], { + key: "missing-checkout", + name: "Missing", + }, "create-flag", state); + const result = renderMockResponse(TEMPLATES["get-flag"], { + flagKey: "missing-checkout", + }, "get-flag", state); + assert.equal(result.key, "missing-checkout"); + assert.equal(result.error, undefined); +}); + +test("list-flags in exp-setup-missing-flag returns similar keys for any query", () => { + const state = createMockState(); + for (const query of ["flags", "Checkout", "missing-checkout"]) { + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "exp-setup-missing-flag", + query, + }, "list-flags", state); + assert.deepEqual( + result.flags.map((flag) => flag.key), + ["checkout-flow", "new-checkout"], + ); + } +}); + +test("list-flags checkout query in any other project returns the shared template", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "my-project", + query: "Checkout", + }, "list-flags", state); + assert.deepEqual(result, TEMPLATES["list-flags"]); +}); + +test("list-feature-flags is unchanged for a checkout query and the sentinel project", () => { + const state = createMockState(); + const checkout = renderMockResponse(TEMPLATES["list-feature-flags"], { + projectKey: "my-project", + query: "checkout", + }, "list-feature-flags", state); + const sentinel = renderMockResponse(TEMPLATES["list-feature-flags"], { + projectKey: "exp-setup-missing-flag", + query: "missing-checkout", + }, "list-feature-flags", state); + assert.deepEqual(checkout, TEMPLATES["list-feature-flags"]); + assert.deepEqual(sentinel, TEMPLATES["list-feature-flags"]); + assert.equal(checkout.flags, undefined); + assert.ok(Array.isArray(checkout.items)); +}); + +test("list-flags without that query keeps the shared flag list", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "web-app", + query: "search-v2", + }, "list-flags", state); + assert.deepEqual( + result.flags.map((flag) => flag.key), + ["enable-notifications", "search-v2"], + ); +}); + +test("search-contexts suggests user only when no other kind is stored", () => { + const state = createMockState(); + const users = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + kind: "user", + }, "search-contexts", state); + const others = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + filter: 'kind notEquals "user"', + }, "search-contexts", state); + assert.equal(users.totalCount, 1); + assert.equal(users.items[0].kind, "user"); + assert.equal(others.totalCount, 0); +}); + +test("search-contexts with no kind filter lists every stored kind", () => { + const state = createMockState(); + const onlyUser = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + }, "search-contexts", state); + assert.deepEqual(onlyUser.items.map((item) => item.kind), ["user"]); + assert.equal(onlyUser.totalCount, 1); + + const mixed = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "mixed-contexts", + environmentKey: "production", + }, "search-contexts", state); + assert.deepEqual( + mixed.items.map((item) => item.kind).sort(), + ["organization", "user"], + ); + assert.equal(mixed.totalCount, 2); +}); + +test("search-contexts on mixed-contexts returns another kind", () => { + const state = createMockState(); + const others = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "mixed-contexts", + environmentKey: "production", + filter: 'kind notEquals "user"', + }, "search-contexts", state); + assert.ok(others.totalCount > 0); + assert.equal(others.items[0].kind, "organization"); +}); + +test("list-metric-events default returns the experiment-setup template unchanged", () => { + const state = createMockState(); + const template = SUITE_TEMPLATES["list-metric-events"]; + const result = renderMockResponse(template, { + projectKey: "my-project", + }, "list-metric-events", state); + assert.deepEqual(result, template); + assert.deepEqual( + result.events.map((event) => event.eventKey), + ["checkout-completed", "page-load-ms", "cta-click", "cta-clicked-v2"], + ); +}); + +test("list-metric-events returns the experiment-orchestration template unchanged", () => { + const state = createMockState(); + const template = ORCH_TEMPLATES["list-metric-events"]; + const result = renderMockResponse(template, { + projectKey: "my-project", + environmentKey: "production", + query: "checkout-completed", + }, "list-metric-events", state); + assert.deepEqual(result, template); + assert.equal(result.events[0].eventKey, "checkout-completed"); + assert.equal(result.events[0].lastSeen, "2026-10-28T12:00:00Z"); + assert.equal(result.totalCount, 1); + assert.equal(result.count, undefined); +}); + +test("list-metric-events query cta-clicked returns similar keys only", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["list-metric-events"], { + projectKey: "my-project", + environmentKey: "production", + query: "cta-clicked", + }, "list-metric-events", state); + assert.deepEqual(result, { + events: [ + { eventKey: "cta-click", eventKind: "custom", lastSeen: "1792281600000" }, + { eventKey: "cta-clicked-v2", eventKind: "custom", lastSeen: "1792281600000" }, + ], + count: 2, + _note: "2 event key(s) found with recent activity.", + }); +}); + +test("get-experiment returns a running iteration for checkout-running", () => { + const state = createMockState(); + const running = renderMockResponse(TEMPLATES["get-experiment"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-running", + }, "get-experiment", state); + const draft = renderMockResponse(TEMPLATES["get-experiment"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-draft", + }, "get-experiment", state); + assert.equal(running.currentIteration.status, "running"); + assert.equal(draft.currentIteration.status, "not_started"); +}); + // --------------------------------------------------------------------------- // Release Autopilot stateful reads (templates from the suite's own mocks)