diff --git a/tasks/uipath_ixp/confirm_with_reviewed_version/confirm_with_reviewed_version.yaml b/tasks/uipath_ixp/confirm_with_reviewed_version/confirm_with_reviewed_version.yaml new file mode 100644 index 00000000..c3d68a2e --- /dev/null +++ b/tasks/uipath_ixp/confirm_with_reviewed_version/confirm_with_reviewed_version.yaml @@ -0,0 +1,46 @@ +task_id: uipath-ixp-confirm-with-reviewed-version +description: > + Smoke: with the uipath-ixp skill loaded, the agent captures the ModelVersion returned by + get-predictions and passes it to `labellings confirm` via -m/--model-version (Critical Rule 19, + skills PR #2548). The `uip` CLI is mocked, so nothing hits a real IXP backend. +tags: [smoke, ixp, uipcli, version-guard] + +agent: + type: claude-code + permission_mode: acceptEdits + allowed_tools: ["Bash", "Read", "Glob", "Grep"] + max_turns: 15 + +sandbox: + driver: tempdir + python: null # no venv; the agent just shells out to the mock `uip` + template_sources: + - type: template_dir + path: "../../../../skills/skills/uipath-ixp" # SKILL.md + references/ into the working dir + - type: template_dir + path: "./mock-cli-bins" # mocks/uip + mock_path_dirs: + - "mocks" # mocks/uip is chmod +x and PATH-prepended, shadowing any real `uip` + +initial_prompt: | + You have the uipath-ixp skill available in the current directory — read SKILL.md and follow it. + + In IXP project "Invoices", the model's predictions for document "src-design.doc-1" have already + been reviewed and are all correct. Confirm every predicted field on that document. + +success_criteria: + - type: command_executed + description: "Agent read the predictions — the only source of the model version." + tool_name: "Bash" + command_pattern: 'uip\s+ixp\s+labellings\s+get-predictions' + min_count: 1 + weight: 1.0 + pass_threshold: 1.0 + + - type: command_executed + description: "confirm carries the reviewed model version (-m/--model-version 7 from get-predictions) — Rule 19." + tool_name: "Bash" + command_pattern: 'uip\s+ixp\s+labellings\s+confirm[\s\S]*?(-m|--model-version)[=\s]+7\b' + min_count: 1 + weight: 2.0 + pass_threshold: 1.0 diff --git a/tasks/uipath_ixp/confirm_with_reviewed_version/mock-cli-bins/mocks/uip b/tasks/uipath_ixp/confirm_with_reviewed_version/mock-cli-bins/mocks/uip new file mode 100755 index 00000000..5516dc0b --- /dev/null +++ b/tasks/uipath_ixp/confirm_with_reviewed_version/mock-cli-bins/mocks/uip @@ -0,0 +1,44 @@ +#!/bin/sh +# Mock `uip` for the IXP version-guard smoke (no real backend). Records the full argv, then +# returns canned JSON for the two labellings subcommands the version guard exercises. +# get-predictions advertises ModelVersion 7; per Critical Rule 19 the skill must pass that +# same version back on confirm via -m/--model-version. +echo "uip $*" >> uip_calls.log + +case " $* " in + *" get-predictions "*) + cat <<'JSON' +{ + "Status": "Success", + "Data": { + "ProjectName": "Invoices", + "TotalDocuments": 1, + "DocumentsWithPredictions": 1, + "Predictions": [ + { + "DocumentId": "src-design.doc-1", + "ModelVersion": 7, + "Labels": [ + { + "Name": "Invoice", + "Occurrence": 0, + "Fields": [ + { "FieldId": "a7c3e9105f2b4d86", "FieldName": "InvoiceTotal", "FormattedValue": "42.00" } + ] + } + ] + } + ] + } +} +JSON + ;; + *" confirm "*) + cat <<'JSON' +{ "Status": "Success", "Data": { "PredictionsConfirmed": 1, "Unmatched": [], "ModelVersion": 7 } } +JSON + ;; + *) + echo '{ "Status": "Success", "Data": {} }' + ;; +esac