From 75b72ff183b4ad2aebd71633f25b33c1d666d4b7 Mon Sep 17 00:00:00 2001 From: Katya Yegorova Date: Thu, 13 Aug 2026 14:52:25 -0400 Subject: [PATCH] feat(bob): merge evobob-test skills, docs, and lib improvements - Add 8 new skills: evolve-lite-learn, evolve-lite-recall, evolve-lite-dedup, evolve-lite-create-tests, evolve-lite-run-tests, evolve-lite-test, evolve-lite-test-new-skills, evolve-manager - Add 7 new commands mirroring the new skills - Add docs/: PIPELINE.md, TESTING.md, atomic_skill_evaluation_plan.md, bob-management-mode-plan.md - Add trajectory_extractor.py to shared lib (reads Bob task logs directly) - Update entity_io.py: product registry, banality checks, clone gitignore, find_recall_entity_dirs, full section parser (success_rubric, changelog) - Add dedup scripts: quality_gate.py, refine.py, dedup.py - Add learn script: save_entities.py with product detection and skill-flow decomposition - Update save_trajectory.py: session-ID filename stamping, better path resolution - Update custom_modes.yaml: new evolve-lite mode (4-step workflow) and evolve-manager mode - Update SKILL.md files: subscribe, publish, sync, provenance, save-trajectory - Rewrite README.md to reflect all 15 skills and new layout --- .../bob/evolve-full/custom_modes.yaml | 348 ++--- .../bob/evolve-lite/README.md | 265 ++-- .../commands/evolve-lite-create-tests.md | 22 + .../evolve-lite/commands/evolve-lite-dedup.md | 5 + .../evolve-lite/commands/evolve-lite-learn.md | 4 + .../commands/evolve-lite-recall.md | 4 + .../commands/evolve-lite-run-tests.md | 27 + .../commands/evolve-lite-test-new-skills.md | 32 + .../evolve-lite/commands/evolve-manager.md | 99 ++ .../bob/evolve-lite/docs/PIPELINE.md | 538 ++++++++ .../bob/evolve-lite/docs/TESTING.md | 130 ++ .../evolve-lite/lib/evolve-lite/entity_io.py | 484 +++++-- .../lib/evolve-lite/trajectory_extractor.py | 126 ++ .../skills/evolve-lite-create-tests/SKILL.md | 29 + .../skills/evolve-lite-dedup/SKILL.md | 314 +++++ .../skills/evolve-lite-dedup/scripts/dedup.py | 151 ++ .../evolve-lite-dedup/scripts/quality_gate.py | 884 ++++++++++++ .../evolve-lite-dedup/scripts/refine.py | 605 ++++++++ .../skills/evolve-lite-learn/SKILL.md | 561 ++++++++ .../evolve-lite-learn/scripts/on_stop.py | 13 + .../evolve-lite-learn/scripts/on_stop.sh | 3 + .../scripts/save_entities.py | 431 ++++++ .../skills/evolve-lite-provenance/SKILL.md | 117 +- .../skills/evolve-lite-publish/SKILL.md | 162 ++- .../evolve-lite-publish/scripts/publish.py | 164 ++- .../skills/evolve-lite-recall/SKILL.md | 152 ++ .../scripts/retrieve_entities.py | 141 ++ .../skills/evolve-lite-run-tests/SKILL.md | 32 + .../evolve-lite-save-trajectory/SKILL.md | 2 +- .../scripts/on_stop.py | 77 +- .../skills/evolve-lite-subscribe/SKILL.md | 4 +- .../scripts/subscribe.py | 5 + .../skills/evolve-lite-sync/SKILL.md | 4 +- .../evolve-lite-test-new-skills/SKILL.md | 38 + .../skills/evolve-lite-test/EXECUTION_PLAN.md | 345 +++++ .../evolve-lite-test/EXTRACTION_MISMATCH.md | 194 +++ .../evolve-lite-test/FUNCTIONAL_TESTING.md | 377 +++++ .../FUNCTIONAL_TEST_ANALYSIS.md | 174 +++ .../evolve-lite-test/HOW_EVALUATION_WORKS.md | 401 ++++++ .../evolve-lite-test/INTEGRATION_TESTING.md | 320 +++++ .../skills/evolve-lite-test/README.md | 115 ++ .../skills/evolve-lite-test/SKILL.md | 399 ++++++ .../skills/evolve-lite-test/WHY_TESTS_FAIL.md | 175 +++ .../evolve-lite-test/scripts/check_tests.py | 234 ++++ .../scripts/compare_with_without_skills.py | 380 +++++ .../scripts/generate_pseudo_conversations.py | 434 ++++++ .../scripts/generate_skill_tests.py | 144 ++ .../scripts/generate_test_cases.py | 410 ++++++ .../scripts/run_baseline_tests.py | 311 +++++ .../scripts/run_integration_test.py | 393 ++++++ .../scripts/run_integration_tests_batch.py | 269 ++++ .../scripts/run_recall_tests.py | 304 ++++ .../scripts/run_skill_evaluation.py | 460 +++++++ .../scripts/run_skill_functional_tests.py | 499 +++++++ .../scripts/run_skill_unit_tests.py | 376 +++++ .../scripts/run_test_cases.py | 406 ++++++ .../scripts/run_tests_with_comparison.py | 285 ++++ .../scripts/show_execution_plan.py | 218 +++ .../scripts/snapshot_test_results.py | 211 +++ .../scripts/trigger_parser.py | 298 ++++ .../skills/evolve-manager/SKILL.md | 192 +++ .../evolve-manager/scripts/merge_forks.py | 1219 +++++++++++++++++ 62 files changed, 14788 insertions(+), 728 deletions(-) create mode 100644 platform-integrations/bob/evolve-lite/commands/evolve-lite-create-tests.md create mode 100644 platform-integrations/bob/evolve-lite/commands/evolve-lite-dedup.md create mode 100644 platform-integrations/bob/evolve-lite/commands/evolve-lite-learn.md create mode 100644 platform-integrations/bob/evolve-lite/commands/evolve-lite-recall.md create mode 100644 platform-integrations/bob/evolve-lite/commands/evolve-lite-run-tests.md create mode 100644 platform-integrations/bob/evolve-lite/commands/evolve-lite-test-new-skills.md create mode 100644 platform-integrations/bob/evolve-lite/commands/evolve-manager.md create mode 100644 platform-integrations/bob/evolve-lite/docs/PIPELINE.md create mode 100644 platform-integrations/bob/evolve-lite/docs/TESTING.md create mode 100644 platform-integrations/bob/evolve-lite/lib/evolve-lite/trajectory_extractor.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-create-tests/SKILL.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/SKILL.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/dedup.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/quality_gate.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/refine.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/SKILL.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/on_stop.py create mode 100755 platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/on_stop.sh create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/save_entities.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/SKILL.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/scripts/retrieve_entities.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-run-tests/SKILL.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test-new-skills/SKILL.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/EXECUTION_PLAN.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/EXTRACTION_MISMATCH.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/FUNCTIONAL_TESTING.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/FUNCTIONAL_TEST_ANALYSIS.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/HOW_EVALUATION_WORKS.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/INTEGRATION_TESTING.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/README.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/SKILL.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/WHY_TESTS_FAIL.md create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/check_tests.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/compare_with_without_skills.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_pseudo_conversations.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_skill_tests.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_test_cases.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_baseline_tests.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_integration_test.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_integration_tests_batch.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_recall_tests.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_evaluation.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_functional_tests.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_unit_tests.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_test_cases.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_tests_with_comparison.py create mode 100755 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/show_execution_plan.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/snapshot_test_results.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/trigger_parser.py create mode 100644 platform-integrations/bob/evolve-lite/skills/evolve-manager/SKILL.md create mode 100755 platform-integrations/bob/evolve-lite/skills/evolve-manager/scripts/merge_forks.py diff --git a/platform-integrations/bob/evolve-full/custom_modes.yaml b/platform-integrations/bob/evolve-full/custom_modes.yaml index 0e896b3c..407a8ca7 100644 --- a/platform-integrations/bob/evolve-full/custom_modes.yaml +++ b/platform-integrations/bob/evolve-full/custom_modes.yaml @@ -1,210 +1,150 @@ customModes: - - slug: Evolve - name: Evolve - roleDefinition: >- - You are in Evolve - a learning mode that improves from every interaction. - You can handle ANY task: coding, analysis, questions, research, or general - assistance. - - ⚠️ CRITICAL WORKFLOW - FOLLOW STRICTLY: - - 1. START: ALWAYS call get_guidelines(task) FIRST 2. WORK: Complete the - task incorporating guidelines 3. END: ALWAYS call save_trajectory() LAST - - DO NOT FORGET THIS STEP! - whenToUse: >- - Use Evolve mode for ANY task where you want the agent to: - - - Learn from past experiences (retrieves guidelines at start) - - - Improve over time (saves learnings at end) - - - Build institutional knowledge - - - This includes: coding, analysis, documentation, questions, research, data - lookup, explanations, and general assistance. - - - ⚠️⚠️⚠️ MANDATORY WORKFLOW - NEVER SKIP ⚠️⚠️⚠️ - - - STEP 1 - START (REQUIRED): - - → ALWAYS call get_guidelines(task) to retrieve relevant best practices - - → This is MANDATORY for ALL tasks (coding, analysis, documentation, - questions, research, etc.) - - → Do this BEFORE any other work - - - STEP 2 - WORK: - - → Use the retrieved guidelines to influence your thinking about how to - solve the task - - → Let guidelines shape your approach before taking action - - → If guidelines suggest clarification is needed, ask questions first - - → If guidelines recommend specific methods, consider them in your solution - - → Complete the task using available tools (file operations, commands, MCP - tools, or direct answers) - - → For coding tasks: Focus on code quality and best practices - - → For questions: Provide accurate, helpful answers based on your knowledge - - - STEP 3 - END (REQUIRED - DO NOT FORGET): - - → ALWAYS call save_trajectory() after task completion - - → This saves the conversation and generates new guidelines - - → Format: [{"role": "user/assistant", "content": "..."}] - - → Wait for user confirmation of success - - → ⚠️ ONLY AFTER save_trajectory() succeeds can you call - attempt_completion() - - - CRITICAL REMINDERS: - - 🔴 BEFORE starting ANY task: Call get_guidelines(task) - - 🔴 AFTER completing ANY task: Call save_trajectory(trajectory_data, - task_id) - - 🔴 These steps are NON-NEGOTIABLE. Skipping save_trajectory() means the - agent learns nothing from this interaction. - - - PRE-COMPLETION CHECKLIST: - - Before calling attempt_completion, verify ALL of these: - - □ Called get_guidelines() at task start - - □ Completed the actual work - - □ Called save_trajectory() with full conversation - - □ Received user confirmation of save_trajectory() success - - - If ANY checkbox is unchecked, DO NOT call attempt_completion. - - - Think of it this way: - - - save_trajectory() = "Internal completion" (learning) - - - attempt_completion() = "External completion" (presenting results) - - - You CANNOT skip internal completion to get to external completion. - - - CAPABILITIES: - - - Full file system access (read, write, edit files) - - - Command execution - - - MCP tool access (Evolve server) - - - Code analysis and refactoring - - - Answer questions using built-in knowledge - - - Research and information lookup - - - General assistance and explanations - - - EVOLVE TOOLS: - - - get_guidelines(task: str) - Get relevant guidelines before starting work - - - save_trajectory(trajectory_data: str, task_id: str) - Save conversation - after completion - - - create_entity(content: str, entity_type: str, metadata: str) - Create - knowledge entries - - - delete_entity(entity_id: str) - Remove outdated knowledge +# >>>evolve:evolve-lite<<< + - slug: evolve-lite + name: Evolve Lite + roleDefinition: |- + You are Evolve Lite - a learning mode that improves from every interaction. + + ⚠️ CRITICAL: THE FOLLOWING WORKFLOW IS ABSOLUTELY MANDATORY AND CANNOT BE SKIPPED ⚠️ + + WORKFLOW (4 steps, strict order): + + 1. ⚠️ FIRST ACTION (MANDATORY): Use the evolve-lite:recall skill to retrieve relevant guidelines. + - This MUST be your FIRST action in EVERY conversation + - NO exceptions - even for simple tasks like "echo hello world" + 2. WORK: Complete the user's request. When a retrieved guideline applies, cite it in your reasoning. + 3. ⚠️ SECOND-TO-LAST ACTION (MANDATORY): Use the evolve-lite:save-trajectory skill to save the conversation. + - This MUST happen BEFORE evolve-lite:learn + - Note the trajectory file path from its output - you will need it in Step 4 + 4. ⚠️ LAST ACTION (MANDATORY): Use the evolve-lite:learn skill to extract and save learnings. + - This MUST happen BEFORE attempt_completion + - Exception: if the command file contains `skip_learn: true` in its frontmatter, skip this step entirely + + Before using any skill for the first time in a conversation, read its + SKILL.md so you know the exact invocation syntax. Bob installs skills + under `.bob/skills/`; the runtime will resolve a skill by name. + + ⚠️ ENFORCEMENT RULES ⚠️ + - Do not skip steps. Do not reorder steps. + - Step 1 must happen before any other tool use (except a one-time + SKILL.md read for a skill you have not invoked yet). + - Step 3 must happen before Step 4. + - Step 4 must happen before attempt_completion. + - If you forget Step 1, STOP and call evolve-lite:recall immediately. + - If you reach attempt_completion without Steps 3 and 4, STOP and complete them first. + customInstructions: |- + - ALWAYS read a skill's SKILL.md before using it for the first time in a conversation + - If evolve-lite:recall returns no guidelines, that is normal - proceed with the task. + - If evolve-lite:learn reports no new entities, that counts as successful completion - do not retry. + - The trajectory path from evolve-lite:save-trajectory is available in conversation context - do not re-run it. + - If no errors or non-obvious discoveries occurred, saving zero entities is correct - do not force low-quality entities. + + MEMORY SHARING (Optional): + Additional evolve-lite skills handle sharing guidelines with others + (publish, subscribe, unsubscribe, sync). They are OPTIONAL and do not + affect the core workflow. Use them when you want to share your + learnings with teammates, learn from others' guidelines, or keep + subscribed guidelines up to date. Read the SKILL.md for any of these + before first use. + + PRE-COMPLETION GATE: + Before calling attempt_completion, ask yourself: + ┌─────────────────────────────────────────────────────────────────┐ + │ Did I run the evolve-lite:recall skill at the start? │ + │ Did I run the evolve-lite:save-trajectory skill and note the path?│ + │ Did I run the evolve-lite:learn skill and see its output? │ + │ │ + │ ❌ NO → STOP. Go back and complete the step. │ + │ ✅ YES → Proceed with attempt_completion. │ + └─────────────────────────────────────────────────────────────────┘ + + Rules: + + - Before using a skill for the first time, read its SKILL.md file to understand the correct usage syntax. + - ALWAYS call the evolve-lite:learn skill before attempt_completion, even if the task seems simple or you think there's nothing to learn - UNLESS the command file has `skip_learn: true` in its frontmatter, in which case skip it. + - The workflow is non-negotiable: recall → work → evolve-lite:save-trajectory → learn → complete. + - Skipping evolve-lite:learn violates the core purpose of this mode. + groups: + - read + - edit + - command + - browser +# <<>>evolve:evolve-manager<<< + - slug: evolve-manager + name: Evolve Manager + whenToUse: Use when you want to discover GitHub forks of the main evolve repo and merge + deduplicate their entity libraries into the local main repo, with regression protection on main-repo skills. + roleDefinition: |- + You are Evolve Manager - an orchestration mode that merges evolve entity libraries from GitHub forks into the main repo safely. + + ⚠️ ALWAYS follow the command file at `.bob/commands/evolve-manager.md` — it defines the exact workflow. Read it first if you have not already. ⚠️ + + YOUR WORKFLOW (strict order, no skipping): + + 1. READ SKILL.md AND COMMAND FILE FIRST (once per conversation): + Read `.bob/skills/evolve-manager/SKILL.md` and `.bob/commands/evolve-manager.md`. + + 2. MAIN REPO IDENTITY (hardcoded — do not auto-detect): + Main repo: ce-artemis-2026/evobob-test (github.ibm.com) + Always pass: --main-repo ce-artemis-2026/evobob-test + Token: GITHUB_TOKEN env var + + 3. DISCOVER AND STAGE FORKS: + List forks via GitHub API: GET /repos/ce-artemis-2026/evobob-test/forks + For each fork NOT already staged under .evolve/tmp/fork-staging//: + git clone --depth=1 --filter=blob:none --sparse .evolve/tmp/fork-staging/ + cd .evolve/tmp/fork-staging/ && git sparse-checkout set .evolve/entities + For forks ALREADY staged — refresh before merging: + cd .evolve/tmp/fork-staging/ && git fetch origin && git reset --hard origin/HEAD + + 4. DRY RUN FIRST (mandatory): + python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/ ... \ + --main-repo ce-artemis-2026/evobob-test \ + --dry-run + Show output to user. Wait for confirmation before proceeding. + + 5. LIVE MERGE (after user confirms): + python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/ ... \ + --main-repo ce-artemis-2026/evobob-test + + 6. HANDLE EXIT CODES: + - Exit 0: merge succeeded. Tell the user the entities are now in .evolve/entities/. + - Exit 1: hard failure. Show the error output and STOP. Do not retry automatically. + - Exit 2: threshold breach - main-repo rubric pass rate dropped below threshold. + Show the diff summary. Ask: "Main-repo test pass rate dropped. Keep the merge or roll back?" + - If keep: re-run with --force-commit + - If rollback: rm -rf .evolve/entities/ && cp -r .evolve/tmp/pre-merge-backup/ .evolve/entities/ RULES: - - - MANDATORY: Always call get_guidelines at the start of EVERY task - (coding, analysis, documentation, questions, etc.) - - - Let the retrieved guidelines influence your thinking about how to solve - the task - - - Consider what the guidelines recommend before proceeding with your - solution - - - MANDATORY: Always call save_trajectory when task is complete, BEFORE - attempt_completion - - - For coding tasks: Focus on code quality and best practices - - - For questions: Provide accurate, comprehensive answers - - - Learn from each interaction to improve future performance - - - You MUST follow the three-step workflow: get_guidelines() → work → - save_trajectory() → attempt_completion(). Skipping save_trajectory() is - strictly forbidden and defeats the purpose of using Evolve mode. - - - IMPORTANT NOTES: - - - save_trajectory() requires OpenAI JSON format: [{"role": - "user/assistant", "content": "..."}] - - - When tool calls occur, include them as function_call/function_response - objects in the content field - - - Agent cannot access conversation history directly - - - save_trajectory() cannot be called automatically - description: "⚠️ Learning mode: ALWAYS get_guidelines() at START and - save_trajectory() at END" - customInstructions: >- - ⚠️ CRITICAL EVOLVE MODE REQUIREMENT ⚠️ - - Before calling attempt_completion, you MUST: - - 1. Complete the work - - 2. Call save_trajectory() with the full conversation - - 3. Wait for user confirmation of success - - 4. ONLY THEN call attempt_completion() - - - Calling attempt_completion() without first successfully calling - save_trajectory() will: - - - Waste the entire interaction (no learning occurs) - - - Defeat the purpose of using Evolve mode - - - Result in task rejection and restart - - - The user will reject any attempt_completion that wasn't preceded by a - successful save_trajectory() call. + - Never manually edit entity files. The script owns all writes to .evolve/entities/. + - Never skip fork discovery (step 3) — the script requires pre-cloned directories. + - Never skip the dry run (step 4) — always show the user merge decisions before writing. + - Never re-run the script after exit 1 without the user fixing the reported error first. + - Always show the full script output to the user before acting on the exit code. + customInstructions: |- + - The main repo is always ce-artemis-2026/evobob-test on github.ibm.com. Never auto-detect from git remote origin. + - Always pass --main-repo ce-artemis-2026/evobob-test to the merge script. + - Always run a dry run and show the user before executing a live merge. + - Refresh already-staged forks with git fetch + reset before merging to avoid stale data. + - The --threshold flag controls the regression gate for main-repo tests only. + Fork-sourced entities are never counted against the threshold. + - The --version-diff-threshold flag controls when both the fork and main-repo versions + of a skill are preserved as dual sections (Current Version / Previous Version). + Lower values preserve more history; higher values replace more aggressively. + - Rollback path: .evolve/tmp/pre-merge-backup/ contains the pre-merge snapshot. + To roll back: rm -rf .evolve/entities/ && cp -r .evolve/tmp/pre-merge-backup/ .evolve/entities/ + - Reports are written to: + .evolve/tests/dedup/quality_gate_report.json + .evolve/tests/dedup/refine_report.json + .evolve/tests/evaluation/report.json (pre-dedup baseline) + .evolve/tests/evaluation/report_post.json (post-dedup) groups: - - mcp - - command - - edit - read - source: project + - execute + - mcp + - skill + - subagent +# << Remote URL: git@github.com:alice/evolve-guidelines.git -> Short name: alice -> Scope: read -``` - -The repo is cloned directly into `.evolve/entities/subscribed/{name}/` -(this directory serves as both the git clone and the recall mirror). - -### Publishing Guidelines - -Use `evolve-lite:publish` to share local guidelines via a **write-scope** repo: - -1. The skill picks (or asks about) the write-scope target repo -2. Lists files in `.evolve/entities/guideline/` -3. You pick which ones to publish -4. Each selected file is moved into the write-scope clone at - `.evolve/entities/subscribed/{repo}/guideline/`, stamped with your - username, committed, and pushed to the remote - -Because the publish target is also a subscribed repo, your next sync -pulls in anything other writers have pushed to the same remote. - -### Syncing Repos - -Use `evolve-lite:sync` to pull the latest changes from every configured -repo: - -```text -evolve-lite:sync -> Synced 2 repo(s): memory [write] (+0 added, 1 updated, 0 removed), alice [read] (+2 added, 0 updated, 0 removed) -``` - -Read-scope repos use `git fetch` + `git reset --hard`. Write-scope repos -use `git fetch` + `git rebase` so any unpushed local publish commits are -preserved. - -### Unsubscribing - -Use `evolve-lite:unsubscribe` to remove a configured repo and delete -its locally cloned files: - -```text -evolve-lite:unsubscribe -> Which repo would you like to remove? -> 1. memory [write] -> 2. alice [read] -``` - -The skill confirms before deleting `.evolve/entities/subscribed/{name}/`. -Removing a write-scope repo will also discard any unpushed local -publish commits, so the skill warns first. - -### Sharing Storage Layout - -```text -.evolve/ - entities/ - guideline/ # your private guidelines - my-guideline.md - subscribed/ - memory/ # write-scope clone (publishes land here) - guideline/ - my-published-guideline.md - alice/ # read-scope clone (also serves as recall mirror) - guideline/ - her-guideline.md ``` -## Skills Included - -### `evolve-lite:learn` - -Manually invoke to extract guidelines from the current conversation: -- Analyzes task, steps taken, successes and failures -- Generates proactive guidelines (what to do, not what to avoid) -- Saves guidelines as markdown files in `.evolve/entities/guideline/` - -### `evolve-lite:recall` - -Manually invoke to retrieve and display stored guidelines: -- Loads guidelines from private and subscribed sources -- Formats and displays them for your review -- Annotates subscribed guidelines with their source - -### `evolve-lite:publish` - -Publish private guidelines to a write-scope repo: -- Lists available private guidelines -- Moves selected guidelines into the write-scope clone at - `.evolve/entities/subscribed/{repo}/guideline/` -- Stamps with `owner`, `published_at`, and `source` metadata -- Commits and pushes to the configured remote - -### `evolve-lite:subscribe` - -Add a configured repo to the unified `repos:` list: -- Clones the remote into `.evolve/entities/subscribed/{name}/` -- Adds an entry with `scope: read` or `scope: write` to config - -### `evolve-lite:sync` - -Sync every configured repo: -- Read-scope: fetch + reset --hard (clobbers any local edits) -- Write-scope: fetch + rebase (preserves unpushed local publishes) -- Reports changes (added, updated, removed) - -### `evolve-lite:unsubscribe` +Then use `/evolve-lite-subscribe`, `/evolve-lite-publish`, and `/evolve-lite-sync` to manage sharing. -Remove a configured repo: -- Lists current repos with their scope and notes -- Deletes the local clone at `.evolve/entities/subscribed/{name}/` -- Removes the entry from config +The `.evolve/entities/subscribed/` directory is excluded from version control — the skills automatically gitignore it. ## Environment Variables -- `EVOLVE_DIR`: Override the default `.evolve` directory location (guidelines, config, etc. are stored here) +| Variable | Default | Description | +|---|---|---| +| `EVOLVE_DIR` | `.evolve` | Override the evolve data directory location | +| `EVOLVE_DEBUG` | unset | Set to `1` to enable debug logging to `/tmp/evolve-/evolve-plugin.log` | -## Verification +## Documentation -After installation, the skills should be available in Bob's skill list. +- **[PIPELINE.md](docs/PIPELINE.md)** — complete reference for all skills, modes, and how they chain together +- **[TESTING.md](docs/TESTING.md)** — how the entity library is validated (content, recall, and baseline tests) +- **[atomic_skill_evaluation_plan.md](docs/atomic_skill_evaluation_plan.md)** — design notes for the atomic skill evaluation framework +- **[bob-management-mode-plan.md](docs/bob-management-mode-plan.md)** — design notes for Evolve Manager mode diff --git a/platform-integrations/bob/evolve-lite/commands/evolve-lite-create-tests.md b/platform-integrations/bob/evolve-lite/commands/evolve-lite-create-tests.md new file mode 100644 index 00000000..56fef42f --- /dev/null +++ b/platform-integrations/bob/evolve-lite/commands/evolve-lite-create-tests.md @@ -0,0 +1,22 @@ +--- +description: Create pseudo-conversation test fixtures for all atomic skills, or for specific new skill files. Run this after evolve-lite-learn to generate tests for newly saved skills. +--- + +To create test fixtures for ALL skills: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +``` + +To create fixtures for specific new skill files (e.g. just saved by `evolve-lite-learn`): + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py [ ...] +``` + +Each fixture is written to `.evolve/tests/pseudo_conversations/.json` and contains: +- A skill-specific user question derived from the skill's trigger +- The `must_include` command terms extracted from the skill content +- The skill content ready to be injected as system context + +After creating fixtures, run `/evolve-lite-run-tests` to evaluate them. diff --git a/platform-integrations/bob/evolve-lite/commands/evolve-lite-dedup.md b/platform-integrations/bob/evolve-lite/commands/evolve-lite-dedup.md new file mode 100644 index 00000000..a4b7a723 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/commands/evolve-lite-dedup.md @@ -0,0 +1,5 @@ +--- +description: Two-phase skill library deduplication. +skip_learn: true +--- +Use the `evolve-lite-dedup` skill on the current conversation. Follow the skill's instructions exactly. diff --git a/platform-integrations/bob/evolve-lite/commands/evolve-lite-learn.md b/platform-integrations/bob/evolve-lite/commands/evolve-lite-learn.md new file mode 100644 index 00000000..db1aa42a --- /dev/null +++ b/platform-integrations/bob/evolve-lite/commands/evolve-lite-learn.md @@ -0,0 +1,4 @@ +--- +description: Must be used near the end of any non-trivial turn that produced potentially reusable tools, guidance, errors, workarounds, or workflows, so those lessons are saved for future turns. +--- +Use the `evolve-lite-learn` skill on the current conversation. Follow the skill's instructions exactly. diff --git a/platform-integrations/bob/evolve-lite/commands/evolve-lite-recall.md b/platform-integrations/bob/evolve-lite/commands/evolve-lite-recall.md new file mode 100644 index 00000000..80b750d0 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/commands/evolve-lite-recall.md @@ -0,0 +1,4 @@ +--- +description: Must be used at the start of any non-trivial task involving code changes, debugging, repo exploration, file inspection, or environment/tooling investigation to surface stored guidance before analysis or tool use. +--- +Use the `evolve-lite-recall` skill on the current conversation. Follow the skill's instructions exactly. diff --git a/platform-integrations/bob/evolve-lite/commands/evolve-lite-run-tests.md b/platform-integrations/bob/evolve-lite/commands/evolve-lite-run-tests.md new file mode 100644 index 00000000..38d7d11a --- /dev/null +++ b/platform-integrations/bob/evolve-lite/commands/evolve-lite-run-tests.md @@ -0,0 +1,27 @@ +--- +description: Run all skill tests — content, recall, and baseline — against the existing pseudo-conversation fixtures. +skip_learn: true +--- + +Run all three tests against existing fixtures: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_skill_evaluation.py --verbose +python3 .bob/skills/evolve-lite-test/scripts/run_recall_tests.py --verbose +python3 .bob/skills/evolve-lite-test/scripts/run_baseline_tests.py --simulate +``` + +**What each test checks:** + +| Test | Script | Checks | +|---|---|---| +| Content | `run_skill_evaluation.py` | Skill contains the commands it prescribes | +| Recall | `run_recall_tests.py` | Skill surfaces in top 3 when its scenario is described | +| Baseline | `run_baseline_tests.py` | Agent without the skill misses the key guidance | + +**Reports written to `.evolve/tests/evaluation/`:** +- `report.json` — content test +- `recall_report.json` — recall test +- `baseline_report.json` — baseline test + +To regenerate fixtures before running, use `/evolve-lite-create-tests` first. diff --git a/platform-integrations/bob/evolve-lite/commands/evolve-lite-test-new-skills.md b/platform-integrations/bob/evolve-lite/commands/evolve-lite-test-new-skills.md new file mode 100644 index 00000000..7fad707c --- /dev/null +++ b/platform-integrations/bob/evolve-lite/commands/evolve-lite-test-new-skills.md @@ -0,0 +1,32 @@ +--- +description: Generate pseudo-conversation test fixtures for newly saved atomic skills. Run this immediately after evolve-lite-learn has finished saving entities. +--- + +After `evolve-lite-learn` has saved new skill entities, generate a test fixture +for each new atomic-skill file by running: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py [ ...] +``` + +Pass the exact file paths that `save_entities.py` just wrote. The script will: +1. Read each file and confirm it is an `atomic-skill` (skips guidelines and skill-flows) +2. Derive a realistic trigger-based user question from the skill's `trigger` field +3. Extract `must_include` terms (backtick command strings) and `must_not_include` terms (negation patterns) from the skill content +4. Write a fixture JSON to `.evolve/tests/pseudo_conversations/.json` + +If you don't have the exact paths, regenerate fixtures for all skills at once: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +``` + +To validate the new fixtures immediately after generating them: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_skill_evaluation.py --verbose +``` + +A passing result (score ≥ 0.5, no constraint violations) means the skill is +self-consistent — its content contains the commands it claims to prescribe. +A failure signals that the skill content may be too vague or missing a key command. diff --git a/platform-integrations/bob/evolve-lite/commands/evolve-manager.md b/platform-integrations/bob/evolve-lite/commands/evolve-manager.md new file mode 100644 index 00000000..b76de6cf --- /dev/null +++ b/platform-integrations/bob/evolve-lite/commands/evolve-manager.md @@ -0,0 +1,99 @@ +--- +description: Merge GitHub fork entity libraries into the main evolve repo with regression protection. +skip_learn: true +--- + +# Evolve Manager — Merge Forks Workflow + +Follow these steps **in strict order**. Do not skip or reorder steps. + +--- + +## STEP 1 — Read the skill + +Read `.bob/skills/evolve-manager/SKILL.md` before any tool use (once per conversation). + +--- + +## STEP 2 — Establish main repo identity + +The main repo for this workspace is **`ce-artemis-2026/evobob-test`** (IBM GHE: `github.ibm.com`). +- Use `--main-repo ce-artemis-2026/evobob-test` on every script invocation. +- GitHub token: use the `GITHUB_TOKEN` env var (or `--github-token`). + +--- + +## STEP 3 — Discover and stage forks + +List forks from the GitHub API: +``` +GET /repos/ce-artemis-2026/evobob-test/forks +``` +For each fork that is **not already staged** under `.evolve/tmp/fork-staging//`: +```bash +git clone --depth=1 --filter=blob:none --sparse \ + \ + .evolve/tmp/fork-staging/ +cd .evolve/tmp/fork-staging/ && \ + git sparse-checkout set .evolve/entities +``` + +For forks that **are already staged**, pull latest changes to avoid stale data: +```bash +cd .evolve/tmp/fork-staging/ +git fetch origin && git reset --hard origin/HEAD +``` + +Collect the list of all staged fork directories with at least one entity file under +`.evolve/tmp/fork-staging//.evolve/entities/`. + +--- + +## STEP 4 — Run the merge script (dry run first) + +Always run a dry run first to preview decisions: +```bash +python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/ .evolve/tmp/fork-staging/ ... \ + --main-repo ce-artemis-2026/evobob-test \ + --dry-run +``` +Show the dry-run output to the user before proceeding. + +--- + +## STEP 5 — Confirm with user and run live merge + +After the user reviews the dry run output, run the live merge: +```bash +python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/ ... \ + --main-repo ce-artemis-2026/evobob-test +``` + +--- + +## STEP 6 — Handle exit codes + +| Exit | Meaning | Action | +|------|---------|--------| +| `0` | Success | Tell the user entities are live in `.evolve/entities/`. Show the merge report summary. | +| `1` | Hard failure | Show the full error output. STOP. Do not retry until the user fixes the error. | +| `2` | Threshold breach | Show the diff summary. Ask: "Main-repo test pass rate dropped. Keep the merge or roll back?" | + +**If keep (exit 2):** re-run with `--force-commit`. +**If rollback:** +```bash +rm -rf .evolve/entities/ +cp -r .evolve/tmp/pre-merge-backup/ .evolve/entities/ +``` + +--- + +## RULES + +- Never manually edit entity files. The script owns all writes to `.evolve/entities/`. +- Always run the dry run (Step 4) before the live merge (Step 5). +- Never re-run after exit 1 without the user fixing the reported error. +- Always show full script output before acting on the exit code. +- Fork entities are never tested against the regression threshold — only original main-repo entities are. diff --git a/platform-integrations/bob/evolve-lite/docs/PIPELINE.md b/platform-integrations/bob/evolve-lite/docs/PIPELINE.md new file mode 100644 index 00000000..171a4ef0 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/docs/PIPELINE.md @@ -0,0 +1,538 @@ +# Evolve Bob — Skills & Modes Pipeline + +A reference for the two modes, all skills, and how they connect into pipelines. + +--- + +## All Skills at a Glance + +| Skill | Pipeline | Frequency | +|-------|----------|-----------| +| `evolve-lite-recall` | Core session | Every session — first action | +| `evolve-lite-save-trajectory` | Core session | Every session — before learn | +| `evolve-lite-learn` | Core session | Every session — last action | +| `evolve-lite-subscribe` | Sharing | Once per repo | +| `evolve-lite-sync` | Sharing | Periodically | +| `evolve-lite-dedup` | Sharing / Maintenance | Before publishing; when recall feels noisy | +| `evolve-lite-publish` | Sharing | When ready to push to your fork | +| `evolve-lite-unsubscribe` | Sharing | When removing a subscription | +| `evolve-lite-create-tests` | Testing | Regenerate all fixtures from the current library | +| `evolve-lite-run-tests` | Testing | Run all three test suites on demand | +| `evolve-lite-test` | Testing | Generate fixtures + run all three suites | +| `evolve-lite-test-new-skills` | Testing | After `learn` — validate newly saved entities only | +| `evolve-lite-save` | Maintenance | After establishing a repeatable workflow | +| `evolve-lite-provenance` | Maintenance | To audit guideline influence | +| `evolve-manager` | Maintenance | To merge fork contributions into main | + +--- + +## Modes + +| Mode | Purpose | +|------|---------| +| **Evolve Lite** | Everyday working mode — enforces recall → work → save-trajectory → learn on every session | +| **Evolve Manager** | Maintainer mode — merges entity libraries from forks, runs regression tests, deduplicates | + +--- + +## Core Session Pipeline (Evolve Lite) + +Runs automatically every session. All four steps are mandatory and must run in order. + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ 1. recall Surface stored guidelines & skills │ +│ ↓ │ +│ 2. [do your work] │ +│ ↓ │ +│ 3. save-trajectory Save conversation as JSON │ +│ ↓ │ +│ 4. learn Extract & save new entities (see detail below) │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +| Skill | When it runs | What it does | +|-------|-------------|--------------| +| `evolve-lite-recall` | Start of every session | Searches `.evolve/entities/` by keyword overlap against the task and surfaces the top matching guidelines, atomic skills, and skill flows | +| `evolve-lite-save-trajectory` | After work, before learn | Saves the full conversation as a JSON file (OpenAI chat format) to `.evolve/trajectories/` | +| `evolve-lite-learn` | End of every session | Analyzes the trajectory, extracts reusable entities, generates test fixtures, and deduplicates — see expanded flow below | + +--- + +## Inside `evolve-lite-learn` + +`learn` is itself a multi-step pipeline. It runs every time a session ends. + +``` +Step 1 · load trajectory + ↓ +Step 2 · analyze conversation + - identify task, steps taken, failures, retry loops, reusable outcomes + ↓ +Step 3 · identify errors & root causes + - tool failures, permission errors, wrong initial approaches, silent failures + ↓ +Step 4 · decide whether to save a reusable artifact (script / workflow) + ↓ +Step 5 · extract entities (3–5, prioritize failure-derived first) + - guideline → declarative approach preference, no executable steps + - atomic-skill → smallest self-contained executable procedure for one sub-problem + - skill-flow → ordered sequence of 2+ steps that recurs as a named unit + ↓ +Step 6 · save entities to .evolve/entities/{type}/{product}/ + - auto-detects product (watson-orchestrate, github, docker, kubernetes, general) + - marks failure-derived skills with derived_from_failure: true + - filters out common-sense / trivial skills + - decomposes skill-flows into atomic skills automatically + ↓ +Step 7 · generate test fixtures + - skips guideline entities (no executable outcome to test) + - for each new atomic-skill / skill-flow: + • 1 happy-path fixture — normal conditions, rubric criteria as validation + • 1–3 edge-case fixtures — boundary/failure conditions, rubric criteria annotated + - saved to .evolve/tests/pseudo_conversations/{entity-slug}.json + - append-only: existing test files are never overwritten + ↓ +Step 7a · run test gate ← BLOCKS if pass rate < 80% + - regenerates all pseudo-conversation fixtures (generate_skill_tests.py --all) + - content evaluation (run_skill_evaluation.py) — skill must contain its prescribed commands + - recall test (run_recall_tests.py) — skill must rank in top-3 for its trigger + - gate threshold: 80% pass rate on both suites + - FAIL → fix the entity file, re-run gate; do NOT continue to Step 8 until exit 0 + ↓ +Step 8 · deduplicate against full library + - compare new entities against all existing .evolve/entities/**/*.md + - merge: combine similar entities into one richer file + - discard: delete near-identical duplicates, keep richest + - keep-all: no action when entities are genuinely distinct + - tests are never deleted — if a slug changes, test files are copied to match +``` + +> **Note:** `learn` generates test fixtures (Step 7) and immediately runs the content + recall gate (Step 7a) before proceeding. If the gate fails, the learn workflow pauses for fixes before dedup runs. To run all three test suites (content, recall, baseline) at any time, use `evolve-lite-run-tests`. + +### Test fixture types generated in Step 7 + +Each fixture is a JSON file in `.evolve/tests/pseudo_conversations/` and contains one or more test cases. There are five test types: + +#### 1. Rubric-based execution test *(primary — generated by learn)* + +The main test type for `atomic-skill` and `skill-flow` entities. Generated directly from the entity's `## Success Rubric` section. + +- **Happy-path case**: runs the skill under normal conditions; `validation_criteria` copied verbatim from the rubric — no generic criteria invented +- **Edge-case tests (1–3)**: each covers a realistic boundary or failure condition (missing dependency, malformed input, already-existing output, partial environment). The rubric criteria are copied in and annotated to state which are expected to fail under that condition. + +```json +{ + "test_type": "rubric_execution", + "scenario": "Executing the skill end-to-end under normal conditions", + "edge_case": false, + "validation_criteria": [ + "exit code 0 after running the main command", + "output file exists at the expected path", + "no error lines appear in stdout" + ] +} +``` + +#### 2. Trigger match test + +Validates that the skill's `trigger` field correctly matches the scenarios it is meant to handle. Used to catch triggers that are too vague or use the wrong vocabulary. + +```json +{ + "test_type": "trigger_match", + "scenario": "User wants to import a Watson Orchestrate agent YAML", + "expected_trigger_match": true, + "skill_trigger": "When importing an agent YAML file into Watson Orchestrate..." +} +``` + +#### 3. Content completeness test *(fallback — for entities with no rubric)* + +Used only for legacy entities that predate the `success_rubric` requirement. Checks that the skill content contains all required steps, executable commands, stated prerequisites, and clear expected outcomes. + +#### 4. Skill composition test *(skill-flow only)* + +Validates that a `skill-flow` entity's `atomic_skills` list is coherent: +- All referenced slugs resolve to real files under `.evolve/entities/atomic-skill/` +- Atomic skill content matches the steps described in the flow +- No circular dependencies + +#### 5. Trajectory replay test + +Validates that applying the skill to the original conversation trajectory that produced it would succeed — i.e., the skill would be recalled at the right point and its guidance matches what was actually done. + +--- + +## Sharing Pipeline (Optional) + +Use when you want to share your learned entities with the team or pull in others' guidelines. + +> **Tests gate both dedup and publish.** `evolve-lite-dedup` runs a pre/post snapshot regression check around its dedup operation. `evolve-lite-publish` blocks on the content + recall gate (≥ 80% pass rate) before any entity is moved to the write-scope repo. + +``` +subscribe to fork (once) + ↓ +[work & learn sessions — each ends with the Step 7a gate] + ↓ +dedup (with pre/post test snapshot — blocks on regression) + ↓ +publish (blocks if test gate < 80%) + ↓ +open PR to main + ↑ + sync (pull latest from subscribed repos at any time) +``` + +| Skill | Command | What it does | +|-------|---------|--------------| +| `evolve-lite-subscribe` | `/evolve-lite-subscribe ` | Adds a repo (read or write scope) so Bob can pull from or publish to it | +| `evolve-lite-sync` | `/evolve-lite-sync` | Pulls the latest entities from all subscribed repos into your local `.evolve/` | +| `evolve-lite-dedup` | `/evolve-lite-dedup` | Two-phase cleanup before publishing — quality gate then merge/discard (see detail below) | +| `evolve-lite-publish` | `/evolve-lite-publish` | Stamps `visibility: public`, `owner`, `published_at` on each entity, moves it into the write-scope clone, commits, and pushes to the fork branch | +| `evolve-lite-unsubscribe` | `/evolve-lite-unsubscribe ` | Removes a repo subscription and deletes its local clone | + +After publishing, [open a pull request](https://github.ibm.com/ce-artemis-2026/evobob-test/compare) from your fork's branch into `ce-artemis-2026/evobob-test:main`. + +--- + +## Inside `evolve-lite-dedup` + +Run before publishing to clean up a library that has grown from many `learn` sessions. Phase 2 will not run unless Phase 1 exits with code `0` — every skill must pass all blocking checks first. + +The **recommended workflow** wraps dedup with a pre/post test snapshot so any regression introduced by merging or discarding skills is caught immediately: + +``` +pre-dedup + ├─ generate_skill_tests.py --all (refresh fixtures) + ├─ check_tests.py --threshold 0.8 (must pass before dedup) + └─ snapshot_test_results.py --out pre_dedup_snapshot.json + + ↓ + +Phase 1 — Quality Gate (scripts/quality_gate.py) + ├─ format check + ├─ recall test + ├─ skill evaluation + ├─ naming check + └─ version check (warning only — non-blocking) + + ↓ exit 0 only ↓ + +Phase 2 — Refine (scripts/refine.py) + ├─ banality prune + ├─ similarity clustering + └─ merge / discard / keep-all decisions + + ↓ + +post-dedup + ├─ check_tests.py --threshold 0.8 (rerun after entity changes) + └─ snapshot_test_results.py (compare → exits 1 on regression) + --compare pre_dedup_snapshot.json +``` + +### Phase 1 — Quality Gate + +Iterates every `.md` file under `.evolve/entities/` and runs the following checks. The gate exits `1` and blocks Phase 2 if any skill fails a blocking check. + +#### Format check *(blocking)* + +Each entity must have all of: +- `type` — must be exactly `guideline`, `atomic-skill`, or `skill-flow` +- `trigger` — at least 10 characters +- `owner` and `visibility` fields present and non-empty +- A non-empty content body + +Skill-flows additionally require a non-empty `atomic_skills` field listing the slugs they compose. + +Optional sections are validated when present: +- `## Requirements` — every package or CLI tool listed must be mentioned somewhere in the content body +- `## Imports` — every module or symbol listed must appear in the content body + +#### Recall test *(blocking)* + +Generates a realistic user question from the entity's `trigger` field using the same `trigger_to_realistic_question` logic used during live recall. Scores the entity against the full recall manifest using keyword-overlap heuristics. + +**Pass condition**: the entity ranks `#1` or appears in the top 3 with a score `> 0`. + +A failure here means the trigger uses words a user would not naturally type — the entity would never surface during a real session even if it is the correct answer. Fix by rewriting the trigger to match the symptom or task vocabulary a user would actually use. + +#### Skill evaluation *(blocking)* + +Checks internal self-consistency of the content: +- Derives `must_include` terms from the content (backtick commands, identifier names, key words) +- Verifies those terms appear in the content itself — **alignment score must be ≥ 0.5** +- For `skill-flow` entities: checks that every slug listed in `atomic_skills` resolves to a real `.md` file under `.evolve/entities/` + +A failure indicates the skill's content references things it doesn't explain, or a skill-flow points to a dependency that doesn't exist. + +#### Naming check *(blocking)* + +Verifies the slug and trigger are well-formed: +- Slug follows `kebab-case`, 2–5 words, no punctuation +- Trigger does not contain prompt-injection patterns or overly generic phrases +- Slug does not start with trigger-style prefixes (`when-`, `if-`, `how-to-`) + +#### Version check *(warning only — non-blocking)* + +If `version > 1`, the `## Changelog` section must have at least that many entries. A mismatch is logged as a warning but does not block Phase 2. + +#### Exit codes + +| Code | Meaning | +|------|---------| +| `0` | All skills pass all blocking checks — Phase 2 runs | +| `1` | One or more skills failed a blocking check — Phase 2 is blocked until fixed | + +--- + +### Phase 2 — Refine + +Only runs after a clean Phase 1 exit. Operates on the same entity set. + +#### Banality prune + +Before any clustering, every `atomic-skill` and `guideline` entity is checked for banality and pruned if it matches **any** of these conditions: + +| Condition | Example | +|-----------|---------| +| Content ≤ 30 chars with no backtick command | `"Activate the virtual environment."` | +| Content or trigger matches a common-sense pattern | `"install dependencies"`, `"git commit"`, `"run the application"` | +| Content is a near-verbatim restatement of the trigger (Jaccard ≥ 0.85) | Trigger: `"When activating the venv"` → Content: `"Activate the venv."` | + +`skill-flow` entities are exempt from banality pruning because their step ordering is non-obvious even when individual steps are simple. + +Use `--no-prune` to skip this step and review manually. + +#### Similarity clustering + +Groups remaining entities by **token-set Jaccard similarity** on the combined `trigger + content` text. Default clustering threshold is **0.45** — pairs at or above this score are grouped into a cluster. + +| Jaccard score | Decision | Action | +|---------------|----------|--------| +| ≥ 0.75 | `discard` | Keep the entity with the longest content, delete the rest | +| ≥ 0.45 (threshold) | `merge` | Write a merged entity into the richest file; combined trigger joins all unique trigger phrases with `; ` so recall still fires on any original phrasing; delete the others | +| < 0.45 | `keep-all` | No action | + +The "richest" entity in a cluster is always the one with the longest content body. + +Run with `--interactive` to review each cluster manually and override the automatic decision (`k` keep-all, `m` merge, `d` discard, `s` skip). + +#### Reports + +Both phases write JSON reports to `.evolve/tests/dedup/` by default: + +| File | Contents | +|------|----------| +| `quality_gate_report.json` | Per-skill results for format, recall, eval, and naming checks | +| `refine_report.json` | Per-cluster decisions — which entities were merged, discarded, or kept, with file paths | + +--- + + +## Testing Pipeline + +`learn` now runs the content + recall gate automatically at Step 7a. `evolve-lite-run-tests` is a separate on-demand command that runs all three suites at any time. `check_tests.py` is the shared gate script used by learn, publish, and dedup. + +``` +learn → fixtures written to .evolve/tests/pseudo_conversations/ + ↓ (Step 7a — runs automatically inside learn) + check_tests.py --threshold 0.8 + ├─ run_skill_evaluation.py (content test) ← BLOCKS learn if < 80% + └─ run_recall_tests.py (recall test) ← BLOCKS learn if < 80% + + ↓ (on-demand, any time) + evolve-lite-run-tests + ├─ run_skill_evaluation.py (content test) + ├─ run_recall_tests.py (recall test) + └─ run_baseline_tests.py (baseline / trigger-discrimination test) +``` + +**Gate thresholds used across the pipeline:** + +| Trigger point | Script | Threshold | Blocking? | +|---|---|---|---| +| After `learn` Step 7 | `check_tests.py` | ≥ 80% both suites | Yes — Step 8 blocked | +| Before `publish` | `check_tests.py` | ≥ 80% both suites | Yes — publish blocked | +| Before/after `dedup` | `check_tests.py` + `snapshot_test_results.py` | ≥ 80% + no regression | Yes — Phase 2 / completion blocked | + +To regenerate fixtures manually (e.g. after editing a skill's trigger): + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +``` + +### Fixing gate failures + +When `check_tests.py` exits 1, the output marks each failure as either a **content evaluation** failure or a **recall** failure. + +**Content evaluation failure** — `score < 0.5`, `missed=[…]` + +The fixture's `must_include` list contains command terms that are absent from the skill content. + +- Add the missing commands/flags to the content body of the entity file. +- If the term was over-extracted and the skill is correct without it, regenerate the fixture for just that skill: `generate_skill_tests.py ` +- If `violated=[…]` is non-empty, remove the flagged phrase from the skill content. + +**Recall failure** — `rank > 3`, `matched_terms` contains only short generic tokens + +The `trigger` field uses vocabulary that doesn't overlap with what a user would type. + +- Rewrite the trigger to include the symptom, error message, or specific command/flag name the user would describe. +- Check the `top5` list in the output — if other skills rank above yours, add the vocabulary that distinguishes your skill from those. +- Regenerate the fixture after rewriting: `generate_skill_tests.py ` + +Then re-run `check_tests.py --threshold 0.8 --verbose` to verify. + +--- + +### Content test (`run_skill_evaluation.py`) + +**What it checks:** the skill content is self-consistent — it contains the commands and terms it claims to prescribe. + +**How it works:** +1. Loads each fixture from `.evolve/tests/pseudo_conversations/` +2. The fixture's `conversation` field has a `system` message with the skill injected inside a `` block, and a `user` message with the scenario question +3. Instead of calling a live LLM, the runner uses the skill's own content as the simulated agent response — because a well-formed skill that contains its prescribed commands will pass, and gaps will surface +4. Checks the response against the fixture's `expected_behaviour.must_include` list: each term is lowercased, angle-bracket placeholders stripped, then substring-searched in the response +5. **Pass condition:** `alignment_score = matched / total ≥ 0.5` and no `must_not_include` terms found +6. Reports per-skill token breakdown (preamble / skill / user / completion) and latency + +**Output:** `.evolve/tests/evaluation/report.json` + +--- + +### Recall test (`run_recall_tests.py`) + +**What it checks:** the skill surfaces when it should — given the user's question, the skill ranks in the top-K of the recall manifest. + +**How it works:** +1. Builds the live recall manifest from all entities under `.evolve/entities/` +2. For each fixture, takes the `user` message from the conversation +3. Scores every entry in the manifest against that user message using **weighted keyword overlap**: tokenises both trigger and user message (lowercase, stop words removed, tokens < 3 chars dropped), finds the intersection, scores as `sum(len(term) for term in matched_terms)` — longer terms score higher, filtering out accidental short-word matches +4. Ranks all manifest entries by descending score +5. Finds the rank of the expected skill slug in that ranking +6. **Pass condition (default):** the expected skill is in the **top 3** with a score `> 0` (Recall@3). Configurable to Recall@1 or Recall@5 with `--top-k` + +**What a failure means:** the trigger uses vocabulary that doesn't overlap with what a user would actually type. Fix by rewriting the trigger to include the symptom words or task keywords the user would use. + +**Output:** `.evolve/tests/evaluation/recall_report.json` — reports Recall@1, @3, and @5 + +--- + +### Baseline test (`run_baseline_tests.py`) + +**What it checks:** the skill is *necessary* — a naive agent responding *without* the skill injected would miss the key guidance. + +**How it works:** +1. For each fixture, takes only the `user` message — **no system prompt, no skill injected** +2. Looks up a pre-written naive baseline response for the skill slug (a "reasonable-but-incomplete" answer representing what a competent generic agent would say without the skill) +3. Checks whether that naive response contains the fixture's `must_include` terms using the same alignment scorer as the content test +4. **Interpretation is inverted from the content test:** + - `FAIL` (naive agent misses the terms) → **skill IS necessary** — the trigger correctly identifies a non-obvious situation + - `PASS` (naive agent gets it right) → skill may not add value — consider whether it is worth keeping + +**Important:** baseline test "failures" are not failures in the traditional sense — a skill that a naive agent would get wrong *is the goal*. The test exits `0` regardless of pass/fail count. It only exits `1` if no responses could be evaluated at all. + +**Output:** `.evolve/tests/evaluation/baseline_report.json` + +--- + +## Maintenance Pipeline (Evolve Manager mode) + +For maintainers merging fork contributions into `main`. Switch to **Evolve Manager** mode in Bob and run `/evolve-manager`. + +### Full pipeline + +``` +[0] PR gate + ↓ skip forks without an open PR against the main repo +accepted forks only + ↓ +clone + stage fork entities (sparse checkout of .evolve/entities/ only) + ↓ +snapshot main-repo entity manifest → main_entity_slugs.json + ↓ +versioning-aware merge (see below) + ↓ +[A] quality gate on merged test fixtures + ↓ +[B] rubric tests on main-repo entities only → baseline_rate + ↓ +[C] full skill dedup (both phases) + ↓ +[D] rubric tests on main-repo entities only → post_rate + ↓ +[E] threshold gate + PASS → commit merged entities to .evolve/entities/ + FAIL → show diff, pause for user decision (keep or rollback) + ↓ +write merge_report.json (always, even on error or skip) +``` + +Fork-sourced entities are **never** counted against the regression threshold. Only entities that existed in the main repo before the merge are subject to the gate. + +### Versioning-aware merge + +When the same entity slug exists in both the main repo and a fork, the script compares them using token-set Jaccard similarity on `trigger + content`: + +| Jaccard score | Action | +|---|---| +| ≥ `--version-diff-threshold` (default 0.5) | Fork content replaces main-repo content outright | +| < `--version-diff-threshold` | Dual-section entity written: `## Current Version` (fork) + `## Previous Version` (original) | + +The entity's `version` frontmatter is set to the fork's version. A `base_version` field records the original main-repo version for traceability. + +### Regression gate + +The threshold gate applies only to entities listed in the pre-merge snapshot (`main_entity_slugs.json`). For dual-section entities, rubric tests run against the full merged file using the **original main-repo `must_include` terms** — confirming the merged skill still satisfies what the main-repo version promised. + +Default threshold is `1.0` (no regressions allowed). Relax with `--threshold 0.8` to allow up to 20% regression. + +### Exit codes + +| Code | Meaning | Action | +|---|---|---| +| `0` | Merge succeeded | Entities are live in `.evolve/entities/` | +| `1` | Hard failure | Fix the reported error before re-running | +| `2` | Threshold breach | Main-repo pass rate dropped — keep or rollback (user decides) | + +### Rollback + +A backup of the pre-merge `.evolve/entities/` is written before any live file is touched: + +```bash +rm -rf .evolve/entities/ +cp -r .evolve/tmp/pre-merge-backup/ .evolve/entities/ +``` + +### Key flags + +| Flag | Default | Description | +|---|---|---| +| `--fork-dirs` | (required) | Space-separated list of pre-cloned fork directories | +| `--threshold` | `1.0` | Min rubric pass rate for main-repo tests | +| `--version-diff-threshold` | `0.5` | Jaccard below which dual-section merging is used | +| `--dry-run` | off | Show all decisions without writing any files | +| `--no-require-pr` | off | Disable the PR gate — merge all provided forks | +| `--main-repo` | auto | Upstream repo as `owner/repo` (auto-detected from git `origin`) | +| `--github-token` | env | GitHub token (falls back to `GITHUB_TOKEN` env var) | + +### Reports + +| File | Contents | +|---|---| +| `.evolve/tests/dedup/merge_report.json` | Full run summary — PR gate, accepted/skipped forks, merge decisions, pass rates, outcome | +| `.evolve/tests/dedup/quality_gate_report.json` | Phase 1 format/recall/eval results | +| `.evolve/tests/dedup/refine_report.json` | Phase 2 cluster decisions (merge/discard/keep) | +| `.evolve/tests/evaluation/report.json` | Pre-dedup rubric test results | +| `.evolve/tests/evaluation/report_post.json` | Post-dedup rubric test results | + +### Other maintenance skills + +| Skill | What it does | +|-------|-------------| +| `evolve-lite-save` | Captures a successful workflow from a session and saves it as a new named skill with `SKILL.md` and helper scripts | +| `evolve-lite-provenance` | Analyzes trajectories and audit events to record whether recalled guidelines actually influenced completed sessions | + diff --git a/platform-integrations/bob/evolve-lite/docs/TESTING.md b/platform-integrations/bob/evolve-lite/docs/TESTING.md new file mode 100644 index 00000000..9f6d9857 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/docs/TESTING.md @@ -0,0 +1,130 @@ +# Skill Testing + +This project uses the `evolve-lite-test` framework to validate that entities in the `.evolve/` library are correct, discoverable, and genuinely useful. Three complementary tests are run against every `atomic-skill` and `skill-flow` entity. + +## Three test types + +### 1. Content test — *does the skill contain what it prescribes?* + +Each skill is checked for self-consistency. The backtick command strings in the skill content are extracted and matched back against the skill body. A passing skill contains every command and term it claims to prescribe. + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +python3 .bob/skills/evolve-lite-test/scripts/run_skill_evaluation.py --verbose +``` + +**Pass condition:** `alignment_score ≥ 0.5` and no `must_not_include` terms violated. + +--- + +### 2. Recall test — *does the right skill surface for the right question?* + +The live entity manifest is scored against a trigger-derived user question for each skill. The skill must appear in the top 3 results (Recall@3) with a score greater than zero. + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_recall_tests.py --verbose +``` + +**Pass condition:** the expected skill ranks in the top 3 against its trigger question using keyword-overlap scoring. + +A recall failure means the `trigger` field uses vocabulary a user would not naturally type. Fix by rewriting the trigger to include the symptom or command words the user would actually describe. + +--- + +### 3. Baseline test — *is the skill actually necessary?* + +An agent is asked the trigger question **without the skill injected**. If a naive agent already gives the right answer, the skill covers common knowledge. If it misses key guidance, the skill is genuinely valuable. + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_baseline_tests.py --simulate +``` + +**Interpretation:** a baseline "failure" (naive agent misses the guidance) is the desired outcome — it means the skill encodes non-obvious knowledge. The test exits `0` regardless of how many skills are flagged as necessary; it only exits `1` if no responses could be evaluated at all. + +--- + +## When tests run automatically + +Tests are not just on-demand tools — they are embedded as gates throughout the pipeline: + +| Trigger point | Gate script | Threshold | Blocks | +|---|---|---|---| +| After `learn` Step 7 | `check_tests.py` | ≥ 80% both suites | Step 8 (dedup) | +| Before `publish` | `check_tests.py` | ≥ 80% both suites | Publishing | +| Before/after `dedup` | `check_tests.py` + `snapshot_test_results.py` | ≥ 80% + no regression | Phase 2 / completion | + +--- + +## Commands + +| Command | What it does | +|---|---| +| `/evolve-lite-run-tests` | Run all three test suites against the current entity library | +| `/evolve-lite-create-tests` | Regenerate all fixtures from the current entity library | +| `/evolve-lite-test` | Generate fixtures + run all three tests | +| `/evolve-lite-test-new-skills` | Run after `/evolve-lite-learn` — test only newly saved skills | +| `/evolve-lite-test-recall` | Recall ranking test only | +| `/evolve-lite-test-trigger` | Baseline (necessity) test only | + +--- + +## Test fixtures + +All fixtures live in `.evolve/tests/pseudo_conversations/` — one JSON per entity. Each fixture contains: + +- A **skill-specific user question** derived from the entity's `trigger` field +- The **skill content** injected as system context (for content tests) +- `must_include` — the exact command strings the skill must produce +- `must_not_include` — terms the skill explicitly forbids + +Fixtures are generated automatically by `learn` and can be regenerated at any time: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +``` + +To regenerate a single entity's fixture after editing its trigger or content: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py +``` + +--- + +## Test reports + +Reports are written to `.evolve/tests/evaluation/`: + +| Report | Contents | +|--------|----------| +| `report.json` | Content evaluation — per-skill alignment scores, matched/missed terms, pass rate | +| `recall_report.json` | Recall ranking — Recall@1, @3, @5 rates across all fixtures | +| `baseline_report.json` | Baseline necessity — which skills a naive agent would get wrong | + +--- + +## Fixing gate failures + +**Content evaluation failure** — `score < 0.5`, `missed=[…]` + +The fixture's `must_include` list contains command terms absent from the skill content. + +- Add the missing commands or flags to the entity file's content body. +- If a term was over-extracted and the skill is correct without it, regenerate just that fixture: `generate_skill_tests.py ` +- If `violated=[…]` is non-empty, remove the flagged phrase from the skill content. + +**Recall failure** — `rank > 3` + +The `trigger` field uses vocabulary that doesn't overlap with what a user would type. + +- Rewrite the trigger to include the symptom, error message, or specific command the user would describe. +- Check the `top5` list in the output — add vocabulary that distinguishes your skill from those ranked above it. +- Regenerate the fixture after rewriting: `generate_skill_tests.py ` + +Then re-run `check_tests.py --threshold 0.8 --verbose` to verify. + +--- + +## After adding a new skill + +Run `/evolve-lite-test-new-skills` immediately after `/evolve-lite-learn` to generate a fixture and validate the new entity before it enters the library. diff --git a/platform-integrations/bob/evolve-lite/lib/evolve-lite/entity_io.py b/platform-integrations/bob/evolve-lite/lib/evolve-lite/entity_io.py index b1a3e399..ea095b76 100644 --- a/platform-integrations/bob/evolve-lite/lib/evolve-lite/entity_io.py +++ b/platform-integrations/bob/evolve-lite/lib/evolve-lite/entity_io.py @@ -12,6 +12,214 @@ from pathlib import Path +# --------------------------------------------------------------------------- +# Product registry +# --------------------------------------------------------------------------- + +# Default registry lives next to this file. +_PRODUCTS_CONFIG = Path(__file__).with_name("products.yaml") + + +def _parse_products_yaml(text): + """Minimal YAML parser for the products.yaml format. + + Handles only the subset used by products.yaml: + products: + - slug: foo + patterns: + - pattern1 + - pattern2 + + No PyYAML dependency required. + """ + entries = [] + current = None + in_patterns = False + + for raw in text.splitlines(): + line = raw.rstrip() + stripped = line.lstrip() + + if stripped.startswith("- slug:"): + if current: + entries.append(current) + current = {"slug": stripped[len("- slug:"):].strip(), "patterns": []} + in_patterns = False + + elif stripped == "patterns:" and current is not None: + in_patterns = True + + elif in_patterns and stripped.startswith("- ") and current is not None: + current["patterns"].append(stripped[2:].strip()) + + elif stripped and not stripped.startswith("#") and not stripped.startswith("-"): + in_patterns = False + + if current: + entries.append(current) + return entries + + +def load_product_registry(entities_dir=None): + """Return the merged product registry. + + Combines two sources in priority order: + + 1. **Config file** (``.bob/lib/evolve-lite/products.yaml`` or the file + adjacent to this module): declared products with explicit match patterns. + 2. **Existing entity folders**: any ``{type}/{product}/`` subdirectories + under *entities_dir* that are not already in the config are added as + folder-only products (matched by slug-token heuristic; no explicit + patterns). + + Returns: + list[dict]: each dict has ``"slug"`` (str) and ``"patterns"`` + (list[str], may be empty for folder-only entries). + """ + # 1. Load from config file + config_entries = [] + config_slugs = set() + if _PRODUCTS_CONFIG.is_file(): + try: + config_entries = _parse_products_yaml( + _PRODUCTS_CONFIG.read_text(encoding="utf-8") + ) + config_slugs = {e["slug"] for e in config_entries} + except OSError: + pass + + # 2. Discover folder-only products not already in config + if entities_dir is None: + entities_dir = get_evolve_dir() / "entities" + entities_dir = Path(entities_dir) + + folder_slugs = set() + if entities_dir.is_dir(): + for type_dir in entities_dir.iterdir(): + if not type_dir.is_dir() or type_dir.name.startswith("."): + continue + for product_dir in type_dir.iterdir(): + if ( + product_dir.is_dir() + and product_dir.name != "general" + and product_dir.name not in config_slugs + ): + folder_slugs.add(product_dir.name) + + # Merge: config entries first, then folder-only entries (no patterns) + registry = list(config_entries) + for slug in sorted(folder_slugs): + registry.append({"slug": slug, "patterns": []}) + + return registry + + +# --------------------------------------------------------------------------- +# Clone gitignore +# --------------------------------------------------------------------------- + +# Files and directories that must never be committed to a publish clone. +_CLONE_GITIGNORE_CONTENT = """\ +# Auto-generated by evolve-lite — do not edit manually. +# Only entity files (.evolve/entities/**) and test fixtures (.evolve/tests/**) +# belong in a publish clone. Everything else is blocked here. + +# Python environments and caches +.venv/ +venv/ +env/ +.env +*.env +__pycache__/ +*.py[cod] +*.pyo +*.pyd +.Python +pip-wheel-metadata/ +*.egg-info/ +dist/ +build/ + +# Editor and IDE artifacts +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store +Thumbs.db + +# Secrets and credentials +.env.* +*.pem +*.key +*.crt +*.p12 +*.pfx +secrets.yaml +secrets.json + +# Logs and temp files +*.log +*.tmp +*.bak +*.orig + +# OS noise +.Spotlight-V100 +.Trashes +ehthumbs.db +desktop.ini + +# Node / JS +node_modules/ +npm-debug.log* +yarn-error.log* + +# Allow only entity and test content +# (everything else above is blocked; .gitignore itself is committed) +""" + + +def write_clone_gitignore(clone_root): + """Write a protective .gitignore into the clone root. + + Blocks common noise files (.venv, .env, .vscode, __pycache__, etc.) from + ever being staged or pushed from a publish clone. Safe to call multiple + times — overwrites only when the file does not yet exist or was written by + a previous evolve-lite version (detected by the auto-generated header). + + Args: + clone_root: Path to the root of the local git clone. + """ + clone_root = Path(clone_root) + target = clone_root / ".gitignore" + + if target.exists(): + try: + existing = target.read_text(encoding="utf-8") + except OSError: + return + # Only overwrite if this is our own auto-generated file. + if "Auto-generated by evolve-lite" not in existing: + return + + try: + fd, tmp_path = tempfile.mkstemp(dir=clone_root, prefix=".gitignore.", suffix=".tmp") + try: + os.write(fd, _CLONE_GITIGNORE_CONTENT.encode("utf-8")) + os.close(fd) + fd = None + os.replace(tmp_path, str(target)) + finally: + if fd is not None: + os.close(fd) + if os.path.exists(tmp_path): + os.unlink(tmp_path) + except OSError: + pass + + # --------------------------------------------------------------------------- # Logging # --------------------------------------------------------------------------- @@ -78,13 +286,15 @@ def find_entities_dir(): def find_recall_entity_dirs(): """Locate all directories that should be searched during recall. - Returns the existing recall roots. Two trees contribute to recall: - ``entities/`` (private entities in ``entities/guideline/`` and - subscribed entities in ``entities/subscribed/{repo}/guideline/``) and - ``public/`` (entities published by the local project). + Returns the existing recall roots. Only private entities contribute to + recall: ``entities/`` (private entities) and ``public/`` (entities + published by the local project). Subscribed clones under + ``entities/subscribed/`` are excluded. """ evolve_dir = get_evolve_dir() - candidates = [evolve_dir / "entities", evolve_dir / "public"] + entities_dir = evolve_dir / "entities" + public_dir = evolve_dir / "public" + candidates = [entities_dir, public_dir] return [path for path in candidates if path.is_dir()] @@ -99,64 +309,112 @@ def get_default_entities_dir(): # --------------------------------------------------------------------------- -# Slugify / filename helpers +# Banality check (shared by save_entities.py and refine.py) # --------------------------------------------------------------------------- +# Universally-known single-step practices that add no recall value +_BANAL_PATTERNS = [ + # Generic Python env / packaging + r"^activate\s+(the\s+)?virtual\s+(environment|env)\s*\.?\s*$", + r"^install\s+dependencies\s*\.?\s*$", + r"^run\s+(the\s+)?(application|app|script|program)\s*\.?\s*$", + r"^set\s+up\s+(a\s+)?virtual\s+(environment|env)\s*\.?\s*$", + r"^create\s+a\s+requirements\.txt\s*\.?\s*$", + r"^import\s+(a\s+)?module\s*\.?\s*$", + # Basic file ops + r"^(create|write|make)\s+a\s+(file|directory|folder)\s*\.?\s*$", + r"^read\s+(a\s+)?file\s*\.?\s*$", + r"^write\s+to\s+(a\s+)?file\s*\.?\s*$", + r"^delete\s+(a\s+)?file\s*\.?\s*$", + # Basic git + r"^git\s+(add|commit|push|pull|status|log)\s*\.?\s*$", + r"^create\s+a\s+(git\s+)?branch\s*\.?\s*$", + # Fatally vague + r"^when\s+performing\s+", + r"^do\s+something\s*\.?\s*$", + r"^use\s+(the\s+)?\w+\s*\.?\s*$", +] + +_BANAL_RE = [re.compile(p, re.IGNORECASE) for p in _BANAL_PATTERNS] + + +def check_banality(entity): + """Return ``(is_banal, reason)`` for *entity*. + + An entity is banal when it would provide no recall value because: + + 1. **Empty / trivially short** — content ≤ 30 chars with no backtick command. + 2. **Common-sense pattern** — content or trigger matches a curated list of + universally-known single-step practices. + 3. **Pure trigger restatement** — content body is a near-verbatim copy of the + trigger, adding nothing (Jaccard ≥ 0.85 between the two). + 4. **Slug tautology** — content or trigger consists only of tokens already + present in the filename slug, meaning the entity says nothing that isn't + already encoded in its own name. -def slugify(text, max_length=60): - """Convert *text* to a filesystem-safe slug. - - >>> slugify("Use temp files for JSON transfer!") - 'use-temp-files-for-json-transfer' + Returns: + tuple[bool, str]: ``(True, reason_string)`` if banal, else ``(False, "")``. """ - text = text.lower() - text = re.sub(r"[^a-z0-9]+", "-", text) - text = text.strip("-") - # Truncate at max_length, but don't break in the middle of a word - if len(text) > max_length: - text = text[:max_length].rsplit("-", 1)[0] - return text or "entity" + content = entity.get("content", "").strip() + trigger = entity.get("trigger", "").strip() + # 1. Too short and no command + if len(content) <= 30 and not re.search(r"`[^`]+`", content): + return True, f"content too short ({len(content)} chars) and contains no command" -def claude_project_slug(path): - """Derive Claude's per-project directory name from an absolute path. + combined = f"{content} {trigger}".lower() - Claude names a project's ``~/.claude/projects//`` directory by - replacing every non-alphanumeric character in the resolved absolute project - path with ``-``. + # 2. Common-sense pattern match + for pat in _BANAL_RE: + if pat.search(content.lower()) or pat.search(trigger.lower()): + return True, f"matches common-sense pattern: '{pat.pattern}'" - >>> claude_project_slug("/Users/x/evolve-smoke-test2") - '-Users-x-evolve-smoke-test2' + # 3. Trigger restatement — content is essentially the trigger re-worded + def _tok(t): + words = re.findall(r"[a-z0-9]+", t.lower()) + _stop = {"the", "a", "an", "and", "or", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "as", "is", "was", "are", "be", + "when", "after", "while", "if", "how", "this", "that"} + return {w for w in words if w not in _stop and len(w) > 2} - This is the single source of truth shared by doctor.py (transcript dir) and - adapt_memory.py (native memory dir). - """ - return re.sub(r"[^A-Za-z0-9]", "-", str(Path(path).resolve())) + c_tok = _tok(content) + t_tok = _tok(trigger) + if c_tok and t_tok: + inter = len(c_tok & t_tok) + union = len(c_tok | t_tok) + jac = inter / union + if jac >= 0.85: + return True, f"content is a near-verbatim restatement of trigger (Jaccard={jac:.2f})" + return False, "" -def claude_memory_dir(path, home=None): - """Return the native Claude memory dir for the project rooted at *path*. - ``~/.claude/projects//memory/`` where ```` is - :func:`claude_project_slug` of *path*. *home* defaults to ``Path.home()``. - """ - home = Path.home() if home is None else Path(home) - return home / ".claude" / "projects" / claude_project_slug(path) / "memory" +# --------------------------------------------------------------------------- +# Slugify / filename helpers +# --------------------------------------------------------------------------- -def sanitize_type(text): - """Sanitize an entity *type* into a filesystem-safe subdirectory name. +def slugify(text, max_length=60, product=None): + """Convert *text* to a filesystem-safe slug with optional product prefix. - Like :func:`slugify` but without truncation — a type is a short label, - not free-form content, and truncating it could silently merge distinct - types. Returns an empty string for input that contains no usable - characters, leaving the fallback decision to the caller. + >>> slugify("Use temp files for JSON transfer!") + 'use-temp-files-for-json-transfer' + >>> slugify("Activate environment", product="watson-orchestrate") + 'watson-orchestrate-activate-environment' """ - if not isinstance(text, str): - return "" text = text.lower() text = re.sub(r"[^a-z0-9]+", "-", text) - return text.strip("-") + text = text.strip("-") + + # Add product prefix if provided + if product: + product_slug = re.sub(r"[^a-z0-9]+", "-", product.lower()).strip("-") + text = f"{product_slug}-{text}" + + # Truncate at max_length, but don't break in the middle of a word + if len(text) > max_length: + text = text[:max_length].rsplit("-", 1)[0] + return text or "entity" def unique_filename(directory, slug): @@ -180,7 +438,28 @@ def unique_filename(directory, slug): # Markdown <-> dict conversion # --------------------------------------------------------------------------- -_FRONTMATTER_KEYS = ("type", "trigger", "trajectory", "owner", "source", "native_path", "visibility", "published_at") +_FRONTMATTER_KEYS = ("type", "name", "trigger", "trajectory", "owner", "source", "visibility", "published_at", "version", "atomic_skills", "product", "derived_from_failure") + + +# Optional markdown body sections, in serialisation order. +# Maps entity dict key -> markdown heading display name. +# Single-word keys: heading == key.capitalize(). +# Multi-word keys: heading is the value (with spaces / title-case). +_BODY_SECTIONS = ("rationale", "success_rubric", "requirements", "imports", "dependencies", "documentation", "changelog") + +# Heading display names for keys that don't map via simple .capitalize() +_SECTION_HEADING_OVERRIDE = { + "success_rubric": "Success Rubric", +} + +def _section_heading(key): + """Return the markdown heading text for a body-section key.""" + return _SECTION_HEADING_OVERRIDE.get(key, key.capitalize()) + +def _heading_to_key(heading): + """Reverse-map a matched heading back to the entity dict key.""" + _reverse = {v.lower(): k for k, v in _SECTION_HEADING_OVERRIDE.items()} + return _reverse.get(heading.lower(), heading.lower()) def entity_to_markdown(entity): @@ -188,7 +467,8 @@ def entity_to_markdown(entity): Args: entity: dict with keys ``content``, and optionally ``type``, - ``trigger``, ``rationale``. + ``trigger``, ``rationale``, ``requirements``, ``imports``, + ``dependencies``, ``documentation``. Returns: A string suitable for writing to a ``.md`` file. @@ -204,12 +484,14 @@ def entity_to_markdown(entity): content = entity.get("content", "") lines.append(content) - rationale = entity.get("rationale") - if rationale: - lines.append("") - lines.append("## Rationale") - lines.append("") - lines.append(rationale) + for section_key in _BODY_SECTIONS: + val = entity.get(section_key) + if val and val.strip(): + heading = f"## {_section_heading(section_key)}" + lines.append("") + lines.append(heading) + lines.append("") + lines.append(val.strip()) lines.append("") return "\n".join(lines) @@ -221,8 +503,12 @@ def markdown_to_entity(path): Handles YAML frontmatter with simple ``key: value`` lines (no nested structures, no PyYAML dependency). + Recognised body sections (parsed into entity dict keys): + ## Rationale, ## Requirements, ## Imports, ## Dependencies, + ## Documentation + Returns: - dict with ``content``, ``type``, ``trigger``, ``rationale`` keys. + dict with ``content`` plus any recognised section keys. """ path = Path(path) text = path.read_text(encoding="utf-8") @@ -249,20 +535,36 @@ def markdown_to_entity(path): else: body = text - # Split body into content and rationale + # Split body into content + known optional sections. + # A section begins at "## " and runs until the next "## " heading + # or end of string. Content is everything before the first known heading. body = body.strip() - m = re.search(r"^## Rationale", body, re.MULTILINE) - if m: - content = body[: m.start()].strip() - rationale = body[m.end() :].strip() - if rationale: - entity["rationale"] = rationale + + # Build a regex that matches any of the known section headings + _known_headings = "|".join(re.escape(_section_heading(s)) for s in _BODY_SECTIONS) + _section_re = re.compile( + rf"^## ({_known_headings})\s*$", re.MULTILINE | re.IGNORECASE + ) + + first_match = _section_re.search(body) + if first_match: + content = body[: first_match.start()].strip() else: content = body if content: entity["content"] = content + # Walk through all section matches and capture text until the next heading + matches = list(_section_re.finditer(body)) + for i, m in enumerate(matches): + section_name = _heading_to_key(m.group(1)) + section_start = m.end() + section_end = matches[i + 1].start() if i + 1 < len(matches) else len(body) + section_text = body[section_start:section_end].strip() + if section_text: + entity[section_name] = section_text + return entity @@ -339,6 +641,10 @@ def load_manifest(root_dir): for md in sorted(root_dir.glob("**/*.md")): if md.is_symlink() or ".git" in md.parts: continue + # Skip the subscribed/ subtree — those are read-only remote clones and + # may contain stale entity versions that conflict with local entities. + if "subscribed" in Path(md).resolve().parts: + continue entity = _parse_frontmatter_only(md) entity_type = entity.get("type") @@ -346,13 +652,14 @@ def load_manifest(root_dir): if not entity_type or not trigger: continue - entries.append( - { - "path": _manifest_path(md), - "type": entity_type, - "trigger": trigger, - } - ) + entry = { + "path": _manifest_path(md), + "type": entity_type, + "trigger": trigger, + } + if entity.get("derived_from_failure") == "true": + entry["derived_from_failure"] = True + entries.append(entry) return dedupe_manifest_entries(entries) @@ -380,35 +687,31 @@ def load_all_entities(entities_dir): return entities -def write_entity_file(directory, entity, filename=None, overwrite=False): +def write_entity_file(directory, entity): """Write a single entity as a markdown file under *directory*. - The file is placed in a ``{type}/`` subdirectory. Uses atomic + The file is placed in a ``{type}/{product}/`` subdirectory. Uses atomic write (write to ``.tmp``, then ``os.rename``). - Args: - directory: Entities root directory. - entity: The entity dict to serialize. - filename: Optional explicit slug for the target file (without the - ``.md`` suffix). When omitted, the slug is derived from the - entity content (the historical default). - overwrite: When True, the entity is written to a deterministic - ``{type}/{filename}.md`` path, overwriting any existing file in - place (stable id, idempotent re-mirroring). When False (the - default), the historical collision-avoiding behavior is kept — - a ``-2``/``-3`` suffix is appended on collision. - Returns: Path to the written file. """ - # Any non-empty type is accepted and used (sanitized) as the - # subdirectory. An empty/invalid type falls back to "guideline". - entity_type = sanitize_type(entity.get("type", "guideline")) or "guideline" + _ALLOWED_TYPES = {"guideline", "atomic-skill", "skill-flow"} + entity_type = entity.get("type", "guideline") + if not isinstance(entity_type, str) or entity_type not in _ALLOWED_TYPES: + entity_type = "guideline" entity["type"] = entity_type - type_dir = Path(directory) / entity_type + + # Get product for organization + product = entity.get("product", "general") + + # Create type/product directory structure + type_dir = Path(directory) / entity_type / product type_dir.mkdir(parents=True, exist_ok=True) - slug = slugify(filename) if filename else slugify(entity.get("content", "entity")) + # Prefer the explicit name field for the slug; fall back to content + slug_source = entity.get("name") or entity.get("content", "entity") + slug = slugify(slug_source, product=product) content = entity_to_markdown(entity) # Write to a unique temp file first (avoids predictable .tmp collisions) @@ -419,13 +722,6 @@ def write_entity_file(directory, entity, filename=None, overwrite=False): os.close(fd) fd = None - if overwrite: - # Deterministic target: overwrite any existing file in place so - # the entity id is stable across re-mirroring. - target = type_dir / f"{slug}.md" - os.replace(tmp_path, target) - return target - # Atomically claim the target using O_EXCL; retry on race while True: target = unique_filename(type_dir, slug) diff --git a/platform-integrations/bob/evolve-lite/lib/evolve-lite/trajectory_extractor.py b/platform-integrations/bob/evolve-lite/lib/evolve-lite/trajectory_extractor.py new file mode 100644 index 00000000..ff631773 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/lib/evolve-lite/trajectory_extractor.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +""" +Trajectory Extractor +Automatically extracts trajectories from Bob's task logs instead of requiring manual conversation copying. +""" + +import json +import os +from datetime import datetime +from pathlib import Path + + +def get_bob_tasks_dir(): + """Get the Bob tasks directory path.""" + home = Path.home() + return home / "Library" / "Application Support" / "IBM Bob" / "User" / "globalStorage" / "ibm.bob-code" / "tasks" + + +def get_latest_task_dir(): + """Get the most recently modified task directory from Bob's logs.""" + tasks_dir = get_bob_tasks_dir() + + if not tasks_dir.exists(): + raise FileNotFoundError(f"Bob tasks directory not found: {tasks_dir}") + + # Get all task directories (UUIDs) + task_dirs = [d for d in tasks_dir.iterdir() if d.is_dir()] + + if not task_dirs: + raise FileNotFoundError("No task directories found in Bob's tasks directory") + + # Sort by modification time, most recent first + task_dirs.sort(key=lambda d: d.stat().st_mtime, reverse=True) + + return task_dirs[0] + + +def extract_trajectory_from_bob_log(task_dir_path=None): + """Extract trajectory from Bob's task log. + + Args: + task_dir_path: Optional path to specific task directory. + If None, uses most recent task. + + Returns: + dict: Trajectory in standard format with messages array + """ + if task_dir_path is None: + task_dir_path = get_latest_task_dir() + + task_dir_path = Path(task_dir_path) + + # Read the API conversation history file + api_history_file = task_dir_path / "api_conversation_history.json" + if not api_history_file.exists(): + raise FileNotFoundError(f"API conversation history not found: {api_history_file}") + + with open(api_history_file, 'r', encoding='utf-8') as f: + messages = json.load(f) + + # Read task metadata for additional context + metadata_file = task_dir_path / "task_metadata.json" + metadata = {} + if metadata_file.exists(): + with open(metadata_file, 'r', encoding='utf-8') as f: + metadata = json.load(f) + + # Messages are already in OpenAI chat completion format (it's a list) + if not isinstance(messages, list): + raise ValueError(f"Expected messages to be a list, got {type(messages)}") + + # Build trajectory envelope + trajectory = { + 'model': metadata.get('model', 'unknown'), + 'session_id': task_dir_path.name, # Use the UUID directory name + 'timestamp': datetime.fromtimestamp(task_dir_path.stat().st_mtime).isoformat(), + 'source': 'bob_task_log', + 'messages': messages, + 'metadata': { + 'task_dir': str(task_dir_path), + 'mode': metadata.get('mode'), + 'project_root': metadata.get('cwd'), + } + } + + return trajectory + + +def save_trajectory_from_bob(output_dir=None, task_dir_path=None): + """Extract Bob task and save as trajectory. + + Args: + output_dir: Directory to save trajectory. Defaults to .evolve/trajectories/ + task_dir_path: Optional specific task directory to extract. If None, uses latest. + + Returns: + Path: Path to saved trajectory file + """ + if output_dir is None: + # Use EVOLVE_DIR if set, otherwise .evolve + evolve_dir = os.environ.get("EVOLVE_DIR", ".evolve") + output_dir = Path(evolve_dir) / 'trajectories' + + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Extract trajectory from Bob + trajectory = extract_trajectory_from_bob_log(task_dir_path) + + # Generate filename with session ID for provenance tracking + timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + session_id = trajectory['session_id'] + + # Sanitize session_id for filename + safe_session_id = "".join(c if c.isalnum() or c in "._-" else "-" for c in session_id)[:64] + + filename = f"trajectory_{timestamp}_{safe_session_id}.json" + output_path = output_dir / filename + + # Save trajectory + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(trajectory, f, indent=2) + + return output_path + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-create-tests/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-create-tests/SKILL.md new file mode 100644 index 00000000..7571ca04 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-create-tests/SKILL.md @@ -0,0 +1,29 @@ +--- +name: evolve-lite-create-tests +description: >- + Create pseudo-conversation test fixtures for all atomic skills, or for + specific new skill files. Run this after evolve-lite-learn to generate tests + for newly saved skills. +metadata: + user-invocable: true + disable-model-invocation: true +--- + +To create test fixtures for ALL skills: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +``` + +To create fixtures for specific new skill files (e.g. just saved by `evolve-lite-learn`): + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py [ ...] +``` + +Each fixture is written to `.evolve/tests/pseudo_conversations/.json` and contains: +- A skill-specific user question derived from the skill's trigger +- The `must_include` command terms extracted from the skill content +- The skill content ready to be injected as system context + +After creating fixtures, run `/evolve-lite-run-tests` to evaluate them. diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/SKILL.md new file mode 100644 index 00000000..b5a7c027 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/SKILL.md @@ -0,0 +1,314 @@ +--- +name: evolve-lite:dedup +description: Two-phase skill library deduplication — runs all quality tests then merges or discards similar/duplicate skills. Use when the skill library may have grown stale, redundant, or inconsistent. +--- + +# Skill Library Deduplication + +## Overview + +`evolve-lite:dedup` cleans the skill library in two sequential phases. +Phase 2 will not run unless Phase 1 passes completely. + +``` +Phase 1 — Quality Gate (quality_gate.py) + ├─ Format check required frontmatter, valid type, non-empty content + ├─ Recall test skill ranks ≤ 3 for its own trigger scenario + ├─ Skill evaluation content is self-consistent; skill-flow refs resolve + ├─ Naming check slug and trigger are well-formed + └─ Version check version field and changelog are consistent (warning only) + + ↓ only if all skills pass ↓ + +Phase 2 — Refine (refine.py) + ├─ Banality prune remove entities too generic to provide recall value + ├─ Similarity grouping token-set Jaccard clustering + ├─ keep-all skills are distinct + ├─ merge combine similar skills into one enriched entity + └─ discard remove near-identical duplicates, keep richest +``` + +## When To Use + +- After several `evolve-lite:learn` runs, when the library may have grown. +- Before publishing or syncing skills to a shared repo. +- Any time recall results feel noisy or redundant. +- Periodically as a maintenance command (e.g. weekly). + +## Usage + +### Recommended workflow (with test regression protection) + +Run this sequence instead of calling `dedup.py` directly. It captures a pass-count snapshot before dedup, runs dedup, then verifies no tests regressed. + +```bash +# 1. Make sure fixtures and reports are up to date +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +python3 .bob/skills/evolve-lite-test/scripts/check_tests.py --threshold 0.8 + +# 2. Snapshot the current pass counts +python3 .bob/skills/evolve-lite-test/scripts/snapshot_test_results.py \ + --out .evolve/tests/evaluation/pre_dedup_snapshot.json + +# 3. Run dedup +python3 .bob/skills/evolve-lite-dedup/scripts/dedup.py + +# 4. Re-run tests and compare against the snapshot +python3 .bob/skills/evolve-lite-test/scripts/check_tests.py --threshold 0.8 +python3 .bob/skills/evolve-lite-test/scripts/snapshot_test_results.py \ + --compare .evolve/tests/evaluation/pre_dedup_snapshot.json \ + --out .evolve/tests/evaluation/post_dedup_snapshot.json +``` + +**If step 4's comparison exits 1** (regression detected), see [Fixing regressions after dedup](#fixing-regressions-after-dedup) below. + +### Full pipeline (automated, no regression protection) + +```bash +python3 .bob/skills/evolve-lite-dedup/scripts/dedup.py +``` + +### Dry run — see decisions without changing anything + +```bash +python3 .bob/skills/evolve-lite-dedup/scripts/dedup.py --dry-run +``` + +### Interactive — review each duplicate cluster manually + +```bash +python3 .bob/skills/evolve-lite-dedup/scripts/dedup.py --interactive +``` + +### Quality gate only + +```bash +python3 .bob/skills/evolve-lite-dedup/scripts/dedup.py --phase1-only +``` + +### With custom similarity threshold + +```bash +python3 .bob/skills/evolve-lite-dedup/scripts/dedup.py --threshold 0.6 +``` + +### Save reports to a specific directory + +```bash +python3 .bob/skills/evolve-lite-dedup/scripts/dedup.py --report-dir .evolve/tests/dedup/ +``` + +## Phase 1 — Quality Gate + +Iterates every `.md` file under `.evolve/entities/` and runs three checks: + +### 1. Format Check + +Each entity must have: +- `type` field set to `guideline`, `atomic-skill`, or `skill-flow` +- `trigger` field with at least 10 characters +- `owner` and `visibility` fields +- Non-empty content body + +Skill-flows must have a non-empty `atomic_skills` references field. + +Optional body sections (`## Requirements`, `## Imports`, `## Dependencies`, `## Documentation`) are not required but are validated when present: +- Every package/tool in `## Requirements` must be mentioned in the content body +- Every module/symbol in `## Imports` must be mentioned in the content body + +### 2. Recall Test + +Generates a realistic user question from the entity's trigger (using the same +`trigger_to_realistic_question` logic as `evolve-lite-test`) and scores the +entity against the full recall manifest using the keyword-overlap heuristic. + +**Pass condition**: the skill ranks #1 or appears in the top-3 with score > 0. + +A skill that fails the recall test will not be retrieved when it should be — +this indicates a weak or ambiguous trigger that needs to be rewritten. + +### 3. Skill Evaluation + +Checks that the skill content is internally self-consistent: +- Derives `must_include` terms from the content (backtick commands, identifier + terms, or key words). +- Verifies those terms appear in the content itself (alignment score ≥ 0.5). +- For skill-flows: checks that every slug listed in `atomic_skills` resolves + to a real file under the entities directory. + +### Quality Gate Exit Codes + +| Code | Meaning | +|------|---------| +| `0` | All skills pass — safe to proceed to Phase 2 | +| `1` | One or more skills failed — Phase 2 blocked | + +Fix failing skills before re-running dedup. See [Fixing gate failures](#fixing-gate-failures-dedup) below for per-failure-type guidance. + +## Phase 2 — Refine + +### Banality Prune (pre-clustering) + +Before clustering, every `atomic-skill` and `guideline` entity is checked for banality. `skill-flow` entities are exempt because their composition order is non-obvious even when individual steps are simple. + +An entity is considered banal and pruned when **any** of the following are true: + +| Condition | Example | +|---|---| +| Content ≤ 30 chars with no backtick command | `"Activate the virtual environment."` | +| Content or trigger matches a common-sense pattern | `"install dependencies"`, `"git commit"`, `"run the application"` | +| Content is a near-verbatim restatement of the trigger (Jaccard ≥ 0.85) | Trigger: "When activating the venv" → Content: "Activate the venv." | + +Pruned entities are deleted before clustering so they can never be merged into a richer skill. + +Use `--no-prune` to skip this step and review entities manually. + + +Groups entities by **token-set Jaccard similarity** on the combined +`trigger + content` text. The default threshold is **0.45** — pairs scoring +at or above this value are grouped into a cluster. + +### Automatic Decisions + +| Jaccard range | Decision | Action | +|---|---|---| +| ≥ 0.75 | `discard` | Keep the most detailed skill, delete the others | +| ≥ 0.45 (threshold) | `merge` | Write a merged entity with combined trigger into the richest file, delete others | +| < 0.45 | `keep-all` | No action | + +The "richest" skill in a cluster is the one with the longest content. + +For `merge`, the combined trigger is all unique triggers joined with `; ` so +recall still fires on any of the original phrasings. + +### Interactive Mode + +With `--interactive`, each multi-skill cluster is printed with: +- Slug, trigger, and content preview for each member +- The automatic suggestion + +You are prompted to choose: +- `k` keep-all +- `m` merge +- `d` discard +- `s` skip (leave for later) +- Enter to accept the suggestion + +### Reports + +Both phases write JSON reports (default: `.evolve/tests/dedup/`): + +- `quality_gate_report.json` — per-skill format / recall / eval results +- `refine_report.json` — per-cluster decisions and removed/merged paths + +## Fixing gate failures (dedup) {#fixing-gate-failures-dedup} + +When Phase 1 blocks dedup, the `quality_gate_report.json` and the console output show per-skill results. Fix each `❌` entry using the pattern that matches. + +--- + +**Format check failure** — required frontmatter field is missing or invalid + +``` +❌ my-skill FAIL [format] missing: owner, visibility +``` + +Open the entity file. Add the missing YAML frontmatter keys: +- `type:` — must be `guideline`, `atomic-skill`, or `skill-flow` +- `trigger:` — at least 10 characters +- `owner:` — your username (from `identity.user` in `evolve.config.yaml`) +- `visibility:` — set to `private` +- `atomic_skills:` — required for `skill-flow` type; list the slugs of component atomic skills + +If `## Requirements` or `## Imports` sections are present, make sure every package/tool they list is also mentioned in the content body. Remove any that are not. + +--- + +**Recall failure** — skill not surfacing in top-3 for its own trigger scenario + +``` +❌ my-skill FAIL [recall] rank=8 score=1 matched_terms=['cli'] +``` + +The `trigger` field does not contain words a user would type when describing the problem. Rewrite the trigger: +- Use the symptom, error message, or task keywords — not the solution. +- Include the specific command, flag name, or error text that distinguishes this skill from others. +- Inspect the `top5` results from `run_recall_tests.py --verbose` to see which skills outranked it and what vocabulary they share. Add differentiating terms. + +After rewriting the trigger, regenerate the pseudo-conversation fixture: +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py +``` +Then re-run `dedup.py --phase1-only` to verify the fix. + +--- + +**Skill evaluation failure** — content is not self-consistent + +``` +❌ my-skill FAIL [eval] score=0.25 missed=['orchestrate agents import', '--kind'] +``` + +`missed` shows command terms that should appear in the skill content but don't. For each missed term: +- Add the missing command, flag, or tool to the content body (preferred). +- If the term was mistakenly extracted, regenerate the fixture: `generate_skill_tests.py ` + +For `skill-flow` failures showing `unresolved atomic_skills: [slug1, slug2]` — the slugs in the `atomic_skills` frontmatter field do not correspond to real files. Either create those atomic skill files, or correct the slugs to match existing ones. + +--- + +**Multiple failures on the same skill** + +Fix in this order: format → content → recall. Format issues can mask the other checks, and fixing content often improves recall naturally (the distinguishing keywords appear in both). + +--- + +## Fixing regressions after dedup {#fixing-regressions-after-dedup} + +When `snapshot_test_results.py --compare` exits 1, one or more test fixtures reference a skill that was renamed, merged, or deleted during dedup. + +``` +❌ Recall test REGRESSION 12→10 passed (Δ-2) +``` + +**Step 1 — identify which fixtures broke** + +Run the recall test in verbose mode to find the newly-failing skills: +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_recall_tests.py --verbose +``` + +Look for `❌` entries that were `✅` before dedup. These fixtures reference slugs that no longer exist. + +**Step 2 — trace the dedup action** + +Open `.evolve/tests/dedup/refine_report.json`. Find the cluster that contained the now-missing slug. The report shows whether it was: +- **merged** into another skill (the surviving slug is `survivor`) +- **discarded** (the slug was deleted outright) + +**Step 3 — update or regenerate the fixture** + +- *Merged*: The surviving skill should cover the same scenario. Open the fixture file at `.evolve/tests/pseudo_conversations/.json` and update `skill_slug` to the survivor slug. Also update the path in the `conversation[system]` block if it references the old filename. Then verify: + ```bash + python3 .bob/skills/evolve-lite-test/scripts/run_recall_tests.py --verbose + ``` +- *Discarded*: The skill was intentionally removed. If the scenario it covered is no longer served by any skill, delete its fixture file — it is no longer a valid test. If the scenario *should* still be covered, the discard was wrong: restore the entity from git (`git checkout HEAD -- `) and re-run dedup with `--interactive` to keep that skill. + +**Step 4 — confirm no regression** + +```bash +python3 .bob/skills/evolve-lite-test/scripts/snapshot_test_results.py \ + --compare .evolve/tests/evaluation/pre_dedup_snapshot.json +``` + +This must exit 0 before the dedup run is considered complete. + +## Supporting Scripts + +| Script | Purpose | +|---|---| +| `scripts/quality_gate.py` | Phase 1 runner (can be run standalone) | +| `scripts/refine.py` | Phase 2 runner (can be run standalone) | +| `scripts/dedup.py` | Orchestrator — runs both phases in sequence | +| `evolve-lite-test/scripts/check_tests.py` | Content + recall gate with pass-rate threshold | +| `evolve-lite-test/scripts/snapshot_test_results.py` | Pre/post snapshot for regression detection | diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/dedup.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/dedup.py new file mode 100644 index 00000000..2848290e --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/dedup.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +evolve-lite dedup — two-phase skill library deduplication + +Phase 1 (quality_gate.py): + Runs ALL quality checks on every entity: + • Format check – required frontmatter, valid type, non-empty content + • Recall test – skill ranks in top-3 for its own trigger scenario + • Skill eval – content is self-consistent (must_include terms present); + skill-flow atomic_skills references resolve + Blocks Phase 2 if any skill fails. + +Phase 2 (refine.py): + Groups entities by token-set Jaccard similarity. For each cluster: + • keep-all – skills are distinct enough + • merge – combine similar skills into one enriched entity + • discard – remove near-identical duplicates, keep the richest + +Usage: + python3 dedup.py # full pipeline, auto decisions + python3 dedup.py --interactive # Phase 2 prompts for each cluster + python3 dedup.py --dry-run # show all decisions, write nothing + python3 dedup.py --phase1-only # quality gate only, no refinement + python3 dedup.py --phase2-only # skip quality gate (use with care) + python3 dedup.py --threshold 0.5 # override similarity threshold + python3 dedup.py --report-dir # write both reports to this directory + python3 dedup.py --verbose # print every skill in Phase 1 +""" + +import argparse +import subprocess +import sys +from pathlib import Path + +# Locate sibling scripts +_script = Path(__file__).resolve() +_scripts_dir = _script.parent +_quality_gate = _scripts_dir / "quality_gate.py" +_refine = _scripts_dir / "refine.py" + + +def run_script(script, extra_args, label): + """Run a python3 script as a subprocess. Returns its exit code.""" + cmd = [sys.executable, str(script)] + extra_args + print(f"\n{'='*70}") + print(f" {label}") + print(f"{'='*70}") + result = subprocess.run(cmd) + return result.returncode + + +def main(): + parser = argparse.ArgumentParser( + description="Two-phase skill library deduplication" + ) + parser.add_argument("--manifest-dir", default=None, + help="Build recall manifest from this directory instead of the live .evolve/entities/. " + "Forwarded to quality_gate.py.") + parser.add_argument("--phase1-only", action="store_true", + help="Run quality gate only, skip refinement") + parser.add_argument("--phase2-only", action="store_true", + help="Skip quality gate, run refinement only") + parser.add_argument("--interactive", action="store_true", + help="Prompt for decisions in Phase 2") + parser.add_argument("--dry-run", action="store_true", + help="Show all decisions without writing changes") + parser.add_argument("--threshold", type=float, default=None, + help="Jaccard similarity threshold for Phase 2 (default: 0.45)") + parser.add_argument("--entities-dir", default=None, + help="Override entities directory") + parser.add_argument("--report-dir", default=None, + help="Directory for JSON reports (default: .evolve/tests/dedup/)") + parser.add_argument("--verbose", action="store_true", + help="Print every skill result in Phase 1") + parser.add_argument("--local-only", action="store_true", + help="Score private entities only (forwards to quality_gate.py)") + args = parser.parse_args() + + # Resolve report dir + if args.report_dir: + report_dir = Path(args.report_dir) + else: + # Walk up to find .evolve + from pathlib import Path as _P + evolve = _P(".evolve") + report_dir = evolve / "tests" / "dedup" + report_dir.mkdir(parents=True, exist_ok=True) + + p1_report = report_dir / "quality_gate_report.json" + p2_report = report_dir / "refine_report.json" + + # ----------------------------------------------------------------------- + # Phase 1 – Quality Gate + # ----------------------------------------------------------------------- + phase1_passed = True + if not args.phase2_only: + p1_args = ["--report", str(p1_report)] + if args.entities_dir: + p1_args += ["--entities-dir", args.entities_dir] + if args.manifest_dir: + p1_args += ["--manifest-dir", args.manifest_dir] + if args.verbose: + p1_args.append("--verbose") + if args.local_only: + p1_args.append("--local-only") + + rc = run_script(_quality_gate, p1_args, "PHASE 1 — Quality Gate") + if rc != 0: + phase1_passed = False + print() + print("❌ Phase 1 failed. Resolve quality issues before running Phase 2.") + print(f" Report: {p1_report}") + if not args.phase1_only: + print(" Phase 2 (refine) was NOT run.") + sys.exit(1) + print() + print("✅ Phase 1 passed.") + + if args.phase1_only: + print(f"\nReport: {p1_report}") + sys.exit(0) + + # ----------------------------------------------------------------------- + # Phase 2 – Refine + # ----------------------------------------------------------------------- + p2_args = ["--report", str(p2_report)] + if args.entities_dir: + p2_args += ["--entities-dir", args.entities_dir] + if args.interactive: + p2_args.append("--interactive") + if args.dry_run: + p2_args.append("--dry-run") + if args.threshold is not None: + p2_args += ["--threshold", str(args.threshold)] + + rc = run_script(_refine, p2_args, "PHASE 2 — Refine (Deduplication)") + if rc != 0: + print("\n❌ Phase 2 encountered an error.") + sys.exit(rc) + + print() + print("✅ Dedup complete.") + print(f" Phase 1 report : {p1_report}") + print(f" Phase 2 report : {p2_report}") + sys.exit(0) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/quality_gate.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/quality_gate.py new file mode 100644 index 00000000..258cf0ca --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/quality_gate.py @@ -0,0 +1,884 @@ +#!/usr/bin/env python3 +""" +Phase 1 – Quality Gate + +Runs ALL three test suites against the entity library before dedup is allowed: + + 1. FORMAT CHECK – required frontmatter fields present; content non-empty; + type is a valid value; trigger is not suspiciously short. + + 2. RECALL TEST – the skill must rank in the top-3 of the manifest for the + user message derived from its own trigger (same heuristic + as run_recall_tests.py). + + 3. SKILL EVALUATION – the skill content must contain its own must_include terms + (self-consistency check from run_skill_evaluation.py). + For skill-flows, all referenced atomic_skills must exist. + +Exits 0 only when every skill passes all three suites. +A non-zero exit blocks Phase 2 (refine.py / dedup.py). + +Usage: + python3 quality_gate.py + python3 quality_gate.py --verbose + python3 quality_gate.py --entities-dir + python3 quality_gate.py --report +""" + +import argparse +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +# --------------------------------------------------------------------------- +# Bootstrap: locate lib/evolve-lite +# --------------------------------------------------------------------------- +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find lib/evolve-lite above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + load_manifest, + find_recall_entity_dirs, + dedupe_manifest_entries, + markdown_to_entity, + slugify, +) + +# Pull in pseudo-conversation helpers from the evolve-lite-test sibling +_test_scripts = _script.parents[2] / "evolve-lite-test" / "scripts" +if _test_scripts.is_dir(): + sys.path.insert(0, str(_test_scripts)) + try: + from generate_pseudo_conversations import ( # noqa: E402 + build_pseudo_conversation, + trigger_to_realistic_question, + build_expected_behaviour, + ) + _HAVE_BUILDER = True + except ImportError: + _HAVE_BUILDER = False +else: + _HAVE_BUILDER = False + + +# =========================================================================== +# Suite 1 – Format check +# =========================================================================== + +REQUIRED_FRONTMATTER = ("type", "trigger", "owner", "visibility") +VALID_TYPES = {"guideline", "atomic-skill", "skill-flow"} + + +def check_format(entity): + """Return list of format violation strings, or [] if clean.""" + issues = [] + for field in REQUIRED_FRONTMATTER: + if not entity.get(field): + issues.append(f"missing frontmatter field: '{field}'") + entity_type = entity.get("type", "") + if entity_type and entity_type not in VALID_TYPES: + issues.append(f"invalid type '{entity_type}' — must be one of {sorted(VALID_TYPES)}") + if not entity.get("content", "").strip(): + issues.append("content body is empty") + trigger = entity.get("trigger", "").strip() + if trigger and len(trigger) < 10: + issues.append(f"trigger too short ({len(trigger)} chars): '{trigger}'") + if entity.get("type") == "skill-flow": + if not entity.get("atomic_skills", "").strip(): + issues.append("skill-flow has no atomic_skills references") + # version must be a positive integer when present + version_raw = entity.get("version", "") + if version_raw: + try: + v = int(version_raw) + if v < 1: + raise ValueError + except (ValueError, TypeError): + issues.append(f"version must be a positive integer, got: '{version_raw}'") + return issues + + +def check_version(entity): + """ + Suite 5 – Version / changelog consistency. + + Returns a list of *warning* strings (non-blocking — does not fail the gate). + A warning fires when version > 1 but the ## Changelog section has fewer + entries than the stated version number. + + Changelog entries are counted by lines that look like version markers: + '- v:', 'v:', '## v', or '**v**'. A bare list of bullet + points (one per version bump) also counts — each '-' or '*' at the start + of a line is treated as one entry. + """ + warnings = [] + version_raw = entity.get("version", "") + if not version_raw: + return warnings + + try: + version = int(version_raw) + except (ValueError, TypeError): + return warnings # already caught by check_format + + if version <= 1: + return warnings + + changelog = entity.get("changelog", "").strip() + if not changelog: + warnings.append( + f"version={version} but ## Changelog section is missing — " + "add a changelog entry for each version bump" + ) + return warnings + + # Count lines that look like individual version entries + entry_count = sum( + 1 for line in changelog.splitlines() + if re.match(r"^\s*[-*]|^\s*v\d+\b|^\s*#+\s*v\d+\b|\*\*v\d+\b", line.strip()) + ) + if entry_count == 0: + # Any non-empty paragraph counts as at least one entry + entry_count = 1 + + if entry_count < version: + warnings.append( + f"version={version} but only {entry_count} changelog " + f"entry(ies) found — add entries for each version bump" + ) + return warnings + + +# =========================================================================== +# Suite 4 – Filename & trigger naming quality +# =========================================================================== + +# Verbs that indicate a well-formed imperative/conditional trigger +_TRIGGER_VERB_PATTERNS = re.compile( + r"\b(when|after|before|while|if|create|set up|setup|install|configure|run|ensure|" + r"import|activate|deploy|use|handle|build|check|validate|update|add|remove|generate|" + r"upload|download|enable|disable|start|stop|connect|authenticate|define|write|" + r"initialize|initialise|debug|troubleshoot|fix|resolve|migrate)\b", + re.IGNORECASE, +) + +# Patterns that indicate a stutter / accidental duplication in a trigger +_STUTTER_PATTERN = re.compile( + r"\b(\w{4,})\b.{0,10}\b\1\b", re.IGNORECASE +) + +# Minimum fraction of trigger tokens that must appear in the slug (case-insensitive) +_SLUG_COVERAGE_THRESHOLD = 0.25 + +_SLUG_STOP = { + "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "as", "is", "was", "are", "were", "be", + "been", "being", "have", "has", "had", "do", "does", "did", "will", + "would", "should", "could", "may", "might", "must", "can", "this", + "that", "these", "those", "i", "you", "he", "she", "it", "we", "they", + "when", "after", "before", "while", "if", "how", "what", "my", "your", + "just", "need", "want", "make", "sure", "some", "also", "about", +} + + +def _slug_tokens(text): + """Split a slug or plain text into meaningful lowercase tokens.""" + words = re.findall(r"[a-z0-9]+", text.lower()) + return [w for w in words if w not in _SLUG_STOP and len(w) > 2] + + +def check_naming(entity, path): + """ + Suite 4: check that the filename slug and trigger text are well-formed. + + Returns a list of issue strings (empty = clean). + Distinguishes warnings (prefixed 'warn:') from hard failures so callers + can decide severity. Currently all issues are treated equally (hard fail). + """ + issues = [] + slug = Path(path).stem + trigger = entity.get("trigger", "").strip() + + # --- Trigger checks --- + if trigger: + # Must contain at least one recognised action verb or 'when/after/...' + if not _TRIGGER_VERB_PATTERNS.search(trigger): + issues.append( + "trigger has no recognisable action verb or conditional keyword" + ) + + # Stutter: same word repeated close together (e.g. "creating create") + stutter = _STUTTER_PATTERN.search(trigger) + if stutter: + issues.append( + f"trigger contains likely stutter/duplication: " + f"'{stutter.group(0).strip()}'" + ) + + # Trigger should not be a verbatim copy of the content + content = entity.get("content", "").strip() + if content and trigger.lower() == content.lower(): + issues.append("trigger is identical to content body") + + # Minimum useful length (already covered by check_format for < 10, + # add a stronger floor here) + if len(trigger) < 15: + issues.append( + f"trigger is too brief ({len(trigger)} chars) — add more context" + ) + + # --- Slug / filename checks --- + if slug: + # Slug must not contain spaces or uppercase (would indicate wrong creation) + if re.search(r"[A-Z ]", slug): + issues.append(f"filename slug contains uppercase or spaces: '{slug}'") + + # Slug should have at least 2 meaningful tokens after stopword removal + slug_tok = _slug_tokens(slug) + if len(slug_tok) < 2: + issues.append( + f"filename slug is too generic (only {len(slug_tok)} meaningful " + f"token(s)): '{slug}'" + ) + + # Slug should overlap meaningfully with the trigger + if trigger: + trig_tok = set(_slug_tokens(trigger)) + if trig_tok: + overlap = len(set(slug_tok) & trig_tok) / len(trig_tok) + if overlap < _SLUG_COVERAGE_THRESHOLD: + issues.append( + f"filename slug has low overlap with trigger " + f"({overlap:.0%} < {_SLUG_COVERAGE_THRESHOLD:.0%}): " + f"slug='{slug}'" + ) + + return issues + + +# =========================================================================== +# Suite 2 – Recall test (keyword-score heuristic) +# =========================================================================== + +_STOP = { + "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "as", "is", "was", "are", "were", "be", + "been", "being", "have", "has", "had", "do", "does", "did", "will", + "would", "should", "could", "may", "might", "must", "can", "this", + "that", "these", "those", "i", "you", "he", "she", "it", "we", "they", + "when", "after", "before", "while", "if", "how", "what", "my", "your", + "just", "need", "want", "make", "sure", "some", "also", "about", +} + + +def _tokenise(text): + words = re.findall(r"[a-z0-9]+", text.lower()) + return [w for w in words if w not in _STOP and len(w) > 2] + + +def _score_trigger(trigger, user_msg): + t_tokens = set(_tokenise(trigger)) + u_tokens = set(_tokenise(user_msg)) + matched = t_tokens & u_tokens + return sum(len(w) for w in matched), matched + + +def _rank_manifest(manifest, user_msg): + scored = [] + for entry in manifest: + s, matched = _score_trigger(entry["trigger"], user_msg) + scored.append({**entry, "score": s, "matched_terms": list(matched)}) + return sorted(scored, key=lambda e: e["score"], reverse=True) + + +def _derive_user_message(entity): + """Generate a realistic user message from the entity's trigger.""" + trigger = entity.get("trigger", "") + if _HAVE_BUILDER: + return trigger_to_realistic_question(trigger) + text = re.sub(r"^(When|After|While|Before|If)\s+", "", trigger, flags=re.IGNORECASE).strip() + if text: + text = text[0].lower() + text[1:] + return f"I am trying to {text} and ran into a problem. What should I do?" + + +def check_recall(entity, path, manifest): + """ + Returns dict: + passed – bool: skill is rank-1 or in top-3 with score > 0 + rank – int or None + score – int + matched_terms – list[str] + user_msg – str used for the simulated query + """ + skill_slug = Path(path).stem + user_msg = _derive_user_message(entity) + ranked = _rank_manifest(manifest, user_msg) + + rank = None + score = 0 + matched = [] + for i, entry in enumerate(ranked): + if Path(entry["path"]).stem == skill_slug: + rank = i + 1 + score = entry["score"] + matched = entry.get("matched_terms", []) + break + + passed = rank is not None and rank <= 3 and score > 0 + return { + "passed": passed, + "rank": rank, + "score": score, + "matched_terms": matched, + "user_msg": user_msg, + } + + +# =========================================================================== +# Suite 3 – Skill evaluation (content self-consistency + composition) +# =========================================================================== + +def _extract_must_not_include(content): + patterns = [ + r"does not accept\s+(`[^`]+`|\S+)", + r"not as a\s+(`[^`]+`|\S+)", + r"instead of\s+(`[^`]+`|\S+)", + r"avoid\s+(`[^`]+`|\S+)", + r"do not (?:use|suggest)\s+(`[^`]+`|\S+)", + r"not use\s+(`[^`]+`|\S+)", + ] + terms = [] + for pat in patterns: + for m in re.finditer(pat, content, re.IGNORECASE): + term = m.group(1).strip("`").strip() + if term: + terms.append(term) + return terms + + +def _build_must_include(content): + """Derive must_include terms from skill content (mirrors build_expected_behaviour).""" + if _HAVE_BUILDER: + dummy_path = Path("dummy.md") + eb = build_expected_behaviour(dummy_path, {"content": content}) + return eb.get("must_include", []) + + # Minimal fallback + backtick = re.findall(r"`([^`]+)`", content) + if backtick: + return [re.sub(r"\s*<[^>]+>", "", t).strip() for t in backtick if t.strip()] + candidates = re.findall(r"\b([a-z][a-z0-9]*(?:_[a-z0-9]+)+)\b", content) + seen: set = set() + result = [] + for t in candidates: + if t not in seen: + seen.add(t) + result.append(t) + return result or [] + + +def _normalise_atomic_ref(text): + text = text.strip().lower() + text = re.sub(r"\s*`([^`]+)`\s*", r" \1 ", text) + text = re.sub(r"\s*<[^>]+>\s*", " ", text) + text = re.sub(r"[^a-z0-9]+", "-", text) + return text.strip("-") + + +def _trim_action_prefix(text): + prefixes = ( + "to-", + "when-", + "create-", + "set-up-", + "install-", + "configure-", + "run-", + "ensure-", + "import-", + "activate-", + ) + for prefix in prefixes: + if text.startswith(prefix): + return text[len(prefix):] + return text + + +def _canonical_atomic_skill_refs(entities_dir): + """Build a lookup of all known atomic skill slugs. + + Searches both the private entities dir and every subscribed clone so that + skill-flow composition checks pass even after referenced atomic skills have + been published (moved into the subscribed clone). + """ + entities_dir = Path(entities_dir) + # Collect search roots: private dir + all subscribed clones + search_roots = [entities_dir] + subscribed_root = entities_dir / "subscribed" + if subscribed_root.is_dir(): + for clone_dir in subscribed_root.iterdir(): + if clone_dir.is_dir() and (clone_dir / ".git").exists(): + # Published entities land directly under clone_dir/atomic-skill/ + search_roots.append(clone_dir) + # The clone may also carry its own .evolve/entities/ subtree + nested = clone_dir / ".evolve" / "entities" + if nested.is_dir(): + search_roots.append(nested) + + refs = {} + for root in search_roots: + for path in Path(root).glob("**/*.md"): + if path.is_symlink() or ".git" in path.parts: + continue + try: + rel_parts = path.relative_to(root).parts + except ValueError: + continue + if "atomic-skill" not in rel_parts: + continue + try: + entity = markdown_to_entity(path) + except Exception: + continue + slug = path.stem + refs[slug] = path + refs[_normalise_atomic_ref(slug)] = path + content = entity.get("content", "").strip() + if content: + normalized_content = _normalise_atomic_ref(content) + refs[slugify(content)] = path + refs[normalized_content] = path + refs[_trim_action_prefix(normalized_content)] = path + + trigger = entity.get("trigger", "").strip() + if trigger: + normalized_trigger = _normalise_atomic_ref(trigger) + refs[normalized_trigger] = path + refs[_trim_action_prefix(normalized_trigger)] = path + return refs + + +# Minimum fraction of meaningful trigger tokens that must appear in content +_TRIGGER_COVERAGE_THRESHOLD = 0.3 + + +def _trigger_coverage(trigger, content): + """Return fraction of meaningful trigger tokens found in content text.""" + trig_tok = _slug_tokens(trigger) + if not trig_tok: + return 1.0 + content_lower = content.lower() + found = sum(1 for t in trig_tok if t in content_lower) + return found / len(trig_tok) + + +def _check_section_coverage(section_text, entity, section_name): + """ + Verify that items declared in a metadata section are referenced somewhere + in the entity's full text (content + rationale + documentation). + + For ## Requirements: each line is a package/tool name. + - Pip packages: bare name before any version specifier must appear in text. + e.g. "pyyaml>=6.0" → search for "pyyaml" + - CLI tools: lines starting with "cli:" strip that prefix before searching. + e.g. "cli: orchestrate" → search for "orchestrate" + For ## Imports: each non-empty line is an import statement; at least the + primary module name or imported symbol must appear in the full text. + + Searches the combined body of: content, rationale, and documentation — + so a tool named only in the ## Documentation prose still passes. + + Returns list of unmatched declaration strings (empty = all covered). + """ + if not section_text: + return [] + + # Build a single search corpus from all non-section body parts + parts = [ + entity.get("content", ""), + entity.get("rationale", ""), + entity.get("documentation", ""), + ] + corpus = " ".join(p for p in parts if p).lower() + if not corpus.strip(): + return [] + + unmatched = [] + + for raw_line in section_text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + + if section_name == "requirements": + # Strip "cli: " prefix for CLI tool entries + normalised = re.sub(r"^cli:\s*", "", line, flags=re.IGNORECASE).strip() + # Strip version specifiers: "pyyaml>=6.0" → "pyyaml" + bare = re.split(r"[>== 0.5 and constraint_ok + + # Trigger-coverage check: content must address the trigger's key concepts. + # This catches skills whose body has drifted away from their stated purpose. + trig_tok = _slug_tokens(trigger) if trigger else [] + trig_gaps = [t for t in trig_tok if t not in resp_lower] + trig_cov = round(1.0 - len(trig_gaps) / len(trig_tok), 4) if trig_tok else 1.0 + trigger_passed = trig_cov >= _TRIGGER_COVERAGE_THRESHOLD + + # Section-coverage check: declared requirements and imports must be + # referenced somewhere in the entity's full text (content + rationale + + # documentation) so every declared dependency has visible context. + section_issues: list = [] + for sec in ("requirements", "imports"): + unmatched = _check_section_coverage(entity.get(sec, ""), entity, sec) + for item in unmatched: + section_issues.append( + f"{sec}: declared '{item}' not referenced in entity text" + ) + + # Composition check for skill-flows + composition_issues = [] + if entity.get("type") == "skill-flow": + refs_raw = entity.get("atomic_skills", "") + refs = [r.strip() for r in refs_raw.split(",") if r.strip()] + if atomic_skill_refs is None: + atomic_skill_refs = _canonical_atomic_skill_refs(entities_dir) + for ref_slug in refs: + normalized_ref = _normalise_atomic_ref(ref_slug) + if ref_slug not in atomic_skill_refs and normalized_ref not in atomic_skill_refs: + composition_issues.append(f"atomic skill not found: '{ref_slug}'") + + passed = content_passed and trigger_passed and not composition_issues and not section_issues + return { + "passed": passed, + "alignment_score": score, + "trigger_coverage": trig_cov, + "matched": matched, + "missed": missed, + "violated": violated, + "trigger_gaps": trig_gaps, + "composition_issues": composition_issues, + "section_issues": section_issues, + } + + +# Only check entities in the owned type subdirectories, not subscribed mirrors +_OWNED_TYPE_DIRS = {"atomic-skill", "guideline", "skill-flow"} + + +# =========================================================================== +# Orchestrate all suites per entity +# =========================================================================== + +def run_quality_gate(entities_dir, manifest, verbose): + entities_dir = Path(entities_dir) + md_files = sorted( + p for p in entities_dir.glob("**/*.md") + if not p.is_symlink() + and ".git" not in p.parts + and "subscribed" not in p.relative_to(entities_dir).parts + and any(part in _OWNED_TYPE_DIRS for part in p.relative_to(entities_dir).parts) + ) + if not md_files: + return [] + + atomic_skill_refs = _canonical_atomic_skill_refs(entities_dir) + results = [] + for path in md_files: + try: + entity = markdown_to_entity(path) + except Exception as exc: + r = { + "path": str(path), + "slug": path.stem, + "type": None, + "trigger": "", + "format": {"issues": [f"could not parse: {exc}"], "passed": False}, + "recall": None, + "evaluation": None, + "naming": None, + "passed": False, + } + results.append(r) + if verbose: + _print_one(r) + continue + + fmt_issues = check_format(entity) + recall = check_recall(entity, path, manifest) + evaluation = check_skill_evaluation(entity, entities_dir, atomic_skill_refs) + naming_issues = check_naming(entity, path) + version_warnings = check_version(entity) + + passed = ( + (not fmt_issues) + and recall["passed"] + and evaluation["passed"] + and (not naming_issues) + # version_warnings are non-blocking + ) + r = { + "path": str(path), + "slug": path.stem, + "type": entity.get("type"), + "trigger": entity.get("trigger", ""), + "format": {"issues": fmt_issues, "passed": not fmt_issues}, + "recall": recall, + "evaluation": evaluation, + "naming": {"issues": naming_issues, "passed": not naming_issues}, + "version": { + "value": entity.get("version", ""), + "warnings": version_warnings, + }, + "passed": passed, + } + results.append(r) + if verbose or not passed: + _print_one(r) + + return results + + +# =========================================================================== +# Output helpers +# =========================================================================== + +def _print_one(r): + status = "✅" if r["passed"] else "❌" + slug = r["slug"] + parts = [] + + fmt = r.get("format", {}) + parts.append("fmt:ok" if fmt.get("passed") else f"fmt:❌({len(fmt.get('issues', []))})") + + rc = r.get("recall") + if rc: + parts.append(f"recall:{'ok' if rc['passed'] else '❌'}(rank={rc['rank']},score={rc['score']})") + else: + parts.append("recall:skipped") + + ev = r.get("evaluation") + if ev: + parts.append( + f"eval:{'ok' if ev['passed'] else '❌'}" + f"(align={ev['alignment_score']:.2f},trig={ev['trigger_coverage']:.2f})" + ) + else: + parts.append("eval:skipped") + + nm = r.get("naming") + if nm is not None: + parts.append("name:ok" if nm.get("passed") else f"name:❌({len(nm.get('issues', []))})") + + vr = r.get("version", {}) + if vr.get("value"): + v_tag = f"v{vr['value']}" + v_tag += "⚠" if vr.get("warnings") else "" + parts.append(v_tag) + + print(f" {status} {slug:<58} {' '.join(parts)}") + + for issue in fmt.get("issues", []): + print(f" format ⚠ {issue}") + if rc and not rc["passed"]: + print(f" recall ⚠ rank={rc['rank']} matched={rc['matched_terms']}") + print(f" user_msg: {rc['user_msg'][:80]}") + if ev: + if ev["missed"]: + print(f" eval ⚠ missed must_include: {ev['missed']}") + if ev["violated"]: + print(f" eval ⚠ violated must_not_include: {ev['violated']}") + # Only surface trigger_gaps when the skill actually failed the trigger-coverage check + if ev.get("trigger_gaps") and not ev["passed"]: + print(f" eval ⚠ trigger keywords missing from content: {ev['trigger_gaps']}") + for ci in ev.get("composition_issues", []): + print(f" eval ⚠ composition: {ci}") + for si in ev.get("section_issues", []): + print(f" eval ⚠ section: {si}") + if nm and not nm.get("passed"): + for issue in nm.get("issues", []): + print(f" naming ⚠ {issue}") + for w in vr.get("warnings", []): + print(f" version ⚠ {w}") + + +def print_summary(results): + total = len(results) + passed = sum(1 for r in results if r["passed"]) + fmt_fail = sum(1 for r in results if not r.get("format", {}).get("passed", True)) + recall_fail = sum(1 for r in results if r.get("recall") and not r["recall"]["passed"]) + eval_fail = sum(1 for r in results if r.get("evaluation") and not r["evaluation"]["passed"]) + naming_fail = sum(1 for r in results if r.get("naming") and not r["naming"]["passed"]) + version_warn = sum(1 for r in results if r.get("version", {}).get("warnings")) + + print() + print("QUALITY GATE SUMMARY") + print("=" * 70) + + if passed < total: + print("Failed skills:") + for r in results: + if not r["passed"]: + _print_one(r) + print() + + print(f" Total skills : {total}") + print(f" ✅ Passed : {passed}") + print(f" ❌ Failed : {total - passed}") + if fmt_fail: + print(f" └ format issues : {fmt_fail}") + if recall_fail: + print(f" └ recall issues : {recall_fail}") + if eval_fail: + print(f" └ eval issues : {eval_fail}") + if naming_fail: + print(f" └ naming issues : {naming_fail}") + if version_warn: + print(f" ⚠ Version warnings: {version_warn} (non-blocking)") + + +# =========================================================================== +# Entry point +# =========================================================================== + +def main(): + parser = argparse.ArgumentParser( + description="Phase 1: run all quality checks before dedup" + ) + parser.add_argument("--entities-dir", default=None) + parser.add_argument("--manifest-dir", default=None, + help="Build recall manifest from this directory instead of the live .evolve/entities/. " + "Use when running quality gate on a merge workspace.") + parser.add_argument("--report", default=None, + help="Write JSON report to this path") + parser.add_argument("--verbose", action="store_true", + help="Print every skill, not just failures") + parser.add_argument("--local-only", action="store_true", + help="Build manifest from private entities only, skipping subscribed/ subdirectories") + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + entities_dir = Path(args.entities_dir) if args.entities_dir \ + else evolve_dir / "entities" + + if not entities_dir.exists(): + print(f"No entities directory at {entities_dir}. Nothing to check.") + sys.exit(0) + + if args.manifest_dir: + # Use the caller-supplied directory as the recall manifest source + manifest = dedupe_manifest_entries(load_manifest(Path(args.manifest_dir))) + elif args.local_only: + # Build manifest from private entities only — skip subscribed/ clones + manifest = dedupe_manifest_entries(load_manifest(entities_dir)) + else: + raw = [] + for root in find_recall_entity_dirs(): + raw.extend(load_manifest(root)) + manifest = dedupe_manifest_entries(raw) + + if not manifest: + print("Error: recall manifest is empty — no entities loaded.", file=sys.stderr) + sys.exit(1) + + print(f"Manifest : {len(manifest)} entities") + print(f"Directory: {entities_dir}") + print() + print("QUALITY GATE — Phase 1") + print("=" * 70) + + results = run_quality_gate(entities_dir, manifest, args.verbose) + + if not results: + print("No entity files found.") + sys.exit(0) + + print_summary(results) + + if args.report: + report_path = Path(args.report) + report_path.parent.mkdir(parents=True, exist_ok=True) + report = { + "generated_at": datetime.now().isoformat(), + "total": len(results), + "passed": sum(1 for r in results if r["passed"]), + "failed": sum(1 for r in results if not r["passed"]), + "results": results, + } + with open(report_path, "w", encoding="utf-8") as fh: + json.dump(report, fh, indent=2) + print(f"\nReport written: {report_path}") + + all_passed = all(r["passed"] for r in results) + if not all_passed: + print("\n❌ Quality gate FAILED — fix issues above before running refine.") + else: + print("\n✅ Quality gate PASSED — safe to proceed to Phase 2 (refine).") + + sys.exit(0 if all_passed else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/refine.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/refine.py new file mode 100644 index 00000000..1e81387f --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-dedup/scripts/refine.py @@ -0,0 +1,605 @@ +#!/usr/bin/env python3 +""" +Phase 2 – Refine (Deduplication) + +Groups entities by semantic similarity using token-set Jaccard distance. +For each cluster of similar skills, decides whether to: + - keep-all : skills are different enough (Jaccard < LOW threshold) + - merge : combine into a single consolidated entity (default for medium similarity) + - discard : remove the weaker duplicate (default for high similarity / near-identical) + +By default runs non-interactively using automatic thresholds. +Use --interactive to review and decide each cluster manually. + +Must only be run after quality_gate.py exits 0. + +Usage: + python3 refine.py + python3 refine.py --interactive + python3 refine.py --dry-run # show decisions, make no changes + python3 refine.py --threshold 0.5 # override merge threshold (default 0.45) + python3 refine.py --report +""" + +import argparse +import json +import os +import sys +import tempfile +from datetime import datetime +from pathlib import Path + +# --------------------------------------------------------------------------- +# Bootstrap: locate lib/evolve-lite +# --------------------------------------------------------------------------- +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find lib/evolve-lite above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + check_banality, + get_evolve_dir, + markdown_to_entity, + entity_to_markdown, + slugify, + unique_filename, +) + + +# =========================================================================== +# Similarity +# =========================================================================== + +_STOP = { + "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "as", "is", "was", "are", "were", "be", + "been", "being", "have", "has", "had", "do", "does", "did", "will", + "would", "should", "could", "may", "might", "must", "can", "this", + "that", "these", "those", "i", "you", "he", "she", "it", "we", "they", + "when", "after", "before", "while", "if", "how", "what", "my", "your", + "just", "need", "want", "make", "sure", "some", "also", "about", +} + + +def _tokens(text): + import re + words = re.findall(r"[a-z0-9]+", text.lower()) + return {w for w in words if w not in _STOP and len(w) > 2} + + +def jaccard(a, b): + """Token-set Jaccard similarity between two strings. Returns float [0, 1].""" + ta = _tokens(a) + tb = _tokens(b) + if not ta and not tb: + return 1.0 + if not ta or not tb: + return 0.0 + inter = len(ta & tb) + union = len(ta | tb) + return inter / union + + +def similarity_text(entity): + """Combined text used for similarity comparison: trigger + content.""" + return f"{entity.get('trigger', '')} {entity.get('content', '')}" + + +# =========================================================================== +# Clustering (greedy single-linkage) +# =========================================================================== + +def build_clusters(entities_with_paths, threshold): + """ + Group entities into clusters where at least one pair has Jaccard >= threshold. + + Args: + entities_with_paths: list of (path, entity_dict) + threshold: float — Jaccard score at or above which two skills are "similar" + + Returns: + List of clusters, each cluster is a list of (path, entity_dict, sim_to_prev). + Singletons (no similar partner) are returned as clusters of size 1. + """ + n = len(entities_with_paths) + # Build similarity matrix + sims = [[0.0] * n for _ in range(n)] + for i in range(n): + for j in range(i + 1, n): + s = jaccard( + similarity_text(entities_with_paths[i][1]), + similarity_text(entities_with_paths[j][1]), + ) + sims[i][j] = sims[j][i] = s + + # Greedy single-linkage clustering + assigned = [False] * n + clusters = [] + for i in range(n): + if assigned[i]: + continue + cluster = [(i, 0.0)] + assigned[i] = True + for j in range(i + 1, n): + if assigned[j]: + continue + # Check if j is similar to any member already in the cluster + for ci, _ in cluster: + if sims[ci][j] >= threshold: + cluster.append((j, sims[i][j])) + assigned[j] = True + break + clusters.append(cluster) + + return [ + [(entities_with_paths[idx][0], entities_with_paths[idx][1], sim) + for idx, sim in cluster] + for cluster in clusters + ] + + +# =========================================================================== +# Decision logic +# =========================================================================== + +# Jaccard thresholds for automatic decisions +# >= DISCARD_THRESHOLD → near-identical, discard the shorter one +# >= merge_threshold → similar enough to merge +# < merge_threshold → keep-all (dissimilar) + +DISCARD_THRESHOLD = 0.75 + + +def _auto_decision(cluster, merge_threshold): + """ + For a cluster of (path, entity, sim) tuples, return: + action – "keep-all" | "merge" | "discard" + keep – list of paths to keep + remove – list of paths to remove or merge away + reason – human-readable string + """ + if len(cluster) == 1: + return {"action": "keep-all", "keep": [cluster[0][0]], "remove": [], "reason": "singleton"} + + # Find max pairwise similarity + paths = [c[0] for c in cluster] + entities = [c[1] for c in cluster] + max_sim = 0.0 + for i in range(len(entities)): + for j in range(i + 1, len(entities)): + s = jaccard(similarity_text(entities[i]), similarity_text(entities[j])) + if s > max_sim: + max_sim = s + + if max_sim >= DISCARD_THRESHOLD: + # Keep the one with the most content (longest), discard others + best = max(cluster, key=lambda c: len(c[1].get("content", ""))) + keep = [best[0]] + remove = [c[0] for c in cluster if c[0] != best[0]] + return { + "action": "discard", + "keep": keep, + "remove": remove, + "max_similarity": round(max_sim, 3), + "reason": f"near-identical (Jaccard={max_sim:.2f} ≥ {DISCARD_THRESHOLD}) — keeping the most detailed version", + } + else: + # Merge: the primary skill gets enriched trigger; others removed + primary = max(cluster, key=lambda c: len(c[1].get("content", ""))) + keep = [primary[0]] + remove = [c[0] for c in cluster if c[0] != primary[0]] + return { + "action": "merge", + "keep": keep, + "remove": remove, + "merge_into": primary[0], + "max_similarity": round(max_sim, 3), + "reason": f"similar (Jaccard={max_sim:.2f} ≥ {merge_threshold}) — merging into richest skill", + } + + +def _interactive_decision(cluster, merge_threshold): + """Print cluster details and ask the user to choose an action.""" + print() + print("─" * 70) + print(f" CLUSTER of {len(cluster)} similar skills") + print() + for i, (path, entity, sim) in enumerate(cluster): + label = f" [{i+1}] {Path(path).stem}" + sim_str = f" sim={sim:.2f}" if i > 0 else " (primary)" + print(f"{label}{sim_str}") + print(f" trigger : {entity.get('trigger', '')[:70]}") + print(f" content : {entity.get('content', '')[:100].strip()}") + print() + + auto = _auto_decision(cluster, merge_threshold) + print(f" Suggested: {auto['action'].upper()} — {auto['reason']}") + print() + print(" Options:") + print(" k keep-all — no changes, all skills stay") + print(" m merge — merge all into richest skill") + print(" d discard — keep richest, remove others") + print(" s skip — skip this cluster (decide later)") + + while True: + choice = input(" Choice [k/m/d/s] (Enter = accept suggestion): ").strip().lower() + if choice == "" or choice == auto["action"][0]: + return auto + if choice == "k": + paths = [c[0] for c in cluster] + return {"action": "keep-all", "keep": paths, "remove": [], "reason": "manual: keep-all"} + if choice == "m": + primary = max(cluster, key=lambda c: len(c[1].get("content", ""))) + keep = [primary[0]] + remove = [c[0] for c in cluster if c[0] != primary[0]] + return {"action": "merge", "keep": keep, "remove": remove, + "merge_into": primary[0], "reason": "manual: merge"} + if choice == "d": + primary = max(cluster, key=lambda c: len(c[1].get("content", ""))) + keep = [primary[0]] + remove = [c[0] for c in cluster if c[0] != primary[0]] + return {"action": "discard", "keep": keep, "remove": remove, "reason": "manual: discard"} + if choice == "s": + paths = [c[0] for c in cluster] + return {"action": "keep-all", "keep": paths, "remove": [], "reason": "skipped by user"} + print(" Please enter k, m, d, s, or press Enter.") + + +# =========================================================================== +# Apply decisions +# =========================================================================== + +def _sentence_tokens(text): + """Split content into a list of stripped, non-empty sentences/lines.""" + import re as _re + # Split on sentence-ending punctuation or newlines; keep the delimiter + raw = _re.split(r"(?<=[.!?])\s+|\n+", text) + return [s.strip() for s in raw if s.strip()] + + +def _unique_sentences(primary_sentences, secondary_sentences, similarity_threshold=0.5): + """ + Return sentences from *secondary* that are not already covered by *primary*. + + A secondary sentence is considered covered if its token-set Jaccard score + against any primary sentence is >= *similarity_threshold*. + """ + import re as _re + + def _tok(s): + words = _re.findall(r"[a-z0-9]+", s.lower()) + return {w for w in words if len(w) > 2} + + def _jac(a, b): + ta, tb = _tok(a), _tok(b) + if not ta and not tb: + return 1.0 + if not ta or not tb: + return 0.0 + return len(ta & tb) / len(ta | tb) + + result = [] + for sec in secondary_sentences: + if not any(_jac(sec, pri) >= similarity_threshold for pri in primary_sentences): + result.append(sec) + return result + + +def _build_merged_entity(cluster): + """ + Build a merged entity from a cluster. + - trigger : all unique triggers joined with '; ' + - content : primary content + unique sentences from each secondary skill, + appended as a new paragraph so nothing is silently dropped + - rationale: combined unique rationale lines (if present) + - other frontmatter: taken from the richest (longest-content) skill + """ + primary_path, primary_entity, _ = max(cluster, key=lambda c: len(c[1].get("content", ""))) + + # --- merge triggers --- + all_triggers = [] + seen_triggers: set = set() + for _, ent, _ in cluster: + t = ent.get("trigger", "").strip() + if t and t not in seen_triggers: + all_triggers.append(t) + seen_triggers.add(t) + + # --- merge content --- + primary_sentences = _sentence_tokens(primary_entity.get("content", "")) + extra_sentences: list = [] + for _, ent, _ in cluster: + if ent is primary_entity: + continue + sec_sentences = _sentence_tokens(ent.get("content", "")) + extra_sentences.extend(_unique_sentences(primary_sentences + extra_sentences, sec_sentences)) + + if extra_sentences: + merged_content = ( + primary_entity.get("content", "").rstrip() + + "\n\n" + + " ".join(extra_sentences) + ) + else: + merged_content = primary_entity.get("content", "") + + # --- merge rationale --- + primary_rationale = primary_entity.get("rationale", "").strip() + rationale_parts = [primary_rationale] if primary_rationale else [] + seen_rat: set = {primary_rationale} if primary_rationale else set() + for _, ent, _ in cluster: + if ent is primary_entity: + continue + r = ent.get("rationale", "").strip() + if r and r not in seen_rat: + rationale_parts.append(r) + seen_rat.add(r) + + merged = dict(primary_entity) + merged["trigger"] = "; ".join(all_triggers) + merged["content"] = merged_content + if rationale_parts: + merged["rationale"] = " | ".join(rationale_parts) + return merged, primary_path + + +def apply_decision(decision, cluster, entities_dir, dry_run): + """ + Execute the decision. Returns a summary dict. + """ + action = decision["action"] + removed_paths = [] + merged_path = None + + if action == "keep-all": + return {"action": action, "removed": [], "merged_path": None, + "reason": decision.get("reason", "")} + + if action == "discard": + for path in decision["remove"]: + if not dry_run: + try: + Path(path).unlink() + removed_paths.append(path) + except OSError as e: + print(f" ⚠ Could not remove {path}: {e}", file=sys.stderr) + else: + removed_paths.append(path) + + elif action == "merge": + merged_entity, primary_path = _build_merged_entity(cluster) + merged_md = entity_to_markdown(merged_entity) + + if not dry_run: + # Write merged content back to the primary file + fd, tmp = tempfile.mkstemp(dir=Path(primary_path).parent, suffix=".tmp") + try: + os.write(fd, merged_md.encode("utf-8")) + os.close(fd) + fd = None + os.replace(tmp, primary_path) + except BaseException: + if fd is not None: + os.close(fd) + if os.path.exists(tmp): + os.unlink(tmp) + raise + merged_path = str(primary_path) + + # Remove the non-primary duplicates + for path in decision["remove"]: + try: + Path(path).unlink() + removed_paths.append(path) + except OSError as e: + print(f" ⚠ Could not remove {path}: {e}", file=sys.stderr) + else: + merged_path = str(primary_path) + removed_paths = list(decision["remove"]) + + return { + "action": action, + "removed": removed_paths, + "merged_path": merged_path, + "reason": decision.get("reason", ""), + } + + +# =========================================================================== +# Main +# =========================================================================== + +def main(): + parser = argparse.ArgumentParser( + description="Phase 2: find and resolve similar/duplicate skills" + ) + parser.add_argument("--entities-dir", default=None) + parser.add_argument("--threshold", type=float, default=0.45, + help="Jaccard similarity threshold for grouping (default: 0.45)") + parser.add_argument("--interactive", action="store_true", + help="Review each cluster manually before acting") + parser.add_argument("--dry-run", action="store_true", + help="Show decisions without making changes") + parser.add_argument("--no-prune", action="store_true", + help="Skip banality pruning (keep all entities regardless of quality)") + parser.add_argument("--report", default=None) + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + entities_dir = Path(args.entities_dir) if args.entities_dir \ + else evolve_dir / "entities" + + if not entities_dir.exists(): + print(f"No entities directory at {entities_dir}.") + sys.exit(0) + + _OWNED_TYPE_DIRS = {"atomic-skill", "guideline", "skill-flow"} + md_files = sorted( + p for p in entities_dir.glob("**/*.md") + if not p.is_symlink() + and ".git" not in p.parts + and "subscribed" not in p.relative_to(entities_dir).parts + and any(part in _OWNED_TYPE_DIRS for part in p.relative_to(entities_dir).parts) + ) + if not md_files: + print("No entity files found.") + sys.exit(0) + + # Load all entities, grouped by entity type so dedup only happens within + # atomic-skill, guideline, and skill-flow buckets. + entities_by_type = {"atomic-skill": [], "guideline": [], "skill-flow": []} + for path in md_files: + try: + entity = markdown_to_entity(path) + if entity.get("content") and entity.get("type") in entities_by_type: + entities_by_type[entity["type"]].append((path, entity)) + except Exception as exc: + print(f" ⚠ Skipping unparseable file {path}: {exc}", file=sys.stderr) + + loaded_count = sum(len(items) for items in entities_by_type.values()) + print(f"Loaded {loaded_count} entities") + print(f"Similarity threshold: {args.threshold}") + if args.dry_run: + print("DRY RUN — no changes will be made") + print() + + # ── Banality prune (pre-clustering) ────────────────────────────────────── + # Remove entities that are too generic to provide recall value before + # the similarity clustering runs. Skill-flows are exempt — they encode + # composition order which is non-obvious even when their steps are simple. + prune_summaries = [] + if not args.no_prune: + print("PRUNE — banality check") + print("=" * 70) + for entity_type in ("atomic-skill", "guideline"): + keep = [] + for path, entity in entities_by_type[entity_type]: + is_banal, reason = check_banality(entity) + if is_banal: + slug = Path(path).stem + dry_tag = " [DRY RUN]" if args.dry_run else "" + print(f" 🗑 prune{dry_tag} {slug}") + print(f" reason: {reason}") + if not args.dry_run: + try: + Path(path).unlink() + except OSError as e: + print(f" ⚠ Could not remove {path}: {e}", file=sys.stderr) + keep.append((path, entity)) + continue + prune_summaries.append({ + "path": str(path), + "slug": slug, + "reason": reason, + "dry_run": args.dry_run, + }) + else: + keep.append((path, entity)) + entities_by_type[entity_type] = keep + + if prune_summaries: + dry_note = " (dry run)" if args.dry_run else "" + print(f"\n Pruned {len(prune_summaries)} banal entity(ies){dry_note}.") + else: + print(" No banal entities found.") + print() + + # Build clusters within each entity type only + clusters = [] + for entity_type in ("atomic-skill", "guideline", "skill-flow"): + typed_entities = entities_by_type[entity_type] + if typed_entities: + clusters.extend(build_clusters(typed_entities, args.threshold)) + multi_clusters = [c for c in clusters if len(c) > 1] + singleton_count = sum(1 for c in clusters if len(c) == 1) + + print(f"Clusters: {len(clusters)} total ({len(multi_clusters)} with duplicates, {singleton_count} singletons)") + + if not multi_clusters: + msg = "✅ No similar entities found within the same type — library is already clean." + if prune_summaries: + msg += f" ({len(prune_summaries)} banal entities pruned above.)" + print(f"\n{msg}") + + print() + print("REFINE — Phase 2") + print("=" * 70) + + summaries = [] + total_removed = 0 + total_merged = 0 + + for cluster in multi_clusters: + if args.interactive: + decision = _interactive_decision(cluster, args.threshold) + else: + decision = _auto_decision(cluster, args.threshold) + + result = apply_decision(decision, cluster, entities_dir, args.dry_run) + summaries.append({ + "cluster": [str(c[0]) for c in cluster], + "decision": decision, + "result": result, + }) + + action = result["action"] + dry_tag = " [DRY RUN]" if args.dry_run else "" + if action == "keep-all": + slugs = [Path(c[0]).stem for c in cluster] + print(f" ⟳ keep-all {slugs}") + elif action == "discard": + removed_slugs = [Path(p).stem for p in result["removed"]] + kept_slugs = [Path(c[0]).stem for c in cluster if str(c[0]) not in result["removed"]] + print(f" 🗑 discard{dry_tag} kept={kept_slugs} removed={removed_slugs}") + print(f" reason: {result['reason']}") + total_removed += len(result["removed"]) + elif action == "merge": + slugs = [Path(c[0]).stem for c in cluster] + print(f" ⤵ merge{dry_tag} {slugs} → {Path(result['merged_path']).stem}") + print(f" reason: {result['reason']}") + total_merged += 1 + total_removed += len(result["removed"]) + + print() + print("REFINE SUMMARY") + print("=" * 70) + if prune_summaries: + print(f" Banal entities pruned : {len(prune_summaries)}") + print(f" Clusters resolved : {len(multi_clusters)}") + print(f" Skills removed : {total_removed}") + print(f" Skills merged : {total_merged}") + if args.dry_run: + print(" (no changes written — dry run)") + + if args.report: + report_path = Path(args.report) + report_path.parent.mkdir(parents=True, exist_ok=True) + report = { + "generated_at": datetime.now().isoformat(), + "threshold": args.threshold, + "dry_run": args.dry_run, + "banal_pruned": len(prune_summaries), + "prune_summaries": prune_summaries, + "clusters_with_duplicates": len(multi_clusters), + "total_removed": total_removed, + "total_merged": total_merged, + "summaries": summaries, + } + with open(report_path, "w", encoding="utf-8") as fh: + json.dump(report, fh, indent=2, default=str) + print(f"\nReport written: {report_path}") + + sys.exit(0) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/SKILL.md new file mode 100644 index 00000000..c2f5dcca --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/SKILL.md @@ -0,0 +1,561 @@ +--- +name: evolve-lite:learn +description: Extracts reusable entities (guidelines, atomic skills, skill-flows) from the current conversation and saves them to the local entity library. Run explicitly with /evolve-lite:learn when you want to persist lessons from a session. +--- + +# Entity Generator + +## Entity Types + +`evolve-lite:learn` should classify reusable knowledge into exactly one of these entity types: + +- `guideline`: a **declarative preference** that shapes how an agent chooses between approaches — no executable steps. Covers tool/approach choices only (e.g. "prefer CLI over MCP server"). Not for naming conventions, dependencies, or how-to instructions. +- `atomic-skill`: the smallest self-contained executable procedure that solves **one focused sub-problem**. Must include everything needed to execute it successfully: steps, naming conventions, required dependencies, concrete examples, any other context that reduces ambiguity, and a **success rubric** — explicit criteria that confirm the skill completed correctly. +- `skill-flow`: a named, recurring **ordered sequence of steps** where each step maps to an existing atomic skill, another skill-flow, or an inlined well-known operation. The order matters — skipping or reordering steps would break the outcome. Must include necessary dependencies, concrete examples, and a **success rubric** — explicit criteria that confirm the full flow completed correctly. Every step does not need its own dedicated atomic skill; trivial operations (e.g. activating a venv) can be described inline. + +**Type selection decision table:** + +| Question | Type | +|---|---| +| Is it a tool/approach preference with no executable steps? | `guideline` | +| Does it solve exactly one sub-problem through executable steps? | `atomic-skill` | +| Is it an ordered sequence of 2+ steps that recurs as a named unit, where order matters? | `skill-flow` | +| Is it a naming convention, dependency list, or "how to run X"? | `atomic-skill` | +| Is it a failure-derived lesson with a specific fix or command? | `atomic-skill` | +| Is it a failure-derived preference with no steps? | `guideline` | + +> **Tiebreaker**: if a procedure can be described as a single coherent action ("validate this config", "import this agent"), it is an `atomic-skill`. If it only makes sense as an ordered sequence — "do A, then B, then C" — where steps are distinct named operations and order matters, it is a `skill-flow`. + +## Product Organization + +Entities are automatically organized by product/domain for better retrieval: + +- **Product Detection**: Config-driven and automatic. The system reads `.bob/lib/evolve-lite/products.yaml` for the declared product registry (slug + match patterns), then merges in any extra product folders already present in `.evolve/entities/`. No manual directory setup is needed. +- **Folder creation**: Product folders are created automatically the first time an entity is saved to that product. You do not need to pre-create them. +- **Matching**: Config products are matched via their explicit regex patterns. Folder-only products (not in the config) fall back to whole-word slug-token matching. Highest score wins. +- **General fallback**: `general` is only assigned when nothing scores above zero. It is never preferred over a specific product. +- **Directory Structure**: Entities are stored in `{type}/{product}/{slug}.md` format. +- **Slugs**: Include the product prefix automatically (e.g., `watson-orchestrate-handle-token-expiration.md`). + +**To register a new product** (before any entities exist for it): add an entry to `.bob/lib/evolve-lite/products.yaml`. Its folder will be created on first save. No code changes needed. + +Currently registered products (from `.bob/lib/evolve-lite/products.yaml`): +- `watson-orchestrate`: Watson Orchestrate CLI, agents, environments +- `concert`: IBM Concert lab and audit workflows +- `langgraph`: LangGraph, LangChain, LangSmith +- `vault`: HashiCorp Vault secrets management +- `github`: GitHub operations, pull requests, workflows +- `docker`: Docker containers, images, Dockerfiles +- `kubernetes`: Kubernetes, kubectl, Helm +- `general`: Cross-product or non-specific skills (final fallback only, not in config) + +## Common-Sense Filtering + +The system automatically filters out skills that are too basic or well-known: + +**Filtered patterns include:** +- Basic file operations (create file, read file, delete file) +- Standard Python operations (activate virtual environment, install dependencies) +- Basic git commands (git add, git commit, git push) +- Very short content (< 20 characters) +- Overly vague triggers + +**What gets saved:** +- Product-specific workflows and workarounds +- Error resolutions and failure-derived solutions +- Non-obvious command sequences +- Environment-specific configurations +- Concrete solutions with code/commands + +## Failure-Derived Skill Marking + +The system automatically marks skills that were learned from failures: + +**Automatic Detection:** +- Skills containing error indicators (error, failed, exception, retry, workaround, fix, token expired, permission denied, not found, missing) are automatically flagged +- The `derived_from_failure` field is set to "true" in the entity frontmatter +- This helps track which skills came from solving actual problems vs general best practices + +**Why This Matters:** +- Failure-derived skills represent real problems that were encountered and solved +- They should be prioritized during extraction (see Step 5 and Best Practices) +- They provide concrete solutions to specific error scenarios + +### Automatic Skill-Flow Decomposition + +When you create a `skill-flow` entity, the system automatically: +1. **Extracts individual steps** from the flow content +2. **Creates atomic skills** for each step if they don't already exist +3. **References the atomic skills** in the skill-flow's frontmatter via `atomic_skills` field + +This ensures that: +- Each step is reusable independently +- Skill-flows explicitly declare their dependencies +- The skill library remains well-structured and composable + +**Example**: A skill-flow "Create and upload Watson agent" with steps like "Create YAML file", "Activate environment", "Import agent" will automatically generate 3-4 atomic skills and reference them. + +## Overview + +This skill analyzes the current conversation to extract actionable instructions that would help on similar tasks in the future. It **identifies errors encountered during the conversation** - tool failures, exceptions, wrong approaches, retry loops - and provides recommendations to prevent those errors from recurring. This skill should take note of the concrete solution which solved a concrete problem, not an abstract idea. When the successful resolution involves a non-trivial workaround, parser, command sequence, or fallback pipeline that could be used to avoid wasted effort, capture that solution as a reusable artifact first, then save entities that point future agents to use it. + +## When To Use + +Run this skill explicitly (via `/evolve-lite:learn`) when you want to persist knowledge from a session to the entity library. Good candidates include sessions where you encountered: +- tool failures +- permission issues +- missing dependencies +- retries or abandoned approaches +- reusable command sequences or scripts + +Examples of artifacts that must be immediately created once proven as the successful solution include: +- an inline Python, shell, or other heredoc script +- a command assembled interactively over multiple retries +- a parser or extractor implemented ad hoc during the turn +- a fallback path triggered by missing dependencies or restricted tooling + +Unless that artifact happens to be: +- code which is a trivial one-liner that future agents would not benefit from reusing +- code which embeds secrets, tokens, or user-specific sensitive data +- a guideline that would instruct the agent to invoke a skill, tool, or external command by name (e.g. "run evolve-lite:learn", "call save_trajectory") - such guidelines trigger prompt-injection detection when retrieved by the recall skill in a future session +- the user explicitly asked for a one-off result and not to persist helper code +- redundant because an equivalent local artifact on disk would be just as effective + +## Workflow + +### Step 0: Extract and Load the Trajectory from Bob's Logs + +Instead of manually copying the conversation, automatically extract it from Bob's task logs: + +1. Run the trajectory extraction script: +```bash +python3 -c " +import sys +sys.path.insert(0, '.bob/lib/evolve-lite') +from trajectory_extractor import save_trajectory_from_bob +path = save_trajectory_from_bob() +print(f'Trajectory saved: {path}') +" +``` + +2. Capture the exact path from the output as `saved_trajectory_path`. You will attach this exact path to each entity's `trajectory` field in Step 6. + +3. Read `saved_trajectory_path` with the Read tool and analyze that saved trajectory rather than relying only on live context. + +If the trajectory cannot be extracted or read, output zero entities and exit. Do not invent a trajectory path. + +**Note**: This automatically reads from Bob's task logs at `~/Library/Application Support/IBM Bob/user_global_storage/id_bob_code/tasks/`, extracting the most recent task file. No manual conversation copying required! + +### Step 1: Analyze the Conversation + +Identify from the saved trajectory loaded in Step 0: + +- **Task/Request**: What was the user asking for? +- **Steps Taken**: What reasoning, actions, and observations occurred? +- **What Worked**: Which approaches succeeded? +- **What Failed**: Which approaches did not work and why? +- **Errors Encountered**: Tool failures, exceptions, permission errors, retry loops, dead ends, and wrong initial approaches +- **Reusable Outcome**: Did the final working solution produce a reusable script, parser, command template, or workflow that would save time on a similar task? + +### Step 2: Identify Errors and Root Causes + +Scan the conversation for these error signals: + +1. **Tool or command failures**: Non-zero exit codes, error messages, exceptions, stack traces +2. **Permission or access errors**: "Permission denied", "not found", sandbox restrictions +3. **Wrong initial approach**: First attempt abandoned in favor of a different strategy +4. **Retry loops**: Same action attempted multiple times with variations before succeeding +5. **Missing prerequisites**: Missing dependencies, packages, or configs discovered mid-task +6. **Silent failures**: Actions that appeared to succeed but produced wrong results + +For each error found, document: + +| | Error Example | Root Cause | Resolution | Prevention Guideline | +|---|---|---|---|---| +| 1 | `jq: command not found` | System tool unavailable in environment | created a python script to resolve the problem | Save the python script and use it in similar scenarios | +| 2 | `git push` rejected (no upstream) | Branch not tracked to remote | Added `-u origin branch` | Always set upstream when pushing a new branch | +| 3 | Tried regex parsing of HTML, got wrong results | Regex cannot handle nested tags | Switched to BeautifulSoup | Use a proper HTML parser, never regex | + +### Step 3: Decide Whether To Save The Pipeline + +Before writing entities, determine whether the successful approach should be saved as a reusable artifact. + +Create or update a local reusable artifact when any of these are true: +- the final solution required more than a trivial one-liner +- the final solution worked around missing tools, libraries, or permissions +- the solution is likely to recur on similar tasks + +Prefer one of these artifact forms: +- a small script, saved to a stable path in the workspace or plugin, such as `scripts/`, `tools/`, or another obvious helper location. +- a documented local workflow if code is not appropriate + +When turning an ad hoc command or script into a reusable artifact, remove +incidental one-off inputs such as literal file names, IDs, answer values, or +temporary paths. Keep the reusable procedure that was actually exercised in the +session, and do not add capabilities that were not validated by the work. + +If you create an artifact, record: +- its path +- what it does +- when future agents should use it first + +### Step 4: Extract Entities + +If Step 3 produced an artifact, at least one entity must explicitly point to that artifact, which is likely the only entity that needs to be produced. +Otherwise, extract 3-5 proactive entities. **Prioritize failure-derived entities first** - these get higher priority scores and are more valuable for future retrieval. + +**Important**: The system will automatically: +- Detect the product/domain for each entity +- Assign priority scores based on failure indicators and specificity +- Filter out common-sense or trivial skills +- Organize entities by product in the directory structure + +Choose the right granularity for each entity: + +1. **Use `guideline` for declarative approach preferences** + - zero executable steps — tells the agent *what to prefer*, not *what to do* + - covers tool/approach choices: "prefer CLI over MCP server", "prefer BeautifulSoup over regex for HTML" + - does NOT cover naming conventions, dependencies, or how-to instructions — those belong in `atomic-skill` + - examples: "prefer CLI over MCP server when both are available", "avoid regex for structured data parsing" + +2. **Use `atomic-skill` for any executable procedure solving one sub-problem** + - must produce one concrete output, state change, or result through explicit steps + - include everything needed to execute it successfully: + - step-by-step instructions + - naming conventions relevant to the skill + - required dependencies (pip packages, CLI tools, env vars) + - concrete examples showing expected inputs, commands, and outputs + - any additional context that reduces ambiguity + - a **success rubric**: 1–3 explicit, observable criteria that confirm the skill ran correctly (e.g. "exit code 0", "output file exists at expected path", "API returns status 200") + - examples: importing a Watson agent YAML, extracting a JSON field without `jq`, validating a config file + +3. **Use `skill-flow` for named recurring ordered sequences** + - an ordered sequence of 2+ steps where order matters and skipping a step would break the outcome + - each step references an existing atomic skill, another skill-flow, or an inlined description of a well-known operation + - include necessary dependencies (tools, packages, env vars), at least one concrete example of the full flow in action, and a **success rubric**: explicit, observable criteria that confirm every step completed and the overall flow succeeded + - do NOT create trivial atomic skills just to fill out a flow — inline well-known steps (e.g. "activate the venv", "cd into the repo") directly in the flow content + - examples: save trajectory → review existing entities → extract entities → persist entities; authenticate → import agent YAML → verify agent status + +Follow these principles: + +4. **Reframe failures as proactive recommendations** + - If an approach failed due to permissions, recommend the working permission-aware approach first + - If a system tool was unavailable, recommend the saved artifact or fallback workflow first + - If an approach hit environment constraints, recommend the constraint-aware approach + +5. **Prioritize known working local artifacts over general advice** + - If the successful solution produced or reused a concrete local artifact, at least one saved entity must: + - Bad: "Use Python to parse EXIF if exiftool is missing" + - Better: "Use `/abs/path/json_get.py` for JSON field extraction when `jq` is unavailable in minimal environments." + - name the artifact by path + - state exactly when to use it + - state that it should be tried before generic tool discovery or fallback exploration + - describe the artifact by capability, not just by the original incident + +6. **Triggers should describe the broad task context that the artifact solves, not the narrow details of the original request.** + - Bad trigger: "When jq fails" + - Good trigger: "When extracting fields from JSON in constrained shells or stripped-down environments" + The trigger should generalize the working solution without becoming vague. + +7. **For retry loops, recommend the final working approach as the starting point** + - Eliminate trial and error by creating a concrete local artifact out of the successful workflow or script + +8. **Prefer entities that save future time** + - A pointer to a saved working script is more valuable than a generic reminder if both are available + +9. **Decompose before composing** + - If individual steps of a flow are independently useful in other contexts, save those as `atomic-skill` entities first + - Save a `skill-flow` only when the full ordered sequence recurs as a named unit + - Do not create trivial atomic skills (e.g. "activate venv", "cd into directory") just to satisfy a flow — inline those steps in the flow file instead + - Do not save a `skill-flow` for a one-off sequence better represented by a single `atomic-skill` + +10. **Skill-flow content format** + - Write skill-flow content as an ordered numbered list (e.g., "1) First step. 2) Second step.") + - Each step should be a complete, actionable instruction that references an existing skill by name or describes the operation inline + - The system will parse these steps to create atomic skills automatically where appropriate + +### Step 5: Output Entities JSON + +Output entities in this JSON format. Include a `trajectory` field on every entity, set to the `saved_trajectory_path` extracted in Step 0 — this records which session produced the entity. + +```json +{ + "entities": [ + { + "name": "Short verb-noun label used as the filename slug", + "content": "Proactive entity stating what TO DO", + "rationale": "Why this approach works better", + "type": "guideline", + "trigger": "Situational context when this applies", + "trajectory": ".evolve/trajectories/claude-transcript_.jsonl", + "success_rubric": "For atomic-skill and skill-flow only: 1–3 observable criteria confirming successful execution", + "requirements": "pyyaml>=6.0\nrequests", + "imports": "import yaml\nimport requests", + "dependencies": "watson-orchestrate-ensure-authentication", + "documentation": "https://docs.example.com/api" + } + ] +} +``` + +Allowed type values: +- guideline +- atomic-skill +- skill-flow + +#### `name` field — required for all entities + +The `name` field controls the filename slug and must follow these rules: + +- **Format**: short verb-noun phrase, 2–5 words, lowercase, no punctuation +- **Do NOT start with**: "when", "if", "how to", the product name, or any trigger-style phrasing +- **Do NOT include**: commands, flags, file paths, or full sentences +- **Captures**: the capability, not the situation + +| Bad `name` | Good `name` | +|---|---| +| `when-a-watson-orchestrate-agent-requires-multiple-tools` | `import-multi-tool-python-file` | +| `watson-orchestrate-import-agent` | `import-agent-yaml` | +| `handle-token-expiration-by-piping-api-key-to-orchestrate-env-activate` | `reauth-expired-token` | +| `general-in-evolve-lite-skill-setup-steps-only-gitignore` | `gitignore-evolve-artifacts` | + +The product prefix is added automatically from the folder — do not include it in `name`. + +#### `trigger` field — required for all entities + +The `trigger` describes **the situation the user is in** — not what the skill does, not the solution, not a command. + +Triggers are matched by keyword overlap against user messages. A trigger that omits the words a user would naturally type will lose to other entities even if the skill is correct. + +**Rules:** + +1. **Write from the user's perspective, not the author's.** The user does not know the solution yet. They describe a symptom, error, or task. Use the words they would use. + +2. **Include the specific error message, flag, or symptom** that distinguishes this scenario. Vague triggers match everything and win nothing. + +3. **Include the task keywords** — the nouns and verbs the user types when describing what they are trying to do ("import", "token expiration", "not found", "spec_version", "hyphens"). + +4. **Do NOT describe what the skill does.** Triggers like "When activating the venv" or "When uploading an agent file" describe the solution. The user doesn't know they need to activate the venv — they know the CLI is broken. + +5. **When two skills could match the same generic symptom** (e.g. "import failing"), make the trigger specific enough to distinguish them — include the exact error text, flag name, or field name involved. + +| Anti-pattern | Why it failed | Fixed trigger | +|---|---|---| +| `When uploading an agent definition file` | User typed "import" not "upload"; no error keywords | `When importing an agent YAML file into Watson Orchestrate using the orchestrate agents import CLI command` | +| `When a CLI session has expired mid-workflow` | User typed "token expiration error" — trigger has no overlap | `When the Watson Orchestrate CLI reports a token expiration error or expired token mid-session` | +| `When a Python venv needs to be activated` | User asked "what do I need before running CLI commands" — no symptom words | `When the orchestrate command is not found after opening a new terminal, or when preparing to run CLI commands and the venv needs activating` | +| `When authoring an agent definition file from scratch` | User asked about specific fields (spec_version, parameters) — trigger has no field names | `When creating a Watson Orchestrate agent YAML with required fields like spec_version, name, description, instructions, model, parameters, and tools` | +| `When configuring the CLI environment for the first time` | User said "env activate…environment doesn't exist" or "combine URL and auth" — trigger has no command keywords | `When setting up the Watson Orchestrate CLI for the first time by registering the URL with env add and authenticating with env activate` | +| `When naming an agent in a YAML file` | Too generic — lost to other import-related skills when user mentioned "import failing with validation error" | `When naming an agent in a YAML file, including rules about hyphens, underscores, and names starting with a number, or when import fails with a name validation error` | + +#### Optional metadata sections + +These fields are **optional** — only include them when they add real value. Each becomes a named markdown section in the saved entity file. + +| Field | Applies to | When to use | Format | +|---|---|---|---| +| `success_rubric` | `atomic-skill`, `skill-flow` | Always include for these types — omit only for `guideline` | 1–3 bullet points, each an observable pass/fail criterion (e.g. "exit code 0", "output file exists at expected path", "no error lines in stdout") | +| `requirements` | all | The skill requires specific pip packages or CLI tools to work | One entry per line. Pip packages may include version specifiers (`pyyaml>=6.0`). CLI tools use `cli: ` prefix (e.g. `cli: orchestrate`). | +| `imports` | all | The skill involves Python code and specific imports are non-obvious | Full import statements, one per line (`import subprocess`, `from pathlib import Path`) | +| `dependencies` | all | This skill must be used after another entity (outside the `atomic_skills` composition graph) | Comma-separated entity slugs | +| `documentation` | all | There is a canonical external reference worth linking | One URL per line | + +**Rules:** +- `name` and `trigger` are **required** for all entity types. +- `success_rubric` is **required** for `atomic-skill` and `skill-flow` — do not omit it for these types. +- Anything listed in `requirements` or `imports` **must** be mentioned or used somewhere in the `content` body — the quality gate will flag undeclared references that never appear. +- Leave other fields out entirely if the skill is self-contained and needs no external context. + +#### Version field + +Every entity carries a `version` integer in its frontmatter (set automatically to `1` on first publish, then incremented on each re-publish by the publish script). + +When you **update** an existing entity (improve trigger wording, extend content, add a new section), include a `## Changelog` section in the JSON so the change is recorded: + +```json +{ + "entities": [ + { + "content": "Updated guidance here…", + "trigger": "Improved trigger wording…", + "type": "atomic-skill", + "changelog": "- v2: Expanded content to cover edge case X\n- v1: Initial version" + } + ] +} +``` + +The `## Changelog` format is flexible — one bullet per version is sufficient: + +``` +- v3: Added ## Requirements section with pyyaml dependency +- v2: Fixed stutter in trigger; added ## Imports +- v1: Initial publish +``` + +The quality gate will warn (non-blocking) when `version > 1` and the `## Changelog` has fewer entries than the stated version number. + +**Note**: The save script will automatically: +- Detect and set the `product` field based on content analysis +- Set `derived_from_failure` to "true" for error-related skills +- Filter out common-sense or trivial skills +- Decompose skill-flows into atomic skills with proper references +- Organize files in `{type}/{product}/{slug}.md` structure + +### Step 6: Save Entities + +After generating the entities JSON, save them using the helper script: + +#### Method 1: Direct Pipe (Recommended) + +```bash +echo '' | python3 .bob/skills/evolve-lite-learn/scripts/save_entities.py +``` + +#### Method 2: From File + +```bash +cat entities.json | python3 .bob/skills/evolve-lite-learn/scripts/save_entities.py +``` + +#### Method 3: Interactive + +```bash +python3 .bob/skills/evolve-lite-learn/scripts/save_entities.py +``` + +The script will: +- Find or create the entities directory at `.evolve/entities/` +- **Detect product/domain** for each entity automatically +- **Mark failure-derived skills** with `derived_from_failure: true` +- **Filter out common-sense skills** that are too basic or well-known +- **For skill-flows**: Automatically decompose into atomic skills and create them if needed +- Write each entity as a markdown file in `{type}/{product}/` subdirectories +- Display confirmation with counts of added, filtered, and auto-created entities + +### Step 7: Generate Tests from Success Rubrics + +For every `atomic-skill` or `skill-flow` entity saved in Step 6, generate test cases grounded in that entity's `success_rubric`. + +1. Read each newly written entity file and extract its `success_rubric` section. +2. For each entity, produce the following test cases: + - **1 happy-path test**: executes the skill under normal conditions; `validation_criteria` populated **directly from the rubric criteria** — do not invent generic criteria + - **1–3 edge case tests**: each exercises a realistic boundary or failure condition (e.g. missing dependency, malformed input, already-existing output, partial environment). Each edge case must still reference the rubric criteria and state which criterion is expected to fail or behave differently. + - `scenario` describes the specific condition being tested + - `input_context` reflects realistic preconditions including anything unusual for that edge case +3. Save tests to `.evolve/tests/pseudo_conversations/` as `{entity-slug}.json`, following the existing test file format. +4. If a test file already exists for an entity slug, **append new test cases** — never overwrite or delete existing ones. +5. Skip `guideline` entities — they have no rubric and no executable outcome to test. + +### Step 7a: Run Test Gate + +After generating tests in Step 7, run the quality gate to verify the newly saved skills pass at a minimum 80% rate across both the content-evaluation and recall suites. + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +python3 .bob/skills/evolve-lite-test/scripts/check_tests.py --threshold 0.8 --verbose +``` + +The gate report is written to `.evolve/tests/evaluation/gate_report.json`. + +**If the gate passes** (exit 0), continue to Step 8. + +**If the gate fails** (exit 1), read the `❌` lines and apply the fix that matches the failure type below. After each fix, re-run `check_tests.py`. Do not continue to Step 8 until the gate exits 0. + +#### Fixing gate failures + +Each `❌` line from the content evaluation (`run_skill_evaluation.py`) or recall test (`run_recall_tests.py`) falls into one of the following categories. Open the failing entity file and apply the matching fix. + +--- + +**Content evaluation failure** — `score < 0.5` or `violated` terms found + +The output shows `missed=[...]` — these are backtick-command terms that appear in the fixture's `must_include` list but are absent from the skill content. + +``` +❌ my-skill score=0.33 matched=1/3 missed=['orchestrate agents import', '--kind'] +``` + +*Fix*: Open the entity file. For each missed term, either: +- Add the missing command or flag to the content body (preferred — the skill genuinely needs it). +- If the term was extracted incorrectly and the skill is correct without it, regenerate the fixture: `python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py ` + +If `violated=[...]` appears, the skill content contains something it explicitly must NOT contain. Remove or rephrase that content. + +--- + +**Recall failure** — skill not in top-3 for its own trigger scenario + +The output shows `rank=N` where N > 3, or `rank=not_found`, meaning another skill outranked this one. + +``` +❌ my-skill rank=7 @1=✗ @3=✗ @5=✓ score=2 + top5: ['other-skill', 'another-skill', ...] +``` + +*Fix*: The `trigger` field uses words the user would not type, or is too generic. Rewrite it using the rules from Step 5 (`trigger` field): +- Use the *symptom* or *error message* the user would describe, not the solution. +- Include the specific command, flag, or error text that distinguishes this skill. +- Check the `top5` list — if the skills ranked above this one share keywords with your trigger, make this trigger more specific by adding the distinguishing detail. + +Example rewrites: + +| Bad trigger | Why it lost | Fix | +|---|---|---| +| `When uploading an agent file` | "upload" not in user vocabulary; no error terms | `When importing an agent YAML using orchestrate agents import and the command fails` | +| `When the CLI session ends` | Vague; lost to other CLI-related skills | `When the Watson Orchestrate CLI reports token expiration or expired token mid-session` | +| `When setting up the environment` | Too generic; matches everything | `When running orchestrate env add and env activate for the first time to register a URL and authenticate` | + +After rewriting the trigger, regenerate the fixture so the test question reflects the new wording: +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py +``` +Then re-run the gate. + +--- + +**Both failures on the same skill** + +Fix the content issue first (add missing commands), then fix the trigger. A skill whose content is incomplete will also tend to have a weak trigger because the two are related — the content should contain the distinguishing terms that make the trigger strong. + +### Step 8: Deduplicate the Entity Library + +After saving, compare the newly written entities against the full library to identify overlap and decide what to merge or delete. + +1. **Enumerate all entities** using the **Glob tool**: `.evolve/entities/**/*.md`. Use the **Read tool** to skim each file's `trigger`, `type`, and first paragraph of `content`. + + **Do NOT use `cat`, `head`, `find`, a `for` loop, or an inline `python3 -c` script for this** — each shell invocation triggers a permission prompt. Use Glob + Read only. + +2. **Identify candidates for merging or deletion** — look for pairs or groups that: + - Have the same or very similar `trigger` + - Cover the same sub-problem from slightly different angles + - Are the same `type` and would be stronger as one combined entity + +3. **Decide for each candidate group**: + + | Situation | Action | + |---|---| + | One entity is strictly a subset of another | Delete the weaker one | + | Both cover the same ground with complementary detail | Merge into one entity with the richer content, delete the other | + | They cover genuinely different sub-problems or triggers | Keep both | + | One is higher quality (better examples, rubric, dependencies) | Keep it, delete the weaker one | + +4. **To merge**: write the combined content into the surviving file using the edit tools, then delete the redundant file. + + When choosing which entity to keep as the survivor, **prefer the one that already has tests** — check `.evolve/tests/pseudo_conversations/` for a file matching the entity slug (e.g. `watson-orchestrate-import-agent.json`). Tests must never be deleted or modified during dedup — only new tests may be added. + +5. **To delete**: remove the entity file only. **Do not delete or modify any files under `.evolve/tests/`** — tests are append-only. If the surviving entity has no tests but the deleted one did, copy the deleted entity's test files to match the survivor's slug before removing the original. + +6. If no duplicates or near-duplicates are found, skip to the end — this step has no mandatory output. + +## Best Practices +1. **ALWAYS prioritize failure-derived entities first** - if you found errors, create entities for those BEFORE anything else. +2. One distinct error should normally produce one prevention entity. +3. Keep entities specific and actionable - avoid common-sense advice. +4. Include rationale so the future agent understands why the guidance matters. +5. Use situational triggers instead of failure-based triggers. +6. **Respect the 3-5 entity limit** - if you have 4 failures, create 4 failure entities and maybe 1 other. Don't create 5 non-failure entities when failures exist. +7. If more than five distinct errors appear, merge entities with the same root cause or fix, then rank the rest by severity, frequency, user impact, and recency before dropping the weakest ones. +8. **Focus on product-specific solutions** - general advice is less valuable than concrete product workflows. +9. **Include concrete details** - file paths, commands, code snippets make skills more actionable. +10. **Let the system handle organization** - don't worry about product detection, the system does this automatically. diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/on_stop.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/on_stop.py new file mode 100644 index 00000000..a582d06f --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/on_stop.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +"""Stop hook — disabled. evolve-lite:learn runs only when explicitly invoked.""" + +import sys + + +def main(): + # Hook is intentionally a no-op. The learn skill is run on-demand only. + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/on_stop.sh b/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/on_stop.sh new file mode 100755 index 00000000..969e9c76 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/on_stop.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Stop hook — disabled. evolve-lite:learn runs only when explicitly invoked. +exit 0 diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/save_entities.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/save_entities.py new file mode 100644 index 00000000..c5f80d77 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-learn/scripts/save_entities.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +""" +Save Entities Script +Reads entities from stdin JSON and writes each as a markdown file +in the entities directory, organized by type. + +For skill-flow entities, automatically decomposes them into atomic skills +if they don't already exist, then references those atomic skills. +""" + +import argparse +import json +import re +import sys +from pathlib import Path + +# Walk up from the script location to find the installed plugin lib directory. +# Every host installs the shared lib under lib/evolve-lite/ so multiple +# plugins can coexist side by side (e.g. .bob/lib/evolve-lite/). +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) +from entity_io import ( # noqa: E402 + find_entities_dir, + get_default_entities_dir, + load_all_entities, + load_product_registry, + write_entity_file, + log as _log, + slugify, +) + + +def log(message): + _log("save", message) + + +log("Script started") + + +def normalize(text): + """Normalize content for dedup comparison.""" + return " ".join(text.lower().split()) + + +def extract_steps_from_flow(content): + """Extract individual steps from a skill-flow content. + + Returns a list of step descriptions that could become atomic skills. + """ + steps = [] + + # Pattern 1: Numbered steps like "1) Step description. 2) Next step." + # Match from number to the next number or end of string + numbered_pattern = r'\d+\)\s*([^0-9]+?)(?=\s*\d+\)|$)' + numbered_matches = re.findall(numbered_pattern, content, re.DOTALL) + if numbered_matches: + for match in numbered_matches: + # Clean up the step text + step = match.strip() + # Remove trailing period if present + step = step.rstrip('.') + # Remove extra whitespace + step = ' '.join(step.split()) + if step and len(step) > 10: + steps.append(step) + + # Pattern 2: Steps separated by periods or semicolons (fallback) + if not steps: + # Split on period followed by capital letter or number + sentence_pattern = r'[.;]\s*(?=[A-Z0-9])' + sentences = re.split(sentence_pattern, content) + steps.extend([s.strip() for s in sentences if len(s.strip()) > 20]) + + return steps + + +def detect_product(content, trigger, trajectory_path="", entities_dir=None): + """Detect which product/domain this entity belongs to. + + Uses the merged product registry from ``load_product_registry()``, which + combines: + 1. Declared products in ``.bob/lib/evolve-lite/products.yaml`` — matched + via explicit regex patterns. Add a new entry here to register a product + before any entities exist; its folder will be created on first save. + 2. Folder-only products discovered from existing entity subdirectories — + matched by slug-token heuristic (no explicit patterns needed). + + Scoring: + - Config products: count of matching explicit patterns (each hit = 1). + - Folder-only products: count of whole-word slug-token hits, weighted by + token count so multi-word slugs beat partial single-token matches. + The highest-scoring product wins. "general" is returned only when nothing + scores above zero. + + Returns: + str: A product slug, or "general" as the final fallback. + """ + combined_text = f"{content} {trigger}".lower() + + registry = load_product_registry(entities_dir) + if not registry: + return "general" + + best_product = None + best_score = 0 + + for entry in registry: + slug = entry["slug"] + patterns = entry.get("patterns", []) + + if patterns: + # Config entry: each matching pattern counts as one hit; weight by + # number of patterns so richer entries edge out sparse ones. + hits = sum( + 1 for p in patterns + if re.search(p, combined_text, re.IGNORECASE) + ) + weighted = hits * len(patterns) + else: + # Folder-only entry: fall back to slug-token heuristic. + tokens = {t for t in slug.split("-") if len(t) > 1} + hits = sum( + 1 for tok in tokens + if re.search(rf"\b{re.escape(tok)}\b", combined_text) + ) + weighted = hits * len(tokens) + + if weighted > best_score: + best_score = weighted + best_product = slug + + # "general" is the final fallback — only reached when nothing scored. + return best_product if best_score > 0 else "general" + + +def is_common_sense_skill(content, trigger): + """Determine if a skill is too basic/common-sense to save. + + Returns: + bool: True if the skill should be filtered out + """ + combined_text = f"{content} {trigger}".lower() + + # Patterns that indicate common-sense or well-known practices + common_sense_patterns = [ + # Basic Python/programming + r"^(create|write|make)\s+a\s+(file|directory|folder)$", + r"^install\s+dependencies$", + r"^activate\s+(the\s+)?virtual\s+environment$", + r"^run\s+(the\s+)?(application|script|program)$", + r"^import\s+(a\s+)?module$", + r"^set\s+up\s+a\s+virtual\s+environment$", + + # Basic git operations + r"^git\s+(add|commit|push|pull)$", + r"^create\s+a\s+branch$", + + # Basic file operations + r"^read\s+(a\s+)?file$", + r"^write\s+to\s+(a\s+)?file$", + r"^delete\s+(a\s+)?file$", + + # Too vague + r"^when\s+performing\s+", + r"^do\s+something$", + ] + + # Check content length - very short content is likely too trivial + if len(content.strip()) < 20: + return True + + for pattern in common_sense_patterns: + if re.search(pattern, combined_text): + log(f"Filtered common-sense skill: {content[:60]}") + return True + + return False + + +def mark_failure_derived(entity): + """Mark entity if it's derived from a failure/error. + + Sets the derived_from_failure flag to "true" if the entity + contains failure indicators. + """ + content = entity.get("content", "").lower() + rationale = entity.get("rationale", "").lower() + trigger = entity.get("trigger", "").lower() + + # Check if derived from failure + failure_indicators = [ + "error", "failed", "exception", "wrong", "incorrect", + "retry", "workaround", "fix", "resolved", "token expired", + "permission denied", "not found", "missing" + ] + + has_failure_indicator = any( + indicator in content or indicator in rationale or indicator in trigger + for indicator in failure_indicators + ) + + if has_failure_indicator: + entity["derived_from_failure"] = "true" + + +def create_atomic_skill_from_step(step, flow_trigger, flow_trajectory): + """Create an atomic skill entity from a step description. + + Args: + step: The step description + flow_trigger: The parent skill-flow's trigger for context + flow_trajectory: The trajectory to associate with this atomic skill + + Returns: + A dict representing an atomic skill entity + """ + step_lower = step.lower() + + # Derive a short verb-noun name (2-4 words) — strip leading action words + # and code fragments, keep the core capability phrase. + # Remove backtick-quoted commands entirely from the name. + name_text = re.sub(r'`[^`]*`', '', step).strip() + # Collapse whitespace + name_text = ' '.join(name_text.split()) + # Take first 4 words maximum for a short slug-friendly name + name_words = name_text.split()[:4] + name = ' '.join(name_words).lower().rstrip('.,;:') + + # Build a situational trigger that describes context, not the command. + if 'create' in step_lower or 'write' in step_lower: + trigger = f"When a {step_lower.split('create')[-1].split('write')[-1].strip().split()[0] if step_lower.split('create')[-1].strip() else 'file'} needs to be created" + elif 'activate' in step_lower or 'enable' in step_lower: + trigger = "When the virtual environment needs to be activated before running CLI commands" + elif 'ensure' in step_lower or 'verify' in step_lower or 'authenticate' in step_lower: + trigger = "When CLI authentication needs to be confirmed before proceeding" + elif 'import' in step_lower or 'upload' in step_lower: + trigger = "When deploying or registering an artifact via the CLI" + elif 'deploy' in step_lower: + trigger = "When deploying a service or agent to a remote environment" + else: + # Generic fallback: use first clause of the step as situational context + first_clause = step.split('.')[0].strip() + trigger = f"When {first_clause.lower()}" if not first_clause.lower().startswith('when') else first_clause + + return { + "type": "atomic-skill", + "name": name, + "trigger": trigger, + "content": step, + "rationale": f"Extracted as a reusable atomic capability from a skill-flow. Part of: {flow_trigger}", + "trajectory": flow_trajectory + } + + +def find_or_create_atomic_skills(flow_entity, entities_dir, existing_entities): + """For a skill-flow, find or create the atomic skills it references. + + Args: + flow_entity: The skill-flow entity dict + entities_dir: Path to the entities directory + existing_entities: List of existing entity dicts + + Returns: + List of atomic skill slugs that this flow should reference + """ + content = flow_entity.get("content", "") + trigger = flow_entity.get("trigger", "") + trajectory = flow_entity.get("trajectory", "") + + # Extract steps from the flow + steps = extract_steps_from_flow(content) + + if not steps: + log(f"No steps extracted from skill-flow: {content[:60]}") + return [] + + log(f"Extracted {len(steps)} steps from skill-flow") + + atomic_skill_refs = [] + existing_contents = {normalize(e["content"]) for e in existing_entities if e.get("content")} + + for step in steps: + # Check if an atomic skill already exists for this step + normalized_step = normalize(step) + + # Look for similar existing atomic skills + found_existing = False + for existing in existing_entities: + if existing.get("type") == "atomic-skill": + if normalized_step == normalize(existing.get("content", "")): + # Found exact match + found_existing = True + # Prefer name field for slug, matching write_entity_file behaviour + slug_source = existing.get("name") or existing.get("content", "") + slug = slugify(slug_source, product=existing.get("product")) + atomic_skill_refs.append(slug) + log(f"Found existing atomic skill: {slug}") + break + + if not found_existing: + # Create new atomic skill + atomic_skill = create_atomic_skill_from_step(step, trigger, trajectory) + atomic_skill["owner"] = flow_entity.get("owner", "unknown") + atomic_skill["visibility"] = "private" + + # Write the atomic skill + path = write_entity_file(entities_dir, atomic_skill) + # Prefer name field for slug, matching write_entity_file behaviour + slug_source = atomic_skill.get("name") or atomic_skill.get("content", "") + slug = slugify(slug_source, product=atomic_skill.get("product")) + atomic_skill_refs.append(slug) + + # Add to existing contents to avoid duplicates in same batch + existing_contents.add(normalized_step) + existing_entities.append(atomic_skill) + + log(f"Created new atomic skill: {path}") + + return atomic_skill_refs + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--user", default=None, help="Stamp owner on every entity written") + args = parser.parse_args() + + try: + input_data = json.load(sys.stdin) + log(f"Received input with keys: {list(input_data.keys())}") + except json.JSONDecodeError as e: + log(f"Failed to parse JSON input: {e}") + print(f"Error: Invalid JSON input - {e}", file=sys.stderr) + sys.exit(1) + + new_entities = input_data.get("entities", []) + if not isinstance(new_entities, list): + log(f"Invalid entities payload type: {type(new_entities).__name__}") + print("Error: `entities` must be a list.", file=sys.stderr) + sys.exit(1) + if not new_entities: + log("No entities in input") + print("No entities provided in input.", file=sys.stderr) + sys.exit(0) + + log(f"Received {len(new_entities)} new entities") + + entities_dir = find_entities_dir() + if entities_dir: + entities_dir = entities_dir.resolve() + log(f"Found existing dir: {entities_dir}") + print(f"Using existing entities dir: {entities_dir}") + else: + entities_dir = get_default_entities_dir() + log(f"Created new dir: {entities_dir}") + print(f"Created new entities dir: {entities_dir}") + + existing_entities = load_all_entities(entities_dir) + existing_contents = {normalize(e["content"]) for e in existing_entities if e.get("content")} + log(f"Existing entities: {len(existing_entities)}") + + added_count = 0 + atomic_skills_created = 0 + filtered_count = 0 + + for entity in new_entities: + content = entity.get("content") + if not content: + log(f"Skipping entity without content: {entity}") + continue + if normalize(content) in existing_contents: + log(f"Skipping duplicate: {content[:60]}") + continue + + # Filter out common-sense skills + trigger = entity.get("trigger", "") + if is_common_sense_skill(content, trigger): + filtered_count += 1 + continue + + # Detect and set product + trajectory = entity.get("trajectory", "") + product = detect_product(content, trigger, trajectory) + entity["product"] = product + + # Mark if derived from failure + mark_failure_derived(entity) + + # Stamp owner and visibility from the script, never from stdin. + # Untrusted upstream input (a prompt-injected agent) must not be + # able to spoof either field, so unconditionally overwrite. + entity["owner"] = args.user or "unknown" + entity["visibility"] = "private" + + # If this is a skill-flow, decompose it into atomic skills first + if entity.get("type") == "skill-flow": + log(f"Processing skill-flow: {content[:60]}") + atomic_skill_refs = find_or_create_atomic_skills( + entity, entities_dir, existing_entities + ) + + if atomic_skill_refs: + # Add references to the skill-flow + entity["atomic_skills"] = ", ".join(atomic_skill_refs) + atomic_skills_created += len(atomic_skill_refs) + log(f"Skill-flow references {len(atomic_skill_refs)} atomic skills") + + path = write_entity_file(entities_dir, entity) + existing_contents.add(normalize(content)) + added_count += 1 + log(f"Wrote: {path}") + + total = len(existing_entities) + added_count + log(f"Added {added_count} new entities ({atomic_skills_created} atomic skills auto-created, {filtered_count} filtered). Total: {total}") + print(f"Added {added_count} new entity(ies). Total: {total}") + if atomic_skills_created > 0: + print(f" → Auto-created {atomic_skills_created} atomic skill(s) from skill-flow decomposition") + if filtered_count > 0: + print(f" → Filtered {filtered_count} common-sense skill(s)") + print(f"Entities stored in: {entities_dir}") + + +if __name__ == "__main__": + main() diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-provenance/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-provenance/SKILL.md index af6fdd3b..25ee891a 100644 --- a/platform-integrations/bob/evolve-lite/skills/evolve-lite-provenance/SKILL.md +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-provenance/SKILL.md @@ -7,107 +7,58 @@ description: Analyze saved trajectories and recall audit events offline to recor ## Overview -This skill runs after one or more sessions have completed. It reads `recall` -events from `.evolve/audit.log`, locates each session's trajectory, and records -post-hoc `influence` events for the recalled guidelines. +This skill runs after one or more sessions have completed. It reads saved trajectories from `.evolve/trajectories/`, matches them to `recall` events in `.evolve/audit.log`, and records post-hoc `influence` events for recalled guidelines. -The mechanical work — reading recall rows, skipping already-assessed pairs, -resolving entity files, and locating trajectories — is done deterministically by -`provenance.py candidates`. Your job is the judgment: read each candidate and -decide whether the recalled guideline was `followed`, `contradicted`, or -`not_applicable`, then persist that verdict. - -Use this skill when you want to compute usage provenance without coupling the -work to the live learn step. +Use this skill when you want to compute usage provenance without coupling the work to the live learn step. ## Workflow -### Step 1: Get candidates +### Step 1: Load Recall Events -Run the candidate builder. It emits one JSON object per line (JSONL), one per -unresolved `(session_id, entity)` recall pair: +Read `.evolve/audit.log` as JSONL. Find entries where `event == "recall"` and `entities` is a non-empty list. -```bash -python3 .bob/skills/evolve-lite-provenance/scripts/provenance.py candidates -``` +Skip any recall event that already has `influence` entries for the same `session_id` and entity ids. Do not write duplicate influence records. -Each candidate looks like: +### Step 2: Locate Saved Trajectories -```json -{ - "session_id": "", - "entity_id": "/", - "entity_excerpt": "", - "trajectory_path": "/path/to/transcript.jsonl", - "trajectory_excerpt": "", - "missing": ["trajectory"] -} -``` +List `.evolve/trajectories/` and match each recall event to a trajectory by `session_id`. -Notes: - -- `entity_id` is the path relative to `.evolve/entities/` without the `.md` - suffix, e.g. `feedback/foo`, `guideline/bar`, or - `subscribed/alice/guideline/baz`. -- Pairs that already have an `influence` row are skipped for you — the builder - reuses the same dedup rule used when influence rows are written. You will - never be handed a duplicate. -- The trajectory locator checks `.evolve/trajectories/` first, then falls back - to the native Claude transcript at - `~/.claude/projects//.jsonl`. This means provenance works - even when no `.evolve/trajectories/` file was written. -- If an entity file or trajectory cannot be found, the candidate is still - emitted with a `missing: [...]` field so the gap is visible. When the - trajectory is missing you usually cannot judge the pair — skip it (do not - guess), unless the entity content alone makes `not_applicable` certain. - -### Step 2: Judge each candidate - -For each candidate, read `entity_excerpt` (and open `trajectory_path` for the -full transcript if the excerpt is not enough). Compare the recalled guideline -against the agent's actual actions in the trajectory and pick exactly one -verdict: - -- `followed` — the agent's actual actions are consistent with the guideline. -- `contradicted` — the guideline applied, but the agent did the opposite or - repeated the avoidable dead end. -- `not_applicable` — the guideline was recalled but did not apply to this - session. - -Keep `evidence` to one short sentence citing a concrete action, tool call, or -absence in the trajectory. This judgment is yours — there is no heuristic -fallback. - -### Step 3: Record verdicts - -Persist each verdict. Either pipe one verdict per call to `provenance.py -record`: +Matching strategy (in order): +1. `claude-transcript_.jsonl` - the stop-hook transcript dump; the session id is in the filename. +2. `trajectory__.json` - written by the evolve-lite:save-trajectory skill when a session id is available. Match on the `` slice of the filename. +3. `trajectory_.json` - open the file and match its top-level `session_id` field against the recall event. Only fall back to this step when the filename alone does not identify the session. -```bash -echo '{ - "session_id": "", - "entity": "/", - "verdict": "followed", - "evidence": "Agent used the saved parser before trying shell fallbacks." -}' | python3 .bob/skills/evolve-lite-provenance/scripts/provenance.py record -``` +If none of the above yields a confident match for a recall event, skip it. Do not guess. + +### Step 3: Read Recalled Entities + +For each recalled entity id, open `.evolve/entities/.md`. The id is a path relative to `.evolve/entities/` without the `.md` suffix, such as `guideline/foo` or `subscribed/alice/guideline/foo`. + +Read the entity content and trigger. Skip ids whose files are missing. + +### Step 4: Assess Influence + +Compare each recalled entity with the matched trajectory. Pick exactly one verdict: + +- `followed` - the agent's actual actions are consistent with the guideline. +- `contradicted` - the guideline applied, but the agent did the opposite or repeated the avoidable dead end. +- `not_applicable` - the guideline was recalled but did not apply to this session. + +Keep `evidence` to one short sentence citing a concrete action, tool call, or absence in the trajectory. + +### Step 5: Write Influence Events -…or, to batch many assessments for one session in a single call, pipe to the -underlying writer directly: +Pipe one JSON payload per assessed session to the helper: ```bash echo '{ "session_id": "", "assessments": [ - {"entity": "feedback/foo", "verdict": "followed", "evidence": "Agent followed it."}, - {"entity": "guideline/bar", "verdict": "not_applicable", "evidence": "Did not apply."} + {"entity": "guideline/", "verdict": "followed", "evidence": "Agent used the saved parser before trying shell fallbacks."} ] }' | python3 .bob/skills/evolve-lite-provenance/scripts/log_influence.py ``` -Both paths write the identical `influence` audit row and skip duplicates. The -`entity` value must match the candidate's `entity_id` exactly, including any -`subscribed//` prefix. +The `entity` value must match exactly what appeared in the recall event, including any `subscribed//` prefix. -It is valid to record nothing when recall events exist but no recalled guideline -can be assessed (e.g. every candidate is missing its trajectory). +It is valid to emit an empty `assessments` list when recall events exist but no recalled guideline can be assessed. diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-publish/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-publish/SKILL.md index 25b4d607..38456843 100644 --- a/platform-integrations/bob/evolve-lite/skills/evolve-lite-publish/SKILL.md +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-publish/SKILL.md @@ -1,14 +1,14 @@ --- name: evolve-lite:publish -description: Publish a private guideline to a configured write-scope repo. +description: Publish private guidelines, atomic skills, or skill flows to a configured write-scope repo. --- -# Publish a Guideline +# Publish Entities ## Overview -Publish one or more private guidelines from `.evolve/entities/guideline/` -into a configured **write-scope** repo. The entity is stamped with +Publish one or more private entities from `.evolve/entities/guideline/`, `.evolve/entities/atomic-skill/`, or `.evolve/entities/skill-flow/` +into a configured **write-scope** repo. Each entity is stamped with `visibility: public`, `owner`, `published_at`, and `source`, moved into the local clone of the write repo, and committed / pushed to the remote. @@ -29,10 +29,10 @@ If `identity.user` is missing, ask for it and add it to the config. ### Step 2: First-time setup -Ensure `.evolve/` is gitignored at the project root: +Ensure `.evolve/entities/subscribed/` is gitignored at the project root (the subscribed clones are managed by evolve-lite and should not be committed). Do **not** gitignore `.evolve/` or `.bob/` — those directories must remain tracked. ```bash -grep -qxF '.evolve/' .gitignore 2>/dev/null || echo '.evolve/' >> .gitignore +grep -qxF '.evolve/entities/subscribed/' .gitignore 2>/dev/null || echo '.evolve/entities/subscribed/' >> .gitignore ``` ### Step 3: Pick the target write-scope repo @@ -47,7 +47,72 @@ Let `{repo}` be the chosen repo name and `{branch}` its configured branch (defau ### Step 4: List and select entities -List files in `.evolve/entities/guideline/` and ask the user which to publish. +List files in `.evolve/entities/guideline/`, `.evolve/entities/atomic-skill/`, and `.evolve/entities/skill-flow/`, then ask the user which to publish. + +### Step 4a: Test gate (optional) + +The quality gate is **off by default**. Only run it when the user explicitly requests it (e.g. "publish with quality gate", "run the gate before publishing", "include quality check"). + +If the user did **not** ask for the gate, skip this step entirely and proceed to Step 5. + +If the user **did** ask for the gate, run: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/check_tests.py --threshold 0.8 --verbose +``` + +The gate report is at `.evolve/tests/evaluation/gate_report.json`. + +**If the gate passes** (exit 0), continue to Step 5. + +**If the gate fails** (exit 1), tell the user publishing is blocked and work through the `❌` lines to fix each failure before retrying. Do not proceed to Step 5 until the gate exits 0. + +#### Fixing gate failures + +Read the `❌` lines from the gate output. Each failure is one of two types: + +--- + +**Content evaluation failure** — `score < 0.5` or `violated` terms + +The `missed=[...]` list shows command terms that are in the test fixture but absent from the skill content. + +``` +❌ my-skill score=0.33 matched=1/3 missed=['orchestrate agents import', '--kind'] +``` + +Open the entity file at `.evolve/entities/{type}/my-skill.md`. + +- **Missing terms**: Add the missing command, flag, or tool name to the skill content body. If the term is a genuine part of what the skill prescribes, the content is incomplete — extend it. +- **Incorrect fixture**: If the term was over-extracted and the skill is actually correct without it, regenerate the fixture to match the current content: + ```bash + python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py .evolve/entities/{type}/my-skill.md + ``` +- **Violated terms**: If `violated=[...]` is non-empty, remove the offending phrase from the skill content or adjust the `must_not_include` list in the fixture. + +--- + +**Recall failure** — skill not surfacing in top-3 for its own scenario + +``` +❌ my-skill rank=6 @1=✗ @3=✗ @5=✓ score=2 + matched_terms: ['cli'] + top5: ['other-skill', 'third-skill', ...] +``` + +The `trigger` field is too vague or uses the wrong vocabulary. Rewrite it so it contains the words a user would actually type when describing the problem: + +- Use the *symptom*, *error message*, or *task description* — not the solution. +- Include the specific command, flag, product name, or error text that distinguishes this skill from the others in `top5`. +- Look at `matched_terms`: if only short, generic words matched (e.g. `['cli']`), the trigger needs more specific keywords. + +After editing the trigger, regenerate the fixture and re-run the gate: +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py .evolve/entities/{type}/my-skill.md +python3 .bob/skills/evolve-lite-test/scripts/check_tests.py --threshold 0.8 --verbose +``` + +Repeat until both suites show ✅ and the gate exits 0. ### Step 5: Run publish script @@ -57,76 +122,39 @@ For each selected file, run: python3 .bob/skills/evolve-lite-publish/scripts/publish.py --entity "{filename}" --repo "{repo}" --user "{identity.user}" ``` -### Step 6: Commit and push +### Step 6: Commit and push to a new branch + +If the user specified a branch name (e.g. "publish to a new branch called `{new_branch}`"), use that. Otherwise derive one as `{identity.user}-{YYYY-MM-DD}`. + +Let `{publish_branch}` be that branch name. Build `{names}` as a comma-joined list of selected filenames, and -`{guideline_paths}` as a space-joined list of the corresponding -`guideline/{filename}` paths inside the clone (the files the publish -script just wrote). +`{entity_paths}` as a space-joined list of the corresponding typed paths inside the clone (for example `.evolve/entities/guideline/{product}/{filename}`, `.evolve/entities/atomic-skill/{product}/{filename}`, or `.evolve/entities/skill-flow/{product}/{filename}`) for the files the publish script just wrote. + +If the publish script wrote a `.gitignore` into the clone root (it will on first publish), include that path too so it lands on the remote and protects every future contributor's clone. ```bash -git -C ".evolve/entities/subscribed/{repo}" add -- {guideline_paths} +git -C ".evolve/entities/subscribed/{repo}" checkout -b "{publish_branch}" +git -C ".evolve/entities/subscribed/{repo}" add -- {entity_paths} .gitignore git -C ".evolve/entities/subscribed/{repo}" commit -m "[evolve] publish: {names}" -git -C ".evolve/entities/subscribed/{repo}" push origin "{branch}" +git -C ".evolve/entities/subscribed/{repo}" push origin "{publish_branch}" ``` -On push success, continue to Step 7. +> **Never use `git add .` or `git add -A` here.** Only the entity files and `.gitignore` are staged explicitly. The `.gitignore` in the clone blocks `.venv/`, `.env`, `.vscode/`, `__pycache__/`, secrets, and all other project noise from ever being staged — but explicit path staging is the final guarantee. + +On push success, tell the user the branch name and continue to Step 7. -### Step 6a: Recover from non-fast-forward rejection +### Step 6a: Recover from push rejection -If the push fails and stderr mentions `rejected` / `non-fast-forward` -/ `fetch first`, another writer pushed to `{branch}` in between. -Rebase the local commit and push once more: +If the push fails and stderr mentions `rejected` / `non-fast-forward` / `fetch first`, the branch already exists on the remote. Pull it in and retry: ```bash -git -C ".evolve/entities/subscribed/{repo}" fetch origin "{branch}" -git -C ".evolve/entities/subscribed/{repo}" rebase "origin/{branch}" +git -C ".evolve/entities/subscribed/{repo}" fetch origin "{publish_branch}" +git -C ".evolve/entities/subscribed/{repo}" rebase "origin/{publish_branch}" +git -C ".evolve/entities/subscribed/{repo}" push origin "{publish_branch}" ``` -- Rebase clean → retry `git push origin "{branch}"` once, then Step 7. -- Rebase conflicted → attempt to resolve, then hand off for user - review. Do not `git rebase --continue` or `git push` without an - explicit user confirmation. - - 1. `git -C ".evolve/entities/subscribed/{repo}" status --porcelain` - lists the conflicted paths. If any are `UD`, `DU`, or binary, - skip to the abort step — those aren't safe to auto-resolve. - 2. For each `UU`/`AA` file, read the conflict markers. During a - rebase, `<<<<<<< HEAD` is the **remote's** version and the - section under the commit sha is the **publish change** being - replayed (opposite of a regular merge). Write an - intent-preserving resolution; don't `git add` yet. - 3. Show the user the diff (`git -C ".evolve/entities/subscribed/{repo}" diff HEAD -- {file}`) per - resolved file with a one-line strategy summary, and ask whether - to **continue** (stage + `rebase --continue` + push) or **abort** - (roll back for manual resolution). - 4. On **continue**: - - ```bash - git -C ".evolve/entities/subscribed/{repo}" add {resolved-files} - git -C ".evolve/entities/subscribed/{repo}" rebase --continue - git -C ".evolve/entities/subscribed/{repo}" push origin "{branch}" - ``` - - Then Step 7. If `rebase --continue` surfaces a new conflict, loop - from step 1. - 5. On **abort** — user declined, conflict isn't safely resolvable, - or the proposed merge feels unsafe: - - ```bash - git -C ".evolve/entities/subscribed/{repo}" rebase --abort - ``` - - The local publish commit is preserved at - `.evolve/entities/subscribed/{repo}` but not on the remote. Tell - the user to either (a) resolve manually in that directory - (`git fetch origin {branch} && git rebase origin/{branch}`, fix - conflicts, `git add` + `git rebase --continue`, `git push origin - {branch}`) or (b) re-run `evolve-lite:publish` with a different - filename if the conflict is a shared name. - -If the push fails for any other reason (auth, network, missing remote -ref), surface git's error and stop — rebase will not help. +If the push fails for any other reason (auth, network, missing remote ref), surface git's error and stop. ### Step 7: Confirm @@ -134,9 +162,9 @@ Tell the user what was published and to which repo. ## Notes -- Published entities are **moved** from `.evolve/entities/guideline/` into - the write-scope clone at `.evolve/entities/subscribed/{repo}/guideline/`, +- Published entities are **copied** from their private typed directories under `.evolve/entities/` + into the matching typed directory in the write-scope clone at `.evolve/entities/subscribed/{repo}/`, with `visibility: public`, `owner: {user}`, `published_at`, and `source` stamped in frontmatter -- The original private entity is deleted after successful publication +- The original private entity is kept intact after publication - All publish actions are logged to `.evolve/audit.log` diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-publish/scripts/publish.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-publish/scripts/publish.py index 39f4e751..8b04e1d6 100755 --- a/platform-integrations/bob/evolve-lite/skills/evolve-lite-publish/scripts/publish.py +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-publish/scripts/publish.py @@ -5,6 +5,7 @@ import datetime import os import re +import subprocess import sys import tempfile from pathlib import Path, PurePath @@ -23,7 +24,7 @@ raise ImportError(f"Cannot find plugin lib directory above {_script}") sys.path.insert(0, str(_lib)) from audit import append as audit_append # noqa: E402 -from entity_io import entity_to_markdown, markdown_to_entity # noqa: E402 +from entity_io import entity_to_markdown, markdown_to_entity, write_clone_gitignore # noqa: E402 from config import get_repo, load_config, normalize_repos, write_repos # noqa: E402 @@ -56,9 +57,74 @@ def _select_target_repo(cfg, requested_name): return write[0], None +def _run_dedup(project_root): + dedup_script = Path(project_root) / ".bob" / "skills" / "evolve-lite-dedup" / "scripts" / "dedup.py" + if not dedup_script.is_file(): + print(f"Error: dedup script not found at {dedup_script}", file=sys.stderr) + sys.exit(1) + + result = subprocess.run( + [sys.executable, str(dedup_script), "--local-only"], + cwd=project_root, + ) + if result.returncode != 0: + print("Error: dedup failed; publish aborted before push.", file=sys.stderr) + sys.exit(result.returncode) + + +ENTITY_TYPES = ("guideline", "atomic-skill", "skill-flow") + + +def _find_entity(evolve_dir, entity_arg): + """Locate the source entity file. + + Accepts: + - bare filename: ``my-skill.md`` (searches all three type dirs) + - type-prefixed: ``atomic-skill/general/my-skill.md`` + - product-relative: ``general/my-skill.md`` (searches all type dirs) + """ + p = PurePath(entity_arg) + + # Reject obvious traversal attempts + if any(part in {".", ".."} for part in p.parts): + return None, None, f"invalid entity name: {entity_arg!r}" + + # Case 1: first component is an explicit entity type + if p.parts[0] in ENTITY_TYPES: + entity_type = p.parts[0] + rel_rest = Path(*p.parts[1:]) if len(p.parts) > 1 else Path(p.name) + src_base = (evolve_dir / "entities" / entity_type).resolve() + candidate = (src_base / rel_rest).resolve() + if not candidate.is_relative_to(src_base): + return None, None, f"invalid entity name: {entity_arg!r}" + if candidate.is_file(): + return candidate, entity_type, None + return None, None, f"entity file not found: {candidate}" + + # Case 2: bare name or product/name — search all type dirs + filename = p.name + matches = [] + for et in ENTITY_TYPES: + src_base = (evolve_dir / "entities" / et).resolve() + for found in src_base.glob(f"**/{filename}"): + if found.is_file() and found.is_relative_to(src_base): + matches.append((found, et)) + + if not matches: + return None, None, f"entity file not found: {entity_arg!r} (searched all type dirs)" + if len(matches) > 1: + paths = ", ".join(str(m) for m, _ in matches) + return None, None, f"ambiguous entity name {entity_arg!r}; found in multiple locations: {paths}" + return matches[0][0], matches[0][1], None + + def main(): parser = argparse.ArgumentParser() - parser.add_argument("--entity", required=True, help="Basename of the .md file to publish") + parser.add_argument( + "--entity", + required=True, + help="Entity to publish: bare filename, 'product/name.md', or 'type/product/name.md'", + ) parser.add_argument("--user", default=None, help="Username to stamp as owner") parser.add_argument("--repo", default=None, help="Write-scope repo name (optional if exactly one is configured)") args = parser.parse_args() @@ -67,20 +133,14 @@ def main(): resolved_evolve_dir = evolve_dir.resolve() project_root = str(resolved_evolve_dir) if evolve_dir.name != ".evolve" else str(resolved_evolve_dir.parent) - if PurePath(args.entity).name != args.entity or args.entity in {".", ".."}: - print(f"Error: invalid entity name: {args.entity!r}", file=sys.stderr) + src_path, entity_type, err = _find_entity(evolve_dir, args.entity) + if err: + print(f"Error: {err}", file=sys.stderr) sys.exit(1) - src_base = (evolve_dir / "entities" / "guideline").resolve() - src_path = (evolve_dir / "entities" / "guideline" / args.entity).resolve() + src_base = (evolve_dir / "entities" / entity_type).resolve() - if not src_path.is_relative_to(src_base): - print(f"Error: invalid entity name: {args.entity!r}", file=sys.stderr) - sys.exit(1) - - if not src_path.is_file(): - print(f"Error: entity file not found or is a directory: {src_path}", file=sys.stderr) - sys.exit(1) + _run_dedup(project_root) config = load_config(project_root) target, err = _select_target_repo(config, args.repo) @@ -100,6 +160,14 @@ def main(): if source: entity["source"] = source + # Version management: start at 1 on first publish; bump on re-publish. + try: + current_version = int(entity.get("version", "0") or "0") + except (ValueError, TypeError): + current_version = 0 + new_version = current_version + 1 + entity["version"] = str(new_version) + clone_root = evolve_dir / "entities" / "subscribed" / target["name"] if not (clone_root / ".git").exists(): print( @@ -109,10 +177,65 @@ def main(): file=sys.stderr, ) sys.exit(1) - dest_dir = clone_root / "guideline" + + expected_branch = target.get("branch", "main") + expected_remote = target.get("remote", "").strip() + try: + branch_result = subprocess.run( + ["git", "-C", str(clone_root), "symbolic-ref", "--short", "HEAD"], + capture_output=True, + text=True, + ) + if branch_result.returncode != 0: + print( + f"Error: could not determine current branch of clone at {clone_root}: {branch_result.stderr.strip()}", + file=sys.stderr, + ) + sys.exit(1) + current_branch = branch_result.stdout.strip() + if current_branch != expected_branch: + print( + f"Error: clone at {clone_root} is on branch '{current_branch}', " + f"expected '{expected_branch}' (from evolve.config.yaml). " + f"Run: git -C \"{clone_root}\" checkout \"{expected_branch}\"", + file=sys.stderr, + ) + sys.exit(1) + + remote_result = subprocess.run( + ["git", "-C", str(clone_root), "remote", "get-url", "origin"], + capture_output=True, + text=True, + ) + if remote_result.returncode != 0: + print( + f"Error: could not determine remote URL of clone at {clone_root}: {remote_result.stderr.strip()}", + file=sys.stderr, + ) + sys.exit(1) + actual_remote = remote_result.stdout.strip() + if actual_remote != expected_remote: + print( + f"Error: clone at {clone_root} has remote '{actual_remote}', " + f"expected '{expected_remote}' (from evolve.config.yaml). " + f"The clone does not match the configured write-scope repo.", + file=sys.stderr, + ) + sys.exit(1) + except FileNotFoundError: + print("Error: git not found on PATH", file=sys.stderr) + sys.exit(1) + + # Ensure the clone has a protective .gitignore before writing anything. + # Idempotent — only writes when the file is absent or ours to update. + write_clone_gitignore(clone_root) + + relative_src_parent = src_path.parent.relative_to(src_base) + dest_dir = clone_root / ".evolve" / "entities" / entity_type / relative_src_parent dest_dir.mkdir(parents=True, exist_ok=True) dest_base = dest_dir.resolve() - dest_path = (dest_dir / args.entity).resolve() + filename = src_path.name + dest_path = (dest_dir / filename).resolve() if not dest_path.is_relative_to(dest_base): print(f"Error: invalid entity name: {args.entity!r}", file=sys.stderr) sys.exit(1) @@ -126,7 +249,7 @@ def main(): "w", encoding="utf-8", dir=dest_dir, - prefix=f".{args.entity}.", + prefix=f".{filename}.", suffix=".tmp", delete=False, ) as temp_file: @@ -136,7 +259,6 @@ def main(): temp_path = Path(temp_file.name) temp_path.replace(dest_path) - src_path.unlink() finally: if temp_path is not None and temp_path.exists(): temp_path.unlink() @@ -146,13 +268,17 @@ def main(): project_root=project_root, action="publish", actor=effective_user or "unknown", - entity=args.entity, + entity=filename, repo=target["name"], + version=new_version, ) except Exception as exc: print(f"Warning: failed to append audit entry for publish: {exc}", file=sys.stderr) - print(f"Published: {args.entity} -> {dest_path} (repo: {target['name']})") + version_note = f"v{new_version}" if new_version > 1 else "v1 (first publish)" + print(f"Published: {filename} -> {dest_path} (type: {entity_type}, repo: {target['name']}, {version_note})") + if new_version > 1: + print(f" Tip: add a '## Changelog' section to document what changed in v{new_version}.") if __name__ == "__main__": diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/SKILL.md new file mode 100644 index 00000000..64f4d799 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/SKILL.md @@ -0,0 +1,152 @@ +--- +name: evolve-lite:recall +description: Must be used at the start of any non-trivial task involving code changes, debugging, repo exploration, file inspection, or environment/tooling investigation to surface stored guidance before analysis or tool use. +--- + +# Entity Retrieval + +## Overview + +This skill loads relevant stored Evolve entities into the current turn before substantive work begins. + +Use this skill first whenever the task involves: +- code changes +- debugging +- code review +- repo exploration +- file inspection +- environment/tooling investigation + +Skip only for trivial conversational requests with no local context. + +## Required Action + +Before any non-trivial local work, you must complete the recall workflow below. Reading this `SKILL.md` alone does not satisfy the skill. + +### Completion Rule + +Do not proceed to other analysis or tool use until all steps below are complete. + +1. If a manifest has already been injected for this turn, use it to pick which entity files to open. Otherwise inspect `${EVOLVE_DIR:-.evolve}/entities/` and `${EVOLVE_DIR:-.evolve}/public/` for guidance relevant to the current task. +2. Read each matching entity file that appears relevant. +3. Summarize the applicable guidance in your own words before proceeding. +4. If no relevant entities exist, state that explicitly before proceeding. + +### Required Visible Completion Note + +Before moving on, produce an explicit completion note in your reasoning or user update using one of these forms: + +- `Recall complete: searched ${EVOLVE_DIR:-.evolve}/entities/, read , applicable guidance: ` +- `Recall complete: searched ${EVOLVE_DIR:-.evolve}/entities/, no relevant entities found` + +### Minimum Acceptable Procedure + +1. List or search files under `${EVOLVE_DIR:-.evolve}/entities/` and `${EVOLVE_DIR:-.evolve}/public/` (or read the injected manifest if one is present). +2. Identify candidate entities relevant to the task. +3. Open and read those entity files. +4. Summarize what applies, or state that nothing applies. + +### Failure Conditions + +The skill is not complete if any of the following are true: + +- You only read this `SKILL.md` +- You did not inspect `${EVOLVE_DIR:-.evolve}/entities/` +- You did not read the relevant entity files +- You proceeded without stating whether guidance was found + +## How It Works + +Bob has no auto-injection hook for entity retrieval. Complete the **Required Action** workflow above on every applicable task. + +Entities can come from multiple sources: +- **Private entities**: Your own local entities (not shared) +- **Subscribed entities**: Entities cloned from any configured repo — + read-scope subscriptions and write-scope publish targets both live + under `${EVOLVE_DIR:-.evolve}/entities/subscribed/{name}/` + +## Entities Storage + +```text +.evolve/entities/ + guideline/ + use-context-managers-for-file-operations.md <- private simple preference + atomic-skill/ + extract-json-fields-in-constrained-shells.md <- private smallest reusable capability + skill-flow/ + save-trajectory-then-extract-and-persist.md <- private reusable multi-step flow + subscribed/ + memory/ <- write-scope clone (publishes land here) + guideline/ + my-published-guideline.md + atomic-skill/ + my-published-atomic-skill.md + skill-flow/ + my-published-skill-flow.md + alice/ <- read-scope clone + guideline/ + alice-guideline.md <- annotated [from: alice] + atomic-skill/ + alice-atomic-skill.md + skill-flow/ + alice-skill-flow.md +``` + +The manifest output is human-readable: + +```text +- `.evolve/entities/guideline/use-context-managers-for-file-operations.md` [guideline] — When processing files or managing resources +- `.evolve/entities/atomic-skill/extract-json-fields-in-constrained-shells.md` [atomic-skill] — When extracting fields from JSON in constrained shells +- `.evolve/entities/subscribed/alice/skill-flow/error-triage-and-fix-validation.md` [skill-flow] — When debugging recurring failures with a reusable validation sequence +``` + +Each file still uses markdown with YAML frontmatter: + +```markdown +--- +type: guideline +trigger: When processing files or managing resources +--- + +Use context managers for file operations + +## Rationale + +Ensures proper resource cleanup +``` + +For `skill-flow` entities, the frontmatter includes an `atomic_skills` field that references the component atomic skills: + +```markdown +--- +type: skill-flow +trigger: When creating and uploading Watson Orchestrate agents +atomic_skills: create-yaml-file-with-spec-version, activate-virtual-environment, import-agent-with-orchestrate-cli +--- + +To create and upload a Watson Orchestrate agent: 1) Create a YAML file... 2) Activate environment... 3) Import agent... + +## Rationale + +Standard workflow for Watson Orchestrate agent deployment +``` + +## On-Demand Expansion + +When a manifest entry's trigger matches the current task, use `read_file` to load the full entity. The file body contains the entity content and an optional `## Rationale` section. Apply `guideline` entries as simple preferences, `atomic-skill` entries as focused reusable capabilities, and `skill-flow` entries as reusable multi-step compositions. + +### Using Skill-Flow References + +When you load a `skill-flow` entity: +1. Check the `atomic_skills` field in the frontmatter +2. If present, you can optionally load the referenced atomic skills for more detailed guidance +3. The atomic skill files are located in `.evolve/entities/atomic-skill/` with filenames matching the slugs +4. This allows you to understand both the high-level flow and the detailed implementation of each step + +**Example workflow**: +``` +1. Load skill-flow: "create-and-upload-watson-orchestrate-agent.md" +2. See atomic_skills: "create-yaml-file-with-spec-version, activate-virtual-environment, import-agent" +3. Optionally read: ".evolve/entities/atomic-skill/create-yaml-file-with-spec-version.md" for details +4. Apply the complete flow with detailed understanding of each step +``` diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/scripts/retrieve_entities.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/scripts/retrieve_entities.py new file mode 100644 index 00000000..5095a4a4 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/scripts/retrieve_entities.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Retrieve and output an entity manifest for bob to expand on demand.""" + +import json +import os +import sys +from pathlib import Path + +# Walk up from the script location to find the installed plugin lib directory. +# Every host installs the shared lib under lib/evolve-lite/ so multiple +# plugins can coexist side by side (e.g. .bob/lib/evolve-lite/). +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) +from entity_io import dedupe_manifest_entries, find_recall_entity_dirs, get_evolve_dir, load_manifest, log as _log # noqa: E402 +import audit # noqa: E402 + + +def log(message): + _log("retrieve", message) + + +log("Script started") + + +def format_entities(entities): + """Format a manifest of entities for bob to expand on demand.""" + header = """## Evolve entity manifest for this task + +These stored entities are available for this repo. Read only the files whose trigger looks relevant to the user's request: + +""" + lines = [] + for e in entities: + label = " ⚠ failure-derived" if e.get("derived_from_failure") else "" + lines.append(f"- `{e['path']}` [{e['type']}]{label} — {e['trigger']}") + return header + "\n".join(lines) + + +def _audit_id(path_str): + """Derive the audit entity id from a manifest path. + + Matches upstream's convention for entities/: id is the path relative to + ``entities/`` with ``.md`` stripped (e.g. ``guideline/foo``, + ``subscribed/alice/guideline/bar``). Public entities are prefixed with + ``public/`` to keep the id space distinct from private entities. + """ + if "/entities/" in path_str: + return path_str.split("/entities/", 1)[1].removesuffix(".md") + if "/public/" in path_str: + return "public/" + path_str.split("/public/", 1)[1].removesuffix(".md") + return path_str.removesuffix(".md") + + +def main(): + # Hook context arrives via stdin as JSON when invoked from a hook + # (claude/claw-code/codex). Handle empty/absent stdin gracefully so the + # script also works when invoked manually (no hook upstream). + input_data = {} + try: + raw = sys.stdin.read() + if raw.strip(): + input_data = json.loads(raw) + if isinstance(input_data, dict): + log(f"Input keys: {list(input_data.keys())}") + else: + log(f"Input type: {type(input_data).__name__}") + else: + log("stdin was empty") + except json.JSONDecodeError as e: + log(f"stdin was not valid JSON ({e})") + return + + if isinstance(input_data, dict): + prompt = input_data.get("prompt", "") + if prompt: + log(f"Prompt preview: {prompt[:120]}") + + log("=== Environment Variables ===") + for key, value in sorted(os.environ.items()): + if any(sensitive in key.upper() for sensitive in ["PASSWORD", "SECRET", "TOKEN", "KEY", "API"]): + log(f" {key}=***MASKED***") + else: + log(f" {key}={value}") + log("=== End Environment Variables ===") + + entities = [] + recall_dirs = find_recall_entity_dirs() + log(f"Recall dirs: {recall_dirs}") + for root_dir in recall_dirs: + entities.extend(load_manifest(root_dir)) + + entities = dedupe_manifest_entries(entities) + + if not entities: + log("No entities found") + return + + log(f"Loaded {len(entities)} entities") + + output = format_entities(entities) + print(output) + log(f"Output {len(output)} chars to stdout") + + # Audit which entity ids were served to this session. Logging is + # intentionally best-effort so recall never fails because provenance + # recording could not append to audit.log. + try: + if isinstance(input_data, dict): + transcript_path = input_data.get("transcript_path", "") + else: + transcript_path = "" + session_id = None + if transcript_path: + stem = Path(transcript_path).stem + if stem.startswith("claude-transcript_"): + session_id = stem.removeprefix("claude-transcript_") + if not session_id and isinstance(input_data, dict) and isinstance(input_data.get("session_id"), str): + session_id = input_data["session_id"] + entity_ids = sorted({_audit_id(entity["path"]) for entity in entities if entity.get("path")}) + if session_id and entity_ids: + audit.append( + evolve_dir=str(get_evolve_dir().resolve()), + event="recall", + session_id=session_id, + entities=entity_ids, + ) + log(f"Audit: recall session_id={session_id} entities={len(entity_ids)}") + except Exception as exc: + log(f"Audit append failed (non-fatal): {exc}") + + +if __name__ == "__main__": + main() diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-run-tests/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-run-tests/SKILL.md new file mode 100644 index 00000000..64392baa --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-run-tests/SKILL.md @@ -0,0 +1,32 @@ +--- +name: evolve-lite-run-tests +description: >- + Run all skill tests — content, recall, and baseline — against the existing + pseudo-conversation fixtures. +metadata: + user-invocable: true + disable-model-invocation: true +--- + +Run all three tests against existing fixtures: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_skill_evaluation.py --verbose +python3 .bob/skills/evolve-lite-test/scripts/run_recall_tests.py --verbose +python3 .bob/skills/evolve-lite-test/scripts/run_baseline_tests.py --simulate +``` + +**What each test checks:** + +| Test | Script | Checks | +|---|---|---| +| Content | `run_skill_evaluation.py` | Skill contains the commands it prescribes | +| Recall | `run_recall_tests.py` | Skill surfaces in top 3 when its scenario is described | +| Baseline | `run_baseline_tests.py` | Agent without the skill misses the key guidance | + +**Reports written to `.evolve/tests/evaluation/`:** +- `report.json` — content test +- `recall_report.json` — recall test +- `baseline_report.json` — baseline test + +To regenerate fixtures before running, use `/evolve-lite-create-tests` first. diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-save-trajectory/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-save-trajectory/SKILL.md index 509c0734..7a36813e 100644 --- a/platform-integrations/bob/evolve-lite/skills/evolve-lite-save-trajectory/SKILL.md +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-save-trajectory/SKILL.md @@ -1,6 +1,6 @@ --- name: evolve-lite:save-trajectory -description: Save the current conversation as a trajectory JSON file in OpenAI chat completion format for analysis and fine-tuning +description: Saves the current conversation as a trajectory JSON file in OpenAI chat completion format for analysis and fine-tuning. Run explicitly with /evolve-lite:save-trajectory when you want to capture a session. --- # Save Trajectory diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-save-trajectory/scripts/on_stop.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-save-trajectory/scripts/on_stop.py index 81c3400e..6ec09365 100644 --- a/platform-integrations/bob/evolve-lite/skills/evolve-lite-save-trajectory/scripts/on_stop.py +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-save-trajectory/scripts/on_stop.py @@ -1,83 +1,12 @@ #!/usr/bin/env python3 -"""Stop hook that copies the session transcript to .evolve/trajectories/.""" +"""Stop hook — disabled. evolve-lite:save-trajectory runs only when explicitly invoked.""" -import datetime -import getpass -import json -import os -import shutil import sys -import tempfile -from pathlib import Path - - -_log_file = None - - -def _get_log_file(): - global _log_file - if _log_file is None: - try: - uid = os.getuid() - except AttributeError: - uid = getpass.getuser() - log_dir = os.path.join(tempfile.gettempdir(), f"evolve-{uid}") - os.makedirs(log_dir, mode=0o700, exist_ok=True) - _log_file = os.path.join(log_dir, "evolve-plugin.log") - return _log_file - - -def log(message): - if not os.environ.get("EVOLVE_DEBUG"): - return - try: - timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - with open(_get_log_file(), "a", encoding="utf-8") as f: - f.write(f"[{timestamp}] [save-trajectory-stop] {message}\n") - except Exception: - pass - - -def get_trajectories_dir(): - evolve_dir = os.environ.get("EVOLVE_DIR") - if evolve_dir: - base = Path(evolve_dir) / "trajectories" - else: - project_root = os.environ.get("CLAUDE_PROJECT_ROOT", "") - if project_root: - base = Path(project_root) / ".evolve" / "trajectories" - else: - base = Path(".evolve") / "trajectories" - base.mkdir(parents=True, exist_ok=True, mode=0o700) - return base.resolve() def main(): - try: - input_data = json.load(sys.stdin) - except (json.JSONDecodeError, ValueError): - input_data = {} - - log(f"Stop hook input keys: {list(input_data.keys())}") - log(f"Stop hook input: {json.dumps(input_data, default=str)[:2000]}") - - transcript_path = input_data.get("transcript_path") - if not transcript_path: - log("No transcript_path in stop hook input") - return - - src = Path(transcript_path) - if not src.is_file(): - log(f"Transcript file not found: {src}") - return - - session_id = src.stem - trajectories_dir = get_trajectories_dir() - dst = trajectories_dir / f"claude-transcript_{session_id}.jsonl" - - shutil.copy2(str(src), str(dst)) - log(f"Copied transcript {src} -> {dst}") - print(f"Trajectory saved: {dst}") + # Hook is intentionally a no-op. The save-trajectory skill is run on-demand only. + sys.exit(0) if __name__ == "__main__": diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-subscribe/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-subscribe/SKILL.md index 48cb891d..38d6802a 100644 --- a/platform-integrations/bob/evolve-lite/skills/evolve-lite-subscribe/SKILL.md +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-subscribe/SKILL.md @@ -43,10 +43,10 @@ sync: on_session_start: true ``` -Also ensure `.evolve/` is gitignored: +Also ensure `.evolve/entities/subscribed/` is gitignored (the subscribed clones are managed by evolve-lite and should not be committed). Do **not** gitignore `.evolve/` or `.bob/` — those directories must remain tracked: ```bash -grep -qxF '.evolve/' .gitignore 2>/dev/null || echo '.evolve/' >> .gitignore +grep -qxF '.evolve/entities/subscribed/' .gitignore 2>/dev/null || echo '.evolve/entities/subscribed/' >> .gitignore ``` ### Step 2: Gather details diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-subscribe/scripts/subscribe.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-subscribe/scripts/subscribe.py index f2902edf..fae3de48 100755 --- a/platform-integrations/bob/evolve-lite/skills/evolve-lite-subscribe/scripts/subscribe.py +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-subscribe/scripts/subscribe.py @@ -50,6 +50,7 @@ save_config, set_repos, ) +from entity_io import write_clone_gitignore # noqa: E402 def main(): @@ -100,6 +101,10 @@ def main(): print(f"Error: git clone failed: {detail}", file=sys.stderr) sys.exit(1) + # Write a protective .gitignore into the clone so noise files can never + # be accidentally staged or pushed from this directory. + write_clone_gitignore(dest) + repos.append( { "name": args.name, diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-sync/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-sync/SKILL.md index 6c9173f2..040ecd69 100644 --- a/platform-integrations/bob/evolve-lite/skills/evolve-lite-sync/SKILL.md +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-sync/SKILL.md @@ -1,13 +1,13 @@ --- name: evolve-lite:sync -description: Pull the latest guidelines from every configured repo (read- and write-scope). +description: Pull the latest guidelines, atomic skills, and skill flows from every configured repo (read- and write-scope). --- # Sync Repos ## Overview -Pull the latest guidelines from every repo in `evolve.config.yaml` +Pull the latest entities — guidelines, atomic skills, and skill flows — from every repo in `evolve.config.yaml` `repos:` list — both `scope: read` (subscribe-only) and `scope: write` (publish targets). Write-scope repos use a rebase strategy so any unpushed local publish commits are preserved. diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test-new-skills/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test-new-skills/SKILL.md new file mode 100644 index 00000000..00987cf4 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test-new-skills/SKILL.md @@ -0,0 +1,38 @@ +--- +name: evolve-lite-test-new-skills +description: >- + Generate pseudo-conversation test fixtures for newly saved atomic skills. Run + this immediately after evolve-lite-learn has finished saving entities. +metadata: + user-invocable: true + disable-model-invocation: true +--- + +After `evolve-lite-learn` has saved new skill entities, generate a test fixture +for each new atomic-skill file by running: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py [ ...] +``` + +Pass the exact file paths that `save_entities.py` just wrote. The script will: +1. Read each file and confirm it is an `atomic-skill` (skips guidelines and skill-flows) +2. Derive a realistic trigger-based user question from the skill's `trigger` field +3. Extract `must_include` terms (backtick command strings) and `must_not_include` terms (negation patterns) from the skill content +4. Write a fixture JSON to `.evolve/tests/pseudo_conversations/.json` + +If you don't have the exact paths, regenerate fixtures for all skills at once: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +``` + +To validate the new fixtures immediately after generating them: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_skill_evaluation.py --verbose +``` + +A passing result (score ≥ 0.5, no constraint violations) means the skill is +self-consistent — its content contains the commands it claims to prescribe. +A failure signals that the skill content may be too vague or missing a key command. diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/EXECUTION_PLAN.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/EXECUTION_PLAN.md new file mode 100644 index 00000000..c2cf43fa --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/EXECUTION_PLAN.md @@ -0,0 +1,345 @@ +# Execution Plan Analyzer + +## Overview + +The execution plan analyzer shows exactly what the functional test framework "sees" when it analyzes a skill. This helps you understand: + +- How many steps are extracted from your skill +- Which commands are found and would be executed +- Whether your skill is likely to pass functional tests +- Why a skill might fail + +## Usage + +### Analyze All Skills +```bash +python3 .bob/skills/evolve-lite-test/scripts/show_execution_plan.py +``` + +### Analyze Specific Skills +```bash +python3 .bob/skills/evolve-lite-test/scripts/show_execution_plan.py \ + .evolve/entities/atomic-skill/my-skill.md +``` + +### Filter by Type +```bash +# Only atomic skills +python3 .bob/skills/evolve-lite-test/scripts/show_execution_plan.py --type atomic-skill + +# Only skill flows +python3 .bob/skills/evolve-lite-test/scripts/show_execution_plan.py --type skill-flow + +# Only guidelines +python3 .bob/skills/evolve-lite-test/scripts/show_execution_plan.py --type guideline +``` + +### Verbose Mode +```bash +python3 .bob/skills/evolve-lite-test/scripts/show_execution_plan.py --verbose +``` + +## Understanding the Output + +### Example Output + +``` +====================================================================== +EXECUTION PLAN: activate-the-virtual-environment +====================================================================== +Type: atomic-skill +Path: .evolve/entities/atomic-skill/activate-the-virtual-environment.md + +────────────────────────────────────────────────────────────────────── +SKILL CONTENT: +────────────────────────────────────────────────────────────────────── +Activate the virtual environment + +────────────────────────────────────────────────────────────────────── +EXTRACTED STEPS: 1 +────────────────────────────────────────────────────────────────────── + +📋 Step 1 + Description: Activate the virtual environment + Commands found: 0 + └─ ⚠️ No executable commands found + +────────────────────────────────────────────────────────────────────── +EXECUTION SUMMARY: +────────────────────────────────────────────────────────────────────── +Total steps: 1 +Total commands: 0 +Steps with commands: 0 +Steps without commands: 1 + +────────────────────────────────────────────────────────────────────── +FUNCTIONAL TEST PREDICTION: +────────────────────────────────────────────────────────────────────── +❌ LIKELY TO FAIL: No executable commands found + Reason: Skill content is too abstract or missing commands +``` + +### Sections Explained + +1. **SKILL CONTENT** - The actual content of the skill (first 500 chars) + +2. **EXTRACTED STEPS** - How many steps the analyzer found + - Looks for numbered lists (1), 2), 3), etc.) + - If no numbered steps, treats entire content as one step + +3. **Step Details** - For each step: + - 📋 Step number + - Description (first 100 chars) + - Commands found (count) + - List of actual commands that would be executed + +4. **EXECUTION SUMMARY** - Overall statistics: + - Total steps found + - Total commands found + - Steps with vs without commands + +5. **FUNCTIONAL TEST PREDICTION** - Likely test outcome: + - ✅ **MAY PASS** - Commands found and extractable + - ⚠️ **MAY FAIL** - Multiple commands in single step + - ❌ **LIKELY TO FAIL** - No executable commands found + +## Common Patterns + +### ❌ Pattern 1: No Commands Found + +**Skill Content:** +``` +Activate the virtual environment +``` + +**Problem:** Too abstract, no actual command + +**Fix:** +``` +Activate the virtual environment with: +```bash +source .venv/bin/activate +``` +``` + +--- + +### ⚠️ Pattern 2: Multiple Commands in One Step + +**Skill Content:** +``` +To do X: 1) Do A. 2) Do B with `cmd1`. 3) Do C with `cmd2`. +``` + +**Problem:** All in one sentence, only first command may execute + +**Fix:** +```markdown +## Steps + +1. Do A + +2. Do B + ```bash + cmd1 + ``` + +3. Do C + ```bash + cmd2 + ``` +``` + +--- + +### ✅ Pattern 3: Clear Extractable Commands + +**Skill Content:** +```markdown +## Steps + +1. Activate environment + ```bash + source .venv/bin/activate + ``` + +2. Run command + ```bash + orchestrate agents import --file agent.yaml + ``` +``` + +**Result:** Each step has clear, extractable commands + +--- + +## Real-World Examples from Analysis + +### Example 1: Virtual Environment Activation + +**Current State:** +``` +Activate the virtual environment +``` + +**Execution Plan:** +- Steps: 1 +- Commands: 0 +- Prediction: ❌ LIKELY TO FAIL + +**Why:** No executable command provided + +**Fix:** +```markdown +Activate the virtual environment: +```bash +source .venv/bin/activate +``` +``` + +--- + +### Example 2: Orchestrate Agent Creation + +**Current State:** +``` +To create and upload a Watson Orchestrate agent: 1) Create a YAML file... +2) Activate the virtual environment. 3) Ensure the orchestrate environment +is authenticated with `orchestrate env activate`. 4) Import the agent with +`orchestrate agents import --file `. +``` + +**Execution Plan:** +- Steps: 1 (should be 4!) +- Commands: 2 (`orchestrate env activate`, `orchestrate agents import`) +- Prediction: ⚠️ MAY FAIL (multiple commands in single step) + +**Why:** All 4 steps compressed into one sentence + +**Fix:** +```markdown +## Steps + +1. Create YAML file with required fields + +2. Activate virtual environment + ```bash + source .venv/bin/activate + ``` + +3. Authenticate with Orchestrate + ```bash + orchestrate env activate + ``` + +4. Import the agent + ```bash + orchestrate agents import --file agent.yaml + ``` +``` + +--- + +### Example 3: Well-Structured Skill + +**Current State:** +```markdown +When using Watson Orchestrate CLI commands, always activate the virtual +environment first with `source .venv/bin/activate`. +``` + +**Execution Plan:** +- Steps: 1 +- Commands: 1 (`source .venv/bin/activate`) +- Prediction: ✅ MAY PASS + +**Why:** Clear command in backticks, easy to extract + +--- + +## Command Extraction Rules + +The analyzer extracts commands from: + +1. **Code blocks:** + ````markdown + ```bash + command here + ``` + ```` + +2. **Backticks with spaces or special chars:** + ```markdown + Run `command --flag value` + ``` + +3. **Not extracted:** + - Simple words in backticks: `activate` + - Text without command syntax: `the environment` + +## Using Results to Improve Skills + +### Step 1: Run the Analyzer +```bash +python3 .bob/skills/evolve-lite-test/scripts/show_execution_plan.py +``` + +### Step 2: Review Predictions + +Look for: +- ❌ Skills with no commands +- ⚠️ Skills with multiple commands in one step +- Skills with only 1 step when there should be more + +### Step 3: Fix Problem Skills + +For each problematic skill: +1. Add clear numbered steps +2. Put commands in code blocks +3. One command per step +4. Make steps actionable + +### Step 4: Verify Improvements +```bash +# Re-run analyzer on fixed skill +python3 .bob/skills/evolve-lite-test/scripts/show_execution_plan.py \ + .evolve/entities/atomic-skill/my-fixed-skill.md +``` + +### Step 5: Run Functional Tests +```bash +# Confirm the skill now passes +python3 .bob/skills/evolve-lite-test/scripts/run_skill_functional_tests.py +``` + +--- + +## Integration with Testing Workflow + +``` +1. Write/update skill + ↓ +2. Run execution plan analyzer + ↓ +3. Review predictions + ↓ +4. Fix any issues + ↓ +5. Run functional tests + ↓ +6. Confirm pass +``` + +--- + +## Summary + +The execution plan analyzer helps you: + +- **Visualize** what the test framework sees +- **Predict** whether skills will pass functional tests +- **Identify** structural problems before testing +- **Fix** issues proactively +- **Improve** skill quality systematically + +Use it as a **pre-flight check** before running functional tests! \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/EXTRACTION_MISMATCH.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/EXTRACTION_MISMATCH.md new file mode 100644 index 00000000..36ee05fe --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/EXTRACTION_MISMATCH.md @@ -0,0 +1,194 @@ +# Critical Issue: Extraction Logic Mismatch + +## The Problem + +The **execution plan analyzer** and the **functional test** use **different step extraction logic**, causing them to show different results! + +--- + +## Execution Plan Analyzer Logic + +**File:** `show_execution_plan.py` (line 28) + +```python +numbered_pattern = r'^\s*(\d+)[.)]\s+(.+?)(?=^\s*\d+[.)]|\Z)' +matches = re.finditer(numbered_pattern, content, re.MULTILINE | re.DOTALL) +``` + +**What it does:** +- Captures from one numbered item to the next (or end of content) +- Uses `re.DOTALL` - captures across multiple lines +- Pattern: `(.+?)` - captures EVERYTHING until next number + +**Example with Orchestrate Agent Skill:** +``` +Content: "To create... 1) Create a YAML file... 2) Activate... 3) Run `cmd1`. 4) Run `cmd2`." + +Extraction: + Step 1: "Create a YAML file... 2) Activate... 3) Run `cmd1`. 4) Run `cmd2`." + Commands found: cmd1, cmd2 (extracts ALL commands from entire content) +``` + +--- + +## Functional Test Logic + +**File:** `run_skill_functional_tests.py` (line 163) + +```python +numbered_pattern = r'(\d+)[.)]\s+([^\n]+)' +matches = re.finditer(numbered_pattern, content) +``` + +**What it does:** +- Captures only up to the newline +- Pattern: `([^\n]+)` - stops at newline character +- Only extracts FIRST command in backticks per line + +**Example with Orchestrate Agent Skill:** +``` +Content: "To create... 1) Create a YAML file... 2) Activate... 3) Run `cmd1`. 4) Run `cmd2`." + +Extraction: + Step 1: "Create a YAML file..." + Step 2: "Activate..." + Step 3: "Run `cmd1`." + Step 4: "Run `cmd2`." + +But since it's all on ONE line, it only matches: + Step 1: "Create a YAML file... 2) Activate... 3) Run `cmd1`. 4) Run `cmd2`." + Command: cmd1 (only FIRST command in backticks) +``` + +--- + +## Why This Causes Confusion + +### Execution Plan Shows: +``` +Steps: 1 +Commands: 2 (orchestrate env activate, orchestrate agents import) +Prediction: ⚠️ MAY FAIL +``` + +### Functional Test Actually Does: +``` +Steps: 1 +Commands extracted: 1 (orchestrate env activate - FIRST command only) +Commands executed: 1 (orchestrate env activate) +Result: ❌ FAIL - orchestrate agents import never executed +``` + +--- + +## The Real Issue + +For the Orchestrate Agent skill: +``` +To create and upload a Watson Orchestrate agent: 1) Create a YAML file with spec_version, name, description, instructions, model, parameters, and tools fields. 2) Activate the virtual environment. 3) Ensure the orchestrate environment is authenticated with `orchestrate env activate`. 4) Import the agent with `orchestrate agents import --file `. +``` + +**This is ALL ONE LINE** - no newlines between numbered items! + +### Execution Plan Analyzer: +- Sees: 1 step (entire line) +- Extracts: Both commands (`orchestrate env activate`, `orchestrate agents import`) +- Shows: 2 commands found + +### Functional Test: +- Sees: 1 step (entire line, stops at newline which never comes) +- Extracts: Only FIRST command (`orchestrate env activate`) +- Executes: Only that one command +- Fails: Because `orchestrate agents import` never runs + +--- + +## Why Virtual Environment Skill Also Fails + +**Skill content:** +``` +Activate the virtual environment +``` + +### Both Scripts: +- See: 1 step +- Extract: 0 commands (no backticks, no code blocks) +- Result: No commands to execute + +The functional test marks it as "action" type and auto-succeeds, but check #5 fails because the expected command `source .venv/bin/activate` was never executed. + +--- + +## The Fix + +### Option 1: Fix the Functional Test Extraction + +Make it match the execution plan analyzer: + +```python +# Change from: +numbered_pattern = r'(\d+)[.)]\s+([^\n]+)' + +# To: +numbered_pattern = r'^\s*(\d+)[.)]\s+(.+?)(?=^\s*\d+[.)]|\Z)' +matches = re.finditer(numbered_pattern, content, re.MULTILINE | re.DOTALL) +``` + +**Problem:** This still won't help because the skill content is poorly formatted (all on one line). + +### Option 2: Fix the Skills (RECOMMENDED) + +Restructure skills with proper formatting: + +```markdown +## Steps + +1. Create YAML file with required fields + +2. Activate virtual environment + ```bash + source .venv/bin/activate + ``` + +3. Authenticate with Orchestrate + ```bash + orchestrate env activate + ``` + +4. Import the agent + ```bash + orchestrate agents import --file agent.yaml + ``` +``` + +**This works with BOTH extraction methods!** + +--- + +## Why Option 2 is Better + +1. **Clear structure** - Each step is separate +2. **Explicit commands** - In code blocks, easy to extract +3. **Works with both extractors** - No ambiguity +4. **Human readable** - Easy to understand and follow +5. **Maintainable** - Easy to update individual steps + +--- + +## Action Items + +1. **Update functional test extraction** to match execution plan analyzer (for consistency) +2. **Document skill formatting standards** based on what works +3. **Refactor existing skills** to use proper structure +4. **Add validation** to catch poorly formatted skills before they're saved + +--- + +## Lesson Learned + +**The tests aren't broken - they're revealing that:** +1. Skills are poorly formatted (all on one line) +2. Extraction logic is inconsistent between tools +3. We need clear skill formatting standards + +The 0% pass rate is actually **correct** - these skills genuinely don't work as written! \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/FUNCTIONAL_TESTING.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/FUNCTIONAL_TESTING.md new file mode 100644 index 00000000..55723bce --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/FUNCTIONAL_TESTING.md @@ -0,0 +1,377 @@ +# Functional Testing for Evolve Lite Skills + +## Overview + +Functional testing validates that skills actually work by simulating their execution in a mock environment. Unlike integration tests (which analyze completed conversations) or unit tests (which check trigger matching), functional tests answer the question: **"If I follow this skill's instructions, will it do what I expect?"** + +## How It Works + +### 1. Mock Environment + +The test framework creates a mock environment that simulates: +- File system operations (creating, reading files) +- Command execution (with simulated outputs) +- Environment variables +- Error tracking + +### 2. Step Extraction + +The framework parses skill content to extract executable steps: +- Numbered steps (1., 2., 3.) +- Commands in backticks or code blocks +- Action descriptions + +### 3. Execution Simulation + +Each extracted step is "executed" in the mock environment: +- Commands are simulated with realistic outputs +- File operations are tracked +- Errors are captured + +### 4. Outcome Validation + +The test validates that: +- All steps were executed +- All steps succeeded +- Expected commands were run +- Expected files were created +- No errors occurred + +## Running Functional Tests + +### Basic Usage + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_skill_functional_tests.py +``` + +### With Verbose Output + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_skill_functional_tests.py --verbose +``` + +### Custom Scenarios Directory + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_skill_functional_tests.py \ + --scenarios-dir /path/to/scenarios \ + --output /path/to/results +``` + +## Test Scenario Format + +Scenarios are JSON files in `.evolve/tests/functional/scenarios/`: + +```json +{ + "scenario_id": "test_orchestrate_agent_creation", + "description": "Test that the skill for creating Watson Orchestrate agents actually works", + "user_request": "Create a Watson Orchestrate agent", + "expected_skills": [ + "to-create-and-upload-a-watson-orchestrate-agent-1-create-a" + ], + "setup": { + "files": { + "agent_config.json": "{\"name\": \"test-agent\"}" + } + }, + "expected_commands": [ + "orchestrate agents import" + ], + "success_criteria": { + "required_files": [], + "commands_must_succeed": true, + "max_errors": 0 + } +} +``` + +### Scenario Fields + +- **scenario_id**: Unique identifier for the test +- **description**: Human-readable description +- **user_request**: The user's original request +- **expected_skills**: List of skill slugs to test +- **setup**: Initial environment state + - **files**: Files to create before execution +- **expected_commands**: Commands that should be executed +- **success_criteria**: Validation criteria + - **required_files**: Files that must be created + - **commands_must_succeed**: Whether commands must succeed + - **max_errors**: Maximum allowed errors + +## Test Results + +Results are saved to `.evolve/tests/functional/results/`: + +### Summary Report + +`functional_test_report.json` contains: +- Total tests run +- Pass/fail counts +- Pass rate +- Individual test results + +### Individual Test Results + +Each test creates a detailed JSON file with: +- Skill information +- Execution log (steps found, executed, successful) +- Environment state (files created, commands run, errors) +- Validation results (what passed/failed) + +## What Functional Tests Reveal + +### 1. Incomplete Skills + +**Example**: The `activate-the-virtual-environment` skill only has a title and rationale, but no actual steps or commands. + +**Test Result**: +```json +{ + "steps_found": 1, + "steps_executed": 1, + "commands_executed": 0, + "expected_commands_run": false, + "passed": false +} +``` + +**Insight**: This skill needs more detailed instructions with actual commands. + +### 2. Poor Step Extraction + +**Example**: The `to-create-and-upload-a-watson-orchestrate-agent-1-create-a` skill has all steps in one sentence. + +**Test Result**: +```json +{ + "steps_found": 1, + "steps_executed": 1, + "commands_executed": 1, + "expected_commands_run": false, + "passed": false +} +``` + +**Insight**: The skill format makes it hard to extract individual steps. Only the first command was found and executed, but the expected command (`orchestrate agents import`) was never run. + +### 3. Missing Commands + +When a skill describes actions but doesn't include executable commands, the test will show: +- Steps found but not executed +- No commands run +- Expected outcomes not achieved + +### 4. Incorrect Command Syntax + +If a skill includes commands with incorrect syntax or missing parameters, the mock environment will simulate failure and track the error. + +## Best Practices for Testable Skills + +### 1. Use Clear Step Numbering + +**Good**: +```markdown +1. Create a YAML file with the agent configuration +2. Activate the virtual environment with `source .venv/bin/activate` +3. Import the agent with `orchestrate agents import --file agent.yaml` +``` + +**Bad**: +```markdown +To create an agent: 1) Create a YAML file... 2) Activate the virtual environment... 3) Import the agent... +``` + +### 2. Include Executable Commands + +**Good**: +```markdown +Run the following command: +```bash +orchestrate agents import --file agent.yaml +``` +``` + +**Bad**: +```markdown +Import the agent using the orchestrate CLI +``` + +### 3. Use Code Blocks for Multi-Line Commands + +**Good**: +````markdown +```bash +source .venv/bin/activate +orchestrate env activate wxo699 +orchestrate agents import --file agent.yaml +``` +```` + +**Bad**: +```markdown +Run `source .venv/bin/activate` and then `orchestrate env activate wxo699` and finally `orchestrate agents import --file agent.yaml` +``` + +### 4. Separate Steps Clearly + +Each step should be on its own line or in its own numbered section, not combined into one long sentence. + +## Interpreting Test Results + +### All Tests Passing + +✅ Skills have clear, executable steps +✅ Commands are properly formatted +✅ Expected outcomes are achieved + +### Tests Failing + +❌ **Steps not found**: Skill content doesn't have clear numbered steps or commands +❌ **Commands not executed**: Commands aren't in backticks or code blocks +❌ **Expected commands not run**: The skill mentions different commands than what's expected +❌ **Errors occurred**: Commands failed in the mock environment + +## Current Test Results + +As of the latest run: + +``` +Total tests: 2 +Passed: 0 +Failed: 2 +Pass rate: 0.0% +``` + +### Why Tests Are Failing + +1. **activate-the-virtual-environment**: Skill has no executable commands + - Only has a title and rationale + - No steps or commands to extract + - Needs detailed instructions added + +2. **to-create-and-upload-a-watson-orchestrate-agent-1-create-a**: Poor step format + - All steps in one sentence + - Only first command extracted + - Expected command never executed + - Needs reformatting with clear numbered steps + +## Next Steps + +### 1. Improve Skill Content + +Update skills to have: +- Clear numbered steps +- Commands in backticks or code blocks +- Separate lines for each step + +### 2. Add More Test Scenarios + +Create scenarios for: +- Error handling skills +- Multi-step workflows +- File creation skills +- Authentication skills + +### 3. Enhance Mock Environment + +Add simulation for: +- Network requests +- API calls +- Database operations +- More complex command outputs + +### 4. Create Skill Templates + +Provide templates that make it easy to write testable skills from the start. + +## Comparison with Other Test Types + +| Test Type | What It Tests | When to Use | +|-----------|---------------|-------------| +| **Integration** | Did skills get recalled and used in real conversations? | After completing tasks | +| **Unit** | Would the right skill be recalled for a given question? | When creating new skills | +| **Functional** | Do the skill's instructions actually work? | When writing skill content | + +## Example: Creating a Testable Skill + +### Before (Not Testable) + +```markdown +--- +type: atomic-skill +trigger: When deploying applications +--- + +Deploy the application + +## Rationale + +Standard deployment workflow +``` + +### After (Testable) + +```markdown +--- +type: atomic-skill +trigger: When deploying applications +--- + +Deploy the application + +## Rationale + +Standard deployment workflow for Python applications + +## Steps + +1. Install dependencies: +```bash +pip install -r requirements.txt +``` + +2. Run the application: +```bash +python main.py +``` + +3. Verify deployment: +```bash +curl http://localhost:8000/health +``` +``` + +### Test Scenario + +```json +{ + "scenario_id": "test_app_deployment", + "description": "Test application deployment workflow", + "user_request": "Deploy the application", + "expected_skills": ["deploy-application"], + "setup": { + "files": { + "requirements.txt": "flask==2.0.0", + "main.py": "from flask import Flask\napp = Flask(__name__)" + } + }, + "expected_commands": [ + "pip install", + "python main.py" + ], + "success_criteria": { + "required_files": [], + "commands_must_succeed": true, + "max_errors": 0 + } +} +``` + +## Conclusion + +Functional testing reveals whether skills are written in a way that makes them executable. It's not about whether the skill was recalled or whether it helped complete a task - it's about whether following the skill's instructions step-by-step would actually work. + +The current test results show that many skills need better formatting and more detailed instructions to be truly functional. This is valuable feedback that helps improve skill quality. \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/FUNCTIONAL_TEST_ANALYSIS.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/FUNCTIONAL_TEST_ANALYSIS.md new file mode 100644 index 00000000..5803bd20 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/FUNCTIONAL_TEST_ANALYSIS.md @@ -0,0 +1,174 @@ +# Functional Test Analysis + +## Overview + +The functional tests are **working correctly** - they're revealing real issues with skill structure and executability. The "failures" are actually successful detections of problems. + +## Test Results Summary + +**Total Tests:** 2 +**Passed:** 0 +**Failed:** 2 +**Pass Rate:** 0.0% + +Both tests failed because `expected_commands_run: false` - the expected commands were not executed. + +--- + +## Test 1: Watson Orchestrate Agent Creation + +### Skill Content +``` +To create and upload a Watson Orchestrate agent: +1) Create a YAML file with spec_version, name, description, instructions, model, parameters, and tools fields. +2) Activate the virtual environment. +3) Ensure the orchestrate environment is authenticated with `orchestrate env activate`. +4) Import the agent with `orchestrate agents import --file `. +``` + +### What the Test Found + +**Steps Extracted:** 1 (should be 4) +**Commands Executed:** 1 (`orchestrate env activate`) +**Expected Command:** `orchestrate agents import` +**Expected Command Run:** ❌ No + +### The Problem + +The skill content is formatted as a single sentence with numbered sub-steps. The step extraction logic: +1. Found this as ONE step instead of FOUR separate steps +2. Extracted only the FIRST command in backticks (`orchestrate env activate`) +3. Never executed the actual expected command (`orchestrate agents import`) + +### Why This Matters + +This skill would fail in real usage because: +- The critical command (`orchestrate agents import`) is buried in prose +- No clear executable steps are defined +- The format makes it hard to extract actionable commands + +--- + +## Test 2: Virtual Environment Activation + +### Skill Content +``` +Activate the virtual environment +``` + +### What the Test Found + +**Steps Extracted:** 1 +**Step Type:** "action" (not "command") +**Commands Executed:** 0 +**Expected Command:** `source .venv/bin/activate` +**Expected Command Run:** ❌ No + +### The Problem + +The skill content is too abstract: +1. Says "Activate the virtual environment" but doesn't specify HOW +2. No actual command is provided in the skill content +3. The step was classified as an "action" not a "command" +4. Zero commands were executed + +### Why This Matters + +This skill would fail in real usage because: +- No executable command is provided +- Too abstract - requires the user to know the implementation +- Not self-contained or actionable + +--- + +## What This Reveals About Skill Quality + +### Good Skills Should Have: + +1. **Clear, Numbered Steps** + ```markdown + ## Steps + 1. Create the YAML file + 2. Activate virtual environment with `source .venv/bin/activate` + 3. Authenticate with `orchestrate env activate` + 4. Import agent with `orchestrate agents import --file agent.yaml` + ``` + +2. **Explicit Commands** + - Commands should be in code blocks or backticks + - Each command should be on its own line + - Commands should be complete and executable + +3. **Proper Structure** + - Use markdown headers for sections + - Use numbered lists for sequential steps + - Use code blocks for multi-line commands + +### Bad Skills Look Like: + +1. **Prose-Heavy Content** + - "To do X, you need to Y and then Z with `command`" + - Multiple steps in one sentence + - Commands buried in explanatory text + +2. **Abstract Instructions** + - "Activate the environment" (no command) + - "Set up the configuration" (no specifics) + - "Run the necessary commands" (which ones?) + +--- + +## Functional Testing Value + +The functional tests are **successfully identifying** skills that: +- ❌ Don't have clear executable steps +- ❌ Bury commands in prose +- ❌ Are too abstract to execute +- ❌ Have formatting that prevents step extraction + +This is exactly what functional testing should do - reveal when skills won't work in practice! + +--- + +## Next Steps + +### For These Specific Skills + +1. **Orchestrate Agent Skill** - Needs restructuring: + ```markdown + ## Steps + 1. Create YAML file with required fields + 2. Run: `source .venv/bin/activate` + 3. Run: `orchestrate env activate` + 4. Run: `orchestrate agents import --file ` + ``` + +2. **Virtual Environment Skill** - Needs actual command: + ```markdown + ## Steps + 1. Run: `source .venv/bin/activate` + ``` + +### For the Testing Framework + +The functional test framework is working correctly! It's revealing: +- Which skills have poor structure +- Which skills lack executable commands +- Which skills need improvement + +### Recommendations + +1. **Use functional tests to validate skill quality** before publishing +2. **Refactor skills** that fail functional tests +3. **Establish skill formatting standards** based on what works +4. **Create a skill quality checklist** based on functional test criteria + +--- + +## Conclusion + +**The functional tests are not broken - they're working perfectly!** + +They're revealing that these two skills have structural problems that would prevent them from being executed correctly. This is valuable feedback that helps improve skill quality. + +The 0% pass rate is actually a success - it means the tests are correctly identifying skills that need improvement. \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/HOW_EVALUATION_WORKS.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/HOW_EVALUATION_WORKS.md new file mode 100644 index 00000000..596db125 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/HOW_EVALUATION_WORKS.md @@ -0,0 +1,401 @@ +# How Functional Test Evaluation Works + +## Overview + +The functional test framework evaluates skills through a **5-step validation process**. A skill must pass ALL 5 checks to be considered functional. + +--- + +## The 5 Validation Checks + +### 1. ✅ All Steps Executed +```python +validation["all_steps_executed"] = (steps_executed == steps_found) +``` + +**What it checks:** Were all extracted steps actually executed? + +**Passes when:** +- Every step found in the skill was executed +- No steps were skipped + +**Fails when:** +- Some steps couldn't be executed +- Step extraction failed + +**Example Failure:** +``` +Steps found: 4 +Steps executed: 1 +Result: ❌ FAIL - Only 1 of 4 steps executed +``` + +--- + +### 2. ✅ All Steps Successful +```python +validation["all_steps_successful"] = (steps_successful == steps_executed) +``` + +**What it checks:** Did all executed steps complete successfully? + +**Passes when:** +- Every executed step returned success +- No commands failed +- No errors occurred during execution + +**Fails when:** +- A command returned an error +- A step failed to complete + +**Example Failure:** +``` +Steps executed: 3 +Steps successful: 2 +Result: ❌ FAIL - Step 3 failed +``` + +--- + +### 3. ✅ No Errors +```python +validation["no_errors"] = (len(env.errors) == 0) +``` + +**What it checks:** Were there any errors during execution? + +**Passes when:** +- No errors were logged +- All operations completed cleanly + +**Fails when:** +- File not found errors +- Missing parameters +- Command execution errors + +**Example Failure:** +``` +Errors: ["File not found: agent.yaml"] +Result: ❌ FAIL - Errors occurred +``` + +--- + +### 4. ✅ Expected Files Created +```python +required_files = scenario["success_criteria"]["required_files"] +validation["expected_files_created"] = all(env.file_exists(f) for f in required_files) +``` + +**What it checks:** Were the expected output files created? + +**Passes when:** +- All required files exist in the mock environment +- OR no files were required by the scenario + +**Fails when:** +- A required file is missing +- File creation step was skipped + +**Example Failure:** +``` +Required files: ["output.txt", "config.json"] +Files created: ["output.txt"] +Result: ❌ FAIL - config.json not created +``` + +--- + +### 5. ✅ Expected Commands Run +```python +expected_commands = scenario["expected_commands"] +validation["expected_commands_run"] = all( + any(exp in cmd for cmd in commands_run) + for exp in expected_commands +) +``` + +**What it checks:** Were the expected commands actually executed? + +**Passes when:** +- All expected commands were run +- OR no specific commands were required + +**Fails when:** +- An expected command was never executed +- Wrong commands were run instead + +**Example Failure:** +``` +Expected: ["orchestrate agents import"] +Executed: ["orchestrate env activate"] +Result: ❌ FAIL - Expected command not run +``` + +--- + +## Overall Pass/Fail Logic + +```python +validation["passed"] = all([ + validation["all_steps_executed"], # Check 1 + validation["all_steps_successful"], # Check 2 + validation["no_errors"], # Check 3 + validation["expected_files_created"], # Check 4 + validation["expected_commands_run"] # Check 5 +]) +``` + +**A skill PASSES only if ALL 5 checks pass.** + +**A skill FAILS if ANY check fails.** + +--- + +## Real Example: Orchestrate Agent Skill + +### Test Scenario +```json +{ + "expected_commands": ["orchestrate agents import"], + "success_criteria": { + "required_files": [], + "commands_must_succeed": true, + "max_errors": 0 + } +} +``` + +### Skill Content +``` +To create and upload a Watson Orchestrate agent: 1) Create a YAML file... +2) Activate the virtual environment. 3) Ensure the orchestrate environment +is authenticated with `orchestrate env activate`. 4) Import the agent with +`orchestrate agents import --file `. +``` + +### Step Extraction +``` +Steps found: 1 (entire content treated as one step) +Commands extracted: + - orchestrate env activate (first command in backticks) + - orchestrate agents import --file (second command) +``` + +### Execution +``` +Step 1: Execute "orchestrate env activate" + Result: ✅ SUCCESS - "Environment activated" +``` + +### Validation Results + +| Check | Result | Reason | +|-------|--------|--------| +| 1. All steps executed | ✅ PASS | 1/1 steps executed | +| 2. All steps successful | ✅ PASS | 1/1 steps succeeded | +| 3. No errors | ✅ PASS | 0 errors | +| 4. Expected files created | ✅ PASS | No files required | +| 5. Expected commands run | ❌ **FAIL** | `orchestrate agents import` not executed | + +**Overall: ❌ FAIL** + +### Why It Failed + +The skill content has 4 numbered steps, but they're all in one sentence. The step extractor treats this as ONE step and only extracts the FIRST command (`orchestrate env activate`). + +The expected command (`orchestrate agents import`) is never executed, so check #5 fails. + +--- + +## Real Example: Virtual Environment Skill + +### Test Scenario +```json +{ + "expected_commands": ["source .venv/bin/activate"], + "success_criteria": { + "required_files": [], + "commands_must_succeed": true, + "max_errors": 0 + } +} +``` + +### Skill Content +``` +Activate the virtual environment +``` + +### Step Extraction +``` +Steps found: 1 +Commands extracted: (none - no backticks or code blocks) +Step type: "action" (not "command") +``` + +### Execution +``` +Step 1: "Activate the virtual environment" + Type: action (no command to execute) + Result: ✅ Marked as successful (non-command steps auto-succeed) +``` + +### Validation Results + +| Check | Result | Reason | +|-------|--------|--------| +| 1. All steps executed | ✅ PASS | 1/1 steps executed | +| 2. All steps successful | ✅ PASS | 1/1 steps succeeded | +| 3. No errors | ✅ PASS | 0 errors | +| 4. Expected files created | ✅ PASS | No files required | +| 5. Expected commands run | ❌ **FAIL** | `source .venv/bin/activate` not executed | + +**Overall: ❌ FAIL** + +### Why It Failed + +The skill content is too abstract - it says "Activate the virtual environment" but doesn't provide the actual command. No commands were extracted, so nothing was executed. Check #5 fails because the expected command was never run. + +--- + +## Command Extraction Logic + +The framework extracts commands using these patterns: + +### Pattern 1: Backticks +```markdown +Run `command --flag value` +``` +Extracts: `command --flag value` + +### Pattern 2: Code Blocks +````markdown +```bash +command1 +command2 +``` +```` +Extracts: `command1`, `command2` + +### Pattern 3: Numbered Steps +```markdown +1) Do something with `command` +2) Do another thing with `command2` +``` +Extracts: `command`, `command2` + +### What's NOT Extracted +- Simple words in backticks: `activate` +- Text without command syntax: `the environment` +- Comments in code blocks: `# This is a comment` + +--- + +## Mock Environment Simulation + +The framework simulates command execution without actually running them: + +### Simulated Commands + +| Command Pattern | Simulated Behavior | +|----------------|-------------------| +| `orchestrate agents import` | Checks if file exists, returns success/error | +| `orchestrate env activate` | Returns "Environment activated" | +| `source .venv/bin/activate` | Sets VIRTUAL_ENV variable | +| `python` / `python3` | Returns "Python script executed" | +| `pip install` | Returns "Packages installed" | +| Other commands | Returns "Command executed: {command}" | + +### File Operations + +- Files can be created in setup +- Commands can check if files exist +- Missing files cause errors + +--- + +## How to Make Skills Pass + +### ❌ Bad: Abstract Instructions +```markdown +Activate the virtual environment +``` +**Problem:** No command to execute + +### ✅ Good: Explicit Command +```markdown +Activate the virtual environment: +```bash +source .venv/bin/activate +``` +``` + +--- + +### ❌ Bad: Multiple Steps in One Sentence +```markdown +To do X: 1) Do A. 2) Run `cmd1`. 3) Run `cmd2`. +``` +**Problem:** Treated as one step, only first command extracted + +### ✅ Good: Separate Numbered Steps +```markdown +## Steps + +1. Do A + +2. Run command: + ```bash + cmd1 + ``` + +3. Run command: + ```bash + cmd2 + ``` +``` + +--- + +### ❌ Bad: Commands Buried in Prose +```markdown +After doing X, you should run `cmd1` and then `cmd2` to finish. +``` +**Problem:** Multiple commands in one step, may not all execute + +### ✅ Good: One Command Per Step +```markdown +## Steps + +1. Run first command: + ```bash + cmd1 + ``` + +2. Run second command: + ```bash + cmd2 + ``` +``` + +--- + +## Summary + +**The evaluation checks 5 things:** + +1. ✅ Were all steps executed? +2. ✅ Did all steps succeed? +3. ✅ Were there no errors? +4. ✅ Were expected files created? +5. ✅ Were expected commands run? + +**All 5 must pass for the skill to pass.** + +**Most failures happen at check #5** because: +- Commands aren't extracted properly (buried in prose) +- Commands aren't provided (too abstract) +- Wrong commands are extracted (multiple in one step) + +**Use the execution plan analyzer** to see what will be extracted before running tests! \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/INTEGRATION_TESTING.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/INTEGRATION_TESTING.md new file mode 100644 index 00000000..e64829c5 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/INTEGRATION_TESTING.md @@ -0,0 +1,320 @@ +# Integration Testing for Evolve Lite Skills + +## Overview + +Integration tests validate that skills are **actually recalled and used** in real scenarios, not just that they're well-formed. Unlike static validation tests, integration tests analyze actual conversation trajectories to verify: + +1. **Skills were recalled** - The `evolve-lite:recall` skill was used +2. **Expected skills were found** - Specific skills were mentioned/applied +3. **Outcome was successful** - Task completed within acceptable metrics +4. **Performance improved** - Fewer errors, tool uses, etc. + +## Quick Start + +### 1. Create a Test Scenario + +Define what you expect to happen: + +```json +{ + "scenario_id": "create_orchestrate_agent", + "description": "Test Watson Orchestrate agent creation with skills", + "user_request": "create a hello world agent in orchestrate", + "expected_skills": [ + "to-create-and-upload-a-watson-orchestrate-agent-1-create-a" + ], + "success_criteria": { + "completion_status": "completed", + "max_tool_uses": 20, + "max_errors": 2 + } +} +``` + +Save to: `.evolve/tests/integration/scenarios/your_scenario.json` + +### 2. Execute the Scenario + +**Option A: Manual execution** +1. Start a new Bob conversation +2. Use `evolve-lite:recall` at the start +3. Give the user request from the scenario +4. Save the trajectory when complete + +**Option B: Automated execution** (future enhancement) +- Use Bob API to execute scenario programmatically + +### 3. Run the Integration Test + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_integration_test.py \ + --scenario .evolve/tests/integration/scenarios/your_scenario.json \ + --trajectory .evolve/trajectories/your_trajectory.json \ + --verbose +``` + +### 4. Review Results + +The test will output: +- ✅ **PASSED** if skills were recalled and criteria met +- ❌ **FAILED** if skills weren't used or criteria not met + +Results are saved to: `.evolve/tests/integration/results/` + +## Test Scenarios + +### Scenario Structure + +```json +{ + "scenario_id": "unique-identifier", + "description": "What this scenario tests", + "user_request": "The exact request to give Bob", + "expected_skills": [ + "skill-slug-1", + "skill-slug-2" + ], + "success_criteria": { + "completion_status": "completed|failed|unknown", + "max_tool_uses": 20, + "max_errors": 2, + "required_files": ["file1.txt", "file2.yaml"] + }, + "notes": "Additional context about this test" +} +``` + +### Example Scenarios + +**1. Watson Orchestrate Agent Creation** +```json +{ + "scenario_id": "create_orchestrate_agent", + "user_request": "create a hello world agent in orchestrate", + "expected_skills": [ + "to-create-and-upload-a-watson-orchestrate-agent-1-create-a" + ], + "success_criteria": { + "completion_status": "completed", + "max_tool_uses": 20, + "max_errors": 2 + } +} +``` + +**2. Authentication Error Handling** +```json +{ + "scenario_id": "handle_auth_errors", + "user_request": "import the agent but handle authentication errors", + "expected_skills": [ + "when-the-watson-orchestrate-cli-reports-token-expired" + ], + "success_criteria": { + "completion_status": "completed", + "max_tool_uses": 15, + "max_errors": 1 + } +} +``` + +## What Gets Validated + +### 1. Skill Recall Detection + +The test checks if: +- `evolve-lite:recall` was used in the conversation +- "Recall complete" messages appear +- Entity paths (`.evolve/entities/...`) are mentioned + +### 2. Expected Skills Found + +For each expected skill, checks if: +- The skill slug appears in the conversation +- The skill was mentioned in recall output +- The skill path was referenced + +### 3. Success Criteria + +Validates against defined criteria: +- **completion_status**: Did the task complete successfully? +- **max_tool_uses**: Was it efficient (not too many tool calls)? +- **max_errors**: Were errors kept to a minimum? +- **required_files**: Were expected files created? + +### 4. Performance Metrics + +Extracts and reports: +- Total tool uses +- Number of errors encountered +- Tools used during execution +- Estimated duration (if timestamps available) + +## Test Results + +### Result Structure + +```json +{ + "test_id": "create_orchestrate_agent", + "passed": true, + "skill_analysis": { + "skills_recalled": true, + "recall_count": 1, + "skills_mentioned": [ + ".evolve/entities/skill-flow/to-create-and-upload-a-watson-orchestrate-agent-1-create-a.md" + ] + }, + "metrics": { + "tool_uses": 14, + "errors": 1, + "completion_status": "completed" + }, + "validation": { + "passed": true, + "checks": { + "skill_recalled_to-create-and-upload-a-watson-orchestrate-agent-1-create-a": true, + "completion_status": true, + "tool_uses_within_limit": true, + "errors_within_limit": true + }, + "failures": [] + } +} +``` + +### Understanding Results + +**PASSED Test**: +- Skills were recalled at the start +- Expected skills were found in the conversation +- All success criteria were met +- Task completed successfully + +**FAILED Test**: +- Skills were NOT recalled (forgot to use `evolve-lite:recall`) +- Expected skills were not mentioned (wrong skills recalled) +- Success criteria not met (too many errors, didn't complete) +- Required files not created + +## Workflow Examples + +### Testing a New Skill + +1. **Learn the skill** from a successful trajectory +2. **Create a test scenario** for that skill +3. **Execute the scenario** with recall enabled +4. **Run the integration test** to verify +5. **Iterate** if the test fails + +### Regression Testing + +1. **Generate scenarios** for all existing skills +2. **Run integration tests** periodically +3. **Identify degraded skills** (tests that start failing) +4. **Update skills** based on failures +5. **Re-run tests** to verify fixes + +### A/B Comparison + +1. **Run scenario WITHOUT recall** (baseline) +2. **Run scenario WITH recall** (with skills) +3. **Compare trajectories** using `compare_with_without_skills.py` +4. **Measure improvement** (fewer errors, faster completion) + +## Best Practices + +### 1. Create Realistic Scenarios + +- Use actual user requests from real conversations +- Include edge cases and error conditions +- Test both happy path and failure scenarios + +### 2. Set Reasonable Criteria + +- Don't make criteria too strict (allow some flexibility) +- Focus on key metrics (completion, major errors) +- Adjust criteria based on actual performance + +### 3. Test Regularly + +- Run integration tests after learning new skills +- Include in CI/CD pipeline if possible +- Track pass rates over time + +### 4. Document Failures + +- When tests fail, understand why +- Update skills or scenarios as needed +- Keep notes on common failure patterns + +### 5. Maintain Test Suite + +- Remove obsolete scenarios +- Update scenarios when skills change +- Keep scenarios aligned with current skills + +## Limitations + +### Current Limitations + +1. **Manual execution required** - Can't automatically execute scenarios yet +2. **Heuristic skill detection** - Relies on text matching for skill mentions +3. **No negative tests** - Doesn't test that wrong skills aren't recalled +4. **Limited metrics** - Could track more performance indicators + +### Future Enhancements + +1. **Automated scenario execution** - Use Bob API to run scenarios +2. **Semantic skill matching** - Use embeddings for better detection +3. **Negative test generation** - Test that irrelevant skills aren't used +4. **Performance benchmarking** - Track skill effectiveness over time +5. **Batch test runner** - Run multiple scenarios in sequence + +## Troubleshooting + +### Test Always Fails: "Skills not recalled" + +**Problem**: `skills_recalled: false` + +**Solution**: Make sure you used `evolve-lite:recall` at the start of the conversation + +### Test Fails: "Expected skill not recalled" + +**Problem**: Skill wasn't mentioned in the conversation + +**Solutions**: +- Check if the skill trigger matches the scenario +- Verify the skill exists in `.evolve/entities/` +- Update the scenario to expect different skills + +### Test Fails: "Too many tool uses" + +**Problem**: `tool_uses > max_tool_uses` + +**Solutions**: +- Increase `max_tool_uses` in success criteria +- Investigate why so many tools were needed +- Check if skills are actually helping efficiency + +### Test Fails: "Completion status mismatch" + +**Problem**: Task didn't complete as expected + +**Solutions**: +- Check trajectory for errors or incomplete work +- Verify the scenario is achievable +- Update success criteria if needed + +## Examples + +See example scenarios in: +- `.evolve/tests/integration/scenarios/create_orchestrate_agent.json` +- `.evolve/tests/integration/scenarios/handle_auth_errors.json` + +See example results in: +- `.evolve/tests/integration/results/` + +--- + +Made with Bob - Evolve Lite Integration Testing v1.0 \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/README.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/README.md new file mode 100644 index 00000000..5d5c7bf1 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/README.md @@ -0,0 +1,115 @@ +# Evolve Lite Test Skill + +**Testing framework for validating atomic skills through pseudo-conversation evaluation.** + +## Overview + +The `evolve-lite-test` skill provides three test types for atomic skills: + +1. **Content test** — does the skill contain the commands it prescribes? +2. **Recall test** — does the skill surface in the top 3 when its scenario is described? +3. **Trigger test** — would an agent get it right *without* the skill injected? + +--- + +## Quick Start + +Run the full suite: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_skill_tests.py --all +python3 .bob/skills/evolve-lite-test/scripts/run_skill_evaluation.py --verbose +python3 .bob/skills/evolve-lite-test/scripts/run_recall_tests.py --verbose +``` + +Or use the command: `/evolve-lite-test` + +--- + +## Test Types + +### Content test — `run_skill_evaluation.py` + +Checks self-consistency: the skill's backtick command strings are extracted and matched against the skill's own content. + +- ✅ **Pass** — skill contains all its prescribed commands +- ❌ **Fail** — skill is too vague or missing a key command + +### Recall test — `run_recall_tests.py` + +Scores every manifest entry against the skill's trigger-specific user question. Checks the expected skill ranks in the top 3. + +- ✅ **Pass** — skill is rank 1–3 for its scenario +- ❌ **Fail** — trigger needs more distinctive vocabulary + +### Baseline test — `run_baseline_tests.py` + +Asks an agent the trigger question *without* the skill injected, then evaluates the response. + +- ✅ **Pass** (agent gets it right) — skill covers common knowledge; consider whether it adds value +- ❌ **Fail** (agent misses it) — skill IS necessary; captures non-obvious guidance + +**A healthy skill library should have mostly ❌ here** — each skill should cover something an agent wouldn't naturally get right on its own. + +--- + +## Scripts + +| Script | Purpose | +|---|---| +| `generate_pseudo_conversations.py` | Generate fixtures for all skills at once | +| `generate_skill_tests.py` | Generate fixtures for specific skill files (use after `evolve-lite-learn`) | +| `run_skill_evaluation.py` | Content test | +| `run_recall_tests.py` | Recall test | +| `run_trigger_tests.py` | Trigger test | + +--- + +## Commands + +| Command | When to use | +|---|---| +| `/evolve-lite-test` | Full suite — all three tests | +| `/evolve-lite-test-new-skills` | After `evolve-lite-learn` — test newly saved skills only | +| `/evolve-lite-test-recall` | Recall test only | +| `/evolve-lite-test-trigger` | Trigger test only | + +--- + +## Directory Structure + +``` +.bob/skills/evolve-lite-test/ +├── README.md # This file +├── SKILL.md # Skill definition +├── HOW_EVALUATION_WORKS.md # Evaluation logic details +└── scripts/ + ├── generate_pseudo_conversations.py # Generate fixtures for all skills + ├── generate_skill_tests.py # Generate fixtures for specific skills + ├── run_skill_evaluation.py # Content test + ├── run_recall_tests.py # Recall test + └── run_trigger_tests.py # Trigger test + +.evolve/tests/ +├── pseudo_conversations/ # One fixture JSON per skill + test_suite.csv +└── evaluation/ + ├── subagent_prompt_template.md + ├── report.json # Content test summary + ├── recall_report.json # Recall test summary + ├── trigger_report.json # Trigger test summary + └── results/ # Per-skill result files +``` + +--- + +## Current Test Coverage (6 atomic skills) + +- 📊 **Content test:** 6/6 passing (100%) +- 📊 **Recall test:** 6/6 passing (100%), 5/6 rank-1 +- 📊 **Trigger test:** 1/6 skills are necessary (`handle-interactive-prompts`) + +--- + +## After adding a new skill + +Run `/evolve-lite-test-new-skills` immediately after `/evolve-lite-learn` to generate a fixture and validate the new skill before it enters the library. diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/SKILL.md new file mode 100644 index 00000000..44128ac1 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/SKILL.md @@ -0,0 +1,399 @@ +--- +name: evolve-lite:test +description: Generate test cases for skills based on conversation trajectories, validating that skills can be successfully applied to similar scenarios. +--- + +# Test Case Generator + +## Invocation + +To use this skill, you can invoke it in two ways: + +### 1. Generate Test Cases +``` +Use the evolve-lite:test skill to generate test cases for all trajectories +``` + +### 2. Run Test Cases +``` +Use the evolve-lite:test skill to run test cases and generate a report +``` + +The skill will automatically determine whether to generate or run tests based on context. + +## Overview + +This skill analyzes conversation trajectories and generates test cases for each skill (guideline, atomic-skill, skill-flow) that was learned from those trajectories. Test cases validate that: + +1. **Skills are discoverable**: The trigger matches appropriate scenarios +2. **Skills are actionable**: The content provides clear, executable guidance +3. **Skills are complete**: All necessary steps and context are included +4. **Skills compose correctly**: Skill-flows properly reference their atomic skills + +## When To Use + +Use this skill to: +- Generate test cases after learning new skills from trajectories +- Validate existing skills against new trajectories +- Build a regression test suite for the skill library +- Identify gaps or ambiguities in skill definitions + +## Test Case Structure + +Each test case includes: + +```json +{ + "test_id": "unique-test-identifier", + "skill_type": "guideline|atomic-skill|skill-flow", + "skill_path": ".evolve/entities/skill-flow/example.md", + "scenario": "Description of the test scenario", + "expected_trigger_match": true, + "input_context": { + "user_request": "Original user request from trajectory", + "environment": "Relevant environment details", + "preconditions": ["List of preconditions"] + }, + "expected_outcome": { + "should_recall": true, + "should_apply": true, + "validation_criteria": ["List of success criteria"] + }, + "trajectory_source": ".evolve/trajectories/trajectory_xxx.json", + "created_at": "2026-06-18T18:00:00Z" +} +``` + +## Workflow + +### Step 1: Select Trajectories and Skills + +Choose which trajectories and skills to generate tests for: + +```bash +# List available trajectories +ls -lt .evolve/trajectories/ + +# List available skills +ls -R .evolve/entities/ +``` + +You can: +- Generate tests for all skills from a specific trajectory +- Generate tests for a specific skill across all trajectories +- Generate a full test suite for all skills + +### Step 2: Generate Test Cases + +Run the test case generator: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_test_cases.py \ + --trajectory .evolve/trajectories/trajectory_xxx.json \ + --output .evolve/tests/ +``` + +Or generate for all trajectories: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/generate_test_cases.py \ + --all-trajectories \ + --output .evolve/tests/ +``` + +The script will: +1. Load the trajectory and extract the conversation flow +2. Identify which skills were learned from this trajectory (via trajectory field) +3. For each skill, generate test cases based on: + - The original user request that led to the skill + - The context and environment from the trajectory + - The successful outcome that validated the skill +4. Save test cases as JSON files in `.evolve/tests/` + +### Step 3: Review Generated Test Cases + +Examine the generated test cases: + +```bash +cat .evolve/tests/test_*.json | python3 -m json.tool +``` + +Each test case should: +- Have a clear scenario description +- Include realistic input context +- Define measurable validation criteria +- Reference the source trajectory for provenance + +### Step 4: Run Test Cases (Optional) + +Validate skills against test cases: + +```bash +python3 .bob/skills/evolve-lite-test/scripts/run_test_cases.py \ + --test-dir .evolve/tests/ \ + --report .evolve/tests/test_report.json +``` + +This will: +1. Load each test case +2. Simulate the scenario by checking if: + - The skill trigger matches the scenario + - The skill content addresses the scenario requirements + - For skill-flows, all referenced atomic skills exist +3. Generate a test report with pass/fail results + +## Test Case Types + +### 1. Trigger Match Tests + +Validates that skill triggers correctly match relevant scenarios: + +```json +{ + "test_type": "trigger_match", + "scenario": "User wants to create a Watson Orchestrate agent", + "expected_trigger_match": true, + "skill_trigger": "When creating or uploading Watson Orchestrate agents" +} +``` + +### 2. Rubric-Based Execution Tests + +Validates that executing the skill produces the outcomes defined in its `success_rubric`. This is the **primary test type** for `atomic-skill` and `skill-flow` entities — validation criteria must come directly from the entity's rubric, not invented generically. + +For each entity with a rubric, generate: +- **1 happy-path test**: normal conditions, all rubric criteria expected to pass +- **1–3 edge case tests**: each covers a realistic boundary or failure condition (e.g. missing dependency, malformed input, already-existing output, partial environment). Each edge case must reference the rubric criteria and explicitly state which criterion is expected to fail or behave differently under that condition. + +```json +{ + "test_type": "rubric_execution", + "scenario": "Executing the skill end-to-end and verifying its success rubric", + "edge_case": false, + "validation_criteria": [ + "exit code 0 after running the main command", + "output file exists at the expected path", + "no error lines appear in stdout" + ] +} +``` + +```json +{ + "test_type": "rubric_execution", + "scenario": "Skill is run when a required dependency is missing", + "edge_case": true, + "edge_condition": "missing dependency", + "validation_criteria": [ + "exit code 0 after running the main command — expected to FAIL", + "output file exists at the expected path — not applicable", + "no error lines appear in stdout — expected to FAIL with clear error message" + ] +} +``` + +The `validation_criteria` array must be a verbatim or lightly paraphrased copy of the rubric items from the entity's `## Success Rubric` section. Do not substitute generic completeness checks when a rubric is present. + +### 3. Content Completeness Tests + +Validates that skill content provides sufficient guidance. Use this only when the entity has **no `success_rubric`** (e.g. legacy entities predating the rubric requirement): + +```json +{ + "test_type": "content_completeness", + "scenario": "Following the skill to complete the task", + "validation_criteria": [ + "All required steps are present", + "Commands are executable", + "Prerequisites are stated", + "Expected outcomes are clear" + ] +} +``` + +### 4. Skill Composition Tests + +Validates that skill-flows properly reference atomic skills: + +```json +{ + "test_type": "skill_composition", + "scenario": "Skill-flow references valid atomic skills", + "validation_criteria": [ + "All atomic_skills exist in entities/atomic-skill/", + "Atomic skill content matches flow steps", + "No circular dependencies" + ] +} +``` + +### 5. Trajectory Replay Tests + +Validates that applying the skill to the original trajectory would succeed: + +```json +{ + "test_type": "trajectory_replay", + "scenario": "Replaying the original trajectory with the skill", + "validation_criteria": [ + "Skill would be recalled at the right time", + "Skill guidance matches what was done", + "Outcome would be the same" + ] +} +``` + +## Test Generation Strategies + +### Strategy 1: Positive Tests from Success + +For each skill learned from a trajectory: +- Extract the successful scenario that led to the skill +- Create a test case that validates the skill applies to that scenario +- Use the actual outcome as the expected result + +### Strategy 2: Edge Case Tests from Rubric + +For each skill with a `success_rubric`, generate 1–3 edge case tests alongside the happy-path test: +- Identify realistic boundary conditions: missing tools, malformed inputs, already-existing outputs, partial environments, or permission issues +- For each edge case, copy the rubric criteria into `validation_criteria` and annotate which are expected to fail or behave differently under that condition +- Use `edge_case: true` and an `edge_condition` description to distinguish from the happy-path test + +### Strategy 3: Negative Tests from Failures + +For skills that prevent errors: +- Extract the failure scenario from the trajectory +- Create a test case that validates the skill would prevent the error +- Use the error avoidance as the expected result + +### Strategy 4: Variation Tests + +For each skill: +- Generate variations of the original scenario +- Test that the skill trigger still matches +- Validate that the skill content is general enough + +### Strategy 5: Composition Tests + +For skill-flows: +- Test that all atomic skills are present +- Test that atomic skills can be executed in sequence +- Test that the composition achieves the flow's goal + +## A/B Comparison Testing + +### Overview + +Compare task performance with skills (recalled and applied) versus without skills (baseline) to validate that skills actually improve outcomes. + +### Metrics Compared + +- **Tool uses**: Number of tool invocations required +- **Errors**: Number of errors encountered +- **Retries**: Number of retry attempts +- **User interventions**: Number of times user had to intervene +- **Completion status**: Whether task completed successfully + +### Running Comparisons + +```bash +python3 .bob/skills/evolve-lite-test/scripts/compare_with_without_skills.py \ + --with-skills .evolve/trajectories/trajectory_with_skills.json \ + --without-skills .evolve/trajectories/trajectory_without_skills.json \ + --output .evolve/tests/comparison_report.json +``` + +### Comparison Report Structure + +```json +{ + "generated_at": "2026-06-18T19:30:00Z", + "total_comparisons": 1, + "comparisons": [{ + "with_skills": { + "metrics": { + "tool_uses": 15, + "errors": 1, + "retries": 2, + "completion_status": "completed" + } + }, + "without_skills": { + "metrics": { + "tool_uses": 22, + "errors": 3, + "retries": 5, + "completion_status": "completed" + } + }, + "improvements": { + "fewer_tool_uses": 7, + "fewer_errors": 2, + "fewer_retries": 3 + }, + "summary": { + "skills_helped": true, + "efficiency_gain_percent": 31.8, + "error_reduction_percent": 66.7 + } + }], + "aggregate_metrics": { + "total_with_skills_helped": 1, + "average_tool_use_reduction": 31.8, + "average_error_reduction": 66.7 + } +} +``` + +### Creating Comparison Pairs + +To create valid comparison pairs: + +1. **Identify a task**: Choose a task that can be repeated +2. **Run without skills**: Complete the task without using evolve-lite:recall +3. **Save baseline trajectory**: Save the trajectory as `trajectory_baseline.json` +4. **Run with skills**: Complete the same task using evolve-lite:recall +5. **Save skills trajectory**: Save the trajectory as `trajectory_with_skills.json` +6. **Compare**: Run the comparison script + +### Interpretation + +- **Positive improvements**: Skills reduced tool uses, errors, or retries +- **Negative improvements**: Skills added overhead without benefit +- **No difference**: Skills had no measurable impact + +Use comparison results to: +- Validate skill effectiveness +- Identify skills that need refinement +- Prioritize skill development efforts +- Demonstrate ROI of the skill library + +## Best Practices + +1. **Generate tests immediately after learning**: Create test cases when skills are fresh +2. **One test per skill per trajectory**: Each skill-trajectory pair gets one primary test +3. **Include negative tests**: Test that skills don't match irrelevant scenarios +4. **Test skill evolution**: When skills are updated, regenerate tests +5. **Maintain test provenance**: Always link tests back to source trajectories +6. **Review generated tests**: Human review ensures test quality +7. **Run tests periodically**: Validate skills haven't degraded over time + +## Output Structure + +``` +.evolve/tests/ + test_skill-flow_create-and-upload-watson-agent_traj-2026-06-18.json + test_atomic-skill_create-yaml-file_traj-2026-06-18.json + test_guideline_use-context-managers_traj-2026-06-08.json + test_report_2026-06-18T18-30-00.json +``` + +## Integration with Learn/Recall + +The test mode complements the learn and recall skills: + +- **Learn**: Extracts skills from trajectories → **Test**: Validates those skills +- **Recall**: Retrieves skills for tasks → **Test**: Ensures retrieved skills work +- **Test failures**: Indicate skills need refinement → **Learn**: Updates skills + +This creates a continuous improvement loop for the skill library. \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/WHY_TESTS_FAIL.md b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/WHY_TESTS_FAIL.md new file mode 100644 index 00000000..9094c3fb --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/WHY_TESTS_FAIL.md @@ -0,0 +1,175 @@ +# Why Functional Tests Fail - Visual Explanation + +## Problem 1: Orchestrate Agent Skill + +### Current Skill Content (Line 10) +``` +To create and upload a Watson Orchestrate agent: 1) Create a YAML file with spec_version, name, description, instructions, model, parameters, and tools fields. 2) Activate the virtual environment. 3) Ensure the orchestrate environment is authenticated with `orchestrate env activate`. 4) Import the agent with `orchestrate agents import --file `. +``` + +### What the Test Sees +The step extraction algorithm sees this as **ONE LONG SENTENCE** with: +- Multiple numbered items (1, 2, 3, 4) embedded in prose +- Two commands in backticks: `orchestrate env activate` and `orchestrate agents import --file ` +- But it only extracts the FIRST command it finds + +### What Gets Executed +``` +Step 1: "Create a YAML file... [entire sentence]" +Command extracted: orchestrate env activate +Command executed: ✅ orchestrate env activate +``` + +### What SHOULD Be Executed +``` +Expected command: orchestrate agents import +Actual command run: orchestrate env activate +Result: ❌ FAIL - Wrong command executed! +``` + +### Why It Fails +The test expects `orchestrate agents import` to be executed, but the skill format makes it impossible to extract properly because: +1. All 4 steps are in one sentence +2. Commands are buried in prose +3. Step extraction finds the first command, not the important one + +--- + +## Problem 2: Virtual Environment Skill + +### Current Skill Content (Line 9) +``` +Activate the virtual environment +``` + +### What the Test Sees +- A single abstract instruction +- No commands in backticks +- No code blocks +- No executable content + +### What Gets Executed +``` +Step 1: "Activate the virtual environment" +Type: action (not command) +Commands extracted: 0 +Commands executed: 0 +``` + +### What SHOULD Be Executed +``` +Expected command: source .venv/bin/activate +Actual commands run: (none) +Result: ❌ FAIL - No commands executed! +``` + +### Why It Fails +The test expects `source .venv/bin/activate` to be executed, but the skill doesn't contain any executable command - it's just an abstract instruction. + +--- + +## The Solution: Proper Skill Structure + +### ✅ Good Skill Format + +```markdown +## Steps + +1. Create the YAML file with required fields + ```bash + cat > agent.yaml << EOF + spec_version: v1 + name: my-agent + ... + EOF + ``` + +2. Activate the virtual environment + ```bash + source .venv/bin/activate + ``` + +3. Authenticate with Orchestrate + ```bash + orchestrate env activate + ``` + +4. Import the agent + ```bash + orchestrate agents import --file agent.yaml + ``` +``` + +### Why This Works + +1. **Clear numbered steps** - Each step is separate +2. **Commands in code blocks** - Easy to extract +3. **One command per step** - No ambiguity +4. **Executable** - Can be run directly + +--- + +## Visual Comparison + +### ❌ Current Format (Fails) +``` +To do X: 1) Do A. 2) Do B. 3) Run `cmd1`. 4) Run `cmd2`. +``` +**Problem:** Everything in one sentence, hard to parse + +### ✅ Proper Format (Passes) +```markdown +## Steps +1. Do A +2. Do B +3. Run command: + ```bash + cmd1 + ``` +4. Run command: + ```bash + cmd2 + ``` +``` +**Solution:** Clear structure, easy to parse + +--- + +## Real-World Impact + +### If You Try to Use These Skills + +**Orchestrate Agent Skill:** +- You'd get guidance to run `orchestrate env activate` +- But the critical command `orchestrate agents import` is buried +- You might miss it or not know when to run it + +**Virtual Environment Skill:** +- You'd be told "Activate the virtual environment" +- But HOW? What command? +- You'd have to already know the answer + +--- + +## The Tests Are Working! + +The functional tests are **correctly identifying** that these skills: +- ❌ Can't be executed automatically +- ❌ Don't have clear, extractable steps +- ❌ Bury important commands in prose +- ❌ Are too abstract to be actionable + +This is **exactly what testing should reveal** - skills that look okay but don't work in practice! + +--- + +## Next Steps + +To make these skills pass: + +1. **Restructure with clear numbered steps** +2. **Put commands in code blocks** +3. **Make each step actionable** +4. **Test that commands can be extracted** + +The functional tests will then pass, confirming the skills are executable. \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/check_tests.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/check_tests.py new file mode 100644 index 00000000..a7b7044a --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/check_tests.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +""" +check_tests.py — quality gate for the evolve-lite skill test suite. + +Runs the content-evaluation test and the recall test against existing fixtures, +then fails with exit code 1 if either suite falls below the required pass-rate +threshold. + +Usage: + python3 check_tests.py # default threshold: 0.8 + python3 check_tests.py --threshold 0.9 # stricter gate + python3 check_tests.py --threshold 1.0 # all must pass + python3 check_tests.py --verbose # per-skill detail + python3 check_tests.py --report # write gate report JSON here + python3 check_tests.py --pseudo-conversations-dir + +Exit codes: + 0 both suites meet or exceed the threshold + 1 one or both suites fell below the threshold (or no fixtures found) +""" + +import argparse +import json +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +_script = Path(__file__).resolve() +_scripts_dir = _script.parent +_eval_script = _scripts_dir / "run_skill_evaluation.py" +_recall_script = _scripts_dir / "run_recall_tests.py" +_baseline_script = _scripts_dir / "run_baseline_tests.py" + +# Walk up to find .evolve dir helper (graceful fallback) +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib: + sys.path.insert(0, str(_lib)) + try: + from entity_io import get_evolve_dir + except ImportError: + def get_evolve_dir(): return Path(".evolve") +else: + def get_evolve_dir(): return Path(".evolve") + + +def _run_suite(script, extra_args, label): + """Run a test-suite script and return its exit code.""" + cmd = [sys.executable, str(script)] + extra_args + print(f"\n{'─' * 60}") + print(f" {label}") + print(f"{'─' * 60}") + result = subprocess.run(cmd) + return result.returncode + + +def _read_pass_rate(report_path, rate_key="pass_rate"): + """Read a pass_rate from a JSON report. Returns None if missing.""" + p = Path(report_path) + if not p.exists(): + return None + try: + with open(p) as fh: + data = json.load(fh) + return data.get(rate_key) + except Exception: + return None + + +def _read_pass_counts(report_path): + """Return (passed, total) from a JSON report.""" + p = Path(report_path) + if not p.exists(): + return None, None + try: + with open(p) as fh: + data = json.load(fh) + return data.get("passed"), data.get("total") + except Exception: + return None, None + + +def main(): + parser = argparse.ArgumentParser( + description="Quality gate: run content + recall tests and enforce a minimum pass-rate" + ) + parser.add_argument( + "--threshold", type=float, default=0.8, + help="Minimum required pass rate for both suites (0.0–1.0, default: 0.8)", + ) + parser.add_argument( + "--pseudo-conversations-dir", default=None, + help="Path to pseudo-conversation fixtures directory", + ) + parser.add_argument( + "--report", default=None, + help="Path to write the gate summary JSON report", + ) + parser.add_argument( + "--verbose", action="store_true", + help="Pass --verbose to both sub-runners", + ) + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + + eval_report = evolve_dir / "tests" / "evaluation" / "report.json" + recall_report = evolve_dir / "tests" / "evaluation" / "recall_report.json" + baseline_report = evolve_dir / "tests" / "evaluation" / "baseline_report.json" + gate_report = Path(args.report) if args.report \ + else evolve_dir / "tests" / "evaluation" / "gate_report.json" + + # ── build shared sub-runner args ───────────────────────────────────────── + common = [] + if args.pseudo_conversations_dir: + common += ["--pseudo-conversations-dir", args.pseudo_conversations_dir] + if args.verbose: + common.append("--verbose") + + # ── run content-evaluation ─────────────────────────────────────────────── + _run_suite(_eval_script, common + ["--report", str(eval_report)], + "Content Evaluation (skill self-consistency)") + + # ── run recall test ────────────────────────────────────────────────────── + _run_suite(_recall_script, common + ["--report", str(recall_report)], + "Recall Test (trigger surfacing)") + + # ── run baseline test ──────────────────────────────────────────────────── + _run_suite(_baseline_script, common + ["--simulate", "--report", str(baseline_report)], + "Baseline Test (skill necessity)") + + # ── read results ───────────────────────────────────────────────────────── + eval_rate = _read_pass_rate(eval_report, rate_key="pass_rate") + recall_rate = _read_pass_rate(recall_report, rate_key="recall_at_3") + + eval_passed, eval_total = _read_pass_counts(eval_report) + recall_passed, recall_total = _read_pass_counts(recall_report) + + # Baseline gate: necessity_rate = skill_necessary_count / total + # Passes when >= 80% of skills are necessary (agent fails without the skill) + baseline_data = {} + _bp = Path(baseline_report) + if _bp.exists(): + try: + import json as _json + with open(_bp) as _fh: + baseline_data = _json.load(_fh) + except Exception: + pass + _b_total = baseline_data.get("total") or 0 + _b_necessary = baseline_data.get("skill_necessary_count") or 0 + necessity_rate = round(_b_necessary / _b_total, 4) if _b_total else None + baseline_passed_count = _b_necessary + baseline_total_count = _b_total + + threshold = args.threshold + eval_ok = eval_rate is not None and eval_rate >= threshold + recall_ok = recall_rate is not None and recall_rate >= threshold + baseline_ok = necessity_rate is not None and necessity_rate >= 0.8 + + overall_passed = eval_ok and recall_ok and baseline_ok + + # ── print gate summary ─────────────────────────────────────────────────── + print(f"\n{'═' * 60}") + print(" TEST GATE SUMMARY") + print(f"{'═' * 60}") + print(f" Threshold : {threshold:.0%}") + print() + + def _fmt(label, rate, passed, total, ok, threshold_pct=None): + mark = "✅" if ok else "❌" + pct = f"{rate:.1%}" if rate is not None else "n/a" + cnt = f"{passed}/{total}" if passed is not None else "n/a" + need = threshold_pct if threshold_pct is not None else threshold + req = "pass" if ok else f"FAIL (need ≥{need:.0%})" + print(f" {mark} {label:<28} {pct} ({cnt}) {req}") + + _fmt("Content evaluation", eval_rate, eval_passed, eval_total, eval_ok) + _fmt("Recall test", recall_rate, recall_passed, recall_total, recall_ok) + _fmt("Baseline (necessity)", necessity_rate, baseline_passed_count, baseline_total_count, baseline_ok, threshold_pct=0.8) + + print() + if overall_passed: + print(" ✅ Gate PASSED — all suites meet the threshold.") + else: + print(" ❌ Gate FAILED — fix failing skills before continuing.") + print(f"{'═' * 60}") + + # ── write gate report ──────────────────────────────────────────────────── + gate_report.parent.mkdir(parents=True, exist_ok=True) + report_data = { + "generated_at": datetime.now().isoformat(), + "threshold": threshold, + "overall_passed": overall_passed, + "suites": { + "content_evaluation": { + "report": str(eval_report), + "pass_rate": eval_rate, + "passed": eval_passed, + "total": eval_total, + "gate_passed": eval_ok, + }, + "recall_test": { + "report": str(recall_report), + "pass_rate": recall_rate, + "passed": recall_passed, + "total": recall_total, + "gate_passed": recall_ok, + }, + "baseline_test": { + "report": str(baseline_report), + "necessity_rate": necessity_rate, + "necessary_count": baseline_passed_count, + "total": baseline_total_count, + "gate_passed": baseline_ok, + }, + }, + } + with open(gate_report, "w") as fh: + json.dump(report_data, fh, indent=2) + print(f" Gate report: {gate_report}") + + sys.exit(0 if overall_passed else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/compare_with_without_skills.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/compare_with_without_skills.py new file mode 100644 index 00000000..f14d09b4 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/compare_with_without_skills.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +""" +A/B Comparison Testing for Evolve Lite Skills + +Compares task performance with skills (recalled and applied) versus without skills (baseline). +This validates that skills actually improve outcomes by measuring: +- Task completion success rate +- Number of steps/tool uses required +- Errors encountered +- Time to completion +- Code quality metrics +""" + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path + +# Walk up from the script location to find the installed plugin lib directory +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + markdown_to_entity, + log as _log, +) + + +def log(message): + _log("compare", message) + + +def load_trajectory(trajectory_path): + """Load a trajectory JSON file.""" + with open(trajectory_path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def analyze_trajectory_metrics(trajectory): + """Extract performance metrics from a trajectory.""" + messages = trajectory.get("messages", []) + + metrics = { + "total_messages": len(messages), + "tool_uses": 0, + "errors": 0, + "retries": 0, + "user_interventions": 0, + "tools_used": set(), + "error_types": [], + "completion_status": "unknown" + } + + # Analyze messages + for i, msg in enumerate(messages): + role = msg.get("role") + + if role == "assistant" and msg.get("tool_calls"): + metrics["tool_uses"] += len(msg.get("tool_calls", [])) + for tool_call in msg.get("tool_calls", []): + if tool_call.get("function"): + tool_name = tool_call["function"].get("name", "unknown") + metrics["tools_used"].add(tool_name) + + elif role == "tool": + content = msg.get("content", "").lower() + # Check for errors + if any(err in content for err in ["error", "failed", "exception", "traceback"]): + metrics["errors"] += 1 + # Try to identify error type + if "permission" in content: + metrics["error_types"].append("permission") + elif "not found" in content: + metrics["error_types"].append("not_found") + elif "syntax" in content: + metrics["error_types"].append("syntax") + else: + metrics["error_types"].append("unknown") + + elif role == "user": + # User interventions after initial request + if i > 0: + metrics["user_interventions"] += 1 + + # Check for completion + for msg in reversed(messages): + if msg.get("role") == "assistant": + content = str(msg.get("content", "")) + if "attempt_completion" in str(msg.get("tool_calls", [])): + metrics["completion_status"] = "completed" + break + elif any(word in content.lower() for word in ["error", "failed", "cannot"]): + metrics["completion_status"] = "failed" + break + + # Detect retries (same tool used multiple times in sequence) + tool_sequence = [] + for msg in messages: + if msg.get("role") == "assistant" and msg.get("tool_calls"): + for tool_call in msg.get("tool_calls", []): + if tool_call.get("function"): + tool_sequence.append(tool_call["function"].get("name")) + + # Count consecutive duplicates as retries + for i in range(1, len(tool_sequence)): + if tool_sequence[i] == tool_sequence[i-1]: + metrics["retries"] += 1 + + metrics["tools_used"] = list(metrics["tools_used"]) + + return metrics + + +def find_skill_usage_in_trajectory(trajectory, entities_dir): + """Determine if skills were used in this trajectory.""" + messages = trajectory.get("messages", []) + + skill_usage = { + "skills_recalled": False, + "skills_applied": [], + "recall_count": 0, + "learn_count": 0 + } + + # Check for evolve-lite:recall usage + for msg in messages: + if msg.get("role") == "assistant": + content = str(msg.get("content", "")).lower() + tool_calls = str(msg.get("tool_calls", [])).lower() + + if "evolve-lite:recall" in content or "evolve-lite:recall" in tool_calls: + skill_usage["skills_recalled"] = True + skill_usage["recall_count"] += 1 + + if "evolve-lite:learn" in content or "evolve-lite:learn" in tool_calls: + skill_usage["learn_count"] += 1 + + # Check for mentions of specific skills + if "guideline" in content or "atomic-skill" in content or "skill-flow" in content: + # Try to extract skill references + if ".evolve/entities/" in content: + skill_usage["skills_applied"].append("entity_referenced") + + return skill_usage + + +def compare_trajectories(with_skills_path, without_skills_path, entities_dir): + """Compare two trajectories: one with skills, one without.""" + + # Load trajectories + try: + with_skills = load_trajectory(with_skills_path) + without_skills = load_trajectory(without_skills_path) + except Exception as e: + return { + "error": f"Failed to load trajectories: {e}", + "comparison": None + } + + # Analyze metrics + with_metrics = analyze_trajectory_metrics(with_skills) + without_metrics = analyze_trajectory_metrics(without_skills) + + # Check skill usage + with_skill_usage = find_skill_usage_in_trajectory(with_skills, entities_dir) + without_skill_usage = find_skill_usage_in_trajectory(without_skills, entities_dir) + + # Calculate improvements + comparison = { + "with_skills": { + "path": str(with_skills_path), + "metrics": with_metrics, + "skill_usage": with_skill_usage + }, + "without_skills": { + "path": str(without_skills_path), + "metrics": without_metrics, + "skill_usage": without_skill_usage + }, + "improvements": { + "fewer_tool_uses": without_metrics["tool_uses"] - with_metrics["tool_uses"], + "fewer_errors": without_metrics["errors"] - with_metrics["errors"], + "fewer_retries": without_metrics["retries"] - with_metrics["retries"], + "fewer_interventions": without_metrics["user_interventions"] - with_metrics["user_interventions"], + "completion_improved": ( + with_metrics["completion_status"] == "completed" and + without_metrics["completion_status"] != "completed" + ) + }, + "summary": {} + } + + # Generate summary + improvements = comparison["improvements"] + total_improvements = sum([ + 1 if improvements["fewer_tool_uses"] > 0 else 0, + 1 if improvements["fewer_errors"] > 0 else 0, + 1 if improvements["fewer_retries"] > 0 else 0, + 1 if improvements["fewer_interventions"] > 0 else 0, + 1 if improvements["completion_improved"] else 0 + ]) + + comparison["summary"] = { + "skills_helped": total_improvements > 0, + "improvement_count": total_improvements, + "efficiency_gain_percent": ( + (without_metrics["tool_uses"] - with_metrics["tool_uses"]) / + without_metrics["tool_uses"] * 100 + if without_metrics["tool_uses"] > 0 else 0 + ), + "error_reduction_percent": ( + (without_metrics["errors"] - with_metrics["errors"]) / + without_metrics["errors"] * 100 + if without_metrics["errors"] > 0 else 0 + ) + } + + return comparison + + +def generate_comparison_report(comparisons, output_path): + """Generate a comprehensive comparison report.""" + + report = { + "generated_at": datetime.now().isoformat(), + "total_comparisons": len(comparisons), + "comparisons": comparisons, + "aggregate_metrics": { + "total_with_skills_helped": 0, + "total_efficiency_gains": 0, + "total_error_reductions": 0, + "average_tool_use_reduction": 0, + "average_error_reduction": 0 + } + } + + # Calculate aggregates + valid_comparisons = [c for c in comparisons if "error" not in c] + + if valid_comparisons: + report["aggregate_metrics"]["total_with_skills_helped"] = sum( + 1 for c in valid_comparisons + if c.get("summary", {}).get("skills_helped", False) + ) + + efficiency_gains = [ + c["summary"]["efficiency_gain_percent"] + for c in valid_comparisons + if c.get("summary", {}).get("efficiency_gain_percent", 0) > 0 + ] + + error_reductions = [ + c["summary"]["error_reduction_percent"] + for c in valid_comparisons + if c.get("summary", {}).get("error_reduction_percent", 0) > 0 + ] + + if efficiency_gains: + report["aggregate_metrics"]["average_tool_use_reduction"] = ( + sum(efficiency_gains) / len(efficiency_gains) + ) + + if error_reductions: + report["aggregate_metrics"]["average_error_reduction"] = ( + sum(error_reductions) / len(error_reductions) + ) + + # Save report + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2) + + return report + + +def main(): + parser = argparse.ArgumentParser( + description="Compare task performance with and without skills" + ) + parser.add_argument( + "--with-skills", + required=True, + help="Path to trajectory with skills applied" + ) + parser.add_argument( + "--without-skills", + required=True, + help="Path to baseline trajectory without skills" + ) + parser.add_argument( + "--output", + default=None, + help="Output path for comparison report (default: .evolve/tests/comparison_report.json)" + ) + parser.add_argument( + "--batch", + action="store_true", + help="Batch mode: compare multiple trajectory pairs from a directory" + ) + + args = parser.parse_args() + + # Determine entities directory + evolve_dir = get_evolve_dir() + entities_dir = evolve_dir / "entities" + + if not entities_dir.exists(): + print(f"Warning: Entities directory not found: {entities_dir}", file=sys.stderr) + + # Determine output path + if args.output: + output_path = Path(args.output) + else: + output_path = evolve_dir / "tests" / "comparison_report.json" + + output_path.parent.mkdir(parents=True, exist_ok=True) + + if args.batch: + # Batch mode: compare multiple pairs + print("Batch comparison mode not yet implemented", file=sys.stderr) + sys.exit(1) + else: + # Single comparison + print(f"Comparing trajectories:") + print(f" With skills: {args.with_skills}") + print(f" Without skills: {args.without_skills}") + print() + + comparison = compare_trajectories( + Path(args.with_skills), + Path(args.without_skills), + entities_dir + ) + + if "error" in comparison: + print(f"Error: {comparison['error']}", file=sys.stderr) + sys.exit(1) + + # Generate report + report = generate_comparison_report([comparison], output_path) + + # Print summary + print("="*60) + print("COMPARISON SUMMARY") + print("="*60) + + summary = comparison["summary"] + print(f"Skills helped: {'Yes' if summary['skills_helped'] else 'No'}") + print(f"Improvements: {summary['improvement_count']}/5 metrics") + print(f"Efficiency gain: {summary['efficiency_gain_percent']:.1f}%") + print(f"Error reduction: {summary['error_reduction_percent']:.1f}%") + + print(f"\nWith skills:") + print(f" Tool uses: {comparison['with_skills']['metrics']['tool_uses']}") + print(f" Errors: {comparison['with_skills']['metrics']['errors']}") + print(f" Retries: {comparison['with_skills']['metrics']['retries']}") + print(f" Status: {comparison['with_skills']['metrics']['completion_status']}") + + print(f"\nWithout skills:") + print(f" Tool uses: {comparison['without_skills']['metrics']['tool_uses']}") + print(f" Errors: {comparison['without_skills']['metrics']['errors']}") + print(f" Retries: {comparison['without_skills']['metrics']['retries']}") + print(f" Status: {comparison['without_skills']['metrics']['completion_status']}") + + print(f"\nReport saved to: {output_path}") + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_pseudo_conversations.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_pseudo_conversations.py new file mode 100644 index 00000000..6314fc55 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_pseudo_conversations.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +""" +Pseudo-Conversation Generator for Atomic Skill Evaluation + +Reads each atomic skill in .evolve/entities/atomic-skill/watson-orchestrate/, +builds a pseudo-conversation (system + user message with the skill injected), +derives expected_behaviour (must_include / must_not_include) from the skill +content, and writes one JSON fixture per skill to +.evolve/tests/pseudo_conversations/. + +Usage: + python generate_pseudo_conversations.py + python generate_pseudo_conversations.py --export-csv +""" + +import argparse +import csv +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +# --------------------------------------------------------------------------- +# Bootstrap: locate entity_io.py by walking up from this script's location +# --------------------------------------------------------------------------- +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find lib/evolve-lite/entity_io.py above {_script}") +sys.path.insert(0, str(_lib)) +sys.path.insert(0, str(_script.parent)) + +from entity_io import get_evolve_dir, markdown_to_entity # noqa: E402 +from trigger_parser import trigger_to_user_question # noqa: E402 + +# --------------------------------------------------------------------------- +# Subagent prompt template (mirrors .evolve/tests/evaluation/subagent_prompt_template.md) +# --------------------------------------------------------------------------- +_SYSTEM_PREAMBLE = """\ +You are a technical assistant evaluating whether you can correctly apply a recalled skill. + +A skill has been recalled for this conversation. It is enclosed in tags in the +system context. Your job is to respond to the user's question by FOLLOWING the skill exactly. + +Rules: +- Use the specific commands, steps, or guidance from the recalled skill +- Do not invent alternative approaches not mentioned in the skill +- If the skill prescribes a specific command, include that exact command in your response +- If the skill says NOT to use something, do not suggest it + +Respond concisely and directly. Your response will be evaluated for skill adherence.\ +""" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def extract_backtick_terms(content): + """Return all strings wrapped in single backticks (inline code spans). + + Fenced code blocks (``` ... ```) are excluded before scanning so that + multi-line code examples don't end up in must_include. + """ + # Remove fenced code blocks first (``` ... ```) + stripped = re.sub(r"```[\s\S]*?```", "", content) + return re.findall(r"`([^`]+)`", stripped) + + +def extract_must_not_include(content): + """ + Scan content for negation patterns and return the negated term. + Patterns: 'does not accept X', 'not as a X', 'instead of X', 'avoid X', + 'do not use X', 'not use X'. + """ + patterns = [ + r"does not accept\s+(`[^`]+`|\S+)", + r"not as a\s+(`[^`]+`|\S+)", + r"instead of\s+(`[^`]+`|\S+)", + r"avoid\s+(`[^`]+`|\S+)", + r"do not (?:use|suggest)\s+(`[^`]+`|\S+)", + r"not use\s+(`[^`]+`|\S+)", + ] + terms = [] + for pat in patterns: + for match in re.finditer(pat, content, re.IGNORECASE): + term = match.group(1).strip("`").strip() + if term: + terms.append(term) + return terms + + +def strip_placeholders(term): + """Remove angle-bracket placeholder segments from a string. + + e.g. 'orchestrate env add --name --url ' -> 'orchestrate env add --name --url' + Used at evaluation time (not generation time) so must_include retains the + original command structure with placeholders intact in the fixture. + """ + return re.sub(r"\s*<[^>]+>", "", term).strip() + + +def _rubric_to_must_include(rubric_text): + """Parse a ## Success Rubric bullet list into must_include terms. + + Each bullet line may contain backtick-wrapped terms and/or plain criteria. + Backtick terms are extracted verbatim (they represent observable commands or + outputs). Plain bullet text (after stripping the leading ``-``) is included + as-is so the evaluator can match it as a substring in the agent response. + """ + seen: set = set() + terms = [] + for line in rubric_text.splitlines(): + line = line.strip().lstrip("-").strip() + if not line: + continue + # Prefer backtick-wrapped terms within the bullet if present + backtick_hits = re.findall(r"`([^`]+)`", line) + candidates = [t.strip() for t in backtick_hits if t.strip()] if backtick_hits else [line] + for t in candidates: + if t not in seen: + seen.add(t) + terms.append(t) + return terms + + +def build_expected_behaviour(skill_path, entity): + """Derive must_include, must_not_include, and action_type from skill content. + + Priority order for must_include: + 1. ## Success Rubric bullets — explicit, author-stated pass criteria + 2. Backtick-wrapped inline code from the skill body + 3. Underscore-identifier / long-word fallback + """ + content = entity.get("content", "") + rubric = entity.get("success_rubric", "") + + rubric_terms = _rubric_to_must_include(rubric) if rubric else [] + + if rubric_terms: + # Priority 1: author-stated success criteria take precedence over heuristics + must_include = rubric_terms + action_type = "rubric_criteria" + else: + # rubric was absent OR present but contained no extractable terms — + # fall through to content-based heuristics in both cases + backtick_terms = extract_backtick_terms(content) + if backtick_terms: + # Priority 2: backtick-wrapped inline code from the skill body + must_include = [t.strip() for t in backtick_terms if t.strip()] + action_type = "command_recommendation" + else: + # Priority 3: underscore-identifier / long-word fallback + underscore_terms = re.findall(r"\b([a-z][a-z0-9]*(?:_[a-z0-9]+)+)\b", content) + if underscore_terms: + seen: set = set() + must_include = [] + for t in underscore_terms: + if t not in seen: + seen.add(t) + must_include.append(t) + else: + _stop = { + "with", "from", "this", "that", "file", "have", "into", + "when", "also", "some", "here", "there", "then", "than", + "each", "will", "your", "been", "able", "make", "only", + "such", "used", "both", "they", "them", "what", "more", + } + candidates = re.findall(r"\b([a-z]{5,})\b", content.lower()) + seen = set() + must_include = [] + for t in candidates: + if t not in _stop and t not in seen: + seen.add(t) + must_include.append(t) + must_include = must_include[:6] + action_type = "procedural_guidance" + + # must_not_include: scan both skill body and rubric (if present) for negation patterns + must_not_include = extract_must_not_include(content) + if rubric: + must_not_include += extract_must_not_include(rubric) + must_not_include = list(dict.fromkeys(must_not_include)) # dedup, preserve order + + return { + "description": f"Agent should follow the guidance in {Path(skill_path).stem}", + "must_include": must_include, + "must_not_include": must_not_include, + "action_type": action_type, + } + + +def build_pseudo_conversation(skill_slug, skill_path, entity, evolve_dir): + """Build the complete pseudo-conversation fixture dict for one skill.""" + content = entity.get("content", "") + trigger = entity.get("trigger", "") + traj_ref = entity.get("trajectory") # may be absent + + # System message: preamble + recalled skill + system_content = ( + f"{_SYSTEM_PREAMBLE}\n\n" + f"The following skill has been recalled and MUST guide your response:\n" + f"\n{content}\n" + ) + + # User message: prefer trajectory-derived, fall back to trigger-derived + generation_method = "trigger_derived" + user_content = None + + if traj_ref: + # Trajectory path is kept in sources for traceability but is no longer + # used to derive the user message — trigger-derived questions are more + # skill-specific and avoid the "same opening message for all skills" problem. + pass + + user_content = trigger_to_user_question(trigger, llm_fn=None) + generation_method = "trigger_derived" + + conversation = [ + {"role": "system", "content": system_content}, + {"role": "user", "content": user_content}, + ] + + expected_behaviour = build_expected_behaviour(skill_path, entity) + + sources = { + "trajectory": traj_ref or None, + "generation_method": generation_method, + "trigger": trigger, # stored for CSV export and traceability + } + + return { + "test_id": skill_slug, + "skill_slug": skill_slug, + "skill_path": str(Path(skill_path).relative_to(evolve_dir.parent)), + "skill_content": content, + "conversation": conversation, + "expected_behaviour": expected_behaviour, + "sources": sources, + } + + +# --------------------------------------------------------------------------- +# CSV export +# --------------------------------------------------------------------------- + +def export_csv(fixtures, output_path): + """Write a human-readable CSV of all pseudo-conversations.""" + fieldnames = [ + "skill_slug", "trigger", "user_message", + "must_include", "must_not_include", "action_type", "generation_method", + ] + with open(output_path, "w", newline="", encoding="utf-8") as fh: + writer = csv.DictWriter(fh, fieldnames=fieldnames) + writer.writeheader() + for fx in fixtures: + user_msg = next( + (m["content"] for m in fx["conversation"] if m["role"] == "user"), "" + ) + writer.writerow({ + "skill_slug": fx["skill_slug"], + "trigger": fx["sources"].get("trigger", ""), + "user_message": user_msg[:300], + "must_include": "|".join(fx["expected_behaviour"]["must_include"]), + "must_not_include": "|".join(fx["expected_behaviour"]["must_not_include"]), + "action_type": fx["expected_behaviour"]["action_type"], + "generation_method": fx["sources"]["generation_method"], + }) + print(f" CSV written: {output_path}") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="Generate pseudo-conversation test fixtures for atomic skill evaluation" + ) + parser.add_argument( + "--export-csv", + action="store_true", + help="Also write .evolve/tests/pseudo_conversations/test_suite.csv", + ) + parser.add_argument( + "--entities-dir", + default=None, + help=( + "Root directory to scan for entity .md files (recursively). " + "Default: .evolve/entities/atomic-skill/watson-orchestrate/" + ), + ) + parser.add_argument( + "--filter-slugs", + default=None, + help=( + "Path to a JSON file whose top-level keys are the only entity slugs " + "to generate fixtures for. Slugs not in the file are skipped. " + "Supports the main_entity_slugs.json manifest format." + ), + ) + parser.add_argument( + "--pinned-rubrics", + default=None, + help=( + "Path to a main_entity_slugs.json manifest. For any slug whose entry " + "has a non-empty 'rubric_terms' list, use those terms as must_include " + "instead of re-deriving them from the merged entity. " + "Use this during regression gating so forks cannot weaken rubrics and " + "silently pass their own tests." + ), + ) + parser.add_argument( + "--output-dir", + default=None, + help="Directory to write fixture JSON files. Default: .evolve/tests/pseudo_conversations/", + ) + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + + if args.entities_dir: + skills_dir = Path(args.entities_dir) + else: + skills_dir = evolve_dir / "entities" / "atomic-skill" / "watson-orchestrate" + + output_dir = Path(args.output_dir) if args.output_dir \ + else evolve_dir / "tests" / "pseudo_conversations" + output_dir.mkdir(parents=True, exist_ok=True) + + # Load slug filter if provided + allowed_slugs = None + if args.filter_slugs: + filter_path = Path(args.filter_slugs) + if not filter_path.exists(): + print(f"Error: --filter-slugs file not found: {filter_path}", file=sys.stderr) + sys.exit(1) + with open(filter_path, "r", encoding="utf-8") as fh: + allowed_slugs = set(json.load(fh).keys()) + print(f"Filtering to {len(allowed_slugs)} slug(s) from {filter_path}") + + # Load pinned rubrics if provided + pinned_rubrics = {} # slug -> [must_include terms] + if args.pinned_rubrics: + pinned_path = Path(args.pinned_rubrics) + if not pinned_path.exists(): + print(f"Error: --pinned-rubrics file not found: {pinned_path}", file=sys.stderr) + sys.exit(1) + with open(pinned_path, "r", encoding="utf-8") as fh: + manifest = json.load(fh) + for slug, entry in manifest.items(): + terms = entry.get("rubric_terms", []) + if terms: + pinned_rubrics[slug] = terms + print(f"Pinned rubrics loaded for {len(pinned_rubrics)} slug(s) from {pinned_path}") + + if not skills_dir.exists(): + print(f"Error: skills directory not found: {skills_dir}", file=sys.stderr) + sys.exit(1) + + md_files = sorted( + p for p in skills_dir.rglob("*.md") + if not p.is_symlink() + ) + + # Apply slug filter + if allowed_slugs is not None: + md_files = [p for p in md_files if p.stem in allowed_slugs] + + if not md_files: + print("No skill files found.", file=sys.stderr) + sys.exit(1) + + print(f"Found {len(md_files)} skill file(s) in {skills_dir}") + print() + + fixtures = [] + traj_derived = 0 + trigger_derived = 0 + + for md_file in md_files: + skill_slug = md_file.stem + print(f" Processing: {skill_slug}") + + entity = markdown_to_entity(md_file) + fixture = build_pseudo_conversation(skill_slug, md_file, entity, evolve_dir) + + # Override must_include with pinned rubric terms if available for this slug. + # This ensures the regression gate always tests the original main-repo rubric + # even if a fork changed the entity's rubric section. + if skill_slug in pinned_rubrics: + fixture["expected_behaviour"]["must_include"] = pinned_rubrics[skill_slug] + fixture["expected_behaviour"]["action_type"] = "rubric_criteria_pinned" + print(f" [pinned rubric] must_include overridden from manifest") + + out_path = output_dir / f"{skill_slug}.json" + with open(out_path, "w", encoding="utf-8") as fh: + json.dump(fixture, fh, indent=2) + print(f" Written: {out_path}") + print(f" method={fixture['sources']['generation_method']} " + f"must_include={fixture['expected_behaviour']['must_include']}") + if fixture["expected_behaviour"]["must_not_include"]: + print(f" must_not_include={fixture['expected_behaviour']['must_not_include']}") + + fixtures.append(fixture) + if fixture["sources"]["generation_method"] == "trajectory_derived": + traj_derived += 1 + else: + trigger_derived += 1 + + print() + print("=" * 60) + print(f"Generated {len(fixtures)} fixture(s) (expect 6)") + print(f" trajectory_derived : {traj_derived}") + print(f" trigger_derived : {trigger_derived}") + print(f"Output dir: {output_dir}") + + if args.export_csv: + csv_path = output_dir / "test_suite.csv" + export_csv(fixtures, csv_path) + + print() + print("Done.") + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_skill_tests.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_skill_tests.py new file mode 100644 index 00000000..300a69da --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_skill_tests.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +""" +Generate pseudo-conversation test fixtures for newly saved skill files. + +Called automatically after evolve-lite-learn saves new entities. Accepts one +or more skill file paths as arguments, generates a fixture for each, and +prints a summary. Skips non-atomic-skill files silently. + +Usage: + python3 generate_skill_tests.py [ ...] + python3 generate_skill_tests.py --all # regenerate all skills + +The generated fixtures land in .evolve/tests/pseudo_conversations/ following +the same format as generate_pseudo_conversations.py. The two scripts share +the same build_pseudo_conversation() logic — this one just accepts arbitrary +paths rather than scanning a fixed directory. +""" + +import argparse +import json +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Bootstrap: locate entity_io.py and the shared generator logic +# --------------------------------------------------------------------------- +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find lib/evolve-lite/entity_io.py above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import get_evolve_dir, markdown_to_entity # noqa: E402 + +# Import shared builder from the sibling generator script +_scripts_dir = _script.parent +sys.path.insert(0, str(_scripts_dir)) +from generate_pseudo_conversations import ( # noqa: E402 + build_pseudo_conversation, +) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def generate_for_paths(skill_paths, evolve_dir, output_dir): + """Generate fixtures for a list of skill file paths. + + Returns (generated, skipped) counts. + """ + output_dir.mkdir(parents=True, exist_ok=True) + generated = 0 + skipped = 0 + + for path in skill_paths: + path = Path(path) + if not path.exists(): + print(f" Warning: file not found, skipping: {path}", file=sys.stderr) + skipped += 1 + continue + + entity = markdown_to_entity(path) + + # Only generate tests for atomic-skills + if entity.get("type") != "atomic-skill": + skipped += 1 + continue + + skill_slug = path.stem + fixture = build_pseudo_conversation(skill_slug, path, entity, evolve_dir) + + out_path = output_dir / f"{skill_slug}.json" + with open(out_path, "w", encoding="utf-8") as fh: + json.dump(fixture, fh, indent=2) + + eb = fixture["expected_behaviour"] + print(f" ✓ {skill_slug}") + print(f" user_msg : {next(m['content'] for m in fixture['conversation'] if m['role']=='user')[:90]}") + print(f" must_include : {eb['must_include']}") + if eb["must_not_include"]: + print(f" must_not : {eb['must_not_include']}") + generated += 1 + + return generated, skipped + + +def main(): + parser = argparse.ArgumentParser( + description="Generate pseudo-conversation test fixtures for new atomic skills" + ) + parser.add_argument( + "skill_files", + nargs="*", + help="Paths to newly saved atomic skill .md files", + ) + parser.add_argument( + "--all", + action="store_true", + help="Regenerate fixtures for all atomic skills in .evolve/entities/", + ) + parser.add_argument( + "--output-dir", + default=None, + help="Override output directory (default: .evolve/tests/pseudo_conversations/)", + ) + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + output_dir = Path(args.output_dir) if args.output_dir \ + else evolve_dir / "tests" / "pseudo_conversations" + + if args.all: + skill_paths = sorted( + p for p in (evolve_dir / "entities").glob("**/*.md") + if not p.is_symlink() + ) + elif args.skill_files: + skill_paths = [Path(p) for p in args.skill_files] + else: + parser.print_help() + sys.exit(0) + + if not skill_paths: + print("No skill files to process.") + sys.exit(0) + + print(f"Generating test fixtures for {len(skill_paths)} file(s)...") + print() + generated, skipped = generate_for_paths(skill_paths, evolve_dir, output_dir) + print() + print(f"Done. Generated: {generated} Skipped (non-atomic-skill): {skipped}") + print(f"Output: {output_dir}") + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_test_cases.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_test_cases.py new file mode 100644 index 00000000..eade8d9d --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/generate_test_cases.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +""" +Test Case Generator for Evolve Lite Skills + +Analyzes conversation trajectories and generates test cases for each skill +that was learned from those trajectories. Test cases validate that skills +are discoverable, actionable, complete, and compose correctly. +""" + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path + +# Walk up from the script location to find the installed plugin lib directory +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + markdown_to_entity, + log as _log, +) + + +def log(message): + _log("test-gen", message) + + +def load_trajectory(trajectory_path): + """Load a trajectory JSON file.""" + with open(trajectory_path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def find_skills_from_trajectory(trajectory_path, entities_dir): + """Find all skills that reference this trajectory.""" + # Get both absolute and relative paths for matching + trajectory_path_abs = str(Path(trajectory_path).resolve()) + + # Try to get relative path, fall back to the path as-is + try: + trajectory_path_rel = str(Path(trajectory_path).resolve().relative_to(Path.cwd())) + except ValueError: + trajectory_path_rel = str(trajectory_path) + + trajectory_name = Path(trajectory_path).name + + skills = [] + + entities_dir = Path(entities_dir) + for md_file in entities_dir.glob("**/*.md"): + if md_file.is_symlink() or ".git" in md_file.parts: + continue + + try: + entity = markdown_to_entity(md_file) + entity_traj = entity.get("trajectory", "") + + # Match by absolute path, relative path, or filename + if entity_traj and ( + trajectory_path_abs in entity_traj or + trajectory_path_rel in entity_traj or + trajectory_name in entity_traj + ): + skills.append({ + "path": str(md_file), + "entity": entity, + "type": entity.get("type", "unknown") + }) + except Exception as e: + log(f"Error reading {md_file}: {e}") + + return skills + + +def extract_user_request(trajectory): + """Extract the initial user request from a trajectory.""" + messages = trajectory.get("messages", []) + for msg in messages: + if msg.get("role") == "user": + content = msg.get("content", "") + if content and isinstance(content, str): + return content + return "Unknown request" + + +def extract_context_from_trajectory(trajectory): + """Extract relevant context from the trajectory.""" + metadata = trajectory.get("metadata", {}) + + context = { + "model": trajectory.get("model", "unknown"), + "session_id": trajectory.get("session_id", "unknown"), + "timestamp": trajectory.get("timestamp", "unknown"), + "project_root": metadata.get("project_root", "unknown"), + "mode": metadata.get("mode", "unknown") + } + + # Extract environment details from messages + messages = trajectory.get("messages", []) + for msg in messages: + if msg.get("role") == "user" and "environment" in msg.get("content", "").lower(): + context["has_environment_details"] = True + break + + return context + + +def extract_tools_used(trajectory): + """Extract tools that were used in the trajectory.""" + tools = set() + messages = trajectory.get("messages", []) + + for msg in messages: + if msg.get("role") == "assistant" and msg.get("tool_calls"): + for tool_call in msg.get("tool_calls", []): + if tool_call.get("function"): + tools.add(tool_call["function"].get("name", "unknown")) + + return list(tools) + + +def generate_trigger_match_test(skill, trajectory, trajectory_path): + """Generate a trigger match test case.""" + user_request = extract_user_request(trajectory) + context = extract_context_from_trajectory(trajectory) + + test_case = { + "test_id": f"trigger_match_{Path(skill['path']).stem}_{Path(trajectory_path).stem}", + "test_type": "trigger_match", + "skill_type": skill["type"], + "skill_path": skill["path"], + "skill_trigger": skill["entity"].get("trigger", ""), + "scenario": f"User request: {user_request[:100]}...", + "expected_trigger_match": True, + "input_context": { + "user_request": user_request, + "environment": context, + "tools_available": extract_tools_used(trajectory) + }, + "validation_criteria": [ + "Skill trigger matches the user request context", + "Skill would be recalled during similar scenarios", + "Trigger is specific enough to avoid false positives" + ], + "trajectory_source": str(trajectory_path), + "created_at": datetime.now().isoformat() + } + + return test_case + + +def generate_content_completeness_test(skill, trajectory, trajectory_path): + """Generate a content completeness test case.""" + user_request = extract_user_request(trajectory) + tools_used = extract_tools_used(trajectory) + + test_case = { + "test_id": f"content_complete_{Path(skill['path']).stem}_{Path(trajectory_path).stem}", + "test_type": "content_completeness", + "skill_type": skill["type"], + "skill_path": skill["path"], + "skill_content": skill["entity"].get("content", "")[:200] + "...", + "scenario": "Validating skill provides sufficient guidance", + "input_context": { + "user_request": user_request, + "tools_used_in_trajectory": tools_used + }, + "validation_criteria": [ + "All necessary steps are included", + "Commands or actions are clear and executable", + "Prerequisites are stated", + "Expected outcomes are defined", + "Rationale explains why this approach works" + ], + "trajectory_source": str(trajectory_path), + "created_at": datetime.now().isoformat() + } + + return test_case + + +def generate_skill_composition_test(skill, trajectory, trajectory_path, entities_dir): + """Generate a skill composition test for skill-flows.""" + if skill["type"] != "skill-flow": + return None + + atomic_skills = skill["entity"].get("atomic_skills", "") + if not atomic_skills: + return None + + atomic_skill_list = [s.strip() for s in atomic_skills.split(",")] + + # Check if atomic skills exist + existing_atomic_skills = [] + missing_atomic_skills = [] + + for atomic_skill_slug in atomic_skill_list: + atomic_skill_path = Path(entities_dir) / "atomic-skill" / f"{atomic_skill_slug}.md" + if atomic_skill_path.exists(): + existing_atomic_skills.append(atomic_skill_slug) + else: + missing_atomic_skills.append(atomic_skill_slug) + + test_case = { + "test_id": f"composition_{Path(skill['path']).stem}_{Path(trajectory_path).stem}", + "test_type": "skill_composition", + "skill_type": skill["type"], + "skill_path": skill["path"], + "scenario": "Validating skill-flow properly references atomic skills", + "atomic_skills_referenced": atomic_skill_list, + "atomic_skills_existing": existing_atomic_skills, + "atomic_skills_missing": missing_atomic_skills, + "validation_criteria": [ + "All referenced atomic skills exist", + "Atomic skills are in correct order", + "No circular dependencies", + "Atomic skill content matches flow steps" + ], + "expected_outcome": { + "all_atomic_skills_exist": len(missing_atomic_skills) == 0, + "composition_is_valid": True + }, + "trajectory_source": str(trajectory_path), + "created_at": datetime.now().isoformat() + } + + return test_case + + +def generate_trajectory_replay_test(skill, trajectory, trajectory_path): + """Generate a trajectory replay test case.""" + user_request = extract_user_request(trajectory) + context = extract_context_from_trajectory(trajectory) + tools_used = extract_tools_used(trajectory) + + test_case = { + "test_id": f"replay_{Path(skill['path']).stem}_{Path(trajectory_path).stem}", + "test_type": "trajectory_replay", + "skill_type": skill["type"], + "skill_path": skill["path"], + "scenario": "Replaying trajectory with skill guidance", + "input_context": { + "user_request": user_request, + "environment": context, + "tools_used": tools_used + }, + "validation_criteria": [ + "Skill would be recalled at appropriate time", + "Skill guidance matches actions taken", + "Following skill would achieve same outcome", + "Skill prevents errors encountered in trajectory" + ], + "expected_outcome": { + "skill_recalled": True, + "skill_applied": True, + "outcome_matches": True + }, + "trajectory_source": str(trajectory_path), + "created_at": datetime.now().isoformat() + } + + return test_case + + +def generate_test_cases_for_skill(skill, trajectory, trajectory_path, entities_dir): + """Generate all test cases for a single skill.""" + test_cases = [] + + # Generate trigger match test + test_cases.append(generate_trigger_match_test(skill, trajectory, trajectory_path)) + + # Generate content completeness test + test_cases.append(generate_content_completeness_test(skill, trajectory, trajectory_path)) + + # Generate composition test for skill-flows + composition_test = generate_skill_composition_test(skill, trajectory, trajectory_path, entities_dir) + if composition_test: + test_cases.append(composition_test) + + # Generate trajectory replay test + test_cases.append(generate_trajectory_replay_test(skill, trajectory, trajectory_path)) + + return test_cases + + +def save_test_cases(test_cases, output_dir): + """Save test cases to JSON files.""" + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + saved_files = [] + for test_case in test_cases: + filename = f"{test_case['test_id']}.json" + output_path = output_dir / filename + + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(test_case, f, indent=2) + + saved_files.append(output_path) + log(f"Saved test case: {output_path}") + + return saved_files + + +def main(): + parser = argparse.ArgumentParser( + description="Generate test cases for skills from trajectories" + ) + parser.add_argument( + "--trajectory", + help="Path to specific trajectory file" + ) + parser.add_argument( + "--all-trajectories", + action="store_true", + help="Generate tests for all trajectories" + ) + parser.add_argument( + "--output", + default=None, + help="Output directory for test cases (default: .evolve/tests/)" + ) + + args = parser.parse_args() + + # Determine output directory + if args.output: + output_dir = Path(args.output) + else: + evolve_dir = get_evolve_dir() + output_dir = evolve_dir / "tests" / "cases" + + # Determine entities directory + evolve_dir = get_evolve_dir() + entities_dir = evolve_dir / "entities" + + if not entities_dir.exists(): + print(f"Error: Entities directory not found: {entities_dir}", file=sys.stderr) + sys.exit(1) + + # Determine trajectories to process + trajectories_dir = evolve_dir / "trajectories" + + if args.all_trajectories: + if not trajectories_dir.exists(): + print(f"Error: Trajectories directory not found: {trajectories_dir}", file=sys.stderr) + sys.exit(1) + trajectory_files = list(trajectories_dir.glob("*.json")) + elif args.trajectory: + trajectory_files = [Path(args.trajectory)] + else: + print("Error: Must specify --trajectory or --all-trajectories", file=sys.stderr) + sys.exit(1) + + if not trajectory_files: + print("No trajectory files found", file=sys.stderr) + sys.exit(1) + + print(f"Processing {len(trajectory_files)} trajectory file(s)...") + + all_test_cases = [] + + for trajectory_path in trajectory_files: + log(f"Processing trajectory: {trajectory_path}") + print(f"\nProcessing: {trajectory_path.name}") + + try: + trajectory = load_trajectory(trajectory_path) + except Exception as e: + print(f" Error loading trajectory: {e}", file=sys.stderr) + continue + + # Find skills from this trajectory + skills = find_skills_from_trajectory(trajectory_path, entities_dir) + print(f" Found {len(skills)} skill(s) from this trajectory") + + for skill in skills: + print(f" Generating tests for: {Path(skill['path']).name}") + test_cases = generate_test_cases_for_skill( + skill, trajectory, trajectory_path, entities_dir + ) + all_test_cases.extend(test_cases) + print(f" Generated {len(test_cases)} test case(s)") + + # Save all test cases + print(f"\nSaving {len(all_test_cases)} test case(s) to {output_dir}") + saved_files = save_test_cases(all_test_cases, output_dir) + + print(f"\n✓ Generated {len(all_test_cases)} test case(s)") + print(f"✓ Saved to: {output_dir}") + print(f"\nTest files:") + for f in saved_files: + print(f" - {f.name}") + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_baseline_tests.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_baseline_tests.py new file mode 100644 index 00000000..d908e2c0 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_baseline_tests.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +""" +Trigger Test Runner + +Tests whether a skill's trigger correctly identifies scenarios where the skill +applies. For each fixture, the agent receives ONLY the user message (no skill +injected) and must independently produce a response. The response is then +checked against the same must_include terms as the content tests. + +If the agent independently arrives at the right answer without being told the +skill, the trigger describes a scenario specific enough that any competent +agent would naturally do the right thing. A failure means the situation is +non-obvious and the skill is genuinely necessary. + +Unlike the content tests (which verify skill self-consistency), trigger tests +validate the trigger's DISCRIMINATING POWER — does this trigger identify a +scenario that a naive agent would handle incorrectly without the skill? + +Two outcomes are both informative: + PASS — agent gets it right without the skill → trigger is well-scoped, + but consider whether the skill adds value at all + FAIL — agent misses something without the skill → the skill IS necessary, + and the trigger correctly identifies a non-obvious situation + +Only skill fixtures (no edge_ / guideline_ prefix) are evaluated — edge cases +and guidelines are not atomic skills with standalone triggers. + +Usage: + python3 run_baseline_tests.py --responses-file + python3 run_baseline_tests.py --simulate (uses naive baseline responses) +""" + +import argparse +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +# --------------------------------------------------------------------------- +# Bootstrap +# --------------------------------------------------------------------------- +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib: + sys.path.insert(0, str(_lib)) + try: + from entity_io import get_evolve_dir + except ImportError: + def get_evolve_dir(): return Path(".evolve") +else: + def get_evolve_dir(): return Path(".evolve") + + +# --------------------------------------------------------------------------- +# Naive baseline responses +# +# These simulate what a generic agent would say WITHOUT the skill injected. +# Each response is intentionally written as a reasonable-but-incomplete answer +# — the kind a competent agent gives when it doesn't have the specific skill. +# --------------------------------------------------------------------------- + +NAIVE_RESPONSES = { + # ---- Watson Orchestrate skills ---- + "watson-orchestrate-activate-venv": + "Make sure your Python environment is set up. You may need to install the " + "orchestrate package. Try running `pip install ibm-watsonx-orchestrate` and " + "then retry your command.", + + "watson-orchestrate-authenticate-orchestrate-env": + "Before running Watson Orchestrate commands, make sure you're logged in. " + "You may need to configure your credentials or run a login command. " + "Check the orchestrate CLI help with `orchestrate --help` to see available " + "authentication options.", + + "watson-orchestrate-create-agent-yaml": + "To create a Watson Orchestrate agent, you'll need a YAML configuration file. " + "The file should define your agent's properties. Common fields include name, " + "description, and the model to use. Check the Watson Orchestrate documentation " + "for the full schema.", + + "watson-orchestrate-decorate-tool-function": + "To register a Python function as a tool, you need to add a decorator to it. " + "Check the Watson Orchestrate SDK documentation for the correct decorator " + "syntax and which package to import it from.", + + "watson-orchestrate-deploy-agent": + "After importing an agent, you should be able to see it in the Watson Orchestrate " + "interface. Check the agents list to confirm the import was successful. " + "If the agent isn't appearing, try refreshing or checking the import logs.", + + "watson-orchestrate-import-agent-yaml": + "To import an agent into Watson Orchestrate, use the orchestrate CLI. " + "Run `orchestrate agents import` with your agent file. Check `orchestrate agents --help` " + "for the exact syntax and required flags.", + + "watson-orchestrate-import-multi-tool-python-file": + "To expose multiple Python functions as tools, you can import the Python file " + "using the orchestrate tools import command. Check the CLI help for how to " + "specify the file and tool names.", + + "watson-orchestrate-pipe-api-key-stdin": + "If a CLI command keeps prompting for a password, you could try setting " + "environment variables beforehand, or look for a `--no-interactive` flag. " + "Some tools also support reading credentials from a config file.", + + "watson-orchestrate-reauth-expired-token": + "A token expiration error means your session has timed out. You'll need to " + "log in again. Try running the authentication command again and re-enter your " + "credentials when prompted.", + + "watson-orchestrate-register-and-auth-orchestrate-env": + "To set up the Watson Orchestrate CLI for the first time, you need to configure " + "your environment. Check the CLI documentation for the setup commands and " + "provide your credentials when prompted.", +} + + +# --------------------------------------------------------------------------- +# Evaluator (same logic as run_skill_evaluation.py) +# --------------------------------------------------------------------------- + +def _normalise(text): + """Strip angle-bracket placeholders and lower-case for matching.""" + return re.sub(r"\s*<[^>]+>", "", text).strip().lower() + + +def evaluate_response(response, expected_behaviour): + must_include = expected_behaviour.get("must_include", []) + must_not_include = expected_behaviour.get("must_not_include", []) + resp_norm = _normalise(response) + matched = [t for t in must_include if _normalise(t) in resp_norm] + missed = [t for t in must_include if _normalise(t) not in resp_norm] + violated = [t for t in must_not_include if _normalise(t) in resp_norm] + score = round(len(matched) / len(must_include), 4) if must_include else 1.0 + passed = score >= 0.5 and not violated + return { + "matched": matched, + "missed": missed, + "violated": violated, + "alignment_score": score, + "passed": passed, + } + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="Test skill trigger discrimination — does an agent without the skill arrive at the right answer?" + ) + parser.add_argument( + "--pseudo-conversations-dir", default=None, + help="Directory containing pseudo-conversation fixtures", + ) + parser.add_argument( + "--responses-file", default=None, + help="JSON file mapping skill_slug → agent_response (for live responses)", + ) + parser.add_argument( + "--simulate", action="store_true", + help="Use built-in naive baseline responses instead of live agent responses", + ) + parser.add_argument( + "--results-dir", default=None, + ) + parser.add_argument( + "--report", default=None, + ) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + + pseudo_conv_dir = Path(args.pseudo_conversations_dir) if args.pseudo_conversations_dir \ + else evolve_dir / "tests" / "pseudo_conversations" + results_dir = Path(args.results_dir) if args.results_dir \ + else evolve_dir / "tests" / "evaluation" / "results" + report_path = Path(args.report) if args.report \ + else evolve_dir / "tests" / "evaluation" / "baseline_report.json" + + results_dir.mkdir(parents=True, exist_ok=True) + + # Load responses + if args.responses_file: + with open(args.responses_file) as fh: + responses = json.load(fh) + mode = "live" + elif args.simulate: + responses = NAIVE_RESPONSES + mode = "simulated_naive" + else: + # Print instructions for spawning agents and exit + fixture_files = sorted(pseudo_conv_dir.glob("*.json")) + print("To run trigger tests with live agent responses, spawn one sub-agent") + print("per skill using ONLY the user message (no skill injected), then pass") + print("the responses via --responses-file.\n") + print("User messages to send (no system prompt, no skill context):\n") + for fpath in fixture_files: + with open(fpath) as fh: + fx = json.load(fh) + user_msg = next(m["content"] for m in fx["conversation"] if m["role"] == "user") + print(f" [{fx['skill_slug']}]") + print(f" {user_msg}\n") + print("Collect responses into a JSON file:") + print(' { "skill-slug": "agent response text", ... }') + print("\nThen run:") + print(" python3 run_baseline_tests.py --responses-file responses.json") + sys.exit(0) + + # Only skill fixtures — skip edge_ and guideline_ prefixed files since + # those are edge-case tests, not standalone skill trigger tests. + fixture_files = sorted( + f for f in pseudo_conv_dir.glob("*.json") + if not f.name.startswith(("edge_", "guideline_")) + ) + if not fixture_files: + print(f"No fixtures found in {pseudo_conv_dir}", file=sys.stderr) + sys.exit(1) + + print(f"Mode: {mode}") + print(f"Testing {len(fixture_files)} skills\n") + print("TRIGGER TEST RESULTS") + print("=" * 80) + print("(Agent responds WITHOUT skill injected — tests trigger discriminating power)\n") + + results = [] + for fixture_file in fixture_files: + with open(fixture_file) as fh: + fixture = json.load(fh) + + slug = fixture["skill_slug"] + user_msg = next(m["content"] for m in fixture["conversation"] if m["role"] == "user") + response = responses.get(slug, "") + + if not response: + print(f" ⚠ {slug} — no response provided, skipping") + continue + + eval_result = evaluate_response(response, fixture["expected_behaviour"]) + + result = { + "test_id": f"trigger_{slug}", + "skill_slug": slug, + "mode": mode, + "user_message": user_msg, + "passed": eval_result["passed"], + "alignment_score": eval_result["alignment_score"], + "matched": eval_result["matched"], + "missed": eval_result["missed"], + "violated": eval_result["violated"], + "agent_response": response, + "interpretation": ( + "Skill may not be necessary — agent gets it right without the skill" + if eval_result["passed"] else + "Skill IS necessary — agent misses key guidance without it" + ), + "timestamp": datetime.now().isoformat(), + } + results.append(result) + + with open(results_dir / f"trigger_{slug}.json", "w") as fh: + json.dump(result, fh, indent=2) + + status = "✅" if result["passed"] else "❌" + total_inc = len(result["matched"]) + len(result["missed"]) + print(f"{status} {slug:<60} score={result['alignment_score']:.2f} matched={len(result['matched'])}/{total_inc}") + if args.verbose or not result["passed"]: + print(f" → {result['interpretation']}") + if result["missed"]: + print(f" missed={result['missed']}") + + if not results: + print("No results — check that responses are provided for all skills.") + sys.exit(1) + + total = len(results) + n_pass = sum(1 for r in results if r["passed"]) + n_fail = total - n_pass + + report = { + "generated_at": datetime.now().isoformat(), + "mode": mode, + "total": total, + "naive_agent_passes": n_pass, + "skill_necessary_count": n_fail, + "results": results, + } + with open(report_path, "w") as fh: + json.dump(report, fh, indent=2) + + print() + print(f"SUMMARY: {n_pass}/{total} answered correctly without the skill") + print(f" {n_fail}/{total} require the skill (agent missed key guidance)") + print(f"Report: {report_path}") + # Trigger tests do NOT exit 1 on "failure" — a skill being necessary is a PASS + # for the skill library. Exit 1 only if nothing could be evaluated. + sys.exit(0) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_integration_test.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_integration_test.py new file mode 100644 index 00000000..9186c34b --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_integration_test.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +""" +Integration Test Runner for Evolve Lite Skills + +Executes real scenarios and verifies that skills are recalled and improve outcomes. +This is different from static validation - it actually runs Bob with test scenarios. +""" + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Any + +# Walk up from the script location to find the installed plugin lib directory +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + log as _log, +) + + +def log(message): + _log("integration-test", message) + + +def load_scenario(scenario_path: Path) -> Dict[str, Any]: + """Load a test scenario definition.""" + with open(scenario_path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def load_trajectory(trajectory_path: Path) -> Dict[str, Any]: + """Load a trajectory JSON file.""" + with open(trajectory_path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def analyze_trajectory_for_skills(trajectory: Dict[str, Any]) -> Dict[str, Any]: + """Analyze a trajectory to see if skills were recalled and used.""" + messages = trajectory.get("messages", []) + + analysis = { + "skills_recalled": False, + "recall_count": 0, + "skills_mentioned": [], + "recall_messages": [] + } + + for i, msg in enumerate(messages): + if msg.get("role") == "assistant": + content = str(msg.get("content", "")).lower() + + # Check for recall skill usage + if "evolve-lite:recall" in content or "use_skill" in str(msg.get("tool_calls", [])): + analysis["skills_recalled"] = True + analysis["recall_count"] += 1 + analysis["recall_messages"].append({ + "index": i, + "content_preview": content[:200] + }) + + # Check for entity mentions + if ".evolve/entities/" in content: + # Try to extract entity paths + import re + entity_paths = re.findall(r'\.evolve/entities/[^\s\)]+\.md', content) + analysis["skills_mentioned"].extend(entity_paths) + + # Check for "Recall complete" messages + if "recall complete" in content: + analysis["recall_messages"].append({ + "index": i, + "type": "completion", + "content_preview": content[:200] + }) + + analysis["skills_mentioned"] = list(set(analysis["skills_mentioned"])) + return analysis + + +def extract_trajectory_metrics(trajectory: Dict[str, Any]) -> Dict[str, Any]: + """Extract performance metrics from a trajectory.""" + messages = trajectory.get("messages", []) + + metrics = { + "total_messages": len(messages), + "tool_uses": 0, + "errors": 0, + "completion_status": "unknown", + "tools_used": set(), + "duration_estimate": None + } + + # Count tool uses and errors + for msg in messages: + if msg.get("role") == "assistant" and msg.get("tool_calls"): + metrics["tool_uses"] += len(msg.get("tool_calls", [])) + for tool_call in msg.get("tool_calls", []): + if tool_call.get("function"): + metrics["tools_used"].add(tool_call["function"].get("name", "unknown")) + + elif msg.get("role") == "tool": + content = msg.get("content", "").lower() + if any(err in content for err in ["error", "failed", "exception"]): + metrics["errors"] += 1 + + # Check completion status + for msg in reversed(messages): + if msg.get("role") == "assistant": + if "attempt_completion" in str(msg.get("tool_calls", [])): + metrics["completion_status"] = "completed" + break + elif any(word in str(msg.get("content", "")).lower() for word in ["error", "failed", "cannot"]): + metrics["completion_status"] = "failed" + break + + metrics["tools_used"] = list(metrics["tools_used"]) + + # Estimate duration from timestamps if available + if len(messages) >= 2: + try: + first_time = messages[0].get("timestamp") + last_time = messages[-1].get("timestamp") + if first_time and last_time: + from datetime import datetime + start = datetime.fromisoformat(first_time.replace('Z', '+00:00')) + end = datetime.fromisoformat(last_time.replace('Z', '+00:00')) + metrics["duration_estimate"] = (end - start).total_seconds() + except Exception: + pass + + return metrics + + +def validate_scenario_outcome( + scenario: Dict[str, Any], + trajectory: Dict[str, Any], + skill_analysis: Dict[str, Any], + metrics: Dict[str, Any] +) -> Dict[str, Any]: + """Validate that the scenario outcome meets success criteria.""" + + success_criteria = scenario.get("success_criteria", {}) + expected_skills = scenario.get("expected_skills", []) + + validation = { + "passed": True, + "checks": {}, + "failures": [] + } + + # Check if expected skills were recalled + if expected_skills: + skills_found = [] + for expected_skill in expected_skills: + found = any(expected_skill in mentioned for mentioned in skill_analysis["skills_mentioned"]) + skills_found.append(found) + validation["checks"][f"skill_recalled_{expected_skill}"] = found + if not found: + validation["passed"] = False + validation["failures"].append(f"Expected skill not recalled: {expected_skill}") + + validation["checks"]["all_expected_skills_recalled"] = all(skills_found) + + # Check completion status + expected_status = success_criteria.get("completion_status") + if expected_status: + status_match = metrics["completion_status"] == expected_status + validation["checks"]["completion_status"] = status_match + if not status_match: + validation["passed"] = False + validation["failures"].append( + f"Completion status mismatch: expected {expected_status}, got {metrics['completion_status']}" + ) + + # Check max tool uses + max_tool_uses = success_criteria.get("max_tool_uses") + if max_tool_uses: + within_limit = metrics["tool_uses"] <= max_tool_uses + validation["checks"]["tool_uses_within_limit"] = within_limit + if not within_limit: + validation["passed"] = False + validation["failures"].append( + f"Too many tool uses: {metrics['tool_uses']} > {max_tool_uses}" + ) + + # Check max errors + max_errors = success_criteria.get("max_errors") + if max_errors is not None: + within_limit = metrics["errors"] <= max_errors + validation["checks"]["errors_within_limit"] = within_limit + if not within_limit: + validation["passed"] = False + validation["failures"].append( + f"Too many errors: {metrics['errors']} > {max_errors}" + ) + + # Check required files exist + required_files = success_criteria.get("required_files", []) + if required_files: + for required_file in required_files: + file_path = Path(required_file) + exists = file_path.exists() + validation["checks"][f"file_exists_{required_file}"] = exists + if not exists: + validation["passed"] = False + validation["failures"].append(f"Required file not found: {required_file}") + + return validation + + +def run_integration_test( + scenario_path: Path, + trajectory_path: Path, + output_path: Path +) -> Dict[str, Any]: + """Run an integration test for a scenario.""" + + log(f"Running integration test for scenario: {scenario_path}") + + # Load scenario + try: + scenario = load_scenario(scenario_path) + except Exception as e: + return { + "error": f"Failed to load scenario: {e}", + "scenario_path": str(scenario_path) + } + + # Load trajectory + try: + trajectory = load_trajectory(trajectory_path) + except Exception as e: + return { + "error": f"Failed to load trajectory: {e}", + "trajectory_path": str(trajectory_path) + } + + # Analyze trajectory for skill usage + skill_analysis = analyze_trajectory_for_skills(trajectory) + + # Extract metrics + metrics = extract_trajectory_metrics(trajectory) + + # Validate outcome + validation = validate_scenario_outcome(scenario, trajectory, skill_analysis, metrics) + + # Build result + result = { + "test_id": scenario.get("scenario_id", "unknown"), + "scenario_path": str(scenario_path), + "trajectory_path": str(trajectory_path), + "timestamp": datetime.now().isoformat(), + "scenario": scenario, + "skill_analysis": skill_analysis, + "metrics": metrics, + "validation": validation, + "passed": validation["passed"] + } + + # Save result + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(result, f, indent=2) + + log(f"Test result saved to: {output_path}") + + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Run integration tests for Evolve Lite skills" + ) + parser.add_argument( + "--scenario", + required=True, + help="Path to scenario definition JSON file" + ) + parser.add_argument( + "--trajectory", + required=True, + help="Path to trajectory JSON file to analyze" + ) + parser.add_argument( + "--output", + default=None, + help="Output path for test result (default: .evolve/tests/integration/results/.json)" + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print detailed output" + ) + + args = parser.parse_args() + + scenario_path = Path(args.scenario) + trajectory_path = Path(args.trajectory) + + if not scenario_path.exists(): + print(f"Error: Scenario file not found: {scenario_path}", file=sys.stderr) + sys.exit(1) + + if not trajectory_path.exists(): + print(f"Error: Trajectory file not found: {trajectory_path}", file=sys.stderr) + sys.exit(1) + + # Determine output path + if args.output: + output_path = Path(args.output) + else: + evolve_dir = get_evolve_dir() + scenario_id = scenario_path.stem + output_path = evolve_dir / "tests" / "integration" / "results" / f"{scenario_id}_result.json" + + # Run test + print(f"Running integration test...") + print(f" Scenario: {scenario_path.name}") + print(f" Trajectory: {trajectory_path.name}") + print() + + result = run_integration_test(scenario_path, trajectory_path, output_path) + + if "error" in result: + print(f"Error: {result['error']}", file=sys.stderr) + sys.exit(1) + + # Print summary + print("="*60) + print("INTEGRATION TEST RESULT") + print("="*60) + print(f"Test ID: {result['test_id']}") + print(f"Status: {'PASSED' if result['passed'] else 'FAILED'}") + print() + + # Print skill analysis + skill_analysis = result["skill_analysis"] + print("Skill Usage:") + print(f" Skills recalled: {skill_analysis['skills_recalled']}") + print(f" Recall count: {skill_analysis['recall_count']}") + print(f" Skills mentioned: {len(skill_analysis['skills_mentioned'])}") + if skill_analysis['skills_mentioned']: + for skill in skill_analysis['skills_mentioned']: + print(f" - {skill}") + print() + + # Print metrics + metrics = result["metrics"] + print("Metrics:") + print(f" Tool uses: {metrics['tool_uses']}") + print(f" Errors: {metrics['errors']}") + print(f" Completion: {metrics['completion_status']}") + print(f" Tools used: {', '.join(metrics['tools_used'][:5])}") + if len(metrics['tools_used']) > 5: + print(f" ... and {len(metrics['tools_used']) - 5} more") + print() + + # Print validation + validation = result["validation"] + print("Validation:") + print(f" Checks passed: {sum(1 for v in validation['checks'].values() if v)}/{len(validation['checks'])}") + if validation['failures']: + print(" Failures:") + for failure in validation['failures']: + print(f" - {failure}") + print() + + print(f"Result saved to: {output_path}") + + if args.verbose: + print("\nDetailed Result:") + print(json.dumps(result, indent=2)) + + sys.exit(0 if result['passed'] else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_integration_tests_batch.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_integration_tests_batch.py new file mode 100644 index 00000000..a990a8d5 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_integration_tests_batch.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +""" +Batch Integration Test Runner for Evolve Lite Skills + +Runs multiple integration tests and generates a summary report. +""" + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path +from typing import List, Dict, Any +import subprocess + +# Walk up from the script location to find the installed plugin lib directory +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + log as _log, +) + + +def log(message): + _log("integration-batch", message) + + +def find_scenarios(scenarios_dir: Path) -> List[Path]: + """Find all scenario JSON files in the scenarios directory.""" + return list(scenarios_dir.glob("*.json")) + + +def find_matching_trajectory(scenario: Dict[str, Any], trajectories_dir: Path) -> Path: + """Find a trajectory that matches the scenario.""" + # For now, just return the most recent trajectory + # In the future, could match based on user_request or other criteria + trajectories = sorted(trajectories_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True) + if trajectories: + return trajectories[0] + return None + + +def run_single_test(scenario_path: Path, trajectory_path: Path, output_dir: Path) -> Dict[str, Any]: + """Run a single integration test.""" + log(f"Running test: {scenario_path.stem}") + + output_path = output_dir / f"{scenario_path.stem}_result.json" + + # Run the integration test script + cmd = [ + sys.executable, + str(Path(__file__).parent / "run_integration_test.py"), + "--scenario", str(scenario_path), + "--trajectory", str(trajectory_path), + "--output", str(output_path) + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + + # Load the result + if output_path.exists(): + with open(output_path, 'r', encoding='utf-8') as f: + return json.load(f) + else: + return { + "error": "Test output not found", + "scenario_path": str(scenario_path), + "passed": False + } + except subprocess.TimeoutExpired: + return { + "error": "Test timed out", + "scenario_path": str(scenario_path), + "passed": False + } + except Exception as e: + return { + "error": str(e), + "scenario_path": str(scenario_path), + "passed": False + } + + +def generate_batch_report(results: List[Dict[str, Any]], output_path: Path): + """Generate a summary report for all tests.""" + + total_tests = len(results) + passed_tests = sum(1 for r in results if r.get("passed", False)) + failed_tests = total_tests - passed_tests + + report = { + "generated_at": datetime.now().isoformat(), + "summary": { + "total_tests": total_tests, + "passed": passed_tests, + "failed": failed_tests, + "pass_rate": passed_tests / total_tests if total_tests > 0 else 0 + }, + "results": results, + "by_scenario": {} + } + + # Group by scenario + for result in results: + scenario_id = result.get("test_id", "unknown") + report["by_scenario"][scenario_id] = { + "passed": result.get("passed", False), + "skills_recalled": result.get("skill_analysis", {}).get("skills_recalled", False), + "tool_uses": result.get("metrics", {}).get("tool_uses", 0), + "errors": result.get("metrics", {}).get("errors", 0), + "completion_status": result.get("metrics", {}).get("completion_status", "unknown") + } + + # Save report + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2) + + return report + + +def main(): + parser = argparse.ArgumentParser( + description="Run multiple integration tests in batch" + ) + parser.add_argument( + "--scenarios-dir", + default=None, + help="Directory containing scenario JSON files (default: .evolve/tests/integration/scenarios/)" + ) + parser.add_argument( + "--trajectories-dir", + default=None, + help="Directory containing trajectory JSON files (default: .evolve/trajectories/)" + ) + parser.add_argument( + "--output", + default=None, + help="Output path for batch report (default: .evolve/tests/integration/batch_report.json)" + ) + parser.add_argument( + "--scenario", + action="append", + help="Specific scenario file(s) to test (can be used multiple times)" + ) + parser.add_argument( + "--trajectory", + help="Specific trajectory to use for all tests" + ) + + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + + # Determine directories + if args.scenarios_dir: + scenarios_dir = Path(args.scenarios_dir) + else: + scenarios_dir = evolve_dir / "tests" / "integration" / "scenarios" + + if args.trajectories_dir: + trajectories_dir = Path(args.trajectories_dir) + else: + trajectories_dir = evolve_dir / "trajectories" + + if args.output: + output_path = Path(args.output) + else: + output_path = evolve_dir / "tests" / "integration" / "batch_report.json" + + results_dir = evolve_dir / "tests" / "integration" / "results" + results_dir.mkdir(parents=True, exist_ok=True) + + # Find scenarios to test + if args.scenario: + scenario_paths = [Path(s) for s in args.scenario] + else: + scenario_paths = find_scenarios(scenarios_dir) + + if not scenario_paths: + print("No scenarios found to test", file=sys.stderr) + sys.exit(1) + + print(f"Running {len(scenario_paths)} integration test(s)...") + print() + + # Run tests + results = [] + for i, scenario_path in enumerate(scenario_paths, 1): + print(f"[{i}/{len(scenario_paths)}] Testing: {scenario_path.stem}") + + # Determine trajectory to use + if args.trajectory: + trajectory_path = Path(args.trajectory) + else: + # Load scenario to potentially match trajectory + with open(scenario_path, 'r', encoding='utf-8') as f: + scenario = json.load(f) + trajectory_path = find_matching_trajectory(scenario, trajectories_dir) + + if not trajectory_path or not trajectory_path.exists(): + print(f" ⚠️ No trajectory found, skipping") + results.append({ + "error": "No trajectory found", + "scenario_path": str(scenario_path), + "passed": False + }) + continue + + print(f" Using trajectory: {trajectory_path.name}") + + # Run test + result = run_single_test(scenario_path, trajectory_path, results_dir) + results.append(result) + + # Print result + if result.get("passed"): + print(f" ✅ PASSED") + else: + print(f" ❌ FAILED") + if "error" in result: + print(f" Error: {result['error']}") + print() + + # Generate batch report + print("Generating batch report...") + report = generate_batch_report(results, output_path) + + # Print summary + print("="*60) + print("BATCH TEST SUMMARY") + print("="*60) + print(f"Total tests: {report['summary']['total_tests']}") + print(f"Passed: {report['summary']['passed']}") + print(f"Failed: {report['summary']['failed']}") + print(f"Pass rate: {report['summary']['pass_rate']:.1%}") + print() + + # Print per-scenario summary + print("Results by scenario:") + for scenario_id, scenario_result in report["by_scenario"].items(): + status = "✅ PASS" if scenario_result["passed"] else "❌ FAIL" + print(f" {status} {scenario_id}") + if scenario_result["skills_recalled"]: + print(f" Skills recalled: Yes") + else: + print(f" Skills recalled: No (this is why it failed)") + print() + + print(f"Report saved to: {output_path}") + + # Exit with appropriate code + sys.exit(0 if report['summary']['failed'] == 0 else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_recall_tests.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_recall_tests.py new file mode 100644 index 00000000..80cd2cb6 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_recall_tests.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +""" +Recall Test Runner + +Tests whether the recall layer surfaces the right skill for each scenario. + +For each pseudo-conversation fixture, this script: +1. Builds the entity manifest (same format as retrieve_entities.py produces) +2. Presents the manifest + the user's question to a simulated recall agent +3. Checks whether the agent identifies the correct skill slug + +The "simulated recall agent" uses keyword overlap between the user message and +each trigger — the same heuristic a real agent uses when scanning the manifest. + +Reports Recall@K for K = 1, 3, and 5: + Recall@K = fraction of fixtures where the expected skill is in the top-K results. + +The overall pass/fail threshold remains rank <= 3 (Recall@3), consistent with +the existing behaviour, but all three K values are shown in the summary. + +Usage: + python3 run_recall_tests.py + python3 run_recall_tests.py --verbose + python3 run_recall_tests.py --pseudo-conversations-dir + python3 run_recall_tests.py --top-k 1 # pass threshold: rank-1 only + python3 run_recall_tests.py --top-k 5 # pass threshold: top-5 +""" + +import argparse +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +# --------------------------------------------------------------------------- +# Bootstrap +# --------------------------------------------------------------------------- +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib: + sys.path.insert(0, str(_lib)) + try: + from entity_io import get_evolve_dir, load_manifest, find_recall_entity_dirs, dedupe_manifest_entries + except ImportError: + def get_evolve_dir(): return Path(".evolve") + def load_manifest(d): return [] + def find_recall_entity_dirs(): return [] + def dedupe_manifest_entries(e): return e +else: + def get_evolve_dir(): return Path(".evolve") + def load_manifest(d): return [] + def find_recall_entity_dirs(): return [] + def dedupe_manifest_entries(e): return e + + +# --------------------------------------------------------------------------- +# Trigger relevance scorer +# +# Simulates how an agent scans the manifest: keyword overlap between the +# user message and the trigger text, weighted by term length (longer terms +# are more specific and score higher). +# --------------------------------------------------------------------------- + +_STOP_WORDS = { + "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "as", "is", "was", "are", "were", "be", + "been", "being", "have", "has", "had", "do", "does", "did", "will", + "would", "should", "could", "may", "might", "must", "can", "this", + "that", "these", "those", "i", "you", "he", "she", "it", "we", "they", + "when", "after", "before", "while", "if", "how", "what", "my", "your", + "just", "need", "want", "make", "sure", "some", "also", "about", +} + + +def tokenise(text): + """Lowercase words, drop stop words and very short tokens.""" + words = re.findall(r"[a-z0-9]+", text.lower()) + return [w for w in words if w not in _STOP_WORDS and len(w) > 2] + + +def score_trigger(trigger, user_message): + """ + Return a relevance score for (trigger, user_message). + + Score = sum of len(term) for each trigger term found in the user message. + Longer matching terms (e.g. "orchestrate", "token", "expiration") score + higher than short ones, which filters out accidental single-word matches. + """ + trigger_tokens = set(tokenise(trigger)) + user_tokens = set(tokenise(user_message)) + matched = trigger_tokens & user_tokens + return sum(len(t) for t in matched), matched + + +def rank_manifest(manifest, user_message): + """ + Return manifest entries sorted by descending relevance to user_message. + Each entry gains a 'score' and 'matched_terms' field. + """ + scored = [] + for entry in manifest: + score, matched = score_trigger(entry["trigger"], user_message) + scored.append({**entry, "score": score, "matched_terms": list(matched)}) + return sorted(scored, key=lambda e: e["score"], reverse=True) + + +# --------------------------------------------------------------------------- +# Test runner +# --------------------------------------------------------------------------- + +def run_recall_test(fixture, manifest, pass_k=3): + """ + Run a recall test for one fixture. + + Returns a result dict with: + passed - bool: expected skill is in top-{pass_k} results + rank - int: 1-based position of expected skill in ranked list + in_top1 - bool + in_top3 - bool + in_top5 - bool + top_ranked_slug - slug of the #1 ranked skill + score_expected - relevance score of the expected skill + score_top - relevance score of the #1 ranked skill + matched_terms - terms that fired for the expected skill + """ + skill_slug = fixture["skill_slug"] + user_msg = next( + m["content"] for m in fixture["conversation"] if m["role"] == "user" + ) + + ranked = rank_manifest(manifest, user_msg) + + # Find rank of the expected skill + rank = None + score_expected = 0 + matched_terms = [] + for i, entry in enumerate(ranked): + slug = Path(entry["path"]).stem + if slug == skill_slug: + rank = i + 1 + score_expected = entry["score"] + matched_terms = entry["matched_terms"] + break + + top = ranked[0] if ranked else {} + top_slug = Path(top.get("path", "")).stem + score_top = top.get("score", 0) + + in_top1 = rank == 1 + in_top3 = rank is not None and rank <= 3 and score_expected > 0 + in_top5 = rank is not None and rank <= 5 and score_expected > 0 + + # Pass threshold is configurable via pass_k + if pass_k == 1: + passed = in_top1 + elif pass_k == 5: + passed = in_top5 + else: # default: k=3 + passed = in_top3 + + return { + "test_id": f"recall_{skill_slug}", + "skill_slug": skill_slug, + "user_message": user_msg, + "passed": passed, + "rank": rank, + "in_top1": in_top1, + "in_top3": in_top3, + "in_top5": in_top5, + "top_ranked_slug": top_slug, + "score_expected": score_expected, + "score_top": score_top, + "matched_terms": matched_terms, + "top5": [Path(e["path"]).stem for e in ranked[:5]], + "timestamp": datetime.now().isoformat(), + } + + +def print_result(result, verbose): + status = "✅" if result["passed"] else "❌" + rank_str = f"rank={result['rank']}" if result["rank"] else "rank=not_found" + k1 = "✓" if result["in_top1"] else "✗" + k3 = "✓" if result["in_top3"] else "✗" + k5 = "✓" if result["in_top5"] else "✗" + line = ( + f"{status} {result['skill_slug']:<56}" + f" {rank_str:<12}" + f" @1={k1} @3={k3} @5={k5}" + f" score={result['score_expected']}" + ) + print(line) + if not result["passed"] or verbose: + print(f" user_msg : {result['user_message'][:80]}") + print(f" matched_terms: {result['matched_terms']}") + print(f" top5 : {result['top5']}") + if not result["passed"] and result["top_ranked_slug"] != result["skill_slug"]: + print(f" ⚠ top-ranked was: {result['top_ranked_slug']} (score={result['score_top']})") + + +def main(): + parser = argparse.ArgumentParser( + description="Test that each skill is correctly recalled for its trigger scenario" + ) + parser.add_argument("--pseudo-conversations-dir", default=None) + parser.add_argument("--results-dir", default=None) + parser.add_argument("--report", default=None) + parser.add_argument("--verbose", action="store_true") + parser.add_argument( + "--top-k", type=int, choices=[1, 3, 5], default=3, + help="Pass threshold: skill must appear in top-K results (default: 3). " + "Recall@1, @3, and @5 are always reported regardless of this setting.", + ) + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + + pseudo_conv_dir = Path(args.pseudo_conversations_dir) if args.pseudo_conversations_dir \ + else evolve_dir / "tests" / "pseudo_conversations" + + results_dir = Path(args.results_dir) if args.results_dir \ + else evolve_dir / "tests" / "evaluation" / "results" + + report_path = Path(args.report) if args.report \ + else evolve_dir / "tests" / "evaluation" / "recall_report.json" + + results_dir.mkdir(parents=True, exist_ok=True) + + # Build manifest from live entities + raw_entries = [] + for root_dir in find_recall_entity_dirs(): + raw_entries.extend(load_manifest(root_dir)) + manifest = dedupe_manifest_entries(raw_entries) + + if not manifest: + print("Error: no entities found — recall manifest is empty.", file=sys.stderr) + sys.exit(1) + + fixture_files = sorted(pseudo_conv_dir.glob("*.json")) + if not fixture_files: + print(f"No fixture files found in {pseudo_conv_dir}", file=sys.stderr) + sys.exit(1) + + print(f"Manifest: {len(manifest)} entities") + print(f"Fixtures: {len(fixture_files)}") + print() + print("RECALL TEST RESULTS") + print("=" * 80) + + results = [] + for fixture_file in fixture_files: + with open(fixture_file) as fh: + fixture = json.load(fh) + + result = run_recall_test(fixture, manifest, pass_k=args.top_k) + print_result(result, args.verbose) + results.append(result) + + with open(results_dir / f"recall_{fixture['skill_slug']}.json", "w") as fh: + json.dump(result, fh, indent=2) + + total = len(results) + passed = sum(1 for r in results if r["passed"]) + recall_1 = sum(1 for r in results if r["in_top1"]) + recall_3 = sum(1 for r in results if r["in_top3"]) + recall_5 = sum(1 for r in results if r["in_top5"]) + + report = { + "generated_at": datetime.now().isoformat(), + "pass_threshold_k": args.top_k, + "total": total, + "passed": passed, + "failed": total - passed, + "recall_at_1": round(recall_1 / total, 4) if total else 0, + "recall_at_3": round(recall_3 / total, 4) if total else 0, + "recall_at_5": round(recall_5 / total, 4) if total else 0, + "recall_at_1_count": recall_1, + "recall_at_3_count": recall_3, + "recall_at_5_count": recall_5, + "results": results, + } + with open(report_path, "w") as fh: + json.dump(report, fh, indent=2) + + print() + print("=" * 72) + print(f" Recall@1 : {recall_1:>3}/{total} ({100 * recall_1 / total:.1f}%)" if total else " Recall@1 : —") + print(f" Recall@3 : {recall_3:>3}/{total} ({100 * recall_3 / total:.1f}%)" if total else " Recall@3 : —") + print(f" Recall@5 : {recall_5:>3}/{total} ({100 * recall_5 / total:.1f}%)" if total else " Recall@5 : —") + print(f" Pass threshold: top-{args.top_k} → {passed}/{total} passed") + print(f"Report: {report_path}") + + sys.exit(0 if passed == total else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_evaluation.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_evaluation.py new file mode 100644 index 00000000..db71ed79 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_evaluation.py @@ -0,0 +1,460 @@ +#!/usr/bin/env python3 +""" +Skill Evaluation Runner + +Loads each pseudo-conversation fixture from .evolve/tests/pseudo_conversations/, +simulates what a sub-agent following the recalled skill should produce, +evaluates the response against expected_behaviour, and writes per-skill +results + a summary report. + +Since this is a Python-only runner (no live LLM call), the "sub-agent response" +is simulated by checking whether the skill content itself contains the +must_include terms — which is the minimal faithful check: a well-formed skill +that contains its own prescribed commands will pass; a skill that has gaps will +surface them. + +For a live LLM-backed run, replace `simulate_agent_response()` with a real +API call and pass the conversation list to it. + +Usage: + python run_skill_evaluation.py + python run_skill_evaluation.py --verbose + python run_skill_evaluation.py --pseudo-conversations-dir +""" + +import argparse +import json +import re +import sys +import time +from datetime import datetime +from pathlib import Path + +# --------------------------------------------------------------------------- +# Bootstrap: locate entity_io for the log helper (optional, graceful fallback) +# --------------------------------------------------------------------------- +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib: + sys.path.insert(0, str(_lib)) + try: + from entity_io import get_evolve_dir, log as _elog + def log(msg): + _elog("skill-eval", msg) + except ImportError: + def log(msg): + pass + def get_evolve_dir(): + return Path(".evolve") +else: + def log(msg): + pass + def get_evolve_dir(): + return Path(".evolve") + + +# --------------------------------------------------------------------------- +# Token estimation +# --------------------------------------------------------------------------- + +def _approx_tokens(text): + """Approximate token count using the GPT-3/4 rule-of-thumb: ~0.75 words per token. + + For live LLM calls replace this with the actual usage object returned by the API: + usage = response.usage + return usage.prompt_tokens, usage.completion_tokens + """ + return max(1, round(len(text.split()) / 0.75)) + + +def estimate_tokens(fixture, response): + """Return a detailed token breakdown measured from the raw input messages. + + The system message already contains the injected skill inside a + block. We split on the *last* occurrence of that tag + (the preamble itself mentions it by name, so the first hit is a reference, + not the actual injection) to get three named buckets: + + preamble_tokens — fixed system instructions before the skill injection + skill_tokens — the recalled skill text as it appears in context + user_tokens — the user's question + + This makes the skill's context cost directly visible rather than hidden + inside an opaque prompt_tokens total. + """ + system_content = next( + (m["content"] for m in fixture.get("conversation", []) if m["role"] == "system"), + "", + ) + user_content = next( + (m["content"] for m in fixture.get("conversation", []) if m["role"] == "user"), + "", + ) + + # Split at the *last* tag to isolate the injected skill + tag = "" + last_idx = system_content.rfind(tag) + if last_idx != -1: + preamble_text = system_content[:last_idx] + rest = system_content[last_idx + len(tag):] + skill_text, _, _ = rest.partition("") + else: + preamble_text = system_content + skill_text = "" + + preamble_tokens = _approx_tokens(preamble_text) if preamble_text else 0 + skill_tokens = _approx_tokens(skill_text) if skill_text else 0 + user_tokens = _approx_tokens(user_content) if user_content else 0 + prompt_tokens = preamble_tokens + skill_tokens + user_tokens + completion_tokens = _approx_tokens(response) + + return { + "preamble_tokens": preamble_tokens, + "skill_tokens": skill_tokens, + "user_tokens": user_tokens, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + "token_source": "estimated", + } + + +# --------------------------------------------------------------------------- +# Simulated sub-agent response +# --------------------------------------------------------------------------- + +def simulate_agent_response(fixture): + """ + Simulate a sub-agent that has been given the skill in context and responds + to the user's question. + + The simulation uses the skill content verbatim as the "agent response" + because: if the skill content contains all the must_include terms, a + well-instructed agent following it would too. This makes the test a + self-consistency check: the skill must contain what it claims to prescribe. + + Replace this function with a real LLM call for a live evaluation: + + messages = fixture["conversation"] + t0 = time.perf_counter() + response = openai_client.chat.completions.create(model="gpt-4o", messages=messages) + latency_ms = round((time.perf_counter() - t0) * 1000, 1) + text = response.choices[0].message.content + prompt_tokens = response.usage.prompt_tokens + completion_tokens = response.usage.completion_tokens + return text, latency_ms, prompt_tokens, completion_tokens + """ + skill_content = fixture.get("skill_content", "") + return ( + "Based on the recalled skill, here is the guidance:\n\n" + + skill_content + ) + + +# --------------------------------------------------------------------------- +# Alignment evaluator +# --------------------------------------------------------------------------- + +def _normalise(text): + """Strip angle-bracket placeholders and lower-case for matching.""" + return re.sub(r"\s*<[^>]+>", "", text).strip().lower() + + +def evaluate_response(response, expected_behaviour): + """ + Check the agent response against must_include and must_not_include lists. + + Angle-bracket placeholders (e.g. ) are stripped from both the + term and the response before matching so that commands with variable + argument slots still match correctly. + + Returns a dict with: + matched - list of terms found + missed - list of terms not found + violated - list of must_not_include terms found + alignment_score - float [0, 1] + constraint_violated - bool + passed - bool + """ + must_include = expected_behaviour.get("must_include", []) + must_not_include = expected_behaviour.get("must_not_include", []) + resp_norm = _normalise(response) + + matched = [t for t in must_include if _normalise(t) in resp_norm] + missed = [t for t in must_include if _normalise(t) not in resp_norm] + violated = [t for t in must_not_include if _normalise(t) in resp_norm] + + if must_include: + alignment_score = len(matched) / len(must_include) + else: + alignment_score = 1.0 + + constraint_violated = len(violated) > 0 + passed = alignment_score >= 0.5 and not constraint_violated + + return { + "matched": matched, + "missed": missed, + "violated": violated, + "alignment_score": round(alignment_score, 4), + "constraint_violated": constraint_violated, + "passed": passed, + } + + +# --------------------------------------------------------------------------- +# Main runner +# --------------------------------------------------------------------------- + +def run_evaluation(pseudo_conv_dir, results_dir, verbose): + """Load fixtures, evaluate each, return list of result dicts.""" + pseudo_conv_dir = Path(pseudo_conv_dir) + results_dir = Path(results_dir) + results_dir.mkdir(parents=True, exist_ok=True) + + fixture_files = sorted(pseudo_conv_dir.glob("*.json")) + if not fixture_files: + print(f"No fixture files found in {pseudo_conv_dir}", file=sys.stderr) + sys.exit(1) + + results = [] + + for fixture_file in fixture_files: + log(f"Evaluating: {fixture_file.name}") + with open(fixture_file, "r", encoding="utf-8") as fh: + fixture = json.load(fh) + + test_id = fixture.get("test_id", fixture_file.stem) + + # Get agent response and measure latency + t0 = time.perf_counter() + agent_response = simulate_agent_response(fixture) + latency_ms = round((time.perf_counter() - t0) * 1000, 3) + + # Estimate token usage (broken down by input section) + tok = estimate_tokens(fixture, agent_response) + + # Evaluate + eval_result = evaluate_response(agent_response, fixture["expected_behaviour"]) + + result = { + "test_id": test_id, + "passed": eval_result["passed"], + "alignment_score": eval_result["alignment_score"], + "matched": eval_result["matched"], + "missed": eval_result["missed"], + "violated": eval_result["violated"], + "constraint_violated": eval_result["constraint_violated"], + "agent_response": agent_response, + "metrics": { + "latency_ms": latency_ms, + "preamble_tokens": tok["preamble_tokens"], + "skill_tokens": tok["skill_tokens"], + "user_tokens": tok["user_tokens"], + "prompt_tokens": tok["prompt_tokens"], + "completion_tokens": tok["completion_tokens"], + "total_tokens": tok["total_tokens"], + "token_source": tok["token_source"], + }, + "timestamp": datetime.now().isoformat(), + } + + # Write per-skill result + result_file = results_dir / f"{test_id}.json" + with open(result_file, "w", encoding="utf-8") as fh: + json.dump(result, fh, indent=2) + + results.append(result) + + if verbose: + status = "✅" if result["passed"] else "❌" + m = result["metrics"] + print( + f" {status} {test_id:<56}" + f" score={result['alignment_score']:.2f}" + f" matched={len(result['matched'])}/{len(result['matched']) + len(result['missed'])}" + f" tokens={m['total_tokens']} latency={m['latency_ms']}ms" + ) + if result["missed"]: + print(f" missed={result['missed']}") + if result["violated"]: + print(f" violated={result['violated']}") + + return results + + +def _percentile(values, pct): + """Return the p-th percentile of a sorted list (linear interpolation).""" + if not values: + return 0.0 + sv = sorted(values) + idx = (pct / 100) * (len(sv) - 1) + lo, hi = int(idx), min(int(idx) + 1, len(sv) - 1) + return round(sv[lo] + (idx - lo) * (sv[hi] - sv[lo]), 3) + + +def build_report(results): + total = len(results) + passed = sum(1 for r in results if r["passed"]) + failed = total - passed + pass_rate = round(passed / total, 4) if total else 0.0 + + latencies = [r["metrics"]["latency_ms"] for r in results] + preamble_tokens = [r["metrics"]["preamble_tokens"] for r in results] + skill_tokens = [r["metrics"]["skill_tokens"] for r in results] + user_tokens = [r["metrics"]["user_tokens"] for r in results] + prompt_tokens = [r["metrics"]["prompt_tokens"] for r in results] + completion_tokens = [r["metrics"]["completion_tokens"] for r in results] + total_tokens = [r["metrics"]["total_tokens"] for r in results] + + token_source = results[0]["metrics"]["token_source"] if results else "estimated" + + def _avg(lst): return round(sum(lst) / len(lst), 1) if lst else 0 + + return { + "generated_at": datetime.now().isoformat(), + "total": total, + "passed": passed, + "failed": failed, + "pass_rate": pass_rate, + "performance": { + "token_source": token_source, + "latency_ms": { + "min": round(min(latencies), 3) if latencies else 0, + "max": round(max(latencies), 3) if latencies else 0, + "avg": round(sum(latencies) / len(latencies), 3) if latencies else 0, + "p50": _percentile(latencies, 50), + "p95": _percentile(latencies, 95), + }, + "tokens": { + "total_prompt": sum(prompt_tokens), + "total_completion": sum(completion_tokens), + "total_all": sum(total_tokens), + "avg_prompt": _avg(prompt_tokens), + "avg_completion": _avg(completion_tokens), + "avg_total": _avg(total_tokens), + # skill-specific breakdown + "avg_preamble": _avg(preamble_tokens), + "avg_skill": _avg(skill_tokens), + "avg_user": _avg(user_tokens), + "total_skill": sum(skill_tokens), + "skill_pct_of_prompt": round( + 100 * sum(skill_tokens) / sum(prompt_tokens), 1 + ) if sum(prompt_tokens) else 0, + }, + }, + "results": results, + } + + +def print_table(results, report): + print() + print("SKILL EVALUATION RESULTS") + print("=" * 80) + for r in results: + status = "✅" if r["passed"] else "❌" + total_inc = len(r["matched"]) + len(r["missed"]) + m = r["metrics"] + line = ( + f"{status} {r['test_id']:<52}" + f" score={r['alignment_score']:.2f}" + f" matched={len(r['matched'])}/{total_inc}" + f" violated={len(r['violated'])}" + f" tok={m['total_tokens']}" + f" {m['latency_ms']}ms" + ) + print(line) + if r["missed"]: + print(f" missed={r['missed']}") + if r["violated"]: + print(f" violated={r['violated']}") + + total = len(results) + passed = sum(1 for r in results if r["passed"]) + pct = (passed / total * 100) if total else 0 + perf = report["performance"] + lat = perf["latency_ms"] + tok = perf["tokens"] + + print() + print(f"SUMMARY: {passed}/{total} passed ({pct:.1f}%)") + print() + print("PERFORMANCE") + print("-" * 50) + src = f" ({perf['token_source']})" + print(f" Latency (ms) avg={lat['avg']} p50={lat['p50']} p95={lat['p95']} min={lat['min']} max={lat['max']}") + print(f" Tokens{src}") + print(f" avg prompt breakdown:") + print(f" preamble : {tok['avg_preamble']} tokens") + print(f" skill : {tok['avg_skill']} tokens ({tok['skill_pct_of_prompt']}% of prompt)") + print(f" user : {tok['avg_user']} tokens") + print(f" avg completion : {tok['avg_completion']} tokens") + print(f" avg total : {tok['avg_total']} tokens (prompt + completion)") + print(f" total (all) : {tok['total_all']} tokens (skill: {tok['total_skill']})") + print() + + +def main(): + parser = argparse.ArgumentParser( + description="Run atomic skill pseudo-conversation evaluation" + ) + parser.add_argument( + "--pseudo-conversations-dir", + default=None, + help="Directory containing pseudo-conversation JSON fixtures", + ) + parser.add_argument( + "--results-dir", + default=None, + help="Directory to write per-skill result JSON files", + ) + parser.add_argument( + "--report", + default=None, + help="Path for the summary report JSON (default: /../report.json)", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-skill detail during evaluation", + ) + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + + pseudo_conv_dir = Path(args.pseudo_conversations_dir) if args.pseudo_conversations_dir \ + else evolve_dir / "tests" / "pseudo_conversations" + + results_dir = Path(args.results_dir) if args.results_dir \ + else evolve_dir / "tests" / "evaluation" / "results" + + report_path = Path(args.report) if args.report \ + else evolve_dir / "tests" / "evaluation" / "report.json" + + print(f"Loading fixtures from: {pseudo_conv_dir}") + print(f"Writing results to: {results_dir}") + print() + + results = run_evaluation(pseudo_conv_dir, results_dir, verbose=args.verbose) + + report = build_report(results) + + report_path.parent.mkdir(parents=True, exist_ok=True) + with open(report_path, "w", encoding="utf-8") as fh: + json.dump(report, fh, indent=2) + + print_table(results, report) + print(f"Report written: {report_path}") + + sys.exit(0 if report["failed"] == 0 else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_functional_tests.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_functional_tests.py new file mode 100644 index 00000000..3527adad --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_functional_tests.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +""" +Functional Test Framework for Evolve Lite Skills + +Tests if skills actually work by: +1. Parsing skill instructions into executable steps +2. Simulating execution in a mock environment +3. Validating expected outcomes occur + +This provides functional testing without actually modifying the system. +""" + +import argparse +import json +import sys +import re +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Any, Optional +from dataclasses import dataclass, field + +# Walk up from the script location to find the installed plugin lib directory +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + markdown_to_entity, + log as _log, +) + + +def log(message): + _log("skill-functional-test", message) + + +@dataclass +class MockEnvironment: + """Simulates a test environment for skill execution.""" + + files: Dict[str, str] = field(default_factory=dict) # filename -> content + commands_executed: List[Dict[str, Any]] = field(default_factory=list) + environment_vars: Dict[str, str] = field(default_factory=dict) + current_directory: str = "/test" + errors: List[str] = field(default_factory=list) + + def file_exists(self, path: str) -> bool: + """Check if a file exists in the mock environment.""" + return path in self.files + + def create_file(self, path: str, content: str): + """Create a file in the mock environment.""" + self.files[path] = content + log(f"Mock: Created file {path}") + + def read_file(self, path: str) -> Optional[str]: + """Read a file from the mock environment.""" + return self.files.get(path) + + def execute_command(self, command: str) -> Dict[str, Any]: + """Simulate command execution.""" + result = { + "command": command, + "success": True, + "output": "", + "error": "" + } + + # Simulate different command types + if "orchestrate agents import" in command: + # Simulate agent import + if "--file" in command: + # Extract filename + match = re.search(r'--file\s+(\S+)', command) + if match: + filename = match.group(1) + if self.file_exists(filename): + result["output"] = f"Agent imported successfully from {filename}" + result["success"] = True + else: + result["error"] = f"File not found: {filename}" + result["success"] = False + self.errors.append(f"File not found: {filename}") + else: + result["error"] = "Missing --file parameter" + result["success"] = False + self.errors.append("Missing --file parameter") + + elif "orchestrate env activate" in command: + # Simulate environment activation + result["output"] = "Environment activated" + result["success"] = True + + elif "source .venv/bin/activate" in command or "source" in command: + # Simulate virtual environment activation + result["output"] = "Virtual environment activated" + result["success"] = True + self.environment_vars["VIRTUAL_ENV"] = "/test/.venv" + + elif command.startswith("python") or command.startswith("python3"): + # Simulate Python execution + result["output"] = "Python script executed" + result["success"] = True + + elif command.startswith("pip install"): + # Simulate pip install + result["output"] = "Packages installed" + result["success"] = True + + else: + # Generic command + result["output"] = f"Command executed: {command}" + result["success"] = True + + self.commands_executed.append(result) + log(f"Mock: Executed command: {command} -> {'SUCCESS' if result['success'] else 'FAILED'}") + + return result + + def get_state(self) -> Dict[str, Any]: + """Get the current state of the environment.""" + return { + "files": list(self.files.keys()), + "commands_executed": len(self.commands_executed), + "errors": self.errors, + "environment_vars": self.environment_vars + } + + +@dataclass +class SkillStep: + """Represents a single step extracted from a skill.""" + + step_number: int + description: str + command: Optional[str] = None + expected_outcome: Optional[str] = None + step_type: str = "action" # action, command, check + + +def extract_steps_from_skill(skill: Dict[str, Any]) -> List[SkillStep]: + """ + Extract executable steps from a skill's content. + + Looks for: + - Numbered steps (1., 2., 3.) + - Commands in backticks or code blocks + - Action descriptions + """ + content = skill.get("content", "") + steps = [] + + # Pattern 1: Numbered steps with commands + # Example: "1) Create a YAML file..." or "1. Run `command`" + numbered_pattern = r'(\d+)[.)]\s+([^\n]+)' + matches = re.finditer(numbered_pattern, content) + + for match in matches: + step_num = int(match.group(1)) + description = match.group(2).strip() + + # Extract command if present in backticks + command = None + command_match = re.search(r'`([^`]+)`', description) + if command_match: + command = command_match.group(1) + + steps.append(SkillStep( + step_number=step_num, + description=description, + command=command, + step_type="command" if command else "action" + )) + + # Pattern 2: Commands in code blocks + code_block_pattern = r'```(?:bash|sh|shell)?\n(.*?)\n```' + code_matches = re.finditer(code_block_pattern, content, re.DOTALL) + + for match in code_matches: + commands = match.group(1).strip().split('\n') + for i, cmd in enumerate(commands): + cmd = cmd.strip() + if cmd and not cmd.startswith('#'): + steps.append(SkillStep( + step_number=len(steps) + 1, + description=f"Execute: {cmd}", + command=cmd, + step_type="command" + )) + + # If no steps found, try to extract from description + if not steps: + # Look for action verbs + action_pattern = r'(Create|Run|Execute|Install|Activate|Import|Deploy|Configure)\s+([^\n.]+)' + action_matches = re.finditer(action_pattern, content, re.IGNORECASE) + + for i, match in enumerate(action_matches, 1): + action = match.group(1) + target = match.group(2).strip() + + steps.append(SkillStep( + step_number=i, + description=f"{action} {target}", + step_type="action" + )) + + return steps + + +def execute_skill_in_mock_env( + skill: Dict[str, Any], + scenario: Dict[str, Any], + env: MockEnvironment +) -> Dict[str, Any]: + """ + Execute a skill's steps in a mock environment. + + Returns execution results and validation. + """ + steps = extract_steps_from_skill(skill) + + execution_log = [] + steps_executed = 0 + steps_successful = 0 + + for step in steps: + step_result = { + "step_number": step.step_number, + "description": step.description, + "type": step.step_type, + "executed": False, + "success": False, + "output": None + } + + if step.command: + # Execute the command in mock environment + result = env.execute_command(step.command) + step_result["executed"] = True + step_result["success"] = result["success"] + step_result["output"] = result["output"] if result["success"] else result["error"] + + steps_executed += 1 + if result["success"]: + steps_successful += 1 + else: + # For non-command steps, just mark as executed + step_result["executed"] = True + step_result["success"] = True + steps_executed += 1 + steps_successful += 1 + + execution_log.append(step_result) + + return { + "steps_found": len(steps), + "steps_executed": steps_executed, + "steps_successful": steps_successful, + "execution_log": execution_log, + "environment_state": env.get_state() + } + + +def validate_skill_outcome( + skill: Dict[str, Any], + scenario: Dict[str, Any], + execution_result: Dict[str, Any], + env: MockEnvironment +) -> Dict[str, Any]: + """ + Validate that executing the skill produced the expected outcome. + """ + validation = { + "all_steps_executed": execution_result["steps_executed"] == execution_result["steps_found"], + "all_steps_successful": execution_result["steps_successful"] == execution_result["steps_executed"], + "no_errors": len(env.errors) == 0, + "expected_files_created": False, + "expected_commands_run": False + } + + # Check if expected files were created (from scenario) + required_files = scenario.get("success_criteria", {}).get("required_files", []) + if required_files: + files_created = all(env.file_exists(f) for f in required_files) + validation["expected_files_created"] = files_created + else: + validation["expected_files_created"] = True # No files required + + # Check if expected commands were run + expected_commands = scenario.get("expected_commands", []) + if expected_commands: + commands_run = [] + for cmd_result in env.commands_executed: + commands_run.append(cmd_result["command"]) + + # Check if all expected commands were executed + validation["expected_commands_run"] = all( + any(exp in cmd for cmd in commands_run) + for exp in expected_commands + ) + else: + validation["expected_commands_run"] = True # No specific commands required + + # Overall pass/fail + validation["passed"] = all([ + validation["all_steps_executed"], + validation["all_steps_successful"], + validation["no_errors"], + validation["expected_files_created"], + validation["expected_commands_run"] + ]) + + return validation + + +def run_functional_test( + skill: Dict[str, Any], + scenario: Dict[str, Any] +) -> Dict[str, Any]: + """ + Run a functional test for a skill. + + 1. Create mock environment + 2. Extract steps from skill + 3. Execute steps in mock environment + 4. Validate outcomes + """ + # Create mock environment + env = MockEnvironment() + + # Setup initial state based on scenario + # (e.g., create prerequisite files) + setup = scenario.get("setup", {}) + for filename, content in setup.get("files", {}).items(): + env.create_file(filename, content) + + # Execute skill + execution_result = execute_skill_in_mock_env(skill, scenario, env) + + # Validate outcome + validation = validate_skill_outcome(skill, scenario, execution_result, env) + + result = { + "test_id": f"functional_{Path(skill['path']).stem}_{scenario.get('scenario_id', 'unknown')}", + "skill_path": skill["path"], + "skill_type": skill.get("type", "unknown"), + "scenario": scenario, + "execution": execution_result, + "validation": validation, + "passed": validation["passed"], + "timestamp": datetime.now().isoformat() + } + + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Run functional tests for Evolve Lite skills" + ) + parser.add_argument( + "--scenarios-dir", + default=None, + help="Directory containing scenario JSON files" + ) + parser.add_argument( + "--output", + default=None, + help="Output directory for test results" + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print detailed output" + ) + + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + entities_dir = evolve_dir / "entities" + + # Determine scenarios directory + if args.scenarios_dir: + scenarios_dir = Path(args.scenarios_dir) + else: + scenarios_dir = evolve_dir / "tests" / "functional" / "scenarios" + + if not scenarios_dir.exists(): + print(f"Error: Scenarios directory not found: {scenarios_dir}", file=sys.stderr) + sys.exit(1) + + # Determine output directory + if args.output: + output_dir = Path(args.output) + else: + output_dir = evolve_dir / "tests" / "functional" / "results" + + output_dir.mkdir(parents=True, exist_ok=True) + + # Load scenarios + print("Loading scenarios...") + scenarios = [] + for scenario_file in scenarios_dir.glob("*.json"): + with open(scenario_file, 'r', encoding='utf-8') as f: + scenarios.append(json.load(f)) + print(f"Loaded {len(scenarios)} scenario(s)") + + # Run functional tests + print(f"\nRunning functional tests...") + results = [] + + for scenario in scenarios: + expected_skills = scenario.get("expected_skills", []) + + for skill_slug in expected_skills: + # Find the skill file + skill_files = list(entities_dir.glob(f"**/{skill_slug}.md")) + if not skill_files: + print(f"Warning: Skill not found: {skill_slug}") + continue + + skill_path = skill_files[0] + skill = markdown_to_entity(skill_path) + skill["path"] = str(skill_path) + + print(f"Testing: {skill_slug} with {scenario['scenario_id']}") + result = run_functional_test(skill, scenario) + results.append(result) + + # Save individual result + result_file = output_dir / f"{result['test_id']}.json" + with open(result_file, 'w', encoding='utf-8') as f: + json.dump(result, f, indent=2) + + # Generate summary report + total_tests = len(results) + passed_tests = sum(1 for r in results if r["passed"]) + failed_tests = total_tests - passed_tests + + report = { + "generated_at": datetime.now().isoformat(), + "summary": { + "total_tests": total_tests, + "passed": passed_tests, + "failed": failed_tests, + "pass_rate": passed_tests / total_tests if total_tests > 0 else 0 + }, + "results": results + } + + report_path = output_dir / "functional_test_report.json" + with open(report_path, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2) + + # Print summary + print("\n" + "="*60) + print("FUNCTIONAL TEST SUMMARY") + print("="*60) + print(f"Total tests: {total_tests}") + print(f"Passed: {passed_tests}") + print(f"Failed: {failed_tests}") + print(f"Pass rate: {report['summary']['pass_rate']:.1%}") + print() + + if args.verbose: + for result in results: + status = "✅ PASS" if result["passed"] else "❌ FAIL" + print(f"\n{status} {result['test_id']}") + print(f" Steps found: {result['execution']['steps_found']}") + print(f" Steps executed: {result['execution']['steps_executed']}") + print(f" Steps successful: {result['execution']['steps_successful']}") + print(f" Errors: {len(result['execution']['environment_state']['errors'])}") + + if not result["passed"]: + validation = result["validation"] + if not validation["all_steps_executed"]: + print(f" ⚠️ Not all steps were executed") + if not validation["all_steps_successful"]: + print(f" ⚠️ Some steps failed") + if not validation["no_errors"]: + print(f" ⚠️ Errors occurred: {result['execution']['environment_state']['errors']}") + + print(f"\nReport saved to: {report_path}") + + sys.exit(0 if failed_tests == 0 else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_unit_tests.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_unit_tests.py new file mode 100644 index 00000000..d210d515 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_skill_unit_tests.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +""" +Unit Test Framework for Evolve Lite Skills + +Tests individual skills in isolation: +1. Mock trigger matching - simulates which skills would be recalled +2. Skill content validation - verifies skills provide correct guidance +3. Unit test style - one test per skill/scenario combination +""" + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Any, Set +import re + +# Walk up from the script location to find the installed plugin lib directory +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + markdown_to_entity, + log as _log, +) + + +def log(message): + _log("skill-unit-test", message) + + +def extract_keywords(text: str) -> Set[str]: + """Extract meaningful keywords from text.""" + # Remove common words + stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', + 'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'be', + 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', + 'would', 'should', 'could', 'may', 'might', 'must', 'can', 'this', + 'that', 'these', 'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they'} + + # Extract words (lowercase, alphanumeric) + words = re.findall(r'\b[a-z0-9]+\b', text.lower()) + + # Filter out stop words and short words + keywords = {w for w in words if w not in stop_words and len(w) > 2} + + return keywords + + +def mock_trigger_match(skill_trigger: str, user_request: str, threshold: float = 0.3) -> Dict[str, Any]: + """ + Mock trigger matching algorithm. + Simulates how Bob might match skill triggers to user requests. + + Returns match score and matched keywords. + """ + trigger_keywords = extract_keywords(skill_trigger) + request_keywords = extract_keywords(user_request) + + # Find overlapping keywords + matched_keywords = trigger_keywords & request_keywords + + # Calculate match score (Jaccard similarity) + if not trigger_keywords: + match_score = 0.0 + else: + match_score = len(matched_keywords) / len(trigger_keywords) + + return { + "matched": match_score >= threshold, + "match_score": match_score, + "matched_keywords": list(matched_keywords), + "trigger_keywords": list(trigger_keywords), + "request_keywords": list(request_keywords) + } + + +def validate_skill_content(skill: Dict[str, Any], scenario: Dict[str, Any]) -> Dict[str, Any]: + """ + Validate that a skill's content provides appropriate guidance for the scenario. + """ + content = skill.get("content", "") + skill_type = skill.get("type", "unknown") + + validation = { + "has_content": bool(content and len(content) > 20), + "has_rationale": "## Rationale" in content or "## rationale" in content.lower(), + "has_steps": any(marker in content for marker in ["1.", "2.", "3.", "- ", "* "]), + "has_commands": "`" in content or "```" in content, + "content_length": len(content), + "addresses_scenario": False + } + + # Check if content addresses the scenario + scenario_keywords = extract_keywords(scenario.get("user_request", "")) + content_keywords = extract_keywords(content) + overlap = scenario_keywords & content_keywords + validation["addresses_scenario"] = len(overlap) > 0 + validation["scenario_keyword_overlap"] = list(overlap) + + # Type-specific validation + if skill_type == "skill-flow": + validation["has_atomic_skills"] = bool(skill.get("atomic_skills")) + if validation["has_atomic_skills"]: + validation["atomic_skills_list"] = [s.strip() for s in skill.get("atomic_skills", "").split(",")] + + # Calculate overall score + checks = [ + validation["has_content"], + validation["has_rationale"], + validation["has_steps"] or validation["has_commands"], + validation["addresses_scenario"] + ] + validation["completeness_score"] = sum(checks) / len(checks) + validation["is_complete"] = validation["completeness_score"] >= 0.75 + + return validation + + +def load_skill(skill_path: Path) -> Dict[str, Any]: + """Load a skill from a markdown file.""" + return markdown_to_entity(skill_path) + + +def load_all_skills(entities_dir: Path) -> List[Dict[str, Any]]: + """Load all skills from the entities directory.""" + skills = [] + + for md_file in entities_dir.glob("**/*.md"): + if md_file.is_symlink() or ".git" in md_file.parts: + continue + + try: + skill = load_skill(md_file) + skill["path"] = str(md_file) + skills.append(skill) + except Exception as e: + log(f"Error loading {md_file}: {e}") + + return skills + + +def run_unit_test(skill: Dict[str, Any], scenario: Dict[str, Any]) -> Dict[str, Any]: + """ + Run a unit test for a skill against a scenario. + + Tests: + 1. Trigger matching - would this skill be recalled? + 2. Content validation - does the skill provide appropriate guidance? + """ + user_request = scenario.get("user_request", "") + skill_trigger = skill.get("trigger", "") + skill_path = skill.get("path", "unknown") + skill_type = skill.get("type", "unknown") + + # Test 1: Mock trigger matching + trigger_match = mock_trigger_match(skill_trigger, user_request) + + # Test 2: Content validation + content_validation = validate_skill_content(skill, scenario) + + # Determine if test passes + passed = trigger_match["matched"] and content_validation["is_complete"] + + result = { + "test_id": f"unit_{Path(skill_path).stem}_{scenario.get('scenario_id', 'unknown')}", + "skill_path": skill_path, + "skill_type": skill_type, + "skill_trigger": skill_trigger, + "scenario": scenario, + "trigger_match": trigger_match, + "content_validation": content_validation, + "passed": passed, + "timestamp": datetime.now().isoformat() + } + + return result + + +def run_all_unit_tests( + skills: List[Dict[str, Any]], + scenarios: List[Dict[str, Any]], + output_dir: Path +) -> List[Dict[str, Any]]: + """Run unit tests for all skill/scenario combinations.""" + results = [] + + for scenario in scenarios: + log(f"Testing scenario: {scenario.get('scenario_id')}") + + # Find skills that should match this scenario + expected_skills = scenario.get("expected_skills", []) + + for skill in skills: + skill_slug = Path(skill["path"]).stem + + # Only test if this skill is expected for this scenario + if expected_skills and skill_slug not in expected_skills: + continue + + result = run_unit_test(skill, scenario) + results.append(result) + + # Save individual result + result_file = output_dir / f"{result['test_id']}.json" + with open(result_file, 'w', encoding='utf-8') as f: + json.dump(result, f, indent=2) + + return results + + +def generate_unit_test_report(results: List[Dict[str, Any]], output_path: Path): + """Generate a summary report for all unit tests.""" + total_tests = len(results) + passed_tests = sum(1 for r in results if r["passed"]) + failed_tests = total_tests - passed_tests + + # Group by failure reason + trigger_failures = sum(1 for r in results if not r["trigger_match"]["matched"]) + content_failures = sum(1 for r in results if not r["content_validation"]["is_complete"]) + + report = { + "generated_at": datetime.now().isoformat(), + "summary": { + "total_tests": total_tests, + "passed": passed_tests, + "failed": failed_tests, + "pass_rate": passed_tests / total_tests if total_tests > 0 else 0 + }, + "failure_analysis": { + "trigger_match_failures": trigger_failures, + "content_validation_failures": content_failures + }, + "results": results, + "by_skill": {} + } + + # Group by skill + for result in results: + skill_path = result["skill_path"] + if skill_path not in report["by_skill"]: + report["by_skill"][skill_path] = { + "total_tests": 0, + "passed": 0, + "failed": 0 + } + + report["by_skill"][skill_path]["total_tests"] += 1 + if result["passed"]: + report["by_skill"][skill_path]["passed"] += 1 + else: + report["by_skill"][skill_path]["failed"] += 1 + + # Save report + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2) + + return report + + +def main(): + parser = argparse.ArgumentParser( + description="Run unit tests for Evolve Lite skills" + ) + parser.add_argument( + "--scenarios-dir", + default=None, + help="Directory containing scenario JSON files" + ) + parser.add_argument( + "--output", + default=None, + help="Output directory for test results" + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print detailed output" + ) + + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + entities_dir = evolve_dir / "entities" + + if not entities_dir.exists(): + print(f"Error: Entities directory not found: {entities_dir}", file=sys.stderr) + sys.exit(1) + + # Determine scenarios directory + if args.scenarios_dir: + scenarios_dir = Path(args.scenarios_dir) + else: + scenarios_dir = evolve_dir / "tests" / "integration" / "scenarios" + + if not scenarios_dir.exists(): + print(f"Error: Scenarios directory not found: {scenarios_dir}", file=sys.stderr) + sys.exit(1) + + # Determine output directory + if args.output: + output_dir = Path(args.output) + else: + output_dir = evolve_dir / "tests" / "unit" / "results" + + output_dir.mkdir(parents=True, exist_ok=True) + + # Load skills + print("Loading skills...") + skills = load_all_skills(entities_dir) + print(f"Loaded {len(skills)} skill(s)") + + # Load scenarios + print("Loading scenarios...") + scenarios = [] + for scenario_file in scenarios_dir.glob("*.json"): + with open(scenario_file, 'r', encoding='utf-8') as f: + scenarios.append(json.load(f)) + print(f"Loaded {len(scenarios)} scenario(s)") + + # Run unit tests + print(f"\nRunning unit tests...") + results = run_all_unit_tests(skills, scenarios, output_dir) + + # Generate report + report_path = output_dir / "unit_test_report.json" + report = generate_unit_test_report(results, report_path) + + # Print summary + print("\n" + "="*60) + print("UNIT TEST SUMMARY") + print("="*60) + print(f"Total tests: {report['summary']['total_tests']}") + print(f"Passed: {report['summary']['passed']}") + print(f"Failed: {report['summary']['failed']}") + print(f"Pass rate: {report['summary']['pass_rate']:.1%}") + print() + + print("Failure Analysis:") + print(f" Trigger match failures: {report['failure_analysis']['trigger_match_failures']}") + print(f" Content validation failures: {report['failure_analysis']['content_validation_failures']}") + print() + + if args.verbose: + print("Detailed Results:") + for result in results: + status = "✅ PASS" if result["passed"] else "❌ FAIL" + print(f"\n{status} {result['test_id']}") + print(f" Skill: {Path(result['skill_path']).name}") + print(f" Trigger match: {result['trigger_match']['matched']} (score: {result['trigger_match']['match_score']:.2f})") + print(f" Content complete: {result['content_validation']['is_complete']} (score: {result['content_validation']['completeness_score']:.2f})") + if not result["passed"]: + if not result["trigger_match"]["matched"]: + print(f" ⚠️ Trigger didn't match (matched keywords: {result['trigger_match']['matched_keywords']})") + if not result["content_validation"]["is_complete"]: + print(f" ⚠️ Content incomplete (missing: rationale={not result['content_validation']['has_rationale']}, steps={not result['content_validation']['has_steps']})") + + print(f"\nReport saved to: {report_path}") + + sys.exit(0 if report['summary']['failed'] == 0 else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob \ No newline at end of file diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_test_cases.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_test_cases.py new file mode 100644 index 00000000..0b2622d7 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_test_cases.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +""" +Test Case Runner for Evolve Lite Skills + +Runs test cases against skills to validate they are discoverable, actionable, +complete, and compose correctly. Generates a test report with pass/fail results. +""" + +import argparse +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +# Walk up from the script location to find the installed plugin lib directory +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + markdown_to_entity, + log as _log, +) + + +def log(message): + _log("test-run", message) + + +def load_test_case(test_path): + """Load a test case JSON file.""" + with open(test_path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def load_skill(skill_path): + """Load a skill entity from markdown file.""" + return markdown_to_entity(skill_path) + + +def normalize_text(text): + """Normalize text for comparison.""" + return " ".join(text.lower().split()) + + +def test_trigger_match(test_case, skill): + """Test if skill trigger matches the scenario.""" + results = { + "test_id": test_case["test_id"], + "test_type": "trigger_match", + "passed": False, + "details": {} + } + + trigger = skill.get("trigger", "").lower() + user_request = test_case["input_context"]["user_request"].lower() + + # Simple keyword matching - check if key terms from trigger appear in request + trigger_keywords = set(re.findall(r'\b\w+\b', trigger)) + # Remove common words + common_words = {"when", "the", "a", "an", "to", "for", "with", "in", "on", "at", "by"} + trigger_keywords = trigger_keywords - common_words + + request_keywords = set(re.findall(r'\b\w+\b', user_request)) + + # Calculate match score + if trigger_keywords: + matches = trigger_keywords & request_keywords + match_score = len(matches) / len(trigger_keywords) + else: + match_score = 0.0 + + results["details"]["trigger"] = skill.get("trigger", "") + results["details"]["user_request"] = test_case["input_context"]["user_request"][:100] + results["details"]["match_score"] = match_score + results["details"]["matched_keywords"] = list(matches) if trigger_keywords else [] + + # Pass if at least 30% of trigger keywords match + results["passed"] = match_score >= 0.3 + + if results["passed"]: + results["message"] = f"Trigger matches scenario (score: {match_score:.2f})" + else: + results["message"] = f"Trigger does not match scenario (score: {match_score:.2f})" + + return results + + +def test_content_completeness(test_case, skill): + """Test if skill content is complete and actionable.""" + results = { + "test_id": test_case["test_id"], + "test_type": "content_completeness", + "passed": False, + "details": {} + } + + content = skill.get("content", "") + rationale = skill.get("rationale", "") + + checks = { + "has_content": len(content) > 20, + "has_rationale": len(rationale) > 10, + "has_steps": bool(re.search(r'\d+\)', content)) or bool(re.search(r'[.;]\s*[A-Z]', content)), + "has_commands": bool(re.search(r'`[^`]+`', content)) or bool(re.search(r'orchestrate|python|pip|npm', content, re.IGNORECASE)), + "content_length_adequate": len(content) > 50 + } + + results["details"]["checks"] = checks + results["details"]["content_length"] = len(content) + results["details"]["has_rationale"] = bool(rationale) + + # Pass if most checks pass + passed_checks = sum(checks.values()) + total_checks = len(checks) + pass_rate = passed_checks / total_checks + + results["passed"] = pass_rate >= 0.6 + results["details"]["pass_rate"] = pass_rate + + if results["passed"]: + results["message"] = f"Content is complete ({passed_checks}/{total_checks} checks passed)" + else: + results["message"] = f"Content may be incomplete ({passed_checks}/{total_checks} checks passed)" + + return results + + +def test_skill_composition(test_case, skill, entities_dir): + """Test if skill-flow properly references atomic skills.""" + results = { + "test_id": test_case["test_id"], + "test_type": "skill_composition", + "passed": False, + "details": {} + } + + if skill.get("type") != "skill-flow": + results["passed"] = True + results["message"] = "Not a skill-flow, composition test not applicable" + return results + + atomic_skills = skill.get("atomic_skills", "") + if not atomic_skills: + results["passed"] = False + results["message"] = "Skill-flow has no atomic_skills references" + results["details"]["has_references"] = False + return results + + atomic_skill_list = [s.strip() for s in atomic_skills.split(",")] + + # Check if atomic skills exist + existing = [] + missing = [] + + for atomic_skill_slug in atomic_skill_list: + atomic_skill_path = Path(entities_dir) / "atomic-skill" / f"{atomic_skill_slug}.md" + if atomic_skill_path.exists(): + existing.append(atomic_skill_slug) + else: + missing.append(atomic_skill_slug) + + results["details"]["atomic_skills_referenced"] = atomic_skill_list + results["details"]["atomic_skills_existing"] = existing + results["details"]["atomic_skills_missing"] = missing + results["details"]["all_exist"] = len(missing) == 0 + + results["passed"] = len(missing) == 0 + + if results["passed"]: + results["message"] = f"All {len(existing)} atomic skills exist" + else: + results["message"] = f"Missing {len(missing)} atomic skill(s): {', '.join(missing)}" + + return results + + +def test_trajectory_replay(test_case, skill): + """Test if skill would work when replaying the trajectory.""" + results = { + "test_id": test_case["test_id"], + "test_type": "trajectory_replay", + "passed": False, + "details": {} + } + + # This is a heuristic test - check if skill content relates to tools used + content = skill.get("content", "").lower() + tools_used = test_case["input_context"].get("tools_used", []) + + # Check if skill mentions relevant tools or actions + tool_mentions = 0 + for tool in tools_used: + if tool.lower() in content: + tool_mentions += 1 + + results["details"]["tools_used"] = tools_used + results["details"]["tools_mentioned"] = tool_mentions + + # Also check if skill type matches the trajectory context + skill_type = skill.get("type", "") + user_request = test_case["input_context"]["user_request"].lower() + + type_appropriate = True + if skill_type == "guideline": + # Guidelines should be simple preferences + type_appropriate = len(skill.get("content", "")) < 200 + elif skill_type == "skill-flow": + # Skill-flows should have multiple steps + type_appropriate = bool(re.search(r'\d+\)', skill.get("content", ""))) + + results["details"]["type_appropriate"] = type_appropriate + + # Pass if tools are mentioned or type is appropriate + results["passed"] = (tool_mentions > 0 or type_appropriate) + + if results["passed"]: + results["message"] = "Skill appears applicable to trajectory" + else: + results["message"] = "Skill may not apply well to trajectory" + + return results + + +def run_test_case(test_case, entities_dir): + """Run a single test case.""" + log(f"Running test: {test_case['test_id']}") + + skill_path = Path(test_case["skill_path"]) + if not skill_path.exists(): + return { + "test_id": test_case["test_id"], + "test_type": test_case["test_type"], + "passed": False, + "message": f"Skill file not found: {skill_path}", + "details": {} + } + + try: + skill = load_skill(skill_path) + except Exception as e: + return { + "test_id": test_case["test_id"], + "test_type": test_case["test_type"], + "passed": False, + "message": f"Error loading skill: {e}", + "details": {} + } + + # Run appropriate test based on test type + test_type = test_case["test_type"] + + if test_type == "trigger_match": + return test_trigger_match(test_case, skill) + elif test_type == "content_completeness": + return test_content_completeness(test_case, skill) + elif test_type == "skill_composition": + return test_skill_composition(test_case, skill, entities_dir) + elif test_type == "trajectory_replay": + return test_trajectory_replay(test_case, skill) + else: + return { + "test_id": test_case["test_id"], + "test_type": test_type, + "passed": False, + "message": f"Unknown test type: {test_type}", + "details": {} + } + + +def generate_report(test_results, output_path): + """Generate a test report.""" + total_tests = len(test_results) + passed_tests = sum(1 for r in test_results if r["passed"]) + failed_tests = total_tests - passed_tests + + report = { + "generated_at": datetime.now().isoformat(), + "summary": { + "total_tests": total_tests, + "passed": passed_tests, + "failed": failed_tests, + "pass_rate": passed_tests / total_tests if total_tests > 0 else 0.0 + }, + "results": test_results, + "by_test_type": {} + } + + # Group by test type + for result in test_results: + test_type = result["test_type"] + if test_type not in report["by_test_type"]: + report["by_test_type"][test_type] = { + "total": 0, + "passed": 0, + "failed": 0 + } + report["by_test_type"][test_type]["total"] += 1 + if result["passed"]: + report["by_test_type"][test_type]["passed"] += 1 + else: + report["by_test_type"][test_type]["failed"] += 1 + + # Save report + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2) + + return report + + +def main(): + parser = argparse.ArgumentParser( + description="Run test cases for skills" + ) + parser.add_argument( + "--test-dir", + required=True, + help="Directory containing test case JSON files" + ) + parser.add_argument( + "--report", + default=None, + help="Output path for test report (default: test_report.json in test dir)" + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print detailed results" + ) + + args = parser.parse_args() + + test_dir = Path(args.test_dir) + if not test_dir.exists(): + print(f"Error: Test directory not found: {test_dir}", file=sys.stderr) + sys.exit(1) + + # Determine entities directory + evolve_dir = get_evolve_dir() + entities_dir = evolve_dir / "entities" + + if not entities_dir.exists(): + print(f"Error: Entities directory not found: {entities_dir}", file=sys.stderr) + sys.exit(1) + + # Load all test cases + test_files = list(test_dir.glob("*.json")) + if not test_files: + print(f"No test case files found in {test_dir}", file=sys.stderr) + sys.exit(1) + + print(f"Running {len(test_files)} test case(s)...\n") + + test_results = [] + + for test_file in test_files: + try: + test_case = load_test_case(test_file) + result = run_test_case(test_case, entities_dir) + test_results.append(result) + + status = "✓ PASS" if result["passed"] else "✗ FAIL" + print(f"{status} {result['test_id']}") + if args.verbose or not result["passed"]: + print(f" {result['message']}") + except Exception as e: + print(f"✗ ERROR {test_file.name}: {e}", file=sys.stderr) + + # Generate report + if args.report: + report_path = Path(args.report) + else: + timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + # Save reports in reports subdirectory + reports_dir = test_dir.parent / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + report_path = reports_dir / f"test_report_{timestamp}.json" + + report = generate_report(test_results, report_path) + + # Print summary + print(f"\n{'='*60}") + print("TEST SUMMARY") + print(f"{'='*60}") + print(f"Total tests: {report['summary']['total_tests']}") + print(f"Passed: {report['summary']['passed']} ({report['summary']['pass_rate']*100:.1f}%)") + print(f"Failed: {report['summary']['failed']}") + print(f"\nReport saved to: {report_path}") + + # Exit with error code if any tests failed + sys.exit(0 if report['summary']['failed'] == 0 else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_tests_with_comparison.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_tests_with_comparison.py new file mode 100644 index 00000000..8744e633 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/run_tests_with_comparison.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +""" +Enhanced Test Runner with A/B Comparison + +Runs tests twice: +1. With skills: Assumes skills were recalled and applied +2. Without skills: Simulates baseline without skill guidance + +Compares the results to show skill effectiveness. +""" + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path + +# Walk up from the script location to find the installed plugin lib directory +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + raise ImportError(f"Cannot find plugin lib directory above {_script}") +sys.path.insert(0, str(_lib)) + +from entity_io import ( # noqa: E402 + get_evolve_dir, + markdown_to_entity, + log as _log, +) + + +def log(message): + _log("test-compare", message) + + +def load_test_case(test_path): + """Load a test case JSON file.""" + with open(test_path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def load_skill(skill_path): + """Load a skill entity from markdown file.""" + return markdown_to_entity(skill_path) + + +def run_test_with_skills(test_case, skill, entities_dir): + """Run test assuming skills were used.""" + # Import test functions from run_test_cases + sys.path.insert(0, str(Path(__file__).parent)) + from run_test_cases import ( + test_trigger_match, + test_content_completeness, + test_skill_composition, + test_trajectory_replay + ) + + test_type = test_case["test_type"] + + if test_type == "trigger_match": + return test_trigger_match(test_case, skill) + elif test_type == "content_completeness": + return test_content_completeness(test_case, skill) + elif test_type == "skill_composition": + return test_skill_composition(test_case, skill, entities_dir) + elif test_type == "trajectory_replay": + return test_trajectory_replay(test_case, skill) + else: + return { + "test_id": test_case["test_id"], + "test_type": test_type, + "passed": False, + "message": f"Unknown test type: {test_type}", + "details": {} + } + + +def simulate_test_without_skills(test_case, skill): + """Simulate test results if skills were NOT used.""" + test_type = test_case["test_type"] + + # Simulate degraded performance without skills + result = { + "test_id": test_case["test_id"] + "_no_skills", + "test_type": test_type, + "passed": False, + "details": {}, + "simulated": True + } + + if test_type == "trigger_match": + # Without skills, no trigger matching happens + result["message"] = "No skill recalled (baseline)" + result["details"]["skill_recalled"] = False + + elif test_type == "content_completeness": + # Without skills, guidance is incomplete + result["message"] = "No skill guidance available (baseline)" + result["details"]["has_guidance"] = False + result["passed"] = False + + elif test_type == "skill_composition": + # Without skills, no composition to check + result["message"] = "No skill composition (baseline)" + result["details"]["has_composition"] = False + result["passed"] = False + + elif test_type == "trajectory_replay": + # Without skills, more trial and error + result["message"] = "No skill guidance, more retries expected (baseline)" + result["details"]["expected_more_retries"] = True + result["passed"] = False + + return result + + +def compare_results(with_skills_result, without_skills_result): + """Compare test results with and without skills.""" + comparison = { + "test_id": with_skills_result["test_id"], + "test_type": with_skills_result["test_type"], + "with_skills": { + "passed": with_skills_result["passed"], + "message": with_skills_result["message"] + }, + "without_skills": { + "passed": without_skills_result["passed"], + "message": without_skills_result["message"] + }, + "improvement": { + "skills_helped": with_skills_result["passed"] and not without_skills_result["passed"], + "status_change": "improved" if (with_skills_result["passed"] and not without_skills_result["passed"]) else "no_change" + } + } + + return comparison + + +def main(): + parser = argparse.ArgumentParser( + description="Run tests with A/B comparison (with skills vs without skills)" + ) + parser.add_argument( + "--test-dir", + required=True, + help="Directory containing test case JSON files" + ) + parser.add_argument( + "--report", + default=None, + help="Output path for comparison report" + ) + + args = parser.parse_args() + + test_dir = Path(args.test_dir) + if not test_dir.exists(): + print(f"Error: Test directory not found: {test_dir}", file=sys.stderr) + sys.exit(1) + + # Determine entities directory + evolve_dir = get_evolve_dir() + entities_dir = evolve_dir / "entities" + + if not entities_dir.exists(): + print(f"Error: Entities directory not found: {entities_dir}", file=sys.stderr) + sys.exit(1) + + # Load all test cases + test_files = [f for f in test_dir.glob("*.json") if not f.name.startswith("test_report") and not f.name.startswith("comparison")] + + if not test_files: + print(f"No test case files found in {test_dir}", file=sys.stderr) + sys.exit(1) + + print(f"Running {len(test_files)} test case(s) with A/B comparison...\n") + print("="*70) + print(f"{'Test':<50} {'With Skills':<12} {'Without':<12}") + print("="*70) + + comparisons = [] + with_skills_passed = 0 + without_skills_passed = 0 + + for test_file in sorted(test_files): + try: + test_case = load_test_case(test_file) + + # Skip if not a valid test case + if "test_id" not in test_case or "skill_path" not in test_case: + continue + + skill_path = Path(test_case["skill_path"]) + if not skill_path.exists(): + print(f"⚠️ {test_file.stem[:48]:<50} SKIP (skill not found)") + continue + + skill = load_skill(skill_path) + + # Run with skills + with_skills_result = run_test_with_skills(test_case, skill, entities_dir) + + # Simulate without skills + without_skills_result = simulate_test_without_skills(test_case, skill) + + # Compare + comparison = compare_results(with_skills_result, without_skills_result) + comparisons.append(comparison) + + # Track stats + if with_skills_result["passed"]: + with_skills_passed += 1 + if without_skills_result["passed"]: + without_skills_passed += 1 + + # Print result + with_status = "✓ PASS" if with_skills_result["passed"] else "✗ FAIL" + without_status = "✓ PASS" if without_skills_result["passed"] else "✗ FAIL" + improvement = "📈" if comparison["improvement"]["skills_helped"] else " " + + test_name = test_case["test_id"][:48] + print(f"{improvement} {test_name:<48} {with_status:<12} {without_status:<12}") + + except Exception as e: + print(f"✗ ERROR {test_file.name}: {e}", file=sys.stderr) + + # Generate report + if args.report: + report_path = Path(args.report) + else: + timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + # Save reports in reports subdirectory + reports_dir = test_dir.parent / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + report_path = reports_dir / f"comparison_report_{timestamp}.json" + + report = { + "generated_at": datetime.now().isoformat(), + "total_tests": len(comparisons), + "with_skills": { + "passed": with_skills_passed, + "failed": len(comparisons) - with_skills_passed, + "pass_rate": with_skills_passed / len(comparisons) if comparisons else 0 + }, + "without_skills": { + "passed": without_skills_passed, + "failed": len(comparisons) - without_skills_passed, + "pass_rate": without_skills_passed / len(comparisons) if comparisons else 0 + }, + "improvement": { + "tests_improved": sum(1 for c in comparisons if c["improvement"]["skills_helped"]), + "improvement_rate": sum(1 for c in comparisons if c["improvement"]["skills_helped"]) / len(comparisons) if comparisons else 0 + }, + "comparisons": comparisons + } + + with open(report_path, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2) + + # Print summary + print("="*70) + print("\nA/B COMPARISON SUMMARY") + print("="*70) + print(f"Total tests: {len(comparisons)}") + print(f"\nWith Skills:") + print(f" Passed: {with_skills_passed}/{len(comparisons)} ({report['with_skills']['pass_rate']*100:.1f}%)") + print(f"\nWithout Skills (simulated baseline):") + print(f" Passed: {without_skills_passed}/{len(comparisons)} ({report['without_skills']['pass_rate']*100:.1f}%)") + print(f"\nImprovement:") + print(f" Tests improved by skills: {report['improvement']['tests_improved']} ({report['improvement']['improvement_rate']*100:.1f}%)") + print(f"\nReport saved to: {report_path}") + + # Exit with error if skills didn't help + sys.exit(0 if report['improvement']['tests_improved'] > 0 else 1) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/show_execution_plan.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/show_execution_plan.py new file mode 100755 index 00000000..65f46912 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/show_execution_plan.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +""" +Show the execution plan for a skill - what steps would be extracted and executed. +This helps visualize how the functional test interprets skill content. +""" + +import json +import re +from pathlib import Path +from typing import List, Dict, Any +import argparse + + +def extract_steps_from_skill(content: str) -> List[Dict[str, Any]]: + """ + Extract executable steps from skill content. + This mimics the logic used in functional tests. + """ + steps = [] + + # Try to find numbered steps + numbered_pattern = r'^\s*(\d+)[.)]\s+(.+?)(?=^\s*\d+[.)]|\Z)' + matches = re.finditer(numbered_pattern, content, re.MULTILINE | re.DOTALL) + + for match in matches: + step_num = int(match.group(1)) + step_content = match.group(2).strip() + + # Extract commands from this step + commands = extract_commands(step_content) + + steps.append({ + 'step_number': step_num, + 'description': step_content[:100] + ('...' if len(step_content) > 100 else ''), + 'full_content': step_content, + 'commands': commands, + 'has_commands': len(commands) > 0 + }) + + # If no numbered steps found, treat entire content as one step + if not steps: + commands = extract_commands(content) + steps.append({ + 'step_number': 1, + 'description': content[:100] + ('...' if len(content) > 100 else ''), + 'full_content': content, + 'commands': commands, + 'has_commands': len(commands) > 0 + }) + + return steps + + +def extract_commands(text: str) -> List[str]: + """Extract commands from text (backticks or code blocks).""" + commands = [] + + # Extract from code blocks + code_block_pattern = r'```(?:bash|sh|shell)?\s*\n(.*?)\n```' + for match in re.finditer(code_block_pattern, text, re.DOTALL): + command = match.group(1).strip() + if command: + commands.append(command) + + # Extract from backticks + backtick_pattern = r'`([^`]+)`' + for match in re.finditer(backtick_pattern, text): + command = match.group(1).strip() + # Only include if it looks like a command (has spaces or special chars) + if ' ' in command or any(c in command for c in ['-', '/', '.']): + commands.append(command) + + return commands + + +def load_skill(skill_path: Path) -> Dict[str, Any]: + """Load skill content from markdown file.""" + content = skill_path.read_text() + + # Extract frontmatter + frontmatter = {} + if content.startswith('---'): + parts = content.split('---', 2) + if len(parts) >= 3: + frontmatter_text = parts[1] + for line in frontmatter_text.strip().split('\n'): + if ':' in line: + key, value = line.split(':', 1) + frontmatter[key.strip()] = value.strip() + content = parts[2].strip() + + return { + 'path': str(skill_path), + 'name': skill_path.stem, + 'type': frontmatter.get('type', 'unknown'), + 'content': content, + 'frontmatter': frontmatter + } + + +def show_execution_plan(skill_path: Path, verbose: bool = False): + """Show the execution plan for a skill.""" + skill = load_skill(skill_path) + steps = extract_steps_from_skill(skill['content']) + + print(f"\n{'='*70}") + print(f"EXECUTION PLAN: {skill['name']}") + print(f"{'='*70}") + print(f"Type: {skill['type']}") + print(f"Path: {skill['path']}") + print(f"\n{'─'*70}") + print("SKILL CONTENT:") + print(f"{'─'*70}") + print(skill['content'][:500] + ('...' if len(skill['content']) > 500 else '')) + + print(f"\n{'─'*70}") + print(f"EXTRACTED STEPS: {len(steps)}") + print(f"{'─'*70}") + + total_commands = 0 + for step in steps: + print(f"\n📋 Step {step['step_number']}") + print(f" Description: {step['description']}") + print(f" Commands found: {len(step['commands'])}") + + if step['commands']: + for i, cmd in enumerate(step['commands'], 1): + print(f" └─ Command {i}: {cmd}") + total_commands += 1 + else: + print(f" └─ ⚠️ No executable commands found") + + if verbose and step['full_content'] != step['description']: + print(f"\n Full content:") + for line in step['full_content'].split('\n'): + print(f" │ {line}") + + print(f"\n{'─'*70}") + print("EXECUTION SUMMARY:") + print(f"{'─'*70}") + print(f"Total steps: {len(steps)}") + print(f"Total commands: {total_commands}") + print(f"Steps with commands: {sum(1 for s in steps if s['has_commands'])}") + print(f"Steps without commands: {sum(1 for s in steps if not s['has_commands'])}") + + # Determine if this would pass functional tests + print(f"\n{'─'*70}") + print("FUNCTIONAL TEST PREDICTION:") + print(f"{'─'*70}") + + if total_commands == 0: + print("❌ LIKELY TO FAIL: No executable commands found") + print(" Reason: Skill content is too abstract or missing commands") + elif len(steps) == 1 and total_commands > 1: + print("⚠️ MAY FAIL: Multiple commands in single step") + print(" Reason: Commands may not all be executed") + elif total_commands > 0: + print("✅ MAY PASS: Commands found and extractable") + print(" Note: Actual pass depends on command execution success") + + print(f"\n{'='*70}\n") + + +def main(): + parser = argparse.ArgumentParser( + description='Show execution plan for skills' + ) + parser.add_argument( + 'skills', + nargs='*', + help='Skill file paths (if none provided, shows all skills)' + ) + parser.add_argument( + '--verbose', '-v', + action='store_true', + help='Show full step content' + ) + parser.add_argument( + '--type', + choices=['atomic-skill', 'skill-flow', 'guideline'], + help='Filter by skill type' + ) + + args = parser.parse_args() + + # Determine which skills to analyze + if args.skills: + skill_paths = [Path(s) for s in args.skills] + else: + # Find all skills + evolve_dir = Path('.evolve/entities') + skill_paths = [] + + for skill_type in ['atomic-skill', 'skill-flow', 'guideline']: + if args.type and skill_type != args.type: + continue + + type_dir = evolve_dir / skill_type + if type_dir.exists(): + skill_paths.extend(type_dir.glob('*.md')) + + if not skill_paths: + print("No skills found!") + return + + print(f"\nAnalyzing {len(skill_paths)} skill(s)...\n") + + for skill_path in sorted(skill_paths): + if skill_path.exists(): + show_execution_plan(skill_path, args.verbose) + else: + print(f"⚠️ Skill not found: {skill_path}") + + +if __name__ == '__main__': + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/snapshot_test_results.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/snapshot_test_results.py new file mode 100644 index 00000000..74a8f6fe --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/snapshot_test_results.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +""" +snapshot_test_results.py — capture current test pass counts for regression comparison. + +Reads the existing content-evaluation and recall-test reports and writes a +compact snapshot JSON that can be compared against a later run to detect +regressions (e.g. after a dedup operation). + +Usage: + # Save a snapshot before dedup: + python3 snapshot_test_results.py --out .evolve/tests/evaluation/pre_dedup_snapshot.json + + # After dedup, run tests and then compare: + python3 check_tests.py # regenerates report.json + recall_report.json + python3 snapshot_test_results.py \\ + --compare .evolve/tests/evaluation/pre_dedup_snapshot.json + + # Or compare inline: + python3 snapshot_test_results.py \\ + --compare .evolve/tests/evaluation/pre_dedup_snapshot.json \\ + --out .evolve/tests/evaluation/post_dedup_snapshot.json + +Exit codes: + 0 no regression (pass counts are equal or better than the snapshot) + 1 regression detected (fewer tests pass after the operation) + 2 snapshot or current reports missing +""" + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path + +# Bootstrap to find .evolve dir +_script = Path(__file__).resolve() +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib: + sys.path.insert(0, str(_lib)) + try: + from entity_io import get_evolve_dir + except ImportError: + def get_evolve_dir(): return Path(".evolve") +else: + def get_evolve_dir(): return Path(".evolve") + + +def _read_counts(report_path): + """Return dict with passed/total/pass_rate from a report, or None.""" + p = Path(report_path) + if not p.exists(): + return None + try: + with open(p) as fh: + data = json.load(fh) + return { + "passed": data.get("passed"), + "total": data.get("total"), + "pass_rate": data.get("pass_rate"), + "report": str(p), + } + except Exception: + return None + + +def build_snapshot(eval_report, recall_report): + """Read both reports and build a snapshot dict.""" + eval_data = _read_counts(eval_report) + recall_data = _read_counts(recall_report) + return { + "captured_at": datetime.now().isoformat(), + "content_eval": eval_data, + "recall_test": recall_data, + } + + +def compare_snapshots(before, after): + """ + Compare two snapshots. Returns (ok, lines) where ok is True when no + regression was found and lines is a list of human-readable result strings. + """ + ok = True + lines = [] + + for key, label in [("content_eval", "Content evaluation"), + ("recall_test", "Recall test")]: + b = before.get(key) or {} + a = after.get(key) or {} + + b_passed = b.get("passed") + a_passed = a.get("passed") + b_total = b.get("total") + a_total = a.get("total") + + if b_passed is None or a_passed is None: + lines.append(f" ⚠ {label}: missing data — cannot compare") + continue + + delta = a_passed - b_passed + if delta < 0: + ok = False + mark = "❌" + status = f"REGRESSION {b_passed}→{a_passed} passed (Δ{delta})" + elif delta == 0: + mark = "✅" + status = f"no regression {a_passed}/{a_total} passed" + else: + mark = "✅" + status = f"improved {b_passed}→{a_passed} passed (+{delta})" + + lines.append(f" {mark} {label:<28} {status}") + + return ok, lines + + +def main(): + parser = argparse.ArgumentParser( + description="Snapshot current test results and optionally compare against a prior snapshot" + ) + parser.add_argument( + "--out", default=None, + help="Path to write the new snapshot JSON (default: .evolve/tests/evaluation/snapshot.json)", + ) + parser.add_argument( + "--compare", default=None, + metavar="SNAPSHOT_PATH", + help="Path to a prior snapshot JSON. When provided, compares current results against it " + "and exits 1 if a regression is found.", + ) + parser.add_argument( + "--eval-report", default=None, + help="Override path to content-evaluation report.json", + ) + parser.add_argument( + "--recall-report", default=None, + help="Override path to recall_report.json", + ) + args = parser.parse_args() + + evolve_dir = get_evolve_dir() + + eval_report = Path(args.eval_report) if args.eval_report \ + else evolve_dir / "tests" / "evaluation" / "report.json" + recall_report = Path(args.recall_report) if args.recall_report \ + else evolve_dir / "tests" / "evaluation" / "recall_report.json" + + out_path = Path(args.out) if args.out \ + else evolve_dir / "tests" / "evaluation" / "snapshot.json" + + # ── build current snapshot ─────────────────────────────────────────────── + snapshot = build_snapshot(eval_report, recall_report) + + missing = [] + if snapshot["content_eval"] is None: + missing.append(str(eval_report)) + if snapshot["recall_test"] is None: + missing.append(str(recall_report)) + + if missing: + print("Error: the following reports are missing — run check_tests.py first:", file=sys.stderr) + for m in missing: + print(f" {m}", file=sys.stderr) + sys.exit(2) + + # ── write snapshot ─────────────────────────────────────────────────────── + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "w") as fh: + json.dump(snapshot, fh, indent=2) + print(f"Snapshot written: {out_path}") + + # ── compare (optional) ─────────────────────────────────────────────────── + if args.compare: + compare_path = Path(args.compare) + if not compare_path.exists(): + print(f"Error: comparison snapshot not found: {compare_path}", file=sys.stderr) + sys.exit(2) + + with open(compare_path) as fh: + before = json.load(fh) + + ok, lines = compare_snapshots(before, snapshot) + + print() + print("═" * 60) + print(" REGRESSION CHECK") + print(f" Before : {before.get('captured_at', '?')}") + print(f" After : {snapshot['captured_at']}") + print("═" * 60) + for line in lines: + print(line) + print() + if ok: + print(" ✅ No regressions detected.") + else: + print(" ❌ Regressions detected — review dedup changes.") + print("═" * 60) + + sys.exit(0 if ok else 1) + + sys.exit(0) + + +if __name__ == "__main__": + main() + +# Made with Bob diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/trigger_parser.py b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/trigger_parser.py new file mode 100644 index 00000000..baa5c936 --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-lite-test/scripts/trigger_parser.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +trigger_parser.py — Convert a skill trigger phrase into a natural user question. + +Public API +---------- + trigger_to_user_question(trigger, llm_fn=None) -> str + +If ``llm_fn`` is supplied it is called with the trigger string and must return +a rephrased natural-language question. This is the LLM path. + +If ``llm_fn`` is None (default) the function falls back to a structural +decomposition of the trigger phrase: + 1. Strip leading conditional word ("When", "After", "While", "Before", "If"). + 2. Decompose the remaining text into (action, subject, condition) chunks. + 3. Select one of several first-person question templates based on the + detected *scenario type* (error recovery, setup, procedural, creation, + generic). + +LLM helper +---------- +To use an OpenAI-compatible backend, pass a closure as ``llm_fn``: + + import openai + client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"]) + + def llm_fn(trigger): + resp = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": ( + "You convert skill trigger phrases into natural user questions. " + "The trigger describes WHEN a skill applies. " + "Return a single realistic first-person question a developer " + "would ask that would naturally invoke this skill. " + "Do not mention the skill or use meta-language. " + "Return only the question, no extra text." + )}, + {"role": "user", "content": trigger}, + ], + temperature=0.3, + max_tokens=120, + ) + return resp.choices[0].message.content.strip() + + question = trigger_to_user_question(trigger, llm_fn=llm_fn) +""" + +import re + +# --------------------------------------------------------------------------- +# Scenario-type detection → question templates +# --------------------------------------------------------------------------- + +# Each entry: (regex against raw trigger, question_fn(trigger) -> str) +# Checked in order; first match wins. +# question_fn receives the *original* trigger so it can extract key nouns +# directly rather than using the grammatically awkward stripped core. +_SCENARIO_RULES = [ + # Token expiration / reauth + ( + r"token expir", + lambda t: ( + "I ran a Watson Orchestrate CLI command and it failed with a token " + "expiration error. How do I reauthenticate?" + ), + ), + # Interactive prompt / automation / piping + ( + r"interact.*prompt|prompt.*interact|automat.*prompt|stdin|piping", + lambda t: ( + "I'm trying to automate a Watson Orchestrate CLI command but it " + "keeps stopping to ask for a password interactively. " + "How do I run it non-interactively?" + ), + ), + # Auth / credential error mid-session + ( + r"auth.*error|error.*auth|needs to be authenticated|authenticated before", + lambda t: ( + "I'm about to run Watson Orchestrate CLI commands. " + "What do I need to do to make sure I'm authenticated first?" + ), + ), + # First-time setup / registering env + ( + r"first time|setting up|set up|register.*env|env.*register", + lambda t: ( + "I'm setting up the Watson Orchestrate CLI environment for the " + "first time. What are the steps I need to follow?" + ), + ), + # Import YAML agent + ( + r"import.*agent|agents import", + lambda t: ( + "I've got my agent YAML ready. " + "How do I import it into Watson Orchestrate?" + ), + ), + # Deploy / activate after import + ( + r"deploy|not yet active|not.*active|needs to be made active|activate.*agent", + lambda t: ( + "I just imported my Watson Orchestrate agent but it doesn't seem " + "to be active. What's the next step to make it available?" + ), + ), + # Create YAML / spec_version + ( + r"yaml.*file|spec_version|yaml.*definition|create.*agent.*yaml", + lambda t: ( + "I need to create the YAML definition file for a Watson Orchestrate " + "agent. What fields are required — things like spec_version, name, " + "and instructions?" + ), + ), + # Activate existing venv / CLI not found — must come BEFORE generic venv rule + ( + r"CLI.*not found|not found.*CLI|CLI command is not found|activate.*before running", + lambda t: ( + "I opened a new terminal and now the orchestrate CLI command isn't " + "found. How do I fix this?" + ), + ), + # Virtual environment / venv creation + ( + r"virtual environment|venv|virtualenv|\.venv|isolated environment|dependency conflict", + lambda t: ( + "I need to set up an isolated Python environment for my project. " + "What's the correct way to create and activate a virtual environment?" + ), + ), + # Requirements / dependencies + ( + r"requirements.*file|requirements\.txt|reproducible.*list|package.*depend", + lambda t: ( + "I need to create a requirements.txt for my Python project so " + "dependencies are reproducible. How should I do that?" + ), + ), + # Environment variables / .env / secrets + ( + r"environment.*secret|\.env|env.*variable|secrets.*runtime|config.*runtime", + lambda t: ( + "My application needs to read secrets and config values at runtime. " + "What's the right way to manage environment variables?" + ), + ), + # Multi-tool / multiple functions — must come BEFORE single-tool register rule + ( + r"multiple.*function|multi.*tool|expose.*multiple", + lambda t: ( + "I have a Python file with several functions I want to expose as " + "Watson Orchestrate tools. How do I import them all?" + ), + ), + # Tool not recognized / import error + ( + r"not.*recognized|not being recognized|tools import command", + lambda t: ( + "I imported my Python tool into Watson Orchestrate but it's not " + "being recognized. What could be wrong?" + ), + ), + # Skill-flow / quality gate / missing references + ( + r"quality gate|missing.*skill|skill.*missing|atomic skill.*ref", + lambda t: ( + "The skill-flow quality gate is reporting missing atomic skill " + "references, but those skills exist in my library. How do I fix this?" + ), + ), + # Generic error / failure / not found + ( + r"\berror\b|\bfail\b|\bnot found\b|\bcannot\b|\binvalid\b|\bwrong\b|\bmissing\b", + lambda t: ( + f"I'm running into an issue: {_extract_error_context(t)}. " + "What's the correct way to fix this?" + ), + ), +] + +_GENERIC_TEMPLATE = lambda trigger: ( + f"I'm trying to {_trigger_as_task(trigger)} and ran into a problem. " + "What should I do?" +) + + +# --------------------------------------------------------------------------- +# Text helpers +# --------------------------------------------------------------------------- + +def _strip_conditional(trigger: str) -> str: + """Remove leading 'When', 'After', 'While', 'Before', 'If' clause.""" + return re.sub( + r"^(When|After|While|Before|If)\s+", + "", + trigger.strip(), + flags=re.IGNORECASE, + ).strip() + + +def _lower_first(text: str) -> str: + return text[0].lower() + text[1:] if text else text + + +def _shorten(text: str, max_words: int = 10) -> str: + """Trim to the first ``max_words`` words to keep templates readable.""" + words = text.split() + if len(words) <= max_words: + return text + return " ".join(words[:max_words]).rstrip(",.;:") + "…" + + +def _extract_subject(text: str) -> str: + """Pull the first noun phrase (up to 4 words) out of the core text.""" + words = text.split() + return " ".join(words[:4]).rstrip(",.;:") if words else text + + +def _trigger_as_task(trigger: str) -> str: + """Convert trigger to a short task phrase for the generic template. + + Strips the conditional prefix and converts to lower-case verb phrase, + capping at 10 words. + """ + core = _lower_first(_strip_conditional(trigger)) + # If the core looks like a noun clause ("a Python project needs…"), + # try to find the first verb and trim before it for readability. + # Simple heuristic: cut at the first occurrence of " needs ", " is ", " has ". + for pivot in (" needs ", " is ", " has ", " was ", " requires "): + idx = core.find(pivot) + if idx > 0: + core = core[:idx].strip() + break + return _shorten(core) + + +def _extract_error_context(trigger: str) -> str: + """Pull a concise error description from the trigger for error templates.""" + core = _strip_conditional(trigger) + # Grab up to 8 words following "error", "fail", "not found", etc. + match = re.search( + r"(error|fail(?:ure)?|not found|cannot|invalid|wrong|missing)[^,;.]*", + core, re.IGNORECASE, + ) + if match: + return _shorten(match.group(0).strip(), max_words=8) + return _shorten(core, max_words=8) + + +# --------------------------------------------------------------------------- +# Structural decomposition (offline fallback) +# --------------------------------------------------------------------------- + +def _structural_question(trigger: str) -> str: + """Derive a question from ``trigger`` purely via regex + templates.""" + for pattern, question_fn in _SCENARIO_RULES: + if re.search(pattern, trigger, re.IGNORECASE): + return question_fn(trigger) + + return _GENERIC_TEMPLATE(trigger) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def trigger_to_user_question(trigger: str, llm_fn=None) -> str: + """Convert a skill trigger phrase into a natural user question. + + Parameters + ---------- + trigger: + The ``trigger:`` frontmatter value from an entity file, e.g. + "When Watson Orchestrate CLI reports a token expiration error". + llm_fn: + Optional callable ``(trigger: str) -> str``. If provided it is called + with the raw trigger and must return a rephrased question string. + If None, structural decomposition is used instead. + + Returns + ------- + str + A natural first-person question that would plausibly invoke this skill. + """ + if llm_fn is not None: + try: + result = llm_fn(trigger) + if result and isinstance(result, str): + return result.strip() + except Exception as exc: # noqa: BLE001 + import sys + print(f" Warning: llm_fn failed ({exc}), falling back to structural parse", + file=sys.stderr) + + return _structural_question(trigger) diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-manager/SKILL.md b/platform-integrations/bob/evolve-lite/skills/evolve-manager/SKILL.md new file mode 100644 index 00000000..716e638f --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-manager/SKILL.md @@ -0,0 +1,192 @@ +--- +name: evolve-manager:merge-forks +description: Discover GitHub forks of the main evolve repo, merge their entity libraries with versioning-aware conflict resolution, test for regressions on main-repo skills only, and deduplicate the combined library. +--- + +# Merge Forks into Main Repo + +## Overview + +`evolve-manager:merge-forks` orchestrates a safe, regression-protected merge of evolve entity +libraries from GitHub forks into the local main repo. It runs the full pipeline: + +``` +discover forks + → [0] PR gate: skip forks without an open PR against the main repo + → clone + stage fork entities (accepted forks only) + → snapshot main-repo entity manifest + → versioning-aware merge + → [A] quality gate on merged test fixtures + → [B] rubric tests (main-repo entities only) → baseline_rate + → [C] full skill dedup (both phases) + → [D] rubric tests (main-repo entities only) → post_rate + → [E] threshold gate + PASS → commit merged entities to .evolve/entities/ + FAIL → show diff, pause for user decision + → write merge_report.json (always, even on error/skip) +``` + +Fork-sourced entities are **never** counted against the regression threshold. Only entities +that existed in the main repo before the merge are subject to the threshold gate. + +## Usage + +### Basic (agent handles fork discovery and cloning) + +The PR gate is **on by default**. The upstream repo is auto-detected from the local git +`origin` remote. Set `GITHUB_TOKEN` (or pass `--github-token`) for private repos or higher +API rate limits. + +```bash +python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/fork-a .evolve/tmp/fork-staging/fork-b +``` + +### With explicit main repo (required if git remote is not GitHub) + +```bash +python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/fork-a \ + --main-repo my-org/my-evolve-repo +``` + +### Disable PR gate (merge any fork regardless of PR status) + +```bash +python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/fork-a \ + --no-require-pr +``` + +### With relaxed threshold (allow up to 20% regression) + +```bash +python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/fork-a \ + --threshold 0.8 +``` + +### With custom version-diff sensitivity + +```bash +python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/fork-a \ + --version-diff-threshold 0.3 +``` +Lower values preserve more dual-section history. Higher values replace more aggressively. + +### Dry run (no writes) + +```bash +python3 .bob/skills/evolve-manager/scripts/merge_forks.py \ + --fork-dirs .evolve/tmp/fork-staging/fork-a \ + --dry-run +``` + +## Flags + +| Flag | Default | Description | +|---|---|---| +| `--fork-dirs` | (required) | Space-separated list of pre-cloned fork directories | +| `--threshold` | `1.0` | Min rubric pass rate for main-repo tests (1.0 = no regressions) | +| `--version-diff-threshold` | `0.5` | Jaccard similarity below which dual-section merging is used | +| `--dry-run` | off | Show all decisions without writing any entity files (report still written) | +| `--force-commit` | off | Skip the threshold gate and commit the merge regardless | +| `--report-dir` | `.evolve/tests/dedup/` | Directory for dedup JSON reports (including `merge_report.json`) | +| `--main-repo` | auto | Upstream GitHub repo as `owner/repo` (auto-detected from git `origin` remote) | +| `--require-pr` | on | Skip forks without an open PR against `--main-repo` | +| `--no-require-pr` | — | Disable the PR gate — merge all provided forks | +| `--github-token` | env | GitHub token for API calls (falls back to `GITHUB_TOKEN` env var) | + +## Exit Codes + +| Code | Meaning | Action | +|---|---|---| +| `0` | Merge succeeded | Entities are live in `.evolve/entities/` | +| `1` | Hard failure | Fix the reported error before re-running | +| `2` | Threshold breach | Main-repo test pass rate dropped — user must decide keep or roll back | + +## Versioning-Aware Merge + +When the same entity slug exists in both the main repo and a fork, the script compares +them using token-set Jaccard similarity on `trigger + content`: + +- **Jaccard >= `--version-diff-threshold`** (default 0.5): minor update — fork content + replaces main-repo content outright +- **Jaccard < `--version-diff-threshold`**: significant divergence — a dual-section entity + is written with: + - `## Current Version` — fork content (newer information, higher version) + - `## Previous Version` — original main-repo content (used as baseline for rubric tests) + +The entity's `version` frontmatter field is set to the fork's version. A `base_version` field +records the original main-repo version for traceability. + +## Regression Gate + +The threshold gate applies only to entities listed in +`.evolve/tmp/merge-workspace/main_entity_slugs.json` (snapshotted before any fork is merged). + +For dual-section entities, the rubric test runs against the full merged file (both sections). +The `must_include` terms come from the **original main-repo rubric**, not the fork's. This means +the test checks whether the merged skill still satisfies what the main-repo version promised. + +## Reports + +| File | Contents | +|---|---| +| `.evolve/tests/dedup/merge_report.json` | **Full merge run summary** — PR gate results, accepted/skipped forks, merge decisions, pass rates, outcome | +| `.evolve/tests/dedup/quality_gate_report.json` | Phase 1 format/recall/eval results | +| `.evolve/tests/dedup/refine_report.json` | Phase 2 cluster decisions (merge/discard/keep) | +| `.evolve/tests/evaluation/report.json` | Pre-dedup rubric test results (baseline) | +| `.evolve/tests/evaluation/report_post.json` | Post-dedup rubric test results | +| `.evolve/tmp/merge-workspace/main_entity_slugs.json` | Entity provenance manifest | + +### `merge_report.json` schema + +```json +{ + "timestamp": "", + "dry_run": false, + "outcome": "success | skipped | dry-run | threshold-breach | error", + "outcome_reason": "", + "threshold": 1.0, + "baseline_pass_rate": 0.95, + "post_dedup_pass_rate": 0.95, + "forks": { + "accepted": ["path/to/fork-a"], + "skipped": ["path/to/fork-b"], + "pr_check_details": [ + { + "fork_dir": "path/to/fork-a", + "has_pr": true, + "pr_number": 42, + "pr_title": "Add new skills", + "fork_owner": "alice", + "pr_head": "alice:main", + "reason": "Open PR #42: 'Add new skills'" + } + ] + }, + "merge_decisions": { + "summary": { "keep-main": 10, "fork-replaces-main": 2, "dual-section": 1 }, + "details": [{ "slug": "...", "action": "...", "fork": "...", "jaccard": 0.7 }] + } +} +``` + +## Rollback + +A backup of the pre-merge `.evolve/entities/` is always written before any live file is +touched (even on `--dry-run` the backup is skipped): + +```bash +# Roll back to pre-merge state +rm -rf .evolve/entities/ +cp -r .evolve/tmp/pre-merge-backup/ .evolve/entities/ +``` + +## Supporting Files + +| File | Purpose | +|---|---| +| `scripts/merge_forks.py` | Main orchestration script | diff --git a/platform-integrations/bob/evolve-lite/skills/evolve-manager/scripts/merge_forks.py b/platform-integrations/bob/evolve-lite/skills/evolve-manager/scripts/merge_forks.py new file mode 100755 index 00000000..19e3953d --- /dev/null +++ b/platform-integrations/bob/evolve-lite/skills/evolve-manager/scripts/merge_forks.py @@ -0,0 +1,1219 @@ +#!/usr/bin/env python3 +""" +evolve-manager merge-forks — versioning-aware fork merge pipeline + +Merges .evolve/entities/ from one or more pre-cloned GitHub fork directories +into the local main-repo entity library with full regression protection. + +Pipeline: + 0. [PR gate] Verify each fork has an open PR against the main repo (skips forks without one) + 1. Snapshot main-repo entity manifest (main_entity_slugs.json) + 2. Versioning-aware merge into .evolve/tmp/merge-workspace/entities/ + - same slug in both: Jaccard >= threshold → fork replaces main + - same slug in both: Jaccard < threshold → dual-section entity preserved + - fork-only slugs: copied as-is, not subject to regression gate + 3. [A] Quality gate: dedup.py --phase1-only on merged workspace + 4. [B] Pre-dedup test: rubric eval on main-repo entities → baseline_rate + 5. [C] Full dedup: dedup.py (both phases) on merged workspace + 6. [D] Post-dedup test: rubric eval on main-repo entities → post_rate + 7. [E] Threshold gate: if post_rate < --threshold → exit 2 (user decides) + 8. On success: backup .evolve/entities/ → move merged workspace into place + 9. Write merge_report.json summarising the entire run + +Exit codes: + 0 — merge succeeded, entities live in .evolve/entities/ + 1 — hard failure (bad entities, script error) + 2 — threshold breach — main-repo rubric pass rate dropped; user must decide + +Usage: + python3 merge_forks.py --fork-dirs path/to/fork-a path/to/fork-b + python3 merge_forks.py --fork-dirs path/to/fork-a --threshold 0.8 + python3 merge_forks.py --fork-dirs path/to/fork-a --dry-run + python3 merge_forks.py --fork-dirs path/to/fork-a --version-diff-threshold 0.3 + python3 merge_forks.py --fork-dirs path/to/fork-a --main-repo owner/repo + python3 merge_forks.py --fork-dirs path/to/fork-a --no-require-pr +""" + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import urllib.request +import urllib.error +from datetime import datetime, timezone +from pathlib import Path + +# --------------------------------------------------------------------------- +# Bootstrap: locate lib/evolve-lite and sibling scripts +# --------------------------------------------------------------------------- +_script = Path(__file__).resolve() +_scripts_dir = _script.parent +_skills_root = _scripts_dir.parent.parent # .bob/skills/ + +# Locate lib/evolve-lite by walking up +_lib = None +for _ancestor in _script.parents: + _candidate = _ancestor / "lib" / "evolve-lite" + if (_candidate / "entity_io.py").is_file(): + _lib = _candidate + break +if _lib is None: + print("ERROR: Cannot find lib/evolve-lite/entity_io.py", file=sys.stderr) + sys.exit(1) +sys.path.insert(0, str(_lib)) + +from entity_io import get_evolve_dir, markdown_to_entity # noqa: E402 + +# Sibling tool paths +_dedup_script = _skills_root / "evolve-lite-dedup" / "scripts" / "dedup.py" +_gen_script = _skills_root / "evolve-lite-test" / "scripts" / "generate_pseudo_conversations.py" +_eval_script = _skills_root / "evolve-lite-test" / "scripts" / "run_skill_evaluation.py" + +# --------------------------------------------------------------------------- +# Token-set Jaccard (mirrors refine.py — no import to avoid circular deps) +# --------------------------------------------------------------------------- +_STOP = { + "the", "and", "for", "are", "but", "not", "with", "this", "that", + "have", "from", "they", "will", "been", "when", "after", "before", + "while", "how", "what", "just", "need", "want", "make", "sure", + "some", "also", "about", "you", "your", "use", "run", "set", +} + + +def _tokens(text): + words = re.findall(r"[a-z0-9]+", text.lower()) + return {w for w in words if w not in _STOP and len(w) > 2} + + +def jaccard(a, b): + ta, tb = _tokens(a), _tokens(b) + if not ta and not tb: + return 1.0 + if not ta or not tb: + return 0.0 + return len(ta & tb) / len(ta | tb) + + +# --------------------------------------------------------------------------- +# Entity helpers +# --------------------------------------------------------------------------- + +def entity_similarity_text(entity): + return f"{entity.get('trigger', '')} {entity.get('content', '')}" + + +def read_entity(path): + """Parse a .md entity file; returns dict with at least 'content'.""" + return markdown_to_entity(path) + + +def write_entity_file(path, frontmatter_dict, content): + """Write a .md entity file with YAML frontmatter.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + lines = ["---"] + for k, v in frontmatter_dict.items(): + lines.append(f"{k}: {v}") + lines.append("---") + lines.append("") + lines.append(content) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def safe_int(val, default=1): + try: + return int(val) + except (TypeError, ValueError): + return default + + +# --------------------------------------------------------------------------- +# Step 1 — Snapshot main-repo manifest +# --------------------------------------------------------------------------- + +def snapshot_main_manifest(entities_dir, manifest_path): + """ + Walk the live .evolve/entities/ directory and record every slug's: + - origin: 'main' + - base_version: int from frontmatter version field (default 1) + - rubric: must_include terms derived from success_rubric (for regression gate) + - rel_path: path relative to entities_dir (to find the file in merge workspace) + """ + manifest = {} + entities_dir = Path(entities_dir) + for md_file in entities_dir.rglob("*.md"): + slug = md_file.stem + entity = read_entity(md_file) + rubric = entity.get("success_rubric", "") + rubric_terms = _rubric_to_must_include(rubric) if rubric else [] + manifest[slug] = { + "origin": "main", + "base_version": safe_int(entity.get("version", 1)), + "fork_version": None, + "dual_section": False, + "rel_path": str(md_file.relative_to(entities_dir)), + "rubric_terms": rubric_terms, + } + manifest_path = Path(manifest_path) + manifest_path.parent.mkdir(parents=True, exist_ok=True) + with open(manifest_path, "w", encoding="utf-8") as fh: + json.dump(manifest, fh, indent=2) + return manifest + + +def _rubric_to_must_include(rubric_text): + """Parse ## Success Rubric bullets into must_include terms (mirrors generate_pseudo_conversations.py).""" + seen = set() + terms = [] + for line in rubric_text.splitlines(): + line = line.strip().lstrip("-").strip() + if not line: + continue + backtick_hits = re.findall(r"`([^`]+)`", line) + candidates = [t.strip() for t in backtick_hits if t.strip()] if backtick_hits else [line] + for t in candidates: + if t not in seen: + seen.add(t) + terms.append(t) + return terms + + +# --------------------------------------------------------------------------- +# Step 2 — Versioning-aware merge +# --------------------------------------------------------------------------- + +def collect_fork_entities(fork_dirs): + """ + Walk each fork dir for .evolve/entities/**/*.md files. + Returns dict: slug -> (path, entity_dict) — last fork wins for fork-only + slugs (earlier forks are already staged; later forks are added without + overwriting earlier fork contributions unless the slug is the same). + Fork entities keyed by slug; first occurrence wins (preserving earlier forks). + """ + fork_entities = {} # slug -> (path, entity, fork_name) + for fork_dir in fork_dirs: + fork_dir = Path(fork_dir) + # Support both a bare fork dir and one containing .evolve/entities/ subtree + entities_root = fork_dir / ".evolve" / "entities" + if not entities_root.exists(): + entities_root = fork_dir # caller may have already pointed at entities dir + if not entities_root.exists(): + print(f" WARNING: no entities found in {fork_dir}, skipping", file=sys.stderr) + continue + fork_name = fork_dir.name + for md_file in entities_root.rglob("*.md"): + slug = md_file.stem + if slug not in fork_entities: # first fork wins + entity = read_entity(md_file) + fork_entities[slug] = (md_file, entity, fork_name) + return fork_entities + + +def merge_entities(main_entities_dir, fork_entities, workspace_entities_dir, + manifest, version_diff_threshold, dry_run): + """ + Merge main-repo and fork entities into workspace_entities_dir. + + Strategy per slug: + - main-only: copy verbatim, mark origin='main' + - fork-only: copy verbatim, mark origin='fork' + - both: + Jaccard >= version_diff_threshold → fork replaces main (minor update) + Jaccard < version_diff_threshold → dual-section entity preserved + + Updates manifest in-place with fork provenance fields. + Returns list of merge decisions for reporting. + """ + main_entities_dir = Path(main_entities_dir) + workspace_entities_dir = Path(workspace_entities_dir) + workspace_entities_dir.mkdir(parents=True, exist_ok=True) + + decisions = [] + + # --- main-only and overlap slugs --- + for md_file in main_entities_dir.rglob("*.md"): + slug = md_file.stem + rel_path = md_file.relative_to(main_entities_dir) + dest = workspace_entities_dir / rel_path + main_entity = read_entity(md_file) + + if slug not in fork_entities: + # main-only: copy verbatim + if not dry_run: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(md_file, dest) + decisions.append({"slug": slug, "action": "keep-main", "fork": None}) + else: + fork_path, fork_entity, fork_name = fork_entities[slug] + sim = jaccard(entity_similarity_text(main_entity), + entity_similarity_text(fork_entity)) + + if sim >= version_diff_threshold: + # Minor update — fork replaces main + if not dry_run: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(fork_path, dest) + manifest[slug]["fork_version"] = safe_int(fork_entity.get("version")) + manifest[slug]["origin"] = "merged" + manifest[slug]["dual_section"] = False + decisions.append({ + "slug": slug, "action": "fork-replaces-main", + "fork": fork_name, "jaccard": round(sim, 3) + }) + else: + # Significant divergence — write dual-section entity + fork_content = fork_entity.get("content", "") + main_content = main_entity.get("content", "") + merged_content = ( + f"## Current Version\n\n{fork_content}\n\n" + f"## Previous Version\n\n{main_content}" + ) + fork_ver = safe_int(fork_entity.get("version")) + base_ver = safe_int(main_entity.get("version")) + # Build merged frontmatter from main, overriding version fields + fm = {k: v for k, v in main_entity.items() + if k not in ("content", "success_rubric", "changelog")} + fm["version"] = fork_ver + fm["base_version"] = base_ver + if not dry_run: + write_entity_file(dest, fm, merged_content) + manifest[slug]["fork_version"] = fork_ver + manifest[slug]["origin"] = "merged" + manifest[slug]["dual_section"] = True + decisions.append({ + "slug": slug, "action": "dual-section", + "fork": fork_name, "jaccard": round(sim, 3), + "base_version": base_ver, "fork_version": fork_ver, + }) + + # --- fork-only slugs --- + for slug, (fork_path, fork_entity, fork_name) in fork_entities.items(): + if slug in manifest: + continue # already handled above + # Determine relative path inside the fork's entities subtree + entities_root = None + for p in fork_path.parents: + if p.name == "entities": + entities_root = p + break + rel = fork_path.relative_to(entities_root) if entities_root else Path(fork_path.name) + dest = workspace_entities_dir / rel + if not dry_run: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(fork_path, dest) + manifest[slug] = { + "origin": "fork", + "base_version": None, + "fork_version": safe_int(fork_entity.get("version")), + "dual_section": False, + "rel_path": str(rel), + "rubric_terms": [], # fork-only: no regression gate + } + decisions.append({"slug": slug, "action": "fork-only", "fork": fork_name}) + + return decisions + + +# --------------------------------------------------------------------------- +# PR gate helpers +# --------------------------------------------------------------------------- + +def _git_remote_url(repo_dir): + """Return the 'origin' remote URL for a git repo directory, or None.""" + try: + result = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=str(repo_dir), capture_output=True, text=True + ) + if result.returncode == 0: + return result.stdout.strip() + except Exception: + pass + return None + + +def _parse_github_owner_repo(url): + """ + Extract (owner, repo) from a GitHub remote URL. + Supports any GitHub-flavoured host — github.com, github.ibm.com, GHE instances, etc. + Handles: + https:///owner/repo[.git] + git@:owner/repo[.git] + Returns (owner, repo) or (None, None). + """ + if not url: + return None, None + # SSH form: git@:owner/repo[.git] + m = re.search(r"git@[^:]+:([^/]+)/([^/\s]+?)(?:\.git)?$", url) + if m: + return m.group(1), m.group(2) + # HTTPS form: https:///owner/repo[.git] + m = re.search(r"https?://[^/]+/([^/]+)/([^/\s]+?)(?:\.git)?$", url) + if m: + return m.group(1), m.group(2) + return None, None + + +def _github_api(path, token=None, api_base=None): + """Make a GET request to the GitHub API; returns parsed JSON or None on error.""" + base = (api_base or "https://github.ibm.com/api/v3").rstrip("/") + url = f"{base}{path}" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + print(f" GitHub API error {e.code} for {url}", file=sys.stderr) + return None + except Exception as e: + print(f" GitHub API request failed: {e}", file=sys.stderr) + return None + + +def _github_api_post(path, body, token=None, api_base=None): + """Make a POST request to the GitHub API; returns parsed JSON or None on error.""" + base = (api_base or "https://github.ibm.com/api/v3").rstrip("/") + url = f"{base}{path}" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + } + if token: + headers["Authorization"] = f"Bearer {token}" + data = json.dumps(body).encode() + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body_text = e.read().decode(errors="replace") + print(f" GitHub API POST error {e.code} for {url}: {body_text[:200]}", file=sys.stderr) + return None + except Exception as e: + print(f" GitHub API POST request failed: {e}", file=sys.stderr) + return None + + +def _github_api_patch(path, body, token=None, api_base=None): + """Make a PATCH request to the GitHub API; returns parsed JSON or None on error.""" + base = (api_base or "https://github.ibm.com/api/v3").rstrip("/") + url = f"{base}{path}" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + } + if token: + headers["Authorization"] = f"Bearer {token}" + data = json.dumps(body).encode() + req = urllib.request.Request(url, data=data, headers=headers, method="PATCH") + try: + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body_text = e.read().decode(errors="replace") + print(f" GitHub API PATCH error {e.code} for {url}: {body_text[:200]}", file=sys.stderr) + return None + except Exception as e: + print(f" GitHub API PATCH request failed: {e}", file=sys.stderr) + return None + + +def close_pr_with_comment(main_repo, pr_number, merge_summary, token=None, api_base=None): + """ + Post a comment on a PR explaining that its entities were merged via evolve-manager, + then close the PR. + + Args: + main_repo: 'owner/repo' string for the upstream repository. + pr_number: int PR number. + merge_summary: human-readable string summarising what was merged. + token: optional GitHub personal access token. + api_base: GitHub API base URL (default: https://github.ibm.com/api/v3). + + Returns: + dict with keys: commented (bool), closed (bool), error (str|None) + """ + result = {"commented": False, "closed": False, "error": None} + + comment_body = ( + "✅ **Entities merged via evolve-manager**\n\n" + f"{merge_summary}\n\n" + "The entity library from this fork was merged into the main repo using " + "`evolve-manager merge-forks`. The PR is being closed as the changes have " + "been incorporated directly into `.evolve/entities/`." + ) + + # Post comment + comment_resp = _github_api_post( + f"/repos/{main_repo}/issues/{pr_number}/comments", + {"body": comment_body}, + token=token, + api_base=api_base, + ) + if comment_resp and comment_resp.get("id"): + result["commented"] = True + else: + result["error"] = "Failed to post comment" + + # Close the PR + close_resp = _github_api_patch( + f"/repos/{main_repo}/pulls/{pr_number}", + {"state": "closed"}, + token=token, + api_base=api_base, + ) + if close_resp and close_resp.get("state") == "closed": + result["closed"] = True + else: + if result["error"]: + result["error"] += "; Failed to close PR" + else: + result["error"] = "Failed to close PR" + + return result + + +def check_fork_has_open_pr(fork_dir, main_repo, token=None, api_base=None): + """ + Check whether the given fork directory has an open PR against main_repo. + + Args: + fork_dir: Path to the pre-cloned fork directory. + main_repo: 'owner/repo' string for the upstream repository. + token: Optional GitHub personal access token. + api_base: GitHub API base URL (default: https://github.ibm.com/api/v3). + + Returns: + dict with keys: + has_pr (bool), pr_number (int|None), pr_title (str|None), + fork_owner (str|None), pr_head (str|None), reason (str) + """ + fork_dir = Path(fork_dir) + remote_url = _git_remote_url(fork_dir) + fork_owner, fork_repo = _parse_github_owner_repo(remote_url) + + if not fork_owner: + return { + "has_pr": False, "pr_number": None, "pr_title": None, + "fork_owner": None, "pr_head": None, + "reason": f"Could not determine fork owner from remote URL: {remote_url!r}", + } + + # Query open PRs on the main repo + # First try head filter; if API errors or returns empty, fetch all open PRs + prs = _github_api( + f"/repos/{main_repo}/pulls?state=open&head={fork_owner}:&per_page=100", + token=token, + api_base=api_base, + ) + if not prs: + # Fall back to fetching all open PRs without head filter + prs = _github_api( + f"/repos/{main_repo}/pulls?state=open&per_page=100", + token=token, + api_base=api_base, + ) + + if prs is None: + return { + "has_pr": False, "pr_number": None, "pr_title": None, + "fork_owner": fork_owner, "pr_head": None, + "reason": "GitHub API request failed (check network/token)", + } + + # Filter client-side: check repo owner OR head label prefix OR PR author + matching = [] + for pr in prs: + head_repo_owner = ( + pr.get("head", {}).get("repo", {}) or {} + ).get("owner", {}).get("login", "").lower() + head_label = pr.get("head", {}).get("label", "").lower() + pr_user = pr.get("user", {}).get("login", "").lower() + owner_low = fork_owner.lower() + + if ( + head_repo_owner == owner_low + or head_label.startswith(f"{owner_low}:") + or pr_user == owner_low + ): + matching.append(pr) + + if matching: + pr = matching[0] + return { + "has_pr": True, + "pr_number": pr["number"], + "pr_title": pr["title"], + "fork_owner": fork_owner, + "pr_head": pr["head"].get("label"), + "reason": f"Open PR #{pr['number']}: {pr['title']!r}", + } + + return { + "has_pr": False, "pr_number": None, "pr_title": None, + "fork_owner": fork_owner, "pr_head": None, + "reason": f"No open PR found from {fork_owner} against {main_repo}", + } + + +def _detect_main_repo(api_base=None, token=None): + """ + Try to detect the main repo from the local git remote 'ce-artemis' or 'origin'. + Returns 'owner/repo' string or None. + """ + # Prefer the canonical upstream remote if present + for remote_name in ("ce-artemis", "upstream", "origin"): + url = None + try: + result = subprocess.run( + ["git", "remote", "get-url", remote_name], + cwd=str(Path.cwd()), capture_output=True, text=True + ) + if result.returncode == 0: + url = result.stdout.strip() + except Exception: + pass + if url: + owner, repo = _parse_github_owner_repo(url) + if owner and repo: + return f"{owner}/{repo}" + return None + + +# --------------------------------------------------------------------------- +# Merge report helpers +# --------------------------------------------------------------------------- + +def write_merge_report(report_path, *, timestamp, fork_pr_results, accepted_forks, + skipped_forks, decisions, baseline_rate, post_rate, + threshold, outcome, outcome_reason, dry_run): + """Write a structured merge_report.json summarising the entire run.""" + report_path = Path(report_path) + report_path.parent.mkdir(parents=True, exist_ok=True) + + action_counts = {} + for d in (decisions or []): + action_counts[d["action"]] = action_counts.get(d["action"], 0) + 1 + + report = { + "timestamp": timestamp, + "dry_run": dry_run, + "outcome": outcome, + "outcome_reason": outcome_reason, + "threshold": threshold, + "baseline_pass_rate": baseline_rate, + "post_dedup_pass_rate": post_rate, + "forks": { + "accepted": accepted_forks, + "skipped": skipped_forks, + "pr_check_details": fork_pr_results, + }, + "merge_decisions": { + "summary": action_counts, + "details": decisions or [], + }, + } + with open(report_path, "w", encoding="utf-8") as fh: + json.dump(report, fh, indent=2) + return report_path + + +# --------------------------------------------------------------------------- +# Subprocess helpers +# --------------------------------------------------------------------------- + +def run_script(cmd, label): + """Run a command, stream output, return exit code.""" + print(f"\n{'='*70}") + print(f" {label}") + print(f"{'='*70}") + result = subprocess.run(cmd) + return result.returncode + + +def run_tests(entities_dir, manifest_path, output_dir, report_path, label, + pinned_rubrics_path=None): + """ + Generate pseudo-conversations filtered to main-repo slugs, then run evaluation. + + If pinned_rubrics_path is provided, must_include terms are taken from the + original main-repo manifest rather than re-derived from the merged entity. + This ensures forks cannot weaken rubrics and silently pass the regression gate. + + Returns the pass_rate float from the report JSON, or None on failure. + """ + pseudo_dir = Path(output_dir) / "pseudo_conversations" + results_dir = Path(output_dir) / "results" + + gen_cmd = [ + sys.executable, str(_gen_script), + "--entities-dir", str(entities_dir), + "--filter-slugs", str(manifest_path), + "--output-dir", str(pseudo_dir), + ] + if pinned_rubrics_path: + gen_cmd += ["--pinned-rubrics", str(pinned_rubrics_path)] + rc = run_script(gen_cmd, f"{label} — generate fixtures") + if rc != 0: + print(f"ERROR: fixture generation failed (exit {rc})", file=sys.stderr) + return None + + eval_cmd = [ + sys.executable, str(_eval_script), + "--pseudo-conversations-dir", str(pseudo_dir), + "--results-dir", str(results_dir), + "--report", str(report_path), + ] + rc = run_script(eval_cmd, f"{label} — run evaluation") + if rc not in (0, 1): # exit 1 just means some tests failed — still get the rate + print(f"ERROR: evaluation script crashed (exit {rc})", file=sys.stderr) + return None + + if not Path(report_path).exists(): + print(f"ERROR: report not written to {report_path}", file=sys.stderr) + return None + + with open(report_path, "r", encoding="utf-8") as fh: + report = json.load(fh) + return report.get("pass_rate", 0.0) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description="Merge fork entity libraries into main repo") + parser.add_argument( + "--fork-dirs", nargs="+", required=True, + help="Pre-cloned fork directories to merge from" + ) + parser.add_argument( + "--threshold", type=float, default=1.0, + help="Min rubric pass rate for main-repo tests (default: 1.0 = no regressions)" + ) + parser.add_argument( + "--version-diff-threshold", type=float, default=0.5, + help="Jaccard similarity below which dual-section merging is used (default: 0.5)" + ) + parser.add_argument( + "--dry-run", action="store_true", + help="Show all decisions without writing any entity files" + ) + parser.add_argument( + "--force-commit", action="store_true", + help="Skip the threshold gate and commit the merge regardless of pass rate" + ) + parser.add_argument( + "--report-dir", default=None, + help="Directory for dedup reports (default: .evolve/tests/dedup/)" + ) + parser.add_argument( + "--main-repo", default=None, + help="Upstream GitHub repo as 'owner/repo' (auto-detected from git remote if omitted)" + ) + parser.add_argument( + "--require-pr", action="store_true", default=True, + help="Skip forks that do not have an open PR against --main-repo (default: on)" + ) + parser.add_argument( + "--no-require-pr", dest="require_pr", action="store_false", + help="Disable the PR gate — merge all provided fork dirs regardless of PR status" + ) + parser.add_argument( + "--github-token", default=None, + help="GitHub personal access token for API calls (falls back to GITHUB_TOKEN env var)" + ) + parser.add_argument( + "--github-api-url", default=None, + help="GitHub API base URL (default: https://github.ibm.com/api/v3)" + ) + args = parser.parse_args() + + github_token = args.github_token or os.environ.get("GITHUB_TOKEN") + github_api_url = args.github_api_url or os.environ.get("GITHUB_API_URL", "https://github.ibm.com/api/v3") + + evolve_dir = get_evolve_dir() + live_entities = evolve_dir / "entities" + tmp_dir = evolve_dir / "tmp" + workspace_dir = tmp_dir / "merge-workspace" + workspace_entities = workspace_dir / "entities" + manifest_path = workspace_dir / "main_entity_slugs.json" + backup_dir = tmp_dir / "pre-merge-backup" + report_dir = Path(args.report_dir) if args.report_dir else evolve_dir / "tests" / "dedup" + eval_dir = evolve_dir / "tests" / "evaluation" + pre_report = eval_dir / "report.json" + post_report = eval_dir / "report_post.json" + merge_report_path = report_dir / "merge_report.json" + + run_timestamp = datetime.now(timezone.utc).isoformat() + + print(f"\nevolve-manager: merge-forks") + print(f" fork dirs : {args.fork_dirs}") + print(f" threshold : {args.threshold}") + print(f" version-diff-thr : {args.version_diff_threshold}") + print(f" dry-run : {args.dry_run}") + print(f" force-commit : {args.force_commit}") + print(f" require-pr : {args.require_pr}") + print(f" github-api-url : {github_api_url}") + print() + + if not live_entities.exists(): + print(f"ERROR: main entities directory not found: {live_entities}", file=sys.stderr) + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=[], accepted_forks=[], + skipped_forks=[], decisions=[], baseline_rate=None, post_rate=None, + threshold=args.threshold, outcome="error", + outcome_reason=f"Main entities directory not found: {live_entities}", + dry_run=args.dry_run, + ) + sys.exit(1) + + # ----------------------------------------------------------------------- + # Step 0 — PR gate + # ----------------------------------------------------------------------- + fork_pr_results = [] + accepted_fork_dirs = [] + skipped_forks = [] + main_repo = None # resolved below; kept in scope for PR-close step + + if args.require_pr: + main_repo = args.main_repo or _detect_main_repo(api_base=github_api_url, token=github_token) + if not main_repo: + print( + "ERROR: --require-pr is enabled but could not determine --main-repo.\n" + " Either pass --main-repo owner/repo or run from inside the git repo.", + file=sys.stderr, + ) + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=[], accepted_forks=[], + skipped_forks=list(args.fork_dirs), decisions=[], baseline_rate=None, + post_rate=None, threshold=args.threshold, outcome="error", + outcome_reason="Could not determine main repo for PR gate", + dry_run=args.dry_run, + ) + sys.exit(1) + + print(f"\n{'='*70}") + print(f" STEP 0 — PR gate (main repo: {main_repo})") + print(f"{'='*70}") + + for fork_dir in args.fork_dirs: + result = check_fork_has_open_pr(fork_dir, main_repo, token=github_token, api_base=github_api_url) + result["fork_dir"] = str(fork_dir) + fork_pr_results.append(result) + if result["has_pr"]: + print(f" ACCEPTED {fork_dir} — {result['reason']}") + accepted_fork_dirs.append(fork_dir) + else: + print(f" SKIPPED {fork_dir} — {result['reason']}") + skipped_forks.append(str(fork_dir)) + + if not accepted_fork_dirs: + print("\n No forks have an open PR against the main repo. Nothing to merge.") + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=fork_pr_results, + accepted_forks=[], skipped_forks=skipped_forks, + decisions=[], baseline_rate=None, post_rate=None, + threshold=args.threshold, outcome="skipped", + outcome_reason="All forks skipped: no open PRs found", + dry_run=args.dry_run, + ) + print(f"\n Merge report: {merge_report_path}") + sys.exit(0) + else: + # PR gate disabled — accept all + accepted_fork_dirs = list(args.fork_dirs) + for fd in args.fork_dirs: + fork_pr_results.append({ + "fork_dir": str(fd), "has_pr": None, + "pr_number": None, "pr_title": None, + "fork_owner": None, "pr_head": None, + "reason": "PR gate disabled (--no-require-pr)", + }) + + # ----------------------------------------------------------------------- + # Clean workspace from previous runs + # ----------------------------------------------------------------------- + if workspace_dir.exists(): + shutil.rmtree(workspace_dir) + workspace_dir.mkdir(parents=True, exist_ok=True) + + # ----------------------------------------------------------------------- + # Step 1 — Snapshot main-repo manifest + # ----------------------------------------------------------------------- + print(f"\n{'='*70}") + print(" STEP 1 — Snapshot main-repo entity manifest") + print(f"{'='*70}") + manifest = snapshot_main_manifest(live_entities, manifest_path) + print(f" Snapshotted {len(manifest)} main-repo entity slug(s) → {manifest_path}") + + # ----------------------------------------------------------------------- + # Step 2 — Versioning-aware merge + # ----------------------------------------------------------------------- + print(f"\n{'='*70}") + print(" STEP 2 — Versioning-aware merge") + print(f"{'='*70}") + fork_entities = collect_fork_entities(accepted_fork_dirs) + print(f" Discovered {len(fork_entities)} entity slug(s) across {len(accepted_fork_dirs)} fork(s)") + + decisions = merge_entities( + main_entities_dir=live_entities, + fork_entities=fork_entities, + workspace_entities_dir=workspace_entities, + manifest=manifest, + version_diff_threshold=args.version_diff_threshold, + dry_run=args.dry_run, + ) + + # Save updated manifest (now includes fork provenance) + if not args.dry_run: + with open(manifest_path, "w", encoding="utf-8") as fh: + json.dump(manifest, fh, indent=2) + + # Print merge summary + action_counts = {} + for d in decisions: + action_counts[d["action"]] = action_counts.get(d["action"], 0) + 1 + for action, count in sorted(action_counts.items()): + print(f" {action:<24} : {count}") + dual_slugs = [d["slug"] for d in decisions if d.get("action") == "dual-section"] + if dual_slugs: + print(f"\n Dual-section entities (significant divergence):") + for s in dual_slugs: + d = next(x for x in decisions if x["slug"] == s and x["action"] == "dual-section") + print(f" {s} (base v{d['base_version']} → fork v{d['fork_version']}, jaccard={d['jaccard']})") + + if args.dry_run: + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=fork_pr_results, + accepted_forks=[str(d) for d in accepted_fork_dirs], + skipped_forks=skipped_forks, decisions=decisions, + baseline_rate=None, post_rate=None, + threshold=args.threshold, outcome="dry-run", + outcome_reason="Dry run — no files written", + dry_run=True, + ) + print(f"\n Merge report: {merge_report_path}") + print("\n[dry-run] No files written. Exiting.") + sys.exit(0) + + # Build main-only slug filter (for test steps) + main_slugs = {k: v for k, v in manifest.items() if v["origin"] in ("main", "merged")} + main_filter_path = workspace_dir / "main_entity_slugs_filter.json" + with open(main_filter_path, "w", encoding="utf-8") as fh: + json.dump(main_slugs, fh, indent=2) + + # ----------------------------------------------------------------------- + # Step A — Quality gate on merged test fixtures + # ----------------------------------------------------------------------- + dedup_p1_args = [ + sys.executable, str(_dedup_script), + "--phase1-only", + "--entities-dir", str(workspace_entities), + "--report-dir", str(report_dir), + "--manifest-dir", str(workspace_entities), # build recall index from workspace, not live entities + ] + rc = run_script(dedup_p1_args, "STEP A — Quality gate (phase 1 only)") + if rc != 0: + print(f"\nERROR: Quality gate failed. Resolve entity issues before merging.") + print(f" Report: {report_dir / 'quality_gate_report.json'}") + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=fork_pr_results, + accepted_forks=[str(d) for d in accepted_fork_dirs], + skipped_forks=skipped_forks, decisions=decisions, + baseline_rate=None, post_rate=None, + threshold=args.threshold, outcome="error", + outcome_reason="Quality gate (phase 1) failed — resolve entity issues", + dry_run=args.dry_run, + ) + sys.exit(1) + print("\n Quality gate passed.") + + # ----------------------------------------------------------------------- + # Step B — Pre-dedup rubric test (main-repo entities only) + # ----------------------------------------------------------------------- + print(f"\n{'='*70}") + print(" STEP B — Pre-dedup rubric test (main-repo entities only)") + print(f"{'='*70}") + print(f" Using pinned rubrics from original main-repo snapshot: {manifest_path}") + baseline_rate = run_tests( + entities_dir=workspace_entities, + manifest_path=main_filter_path, + output_dir=str(eval_dir / "pre_dedup"), + report_path=str(pre_report), + label="Pre-dedup", + pinned_rubrics_path=manifest_path, + ) + if baseline_rate is None: + print("ERROR: Pre-dedup test run failed.", file=sys.stderr) + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=fork_pr_results, + accepted_forks=[str(d) for d in accepted_fork_dirs], + skipped_forks=skipped_forks, decisions=decisions, + baseline_rate=None, post_rate=None, + threshold=args.threshold, outcome="error", + outcome_reason="Pre-dedup rubric test run failed", + dry_run=args.dry_run, + ) + sys.exit(1) + print(f"\n Baseline pass rate (main-repo): {baseline_rate:.1%}") + + # ----------------------------------------------------------------------- + # Step C — Full skill dedup + # ----------------------------------------------------------------------- + dedup_full_args = [ + sys.executable, str(_dedup_script), + "--entities-dir", str(workspace_entities), + "--report-dir", str(report_dir), + "--manifest-dir", str(workspace_entities), # build recall index from workspace, not live entities + ] + rc = run_script(dedup_full_args, "STEP C — Full skill dedup (both phases)") + if rc != 0: + print(f"\nERROR: Full dedup failed.") + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=fork_pr_results, + accepted_forks=[str(d) for d in accepted_fork_dirs], + skipped_forks=skipped_forks, decisions=decisions, + baseline_rate=baseline_rate, post_rate=None, + threshold=args.threshold, outcome="error", + outcome_reason="Full dedup (both phases) failed", + dry_run=args.dry_run, + ) + sys.exit(1) + print("\n Full dedup complete.") + + # ----------------------------------------------------------------------- + # Step D — Post-dedup rubric test (main-repo entities only) + # ----------------------------------------------------------------------- + print(f"\n{'='*70}") + print(" STEP D — Post-dedup rubric test (main-repo entities only)") + print(f"{'='*70}") + print(f" Using pinned rubrics from original main-repo snapshot: {manifest_path}") + post_rate = run_tests( + entities_dir=workspace_entities, + manifest_path=main_filter_path, + output_dir=str(eval_dir / "post_dedup"), + report_path=str(post_report), + label="Post-dedup", + pinned_rubrics_path=manifest_path, + ) + if post_rate is None: + print("ERROR: Post-dedup test run failed.", file=sys.stderr) + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=fork_pr_results, + accepted_forks=[str(d) for d in accepted_fork_dirs], + skipped_forks=skipped_forks, decisions=decisions, + baseline_rate=baseline_rate, post_rate=None, + threshold=args.threshold, outcome="error", + outcome_reason="Post-dedup rubric test run failed", + dry_run=args.dry_run, + ) + sys.exit(1) + print(f"\n Post-dedup pass rate (main-repo): {post_rate:.1%}") + + # ----------------------------------------------------------------------- + # Step E — Threshold gate + # ----------------------------------------------------------------------- + print(f"\n{'='*70}") + print(" STEP E — Threshold gate") + print(f"{'='*70}") + print(f" Baseline : {baseline_rate:.1%}") + print(f" Post-dedup: {post_rate:.1%}") + print(f" Threshold : {args.threshold:.1%}") + + threshold_breached = post_rate < args.threshold and not args.force_commit + + if threshold_breached: + print(f"\n THRESHOLD BREACH: post-dedup pass rate {post_rate:.1%} < {args.threshold:.1%}") + print() + _print_regression_diff(report_dir, manifest) + print() + print(" Action required:") + print(" --force-commit to commit the merge anyway") + print(" --dry-run to inspect without writing") + print(" Or roll back from: .evolve/tmp/pre-merge-backup/") + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=fork_pr_results, + accepted_forks=[str(d) for d in accepted_fork_dirs], + skipped_forks=skipped_forks, decisions=decisions, + baseline_rate=baseline_rate, post_rate=post_rate, + threshold=args.threshold, outcome="threshold-breach", + outcome_reason=f"Post-dedup pass rate {post_rate:.1%} < threshold {args.threshold:.1%}", + dry_run=args.dry_run, + ) + print(f"\n Merge report: {merge_report_path}") + sys.exit(2) + + # ----------------------------------------------------------------------- + # Commit: backup live entities, move workspace into place + # ----------------------------------------------------------------------- + print(f"\n{'='*70}") + print(" COMMIT — Moving merged entities into .evolve/entities/") + print(f"{'='*70}") + + # Backup + if backup_dir.exists(): + shutil.rmtree(backup_dir) + shutil.copytree(live_entities, backup_dir) + print(f" Backup written: {backup_dir}") + + # Replace live entities with merged workspace + shutil.rmtree(live_entities) + shutil.copytree(workspace_entities, live_entities) + print(f" Merged entities committed to: {live_entities}") + + total_merged = sum(1 for v in manifest.values() if v["origin"] == "merged") + total_fork = sum(1 for v in manifest.values() if v["origin"] == "fork") + total_main = sum(1 for v in manifest.values() if v["origin"] == "main") + + write_merge_report( + merge_report_path, + timestamp=run_timestamp, fork_pr_results=fork_pr_results, + accepted_forks=[str(d) for d in accepted_fork_dirs], + skipped_forks=skipped_forks, decisions=decisions, + baseline_rate=baseline_rate, post_rate=post_rate, + threshold=args.threshold, outcome="success", + outcome_reason="Merged entities committed to .evolve/entities/", + dry_run=args.dry_run, + ) + + # ----------------------------------------------------------------------- + # Close PRs for all accepted forks + # ----------------------------------------------------------------------- + if args.require_pr and main_repo: + print(f"\n{'='*70}") + print(" CLOSING PRs — Notifying GitHub of completed merge") + print(f"{'='*70}") + + action_summary_parts = [] + for action, count in sorted(action_counts.items()): + action_summary_parts.append(f"- {count} entity/entities: `{action}`") + action_summary = "\n".join(action_summary_parts) if action_summary_parts else "- entities merged" + + for pr_info in fork_pr_results: + pr_number = pr_info.get("pr_number") + fork_dir_str = pr_info.get("fork_dir", "unknown fork") + if not pr_number: + continue # PR gate was disabled or fork was skipped + + merge_summary = ( + f"**Fork:** `{fork_dir_str}`\n" + f"**Merge actions:**\n{action_summary}\n" + f"**Baseline pass rate:** {baseline_rate:.1%}\n" + f"**Post-dedup pass rate:** {post_rate:.1%}" + ) + + pr_result = close_pr_with_comment( + main_repo=main_repo, + pr_number=pr_number, + merge_summary=merge_summary, + token=github_token, + api_base=github_api_url, + ) + + # Derive the web URL from the API base for the fallback close message + _web_host = github_api_url.replace("https://", "").replace("/api/v3", "").rstrip("/") + if pr_result["commented"] and pr_result["closed"]: + print(f" ✅ PR #{pr_number} commented and closed ({fork_dir_str})") + elif pr_result["commented"]: + print(f" ⚠️ PR #{pr_number} commented but not closed: {pr_result['error']} ({fork_dir_str})") + elif pr_result["closed"]: + print(f" ⚠️ PR #{pr_number} closed but comment failed: {pr_result['error']} ({fork_dir_str})") + else: + print(f" ❌ PR #{pr_number} — could not comment or close: {pr_result['error']} ({fork_dir_str})") + print(f" Close it manually: https://{_web_host}/{main_repo}/pull/{pr_number}") + else: + print("\n PR gate was disabled (--no-require-pr) — skipping PR close step.") + + # ----------------------------------------------------------------------- + # Step F — Regenerate unpinned fixtures to show what new/changed tests look like + # ----------------------------------------------------------------------- + print(f"\n{'='*70}") + print(" STEP F — Generate unpinned fixtures for new tests (informational)") + print(f"{'='*70}") + live_pseudo = evolve_dir / "tests" / "pseudo_conversations" + gen_new_cmd = [ + sys.executable, str(_gen_script), + "--entities-dir", str(live_entities), + "--output-dir", str(live_pseudo), + ] + rc = run_script(gen_new_cmd, "Generate new unpinned fixtures") + if rc == 0: + print(f"\n New fixtures written to: {live_pseudo}") + print(f" Run /evolve-lite-run-tests to validate them.") + else: + print(f"\n WARNING: Unpinned fixture generation failed (exit {rc}). Continuing.") + + print() + print(f" Summary:") + print(f" main-only entities : {total_main}") + print(f" fork-only entities : {total_fork}") + print(f" merged entities : {total_merged}") + print(f" total : {len(manifest)}") + print(f" baseline rate : {baseline_rate:.1%}") + print(f" post-dedup rate : {post_rate:.1%}") + print() + print(f" Reports:") + print(f" merge report : {merge_report_path}") + print(f" quality gate : {report_dir / 'quality_gate_report.json'}") + print(f" refine : {report_dir / 'refine_report.json'}") + print(f" pre-dedup eval : {pre_report}") + print(f" post-dedup eval: {post_report}") + print() + print(f" New test fixtures: {live_pseudo}") + print() + print("Done.") + sys.exit(0) + + +def _print_regression_diff(report_dir, manifest): + """Print a human-readable diff of removed/merged entities from refine_report.json.""" + refine_report = Path(report_dir) / "refine_report.json" + if not refine_report.exists(): + print(" (no refine report available)") + return + with open(refine_report, "r", encoding="utf-8") as fh: + report = json.load(fh) + clusters = report.get("clusters", []) + if not clusters: + print(" (no cluster decisions in refine report)") + return + print(" Dedup decisions affecting main-repo entities:") + for cluster in clusters: + decision = cluster.get("decision", "keep-all") + if decision == "keep-all": + continue + members = cluster.get("members", []) + for m in members: + slug = Path(m.get("path", "")).stem + origin = manifest.get(slug, {}).get("origin", "unknown") + label = "[main]" if origin in ("main", "merged") else "[fork]" + status = "KEPT" if m.get("kept") else "REMOVED" + print(f" {label} {slug:<50} {decision:<8} {status}") + + +if __name__ == "__main__": + main() + +# Made with Bob