diff --git a/.claude/launch.json b/.claude/launch.json index 58c46cc6d..43a6c535b 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -19,7 +19,8 @@ "main.py" ], "port": 8000, - "cwd": "backend/src/apis/app_api" + "cwd": "backend/src/apis/app_api", + "autoPort": false }, { "name": "inference-api", @@ -28,7 +29,8 @@ "main.py" ], "port": 8001, - "cwd": "backend/src/apis/inference_api" + "cwd": "backend/src/apis/inference_api", + "autoPort": false }, { "name": "docs-site", diff --git a/.claude/skills/cutting-a-release/SKILL.md b/.claude/skills/cutting-a-release/SKILL.md index abf11a98a..fa41973d3 100644 --- a/.claude/skills/cutting-a-release/SKILL.md +++ b/.claude/skills/cutting-a-release/SKILL.md @@ -181,6 +181,67 @@ flag, or skip path**. So a failed two-index deploy costs two patch releases --- +## 1b. Pre-merge prerequisite — data backfills + +A backfill script is not code that runs itself. Someone has to run it, against +each environment, in a window that matters. **If this release adds one, the +release notes must name it** — that is the only place an operator looks. + +### Why this is its own gate + +The failure is silent, and in the same shape as §1. Backfills populate a sparse +index or a new attribute, and the read path does not error when the data is +missing — a sparse index returns **fewer rows**, not an exception. An unrun +backfill therefore looks like a short list, not an outage: the tool catalog +missing half its tools, the artifact library missing the older entries. Nothing +goes red, and no alarm fires. + +`develop` cannot surface it either. Whoever wrote the script ran it against dev +by hand the day they wrote it, so dev is always fine. **Prod is the environment +with nobody assigned**, and the release is the last moment the instruction can +still reach someone. + +### How to check + +CI enforces this on every PR into `main` +(`.github/workflows/pending-backfills.yml`). Locally: + +```bash +node scripts/release/check-pending-backfills.mjs +``` + +It diffs `backend/scripts/backfill_*.py` between `origin/main` and your branch +and fails when a script added in the range is not named in `RELEASE_NOTES.md`. + +It cannot verify the backfill was actually **run** — no CI job can know that. +It guarantees the instruction reaches the person who can. + +### What to write + +Name the script, the command, and the environments. Not "a backfill is +required" — that is the note that sends someone digging through `git log`: + +> **Manual step — run before/with this deploy.** Populates `EntityTypeIndex` +> for tool rows written before it existed. Idempotent; dry-run by default. +> +> ```bash +> AWS_PROFILE= python backend/scripts/backfill_tool_catalog_index.py \ +> --table -app-roles --region us-west-2 --apply +> ``` +> +> Verify `skipped=0 failed=0` and that the index item count matches the tool +> count before considering the deploy complete. + +### Ordering, when the release also switches a read onto the backfilled data + +Deploy → backfill → *then* the read. If the release carries both the backfill +and the code that depends on it, the backfill runs **inside the release +window**, not afterwards. A fresh deployment is exempt only if its seed path +already writes the new keys — check, do not assume: `seed_bootstrap_data.py` +hand-builds its items rather than going through the model. + +--- + ## 2. Branch workflow ```bash diff --git a/.claude/skills/kaizen-research/SKILL.md b/.claude/skills/kaizen-research/SKILL.md index 8b9ae40f9..2e5b6bceb 100644 --- a/.claude/skills/kaizen-research/SKILL.md +++ b/.claude/skills/kaizen-research/SKILL.md @@ -41,10 +41,23 @@ Friday early morning (~6am MT). `kaizen-review-prep` runs ~2 hours later (~8am M We carry custom code that exists only because Strands' caching support is Bedrock/Anthropic-shaped. Upstream is converging on a provider-agnostic - `CacheConfig`, so this is a **subtraction** item: each upstream landing should - delete code here, not add it. Check every run, and check it for *all* cacheable - families — Anthropic, OpenAI/GPT, and any newly cacheable model — not just the - one that prompted this. + `CacheConfig`, so this is *mostly* a **subtraction** item: each upstream landing + should delete code here, not add it. Check every run, and check it for *all* + cacheable families — Anthropic, OpenAI/GPT, and any newly cacheable model — not + just the one that prompted this. + + ⚠️ **Amended 2026-09-11: it is not subtraction-only, and the second half is now + the more valuable one.** Two kinds of drift matter, and only the first deletes + code. **(i) Convergence** — upstream absorbs something we hand-roll; propose the + swap. **(ii) Coverage** — a Bedrock family gains prompt caching that *neither* + we nor upstream will use, because both gate on the same `_cache_strategy` + string test for `"claude"`/`"anthropic"`. Measured that day: **Nova Micro + accepts an explicit system cachePoint and honors it** (7,203 input tokens → 2, + 7,201 cache-written) while `bedrock_cache_points_supported()` refuses it. That + is caching left on the floor, it grows every time AWS ships a cacheable model, + and no test can catch it — a string predicate keeps answering the same thing + while the platform moves underneath it. Coverage findings are **additions**; + file them anyway. Our carried code, and what would retire it: @@ -55,6 +68,7 @@ Friday early morning (~6am MT). `kaizen-review-prep` runs ~2 hours later (~8am M | `build_prompt_cache_key()` in `bedrock_responses.py` | `strands/models/_openai_cache.py::apply_cache_config` — **already on upstream main**, maps `CacheConfig.cache_key` → `prompt_cache_key`. Adopt on the next pin bump. | | `apply_explicit_prompt_cache()` (breakpoints) | No upstream equivalent yet — `apply_cache_config` emits no `prompt_cache_breakpoint`. Ours is OFF by default (measured 57% worse); don't re-enable without re-running the probe. | | `cache_ttl_seconds_for()` in `observability/prompt_cache.py` | A model-derived TTL upstream. Note `apply_cache_config` maps ttl to `prompt_cache_retention` (`in_memory`/`24h`), *not* GPT-5.6's `prompt_cache_options.ttl: "30m"` — so these are not yet the same concept. | + | `bedrock_cache_points_supported()` + the hand-placed system cachePoint | **NOTHING upstream can retire this — settled by measurement 2026-09-11, do not re-propose.** `format_request` copies `system_prompt_content` verbatim and `_apply_system_cache_ttl` never removes a point, so the block reaches Bedrock, which answers `AccessDeniedException` on a model that can't cache. The rejection is Bedrock's, not the SDK's. Only its `tools_ttl` half is redundant (upstream applies the same test at `bedrock.py:579`). | Each run, answer: - Does the pinned Strands version now ship `_openai_cache.py` / a `CacheConfig` @@ -64,7 +78,17 @@ Friday early morning (~6am MT). `kaizen-review-prep` runs ~2 hours later (~8am M that maps onto something we do manually? - Any new Bedrock model family with prompt caching? Confirm which API surface serves it — GPT-5.6 caches **only** over the Responses API, and the same model - over Converse caches not at all. + over Converse caches not at all. Then **measure it, don't read it**: + + cd backend + AWS_PROFILE=dev-ai uv run python scripts/probe_bedrock_cache_point_support.py \ + --model-id + + One command, ~$0.01, and it answers both layers at once — what the pinned + Strands emits for that id, and whether Bedrock accepts and honors an explicit + cachePoint. A row where Bedrock caches but `_cache_strategy` is `None` is a + coverage finding (ii) — file it. Use `--offline-only` for a free check of the + SDK half alone after a pin bump. - Movement on #3546 / #4193, or a new `Usage` convention. Both change what our cost math may assume. @@ -73,6 +97,15 @@ Friday early morning (~6am MT). `kaizen-review-prep` runs ~2 hours later (~8am M in a live measurement. `backend/scripts/probe_gpt56_cache_rates.py` is the gate: beat the current arm, measured, before switching. + ⚠️ **A caching A/B measures nothing if its prefix is under the model's minimum.** + Below that floor Bedrock **silently ignores** the cache point — no error, no + cache buckets, a result indistinguishable from "unsupported". Measured on Haiku + 4.5 in us-west-2 (2026-09-11): 2,351 and 3,911 tokens wrote **nothing**, 5,202 + wrote in full — a real floor of **4,096**, twice the 2,048 the first-party + Anthropic docs publish for Haiku. Do not trust a vendor-documented minimum; + bracket it. This already produced one false "does not cache" against our own + production model. + 3. **Reference repo — `aws-samples/sample-strands-agent-with-agentcore`** - https://github.com/aws-samples/sample-strands-agent-with-agentcore/commits/main - Diff the last 7 days (or "since last research run" — whichever is longer). Identify new patterns, removed approaches, or fixes that map to constructs in this repo: agent setup, tool registration, AgentCore Identity flows, Memory configuration, Gateway/MCP wiring. diff --git a/.claude/skills/tailwind-ui/references/app-conventions.md b/.claude/skills/tailwind-ui/references/app-conventions.md index 1ee596da7..a692583a8 100644 --- a/.claude/skills/tailwind-ui/references/app-conventions.md +++ b/.claude/skills/tailwind-ui/references/app-conventions.md @@ -15,22 +15,146 @@ older boxed-card style in `model-form.page.html`. | Helper & meta text | `text-xs/5` | | Page title (`h1`) | `text-2xl/8 font-bold` | | Section heading (`h2`) | `text-base/7 font-semibold` | -| Accent color | `blue` (600/500) — never `indigo` | -| Focus ring (inputs) | `focus:border-blue-500 focus:outline-none focus:ring-2 focus:ring-blue-500` | -| Focus ring (buttons/links) | `focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-blue-500` | +| Accent color | brand `primary-*` — never raw `blue-*` for an affordance, never `indigo` | +| Solid brand fill / brand text | `primary-accessible` (+ `dark:*-accessible-dark` for text) | +| Chip / badge / icon tile / selected-row fill | `bg-gray-100` + `text-primary-accessible`, `dark:bg-gray-700` + `dark:text-primary-50` | +| Focus ring (inputs) | `focus:border-primary-500 focus:outline-none focus:ring-2 focus:ring-primary-500` | +| Focus ring (buttons/links) | `focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary-500` | Every token has a dark-mode pair (`dark:*`). Test both modes. +**Brand colour, not Tailwind blue.** These rows used to say `blue-600/500`, which +predates the generated brand theme. The palette now comes from `brand.config.ts` +via `styles/generated/brand-theme.css`, which emits an 11-step `--color-primary-*` +scale plus two contrast-guaranteed aliases: `primary-accessible` (AA against the +light surface) and `primary-accessible-dark` (AA against the dark one). Use the +alias for solid fills with white text and for brand-coloured text; use +`primary-500` for focus rings. Raw `blue-*` is a different hue from the brand +(`#2563eb` vs `#0033a0`) and does not follow a rebrand. The code agrees — zero +files use `focus:ring-blue-500` or `focus-visible:outline-blue-500`, against 74 +and 94 respectively for the `primary` equivalents. + +**`primary` is not a tint ramp — never `bg-primary-50/100/200` as a fill.** The +scale is generated from the brand hex by lightness offset alone and keeps full +chroma at every step, so `primary-50` is not the pale wash its name implies: at +`#0033a0` it resolves to `rgb(118, 179, 255)`, a saturated mid-blue. Used as a +chip, badge, icon tile or selected-row background it reads as a blue blob behind +small text, and it fails AA — `text-primary-accessible` on `bg-primary-100` is +4.13:1, and `hover:bg-primary-200` drops it to 3.52:1. `text-gray-500` sub-labels +on `bg-primary-50` are 2.23:1. The `state-*` scales *are* real tints +(`state-success-50` = `rgb(240, 253, 244)`), which is exactly why the pattern +looks safe by analogy and isn't. + +Use a neutral surface and put the brand in the text instead: + +```html + + + + +
+``` + +Two traps that follow from this: + +- **`dark:text-primary-accessible-dark` is guaranteed against the page, not against a + tinted fill.** On `dark:bg-primary-900/30` it measures 4.15:1 and fails. On a neutral + `dark:bg-gray-700` use `dark:text-primary-50` (4.74:1). +- **A fraction is a different thing.** `bg-primary-50/40` composites to + `rgb(200, 225, 255)` — an actual pale wash, and fine for a large transient surface + such as a drag-and-drop target. The ban is on the opaque steps. + +The one sanctioned exception is *decorative* colour that isn't standing in for the +brand — e.g. the agent-detail hero's `bg-linear-to-br from-blue-700 to-sky-500` +backdrop and the near-white pill on top of it. That is a picture, not an +affordance. Anything a user clicks, focuses, or reads as "this is the product's +colour" uses the brand tokens. + ## Page shell ```html
- +
``` +## Top-level user-facing pages + +The pages a non-admin lands on from the sidenav — `/agents` (all three tabs), +`/customize` (all three tabs), `/artifacts`, `/my-skills`, `/schedules`, +`/memory-spaces` — use a **larger header and a wider shell** than the admin +tables above. These are destinations, not records-management screens, and the +`text-2xl/8` admin title reads as a section label rather than a page. + +```html +
+
+ + +
+

+ Title +

+

+ One sentence on what the page is for. +

+
+
+
+``` + +| Element | Token | +|---------|-------| +| Shell | `mx-auto max-w-6xl px-4 py-8 sm:px-6 lg:px-8` | +| Header block | `mt-6 mb-10` (drop `mt-6` when no tab strip sits above it) | +| `h1` | `text-2xl font-bold tracking-tight text-gray-900 sm:text-3xl dark:text-white` | +| Subtitle | `mt-1.5 max-w-2xl text-sm/6 text-gray-600 dark:text-gray-400` | +| Card grid | `grid grid-cols-1 gap-5 sm:grid-cols-2 lg:grid-cols-3` (`gap-4` for the denser Customize toggle cards) | +| Primary button | `inline-flex items-center gap-2 rounded-2xl bg-primary-accessible px-4 py-2.5 text-sm/6 font-semibold text-white shadow-xs transition hover:brightness-95 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary-500` | +| Secondary button | same shell, `rounded-2xl border border-gray-200 bg-white px-3.5 py-1.5 text-sm/6 font-medium text-gray-700` | +| Search field | `block w-full rounded-full border border-gray-300 bg-white py-2.5 pl-10 pr-4 text-sm/6 …` in a `relative max-w-md` wrapper, with the `heroMagnifyingGlass` icon at `left-4` | +| Filter chip | `rounded-full border px-3.5 py-1 text-sm/6 font-medium`; active `border-gray-900 bg-gray-900 text-white dark:border-white dark:bg-white dark:text-gray-900` | + +Use `bg-primary-accessible`, never a raw `bg-primary-500` fill: the two resolve to +the same hex for the current brand, but only the alias is guaranteed AA against +white text after a rebrand. + +### Pill tabs (hub strips and in-page filters) + +One idiom, whether the tabs are routes (`AgentsTabsComponent`, +`CustomizeTabsComponent`) or an in-page filter (the Artifacts All/Yours/Shared +strip). A raised white pill on a recessed gray shell — **not** a solid brand fill, +and **not** an underline: the brand token is a fixed colour with no dark variant, +so an underline in it all but disappears on the dark surface. + +```html + +``` + +`inline-flex`, never `flex`: a strip stretched to the content width reads as a +segmented control over the whole page instead of as N choices. + +The grid/list view toggle is the same idiom one size down — `rounded-xl` shell, +`grid size-8 place-items-center rounded-lg` buttons, same raised-active classes — +and is a `role="radiogroup"` of `role="radio"` buttons, since it is one setting +with two values rather than two independent toggles. The smaller radii are +correct *there* because the control reads as segments inside a shell, not as +buttons. + +**Standalone buttons are `rounded-2xl` everywhere in the app** — user-facing +pages, admin lists and forms, and dialogs alike. There is deliberately no +per-surface exception: the earlier split between a user-facing radius and an +admin radius is what let `rounded-sm`/`rounded-md`/`rounded-lg` buttons drift in +between them. The only radii that are not `rounded-2xl` on a clickable element +are the segment children described above and the `rounded-full` chips, search +field and floating pill CTA. + ## Form pages Flat `
` blocks separated by a top border — **no boxed section cards**. @@ -55,7 +179,7 @@ Field:

Error message

@@ -65,7 +189,7 @@ Select (`rounded-2xl` selects need a custom chevron — see "Selects" below): ```html
- +
``` @@ -74,10 +198,10 @@ Buttons: ```html - + - + @@ -90,7 +214,7 @@ Buttons: Underline tabs inside a dialog or section — use `aria-selected` to drive the active state so styling rides an attribute-selector variant. Do **not** use parallel -`[class.border-b-blue-600]` bindings (see "Common gotchas"). +`[class.border-b-primary-accessible]` bindings (see "Common gotchas"). ```html
@@ -99,7 +223,7 @@ state so styling rides an attribute-selector variant. Do **not** use parallel role="tab" [attr.aria-selected]="active()" (click)="active.set(true)" - class="-mb-px inline-flex items-center gap-1.5 border-b-2 border-b-transparent px-3 py-2 text-sm/6 font-medium text-gray-600 hover:text-gray-900 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-blue-500 aria-selected:border-b-blue-600 aria-selected:font-semibold aria-selected:text-blue-600 dark:text-gray-400 dark:hover:text-white dark:aria-selected:border-b-blue-400 dark:aria-selected:text-blue-400" + class="-mb-px inline-flex items-center gap-1.5 border-b-2 border-b-transparent px-3 py-2 text-sm/6 font-medium text-gray-600 hover:text-gray-900 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary-500 aria-selected:border-b-primary-accessible aria-selected:font-semibold aria-selected:text-primary-accessible dark:text-gray-400 dark:hover:text-white dark:aria-selected:border-b-primary-accessible-dark dark:aria-selected:text-primary-accessible-dark" > Tab label @@ -118,7 +242,7 @@ state so styling rides an attribute-selector variant. Do **not** use parallel ### Conditional Tailwind classes can lose the cascade Two classes that set the same property at the same specificity (`border-b-transparent` -base + `[class.border-b-blue-600]="active()"`) collide. Whichever Tailwind emits **later** +base + `[class.border-b-primary-accessible]="active()"`) collide. Whichever Tailwind emits **later** in the stylesheet wins, regardless of class order in your `class="…"` string. In practice the transparent base wins and the active underline never appears. @@ -129,7 +253,7 @@ base utility (`0,1,0`): ```html + class="border-b-2 border-b-transparent aria-selected:border-b-primary-accessible">… ``` DevTools symptom: the conditional class IS on the DOM, but `getComputedStyle(el).borderBottomColor` returns `rgba(0, 0, 0, 0)`. If you see that, this is the bug. @@ -167,4 +291,7 @@ Empty state: Chip / badge: `inline-flex items-center rounded-2xl px-2.5 py-0.5 text-xs/5 font-medium` plus a tinted `bg-*-100 text-*-800` pair (status: green/yellow/red/blue; role tags: purple). -Spinner: `animate-spin rounded-full border-4 border-gray-300 border-t-blue-600 dark:border-gray-600 dark:border-t-blue-400` (use `border-2` at `size-5` or smaller). +Spinner: use the shared `` (`components/spinner/`), which +92 files already do — hand-rolling one is almost always wrong. If you must inline one: +`animate-spin rounded-full border-4 border-gray-300 border-t-primary-accessible dark:border-gray-600 dark:border-t-primary-accessible-dark` +(use `border-2` at `size-5` or smaller). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2321fe0a8..8701da4d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,11 @@ jobs: with: run_backend: true run_frontend: true + # The frontend suite again under --coverage — the nightly's invocation. + # Coverage changes how the specs are built (see tests.yml), so specs can + # pass here and fail there. Only the PR gate pays for it; the deploy + # workflows keep running the plain suite. + run_frontend_coverage: true run_infra: true # Pure-logic tests for tests/load. Deliberately only on the PR gate: the # deploy workflows run test suites to protect a deploy, and the load diff --git a/.github/workflows/pending-backfills.yml b/.github/workflows/pending-backfills.yml new file mode 100644 index 000000000..86e6fa1db --- /dev/null +++ b/.github/workflows/pending-backfills.yml @@ -0,0 +1,106 @@ +name: "Pending Backfills" + +# Release gate: a release that ships a data backfill must say so in RELEASE_NOTES.md. +# +# A backfill script is not code that runs itself. Someone has to run it, against +# each environment, in a window that matters — and the release notes are the only +# place an operator looks for that. A backfill that ships unannounced is a +# migration nobody performs. +# +# This only triggers on PRs into `main`, for the same reason the GSI gate does: +# whoever writes a backfill runs it against dev by hand the day they write it, so +# `develop` is always fine. Prod is the environment with nobody assigned, and a +# release is the only moment the instruction can still reach someone. +# +# The failure mode is silent. Backfills populate a sparse index or a new +# attribute, and the code that reads them does not error when the data is +# missing — a sparse index returns FEWER ROWS, not an exception. An unrun +# backfill therefore looks like a short list (half the tool catalog, the older +# artifacts missing), not an outage. Nothing goes red. +# +# See .claude/skills/cutting-a-release/SKILL.md §1b. + +on: + workflow_dispatch: + pull_request: + branches: + - main + +permissions: + contents: read + +# Safe to cancel superseded runs: this reads a git diff and a markdown file and +# does no AWS work, so there is no deploy hazard in interrupting it. +concurrency: + group: pending-backfills-${{ github.ref }} + cancel-in-progress: true + +jobs: + pending-backfills: + name: Pending Backfills + runs-on: ubuntu-24.04 + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Fetch main branch + run: | + git fetch origin main + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + + # Diffs backend/scripts/backfill_*.py between origin/main and this PR and + # requires each ADDED script to be named in RELEASE_NOTES.md. No install + # needed — both sides are already in git. + # + # This gate cannot verify the backfill was RUN; no CI job can know that. It + # guarantees the instruction reaches the person who can. + - name: Check pending backfills + id: backfill-check + continue-on-error: true + run: | + node scripts/release/check-pending-backfills.mjs --base origin/main \ + 2>&1 | tee /tmp/backfill-check-output.txt + + # tee masks the script's exit status, so read it back from PIPESTATUS. + exit "${PIPESTATUS[0]}" + + - name: Generate summary + if: always() + run: | + source scripts/common/summary.sh + + if [ "${{ steps.backfill-check.outcome }}" = "success" ]; then + STATUS="success" + else + STATUS="failure" + fi + + write_workflow_header "Pending Backfills" "${STATUS}" + + { + echo '```' + cat /tmp/backfill-check-output.txt 2>/dev/null || echo "(no output captured)" + echo '```' + echo "" + if [ "${STATUS}" != "success" ]; then + echo "This release adds a backfill script that the release notes do not name." + echo "Add it, with the exact command and the environments it must run against," + echo "so whoever deploys knows there is a step to perform. An unrun backfill" + echo "fails **silently** — as a short list, not an error." + fi + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Evaluate result + run: | + if [ "${{ steps.backfill-check.outcome }}" != "success" ]; then + echo "::error::This release adds a backfill script that RELEASE_NOTES.md does not name. Add it with the command to run and the environments it applies to — see .claude/skills/cutting-a-release/SKILL.md §1b." + exit 1 + fi + echo "Pending-backfill check passed." diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index b1dc8791a..9bc5360de 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -250,6 +250,9 @@ jobs: CDK_OBSERVABILITY_ALB_P99_LATENCY_MS: ${{ vars.CDK_OBSERVABILITY_ALB_P99_LATENCY_MS }} CDK_OBSERVABILITY_AGENTCORE_LATENCY_MS: ${{ vars.CDK_OBSERVABILITY_AGENTCORE_LATENCY_MS }} CDK_OBSERVABILITY_AGENTCORE_ERROR_THRESHOLD: ${{ vars.CDK_OBSERVABILITY_AGENTCORE_ERROR_THRESHOLD }} + # Concurrent AgentCore Runtime sessions. A cost signal — Runtime bills + # memory for the whole session lifetime — not a quota one. + CDK_OBSERVABILITY_AGENTCORE_ACTIVE_SESSION_THRESHOLD: ${{ vars.CDK_OBSERVABILITY_AGENTCORE_ACTIVE_SESSION_THRESHOLD }} CDK_OBSERVABILITY_LAMBDA_ERROR_THRESHOLD: ${{ vars.CDK_OBSERVABILITY_LAMBDA_ERROR_THRESHOLD }} CDK_OBSERVABILITY_LAMBDA_DURATION_PERCENT_OF_TIMEOUT: ${{ vars.CDK_OBSERVABILITY_LAMBDA_DURATION_PERCENT_OF_TIMEOUT }} CDK_OBSERVABILITY_DYNAMO_THROTTLE_THRESHOLD: ${{ vars.CDK_OBSERVABILITY_DYNAMO_THROTTLE_THRESHOLD }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 12c16a8c6..21d771988 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,6 +21,11 @@ on: required: false default: false type: boolean + run_frontend_coverage: + description: 'Run the frontend suite a second time under --coverage (the nightly invocation)' + required: false + default: false + type: boolean run_infra: description: 'Run infrastructure jest suite' required: false @@ -82,6 +87,51 @@ jobs: working-directory: frontend/ai.client run: npm run test:ci + # The SAME frontend suite, run the way the nightly runs it: `ng test + # --no-watch --coverage`, via the very script nightly.yml invokes. + # + # Why this is a separate gate rather than a flag on `test:ci`: under + # --coverage, @angular/build's unit-test builder bundles each spec into a + # flattened chunk emitted at the PROJECT ROOT instead of handing Vitest the + # spec at its own source path. `import.meta.url` and the `__dirname` shim + # move with it, so every filesystem-reading spec resolves relative paths + # from a different directory than it does without coverage. That is a real + # failure mode with no watered-down version — a doc-presence spec reads the + # wrong README and a hygiene guard walks generated CSS and "finds" + # violations — and until this job existed NOTHING on a pull request ever + # passed --coverage, so the class could only be discovered by the nightly, + # after merge. It went undetected for seven consecutive nights. + # + # It is its own job, and enabled only on the PR gate (ci.yml), because the + # deploy workflows run `test-frontend` to protect a deploy and gain nothing + # from paying for instrumentation there. Running in parallel with + # test-frontend costs no extra wall-clock on the PR. + # + # Coverage artifacts are deliberately NOT uploaded here: the nightly owns + # coverage *reporting* (analyze-coverage compares runs over time). This job + # only cares that the suite passes under the instrumented build. + test-frontend-coverage: + if: ${{ inputs.run_frontend_coverage }} + name: Test frontend (coverage build) + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: frontend/ai.client/package-lock.json + - name: Install + working-directory: frontend/ai.client + run: npm ci --prefer-offline + - name: Run tests with coverage + run: bash scripts/frontend/test.sh + test-infra: if: ${{ inputs.run_infra }} name: Test infrastructure (jest) diff --git a/.kiro/specs/kb-chunk-inspector/design.md b/.kiro/specs/kb-chunk-inspector/design.md new file mode 100644 index 000000000..2a45fdac2 --- /dev/null +++ b/.kiro/specs/kb-chunk-inspector/design.md @@ -0,0 +1,105 @@ +# KB Chunk Inspector — Design + +**Status:** Draft · Reads only. Built entirely on the existing retrieval facade. + +## 1. The one hard constraint + +Bedrock managed knowledge bases have **no "list the chunks of a document" API**. +The only read primitive is `Retrieve`, which is *query-driven* and *rank-bounded* +by `numberOfResults` (`retrieval_configuration` in +`backend/src/apis/shared/kb_backend/managed_backend.py` sets +`managedSearchConfiguration.numberOfResults`). `GetKnowledgeBaseDocuments`/ +`ListKnowledgeBaseDocuments` list *documents*, not chunks. + +Consequence: we cannot promise a complete, in-order dump of a document's chunks +on the managed engine. We can get "the top-N chunks `Retrieve` returns for this +document, by relevance to a query, filtered to this document_id." That is more +than enough to answer *"did my columns/headers/image survive?"* — you will see +the vision descriptions and the flattened tables — but the spec (Req 3) requires +we present it honestly as "up to N", not "all". + +This is why the feature is framed as an **inspector**, not an **exporter**. + +## 2. Reuse, don't invent + +Everything needed already exists: + +- **Facade search**, both engines: + `async backend.search(kb_ref, query, top_k, retrieval_filter=None) -> List[Chunk]` + (`protocol.py`; managed + legacy adapters). `Chunk` carries `text`, `metadata` + (incl. `document_id`, `filename`/`source`), `s3_key`, `relevance`. +- **Per-document isolation filter**, already proven safe and used by the + retrievability probe: `{"equals": {"key": "document_id", "value": }}` + (`equals` is in `ISOLATION_SAFE_FILTER_OPERATORS`). +- **Engine resolution**: `resolve_engine_for(assistant_id, record=...)`. +- **Permission gate**: `_require_edit_permission(assistant_id, current_user)` in + `backend/src/apis/app_api/documents/routes.py`. +- **Citation display shape** on the frontend already renders + `{documentId, fileName, text}` — the inspector renders the same shape, longer. + +## 3. Backend — one new read endpoint + +`GET /assistants/{assistant_id}/documents/{document_id}/chunks` +in `backend/src/apis/app_api/documents/routes.py`. + +1. `owner_id = await _require_edit_permission(assistant_id, current_user)`. +2. Confirm the `DOC#` row exists and is `complete` (else 409 "still processing" + / 404). +3. `engine, record = resolve_engine_for(...)`; get the backend via the resolver. +4. Enumerate chunks (see §4), filtered to `document_id`. +5. Return `{ documentId, fileName, engine, chunks: [{ text, page?, order, score? }], + complete: bool, returned: N, capReached: bool }`. + +Read-only, no writes, no new ingestion path (Req 5). Bounded `Retrieve` calls +(Req 6). + +## 4. Enumerating chunks (the honest best-effort) + +`Retrieve` needs a query and returns top-N by relevance. To approximate "all +chunks of this document": + +- **Query text:** use a neutral, document-anchored query — the filename, or the + document's own leading text — purely to give the reranker *something*; the + `document_id` `equals` filter is what actually scopes results. Ranking order is + not meaningful here, so the UI presents chunks as an unordered set (or by page + when metadata carries it), not as "chunk 1..N". +- **`numberOfResults`:** request the backend maximum (Bedrock's documented ceiling + is 100 per call). If the count returned equals the ceiling, set + `capReached=true` and surface Req 3's "up to N / more may exist" note. +- **De-dup** by chunk content hash across calls (a query-ranked API can repeat). +- **Pagination:** only if we later find a reliable way to page distinct chunks; + v1 ships single-call, capped, honest. + +> Design note: because completeness is not guaranteed, do **not** compute +> "missing header in chunk 2" server-side — just show what came back and let the +> human eyeball it (matches the task-16.2 "guidance not code" decision). + +Legacy engine: the same facade call works; we own chunking there so results are +closer to complete, but we keep the same "up to N" contract for a uniform shape. + +## 5. Frontend + +- On the document row / detail (where status already shows), add a **"View + extracted content"** action, enabled once status is `complete`. +- Opens a panel listing each chunk's full text (monospace/pre for tables), + page/location when present, and a small header: *"This is what the knowledge + base extracted. If a table or image looks wrong here, the assistant will answer + from this — consider reformatting your source."* (the actionable nudge). +- If `capReached`, show the "showing first N; more may exist" line. +- Reuse the existing citation card component; the only new data is longer text + and the header/nudge. + +## 6. Testing + +- Route: owner sees chunks (managed + legacy), non-owner 403, non-`complete` + document 409, filter is `equals` on `document_id` (assert the exact filter — + mirrors the retrievability-probe test), `capReached` set when N == ceiling. +- Use moto + a fake backend modeling `search(..., retrieval_filter=...)` + (the ingestion-consumer tests already have this fake to copy). +- Mutation guard: dropping the `document_id` filter must fail a test that seeds a + second document's chunks and asserts they never appear. + +## 7. Effort + +Small–medium. One read endpoint + one resolver/facade call + a frontend panel +reusing the citation card. No new pipeline, no infra, no legacy dependency. diff --git a/.kiro/specs/kb-chunk-inspector/requirements.md b/.kiro/specs/kb-chunk-inspector/requirements.md new file mode 100644 index 000000000..0228372e2 --- /dev/null +++ b/.kiro/specs/kb-chunk-inspector/requirements.md @@ -0,0 +1,104 @@ +# KB Chunk Inspector — Requirements + +**Status:** Draft · **Related:** `managed-kb-migration` (§5.41 diagram answer quality, task 16.2) + +## Problem + +The managed backend can extract content a user never sees and cannot verify. A +flowchart or a two-column table is flattened by the vision/parse step at +ingestion, so an answer about "semester 4" or a per-column total is *confidently +wrong* with no signal to the user. The legacy backend failed **silently** (0 +chunks, no answer); the managed backend fails **invisibly** (wrong answer, no +trace). We deliberately will not fix the managed parser (it is a managed service, +and this is a self-service platform — see task 16.2's decision). Instead we make +the extraction **visible**, so a user can look at what the knowledge base +actually holds for their document and decide to change their input (convert a +flowchart to a text table, re-export a scanned PDF, etc.). + +We already ship a *retrieval trace*: the chunks sent to the LLM for a given +answer are streamed to the UI as `citation` events (assistantId, documentId, +fileName, text — capped at 500 chars) from +`backend/src/apis/inference_api/chat/routes.py`, and this already works for both +engines via the facade. What is missing is an **upload-time, per-document, +full-chunk view** that does not require asking a question and is not truncated to +the retrieved top-k. + +## Requirements + +### Requirement 1 — Inspect a document's extracted chunks +**User story:** As an assistant owner, I want to see the content the knowledge +base extracted from a document I uploaded, so I can tell whether my file was +parsed usefully before I rely on it. + +- WHEN an owner or editor opens a document that has reached `complete`, THEN the + system SHALL provide the chunks the knowledge base holds for that document, + each with its full extracted text and its source metadata (document id, + filename, and page/location when the backend supplies it). +- WHEN a chunk was produced from a non-text element (an image or a diagram), THEN + its extracted text (the vision model's description) SHALL be shown verbatim, so + the user sees exactly what the model "read". +- The excerpt SHALL NOT be truncated to the 500-character citation limit; the + inspector shows the full chunk text. + +### Requirement 2 — Engine-agnostic +- WHEN the document belongs to a managed knowledge base, THEN chunks SHALL be + read through the managed backend; WHEN it belongs to a legacy knowledge base, + THEN through the legacy backend. The caller SHALL resolve the engine via + `resolve_engine_for` and never branch on it in the UI. +- The response shape SHALL be identical across engines. + +> **AMENDED BY IMPLEMENTATION 2026-09-11 — managed only.** The clause above assumed +> `backend.search(..., retrieval_filter=...)` was part of the protocol. It is not: +> `retrieval_filter` exists **only** on `ManagedKbBackend.search`. The legacy +> adapter's signature is `search(kb_ref, query, top_k)`, it accepts no filter, and it +> ignores `top_k` — it always asks its index for a fixed five results across the +> **whole** knowledge base. +> +> So on legacy there is no way to scope a retrieval to one document. Running it anyway +> would return five whole-knowledge-base chunks, most or all belonging to *other* +> documents, rendered under this document's filename. That is a cross-document leak — +> exactly what Requirement 4 exists to prevent — not a cosmetic defect to tidy later. +> +> Teaching the legacy adapter to filter was rejected on Requirement 5's own grounds: +> the inspector must not depend on the legacy pipeline continuing to exist, and that +> pipeline is being deprecated. New capability there has a negative lifespan. +> +> **As built:** a legacy document returns `available=false` with an owner-facing +> `reason`, as a **200 rather than an error** — the owner asked a reasonable question +> and "your knowledge base is on the classic engine, which cannot show this" is an +> answer. The response shape is identical either way, so the second clause above holds +> and the UI still never branches on engine, which is what this requirement was +> actually protecting. + +### Requirement 3 — Honest completeness +- Bedrock managed knowledge bases expose **no chunk-enumeration API**; `Retrieve` + is query-ranked and bounded by `numberOfResults`. WHERE the full set of chunks + cannot be guaranteed, the system SHALL label the view as "chunks the knowledge + base returned for this document (up to N)", and SHALL NOT claim to be a + complete or ordered dump. +- WHEN more chunks may exist than were returned, THEN the UI SHALL say so rather + than imply the document has only N chunks. + +### Requirement 4 — Isolation and permission +- The chunk query SHALL be filtered to the requested `document_id` using only an + isolation-safe operator (`equals`) — never a prefix/substring operator, which + over-matches (a filter for `DOC-1` must not admit `DOC-10`). +- Access SHALL require owner or editor permission on the parent assistant + (reuse `_require_edit_permission`); a non-owner SHALL receive 403/404 exactly + as the other document endpoints do. + +### Requirement 5 — No new ingestion path, no legacy dependency +- The inspector SHALL be read-only and SHALL reuse the existing retrieval facade + (`backend.search(...)`). It SHALL NOT introduce a new chunking, parsing, or + ingestion code path, and SHALL NOT depend on the legacy pipeline continuing to + exist (so it survives v1 deprecation). + +### Requirement 6 — Cost and latency are bounded +- A single inspect request SHALL issue a bounded number of `Retrieve` calls + (one, plus pagination up to a hard cap), so opening the view cannot fan out + into an unbounded or expensive scan. + +## Out of scope +- Fixing or re-parsing the document (managed owns parsing). +- A per-question retrieval trace — that already exists (citations). +- Editing/curating chunks. diff --git a/.kiro/specs/kb-chunk-inspector/tasks.md b/.kiro/specs/kb-chunk-inspector/tasks.md new file mode 100644 index 000000000..ed746ba52 --- /dev/null +++ b/.kiro/specs/kb-chunk-inspector/tasks.md @@ -0,0 +1,87 @@ +# KB Chunk Inspector — Tasks + +**Status:** Built (tasks 1-5); task 6 is a manual check against dev · **Requirements:** `requirements.md` · **Design:** `design.md` +**Why now:** it is the tooling half of the task-16.2 decision (`managed-kb-migration` +§5.41). That decision was "guidance, not code" — this is what makes the guidance +possible, because a user cannot act on advice about mangled tables if they cannot see +that their table was mangled. + +**Effort:** small–medium. One read endpoint, one facade call, one frontend panel +reusing the existing citation card. No new pipeline, no infra, no legacy dependency. + +--- + +- [x] 1. Backend read endpoint + - `GET /assistants/{assistant_id}/documents/{document_id}/chunks` in + `backend/src/apis/app_api/documents/routes.py` + - `_require_edit_permission` first, exactly as the sibling document endpoints do; + a non-owner gets the same 403/404 they already get elsewhere (Req 4) + - Confirm the `DOC#` row exists and is `complete`; 409 while it is still + processing, 404 when absent. **A born-managed first upload can be + `provisioning`** — that is a 409 too, not a 404, and its message should say the + knowledge base is still being created rather than implying the file is missing + - Resolve the engine with `resolve_engine_for` and get the backend from the + resolver; never branch on engine in the response shape (Req 2) + - _Requirements: 1, 2, 4, 5_ + +- [x] 2. Chunk enumeration, honest about completeness + - One `search(...)` call through the facade with the `document_id` `equals` + filter, `numberOfResults` at the backend ceiling (Bedrock documents 100) + - Neutral document-anchored query text (the filename, or the row's leading text) + purely to give the reranker something — the filter is what scopes the result + - De-duplicate by content hash: a query-ranked API can repeat a chunk + - Set `capReached=true` when the count returned equals the ceiling + - Present as an unordered set (or by page where metadata carries one), never as + "chunk 1..N" — ranking order is not document order, and implying otherwise is a + lie the UI would be telling + - **Do not** compute "the header is missing from chunk 2" server-side. Show what + came back and let the human judge it; that is the 16.2 decision, not laziness + - _Requirements: 1, 3, 6_ + +- [x] 3. Response contract + - `{ documentId, fileName, engine, chunks: [{ text, page?, order, score? }], + complete: bool, returned: N, capReached: bool }` + - Full chunk text, **not** truncated to the 500-character citation limit — that + truncation is the whole reason the existing citation trace cannot serve this + - _Requirements: 1, 2, 3_ + +- [x] 4. Frontend panel + - A "View extracted content" action on the document row, enabled at `complete` + - Panel lists each chunk's full text, monospace/`pre` so a flattened table's + damage is actually visible, with page/location when present + - Header carries the actionable nudge: *"This is what the knowledge base + extracted. If a table or image looks wrong here, the assistant will answer from + this — consider reformatting your source."* + - When `capReached`, say "showing the first N; more may exist" rather than + implying the document has exactly N chunks (Req 3) + - Reuse the citation card component; the only new data is longer text plus the + header + - _Requirements: 1, 3_ + +- [x] 5. Tests + - Route: owner sees chunks on **both** engines; non-owner 403; non-`complete` + document 409 (cover `provisioning` and `uploading` separately); `capReached` + set when the count equals the ceiling + - Assert the **exact** retrieval filter is `equals` on `document_id`, mirroring + the retrievability-probe test — `ISOLATION_SAFE_FILTER_OPERATORS` exists + because a prefix operator over-matches, and `DOC-1` admitting `DOC-10` is a + cross-document leak, not a display bug + - Fake backend modelling `search(..., retrieval_filter=...)`; copy the one in + `backend/tests/lambdas/test_kb_ingestion_consumer.py` + - **Mutation guard:** drop the `document_id` filter and a test that seeds a + second document's chunks must fail on those chunks appearing + - _Requirements: 4, 6_ + +- [ ] 6. Verify against the real §5.41 corpus + - Open the inspector on the diagram documents that produced §5.41 + (`4-yr-flowchart-v2026.pdf`, `sustainable-farming.pdf` on the dev retain + assistant) and confirm the flattening is *visible* to a human reader + - This is the acceptance test for the whole feature: if a user still cannot tell + from this panel why their per-semester answer was wrong, it has not delivered + what 16.2 promised + - _Requirements: 1_ + +## Out of scope +- Re-parsing or fixing a document — managed owns parsing. +- A per-question retrieval trace — that already ships as citation events. +- Editing or curating chunks. diff --git a/.kiro/specs/managed-kb-migration/HANDOFF.md b/.kiro/specs/managed-kb-migration/HANDOFF.md index e1f2ab8ca..0391490b7 100644 --- a/.kiro/specs/managed-kb-migration/HANDOFF.md +++ b/.kiro/specs/managed-kb-migration/HANDOFF.md @@ -10,12 +10,18 @@ re-deriving anything. Read this, then `tasks.md`. ## 0. Read this first +**Where this stands, in one line:** the build is finished and running in dev with the +flags on there; production has every flag off, and what remains is the group-15 +probes plus turning the flags on. Ladder step 2 (born managed) is the next real +move — see §6 "Do these first". If you read nothing else, read that table. + Four things invalidate earlier versions of this document: 1. **It is deployed.** The feature shipped to production in release 1.16.0 and the platform deploy succeeded on 2026-08-28, so `GSI7`, the Bedrock service role and - the four Lambdas exist in **both** dev and prod. Earlier revisions of this file - said "Nothing deployed"; that is no longer true. + the Lambdas exist in **both** dev and prod. Earlier revisions of this file + said "Nothing deployed"; that is no longer true. There are now **five** migration + Lambdas, not four — task 16.5 added the document reconciler. 2. **Sixteen defects were found only by running it**, each reviewed clean and deployed clean. They are §5 items 25–41 and they are the most useful part of this document. Three clusters: 32–36 trace to the two engines never being made @@ -47,13 +53,17 @@ Four things invalidate earlier versions of this document: | | | |---|---| -| Spec | Complete. Requirement **8.5 was amended by measurement on 2026-08-31** — see §5.29 | -| Implementation | Groups 1–14 except 14.5. A migration has completed `shadow → verify → promote → retain` in dev and serves from the managed backend | -| Tests | 640 infra (jest) · ~6,840 backend (pytest) · 1,936 frontend (vitest) · 5 pre-existing unrelated Strands failures | -| Deployed | **dev and prod.** Flags off in prod; `migrationEnabled` on in dev | -| Open PRs | **#908** — the filtered retrievability probe (§5.38) and `TEXT_INDEXED` (§5.39), plus this document. · merged: #898 `ef2f4c9e`, #899, #900 `df93471c`, #901 `a4b660ba` | +| Spec | Complete. Requirement **8.5 was amended by measurement on 2026-08-31** — see §5.29. Requirement 3.2 amended by task 16.1; Requirement 12.11 completed by the upload-time byte cap | +| Implementation | **Build phase done.** Groups 1–14 except 14.4 (retry endpoint) and 14.5 (admin surface); all of group 16; born-managed provision-then-ingest. A migration has completed `shadow → verify → promote → retain` in dev and serves from the managed backend | +| Tests | Counts drift with every merge — read the latest `develop` run rather than a number pasted here. CI is green on `develop` (backend pytest, frontend, infra jest, load suite, scan) | +| Deployed | **dev and prod.** In dev: `migrationEnabled` on, everything else off. In prod: **every flag off**, so none of it can fire | +| Open PRs | **none** for this feature. Last merged: **#1027** born-managed (`f1e11bd3`, 2026-09-10), **#1019** upload-time byte cap, **#1018** doc-reconciler flag forwarding, **#1017** task-16.2 docs | | Uncommitted | none | +**What is left is not build work.** It is (a) the three pre-promotion probes in +group 15, (b) turning flags on in prod, and (c) two deferred surfaces (14.4, 14.5). +See §6. + ### Flag state (GitHub Environment variables) | Flag | development | production | @@ -61,76 +71,104 @@ Four things invalidate earlier versions of this document: | `CDK_MANAGED_KB_MIGRATION_ENABLED` | `true` | `false` | | `CDK_MANAGED_KB_NEW_DEFAULT` | `false` | `false` | | `CDK_MANAGED_KB_RECONCILER_ARMED` | `false` | `false` | +| `CDK_MANAGED_KB_DOC_RECONCILER_ARMED` | `false` | `false` | | `CDK_TAG_ENVIRONMENT` | `dev` | `prod` | -`newDefault` has **no reader anywhere in `backend/src`** — "new knowledge bases are -created managed" is design §14.7 steps 5–8, a follow-up spec. Setting it does -nothing, which is worth knowing before someone flips it expecting an effect. +`CDK_MANAGED_KB_DOC_RECONCILER_ARMED` is the fourth flag, added by task 16.5. It was +settable only by hand-editing `cdk.context.json` until PR #1018 added the missing +`platform.yml` line — the wiring existed end to end *except* for forwarding the +GitHub variable. + +**`newDefault` now has a reader** (PR #1027). An earlier revision of this section +said it had none anywhere in `backend/src` and that setting it did nothing; that was +true then and is false now. Turning it on makes an agent's knowledge base provision +on the managed backend **when its first document is uploaded** — not at agent +creation, which would spend a Bedrock knowledge base on every prompt-only agent and +abandoned draft. See `.kiro/specs/managed-kb-migration/born-managed-provision-then-ingest.md` +for the flow, the failure/rollback path, and why the engine must be declared before +the object lands. ⚠️ **Production carries every defect fixed after 1.16.0 shipped.** It cannot fire, -because nothing enrols while `migrationEnabled` is false. Do not turn that flag on -in prod until #898 and the uncommitted work have landed and shipped. +because nothing enrols and nothing provisions while all four flags are false. The +release-order rule still holds: ship the code, confirm the deploy, *then* consider a +flag — never both in one motion. ### Commits -**On `fix/kb-legacy-pipeline-engine-gate` — PR #900, open:** +Deliberately **not** an exhaustive log any more. It had drifted into listing +branches that merged weeks ago as "open", which is worse than having no list — a +reader trusts it and then chases a PR that no longer exists. `git log --oneline +origin/develop -- .kiro/specs/managed-kb-migration backend/src/apis/app_api/kb_migration +backend/src/apis/shared/kb_backend` is authoritative and never stale. -``` -e3398f30 propagate document deletion to the managed knowledge base (§5.36) -8a35dbc6 the legacy pipeline stands down for a promoted knowledge base (§5.32, §5.34) -``` - -**Merged as `ef2f4c9e` (was PR #898):** - -``` -fdf15d21 grant bedrock:StartIngestionJob, which authorizes direct ingestion (§5.31) -6420f148 drop the embedding pin, defer verify, complete a migration in dev (§5.29, §5.30) -7542d907 record the knowledge base id before anything else can fail (§5.28) -``` +The landmarks worth knowing by number, newest first: -**Already merged (16, on develop):** - -``` -45239838 one source of truth for the managed KB tag contract -4acaa8f2 handoff reflects group 14 backend half and three more defects -e59f771c register the managed backend, fleet metrics, tagged teardown (group 14 backend) -d5e56f31 handoff reflects group 13 and four more defects -ee091971 migration dispatcher and the shadow/verify/promote/retain worker (group 13) -53476544 handoff reflects groups 11-12 and two new defects -a361fdd4 opt-in dual-read pilot that legacy always wins (group 12) -a43d80bf app-side authorization, IAM-enforced sharing, publication (group 11) -58f0c6b6 handoff document and accurate task-list state -8079f7e2 tombstone deletion sagas and the report-only reconciler (group 10) -e6936b0b ingestion consumer with exclusive engine routing (group 9) -620fa49c managed KB provisioning, retrieval and direct ingestion (group 8) -d433d6f1 per-owner byte cap with atomic reserve/commit/release (group 7) -f2e86afe clamp retrieval queries and fail closed on status (groups 5, 6) -24689de1 backend abstraction seam behind the retrieval entry point (group 4) -ffa7a408 KB_Record data layer with conditional state transitions (group 3) -5f2c98b1 spec, schema and worker platform (groups 1, 2) -``` - -**Working tree: clean.** An earlier revision listed the 14.3 upgrade surface and -then the 8.5 amendment as uncommitted; both have landed. `apis/app_api/kb_upgrade/` -is merged — see §7 for the file map and §2 for how to run it. +| PR | What | +|---|---| +| **#1027** `f1e11bd3` | Born-managed: provision on first upload, `MANAGED_KB_NEW_DEFAULT` gets a reader | +| **#1019** | Byte cap enforced at document upload time — completes Req 12.11 | +| **#1018** | Forward `CDK_MANAGED_KB_DOC_RECONCILER_ARMED` through `platform.yml` | +| **#1007 / #1008** | Dead-letter document reconciler (task 16.5) — backend, then Lambda + nightly schedule + IAM | +| **#1006** | Engine visibility: Managed/Classic badge, engine-aware status vocabulary (task 16.4) | +| **#998** | Fail-closed status filter (§5.33) | +| **#997** | Engine-aware context cap, managed 8,000 / legacy 2,000 (task 16.1, §5.40) | +| **#900** `df93471c` | Legacy pipeline stands down for a promoted KB; deletion propagates (§5.32, §5.34, §5.36) | +| **#898** `ef2f4c9e` | `bedrock:StartIngestionJob` grant, defer-verify, record the KB id first (§5.28–§5.31) | + +**Working tree: clean.** ### Is the feature reachable yet? -**Yes, end to end, once the flag is on — except that nothing performs the work.** -Group 14.3 closed the last gap in the *control* path: a user can now enrol a -knowledge base, which writes a `KB#` record in `shadow` with the GSI7 work keys. -Before it, nothing wrote either, so every group could have been finished with the -feature unreachable (§5 defect 21). - -The worker's image **is deployed** — PR #886 shipped `Dockerfile.kb-migration` and -all four Lambdas run real handlers. An earlier revision of this section said the -image was undeployed and a `shadow` record would sit forever; that is no longer -true. The dispatcher's rule is `ENABLED` and ticking every 15 minutes in dev. - -⚠️ **That tick is a hazard while #898 is unmerged.** The deployed worker predates -it, so an enrolled record can be picked up by pre-fix code and failed at `verify`. -Always drive a local migration with `--break-lease`, which defers `dueAt` 20 minutes -out so the deployed dispatcher skips it. +**Yes, end to end, and the work actually happens.** Two paths reach the managed +backend now: + +1. **Upgrade** (opt-in, existing corpus). A user enrols; the record enters `shadow` + with GSI7 work keys; the dispatcher hands it to the worker, which runs + `shadow → verify → promote → retain`. Gated on `MANAGED_KB_MIGRATION_ENABLED`. +2. **Born managed** (new agent, no corpus). The first document upload declares the + engine managed and queues a provisioning job; the worker builds the knowledge + base and ingests the waiting document itself. Gated on `MANAGED_KB_NEW_DEFAULT`. + +The dispatcher's rule is enabled when **either** flag is on, and the Python gates +each work state on its own flag — so step 2 of the rollout ladder provisions new +agents without touching a single existing knowledge base. + +All five Lambdas run real handlers from `Dockerfile.kb-migration`. The dispatcher +ticks every 15 minutes in dev. + +⚠️ **That 15-minute tick is also born-managed's pickup latency.** A first upload can +sit at "Provisioning knowledge base…" for up to a whole interval before the real +47–124 s create even begins. It is recorded as a known cost, not a bug, with the fix +(provision inline in the ingestion consumer, keeping the queue as the fallback) +written up in the born-managed spec. + +When driving a migration by hand, still use `--break-lease`: it defers `dueAt` 20 +minutes out so the deployed dispatcher does not race your local run. + +### The production rollout ladder + +Four flags, four rungs, in this order. Each is a GitHub per-environment Variable fed +through `platform.yml`; all default OFF and unset means off. **Deploy, confirm the +deploy, then set a variable — never both in one motion.** + +| Rung | Flag | What changes | Risk | +|---|---|---|---| +| **1** | *none* — just deploy | Everything dark. Both reconcilers run report-only, which is deliberate: Req 14.7 makes report-only the initial deployed mode so their judgement can be audited against real data before either is allowed to act | none | +| **2** | `CDK_MANAGED_KB_NEW_DEFAULT` | New agents are born managed on their **first document upload**. No existing knowledge base is touched | **low** — each agent is independent, so a problem affects one agent | +| **3** | `CDK_MANAGED_KB_MIGRATION_ENABLED` | The Upgrade card appears, and the background worker migrates enrolled knowledge bases `shadow → verify → promote → retain` | **medium** — touches existing corpora. Needs 15.2 and 14.5 first | +| **4a** | `CDK_MANAGED_KB_RECONCILER_ARMED` | The KB reconciler starts **deleting** orphaned knowledge bases instead of reporting them | **high** — only after a clean report-only period you have actually read | +| **4b** | `CDK_MANAGED_KB_DOC_RECONCILER_ARMED` | The document reconciler starts correcting stranded `DOC#` rows instead of reporting them | medium — after an audit | + +Rung 2 is deliberately reachable on its own: the dispatcher's schedule is enabled by +*either* flag and the Python gates each work state on its own, so `NEW_DEFAULT` alone +provisions new agents and migrates nothing. + +**The intended end state**, and the one genuinely dangerous step: once the fleet is +fully migrated, retire the legacy pipeline. That step needs a hard **zero-legacy- +knowledge-bases gate**, because the legacy code does not merely *create* old +knowledge bases — it also **serves retrieval** for every un-migrated one. Removing it +early breaks every agent that has not moved. For a public repo with forks that sync, +make it a loud version boundary with a long deprecation window, never a quiet sync. ### What works today, verified live in dev @@ -537,11 +575,13 @@ saying why that number is a property of AWS rather than a knob. `app-api-environment.test.ts`. The mutation — deleting the line, which is precisely what the defect was — is caught. - ⚠️ `managedKb.newDefault` has **no reader anywhere in `backend/src`**. It is - set on the Lambdas' environment and consumed by nothing, because - "new knowledge bases are created managed" is a follow-up spec (design §14.7 - steps 5–8), not this phase. Leave it off; turning it on is a no-op that reads - like a behaviour change. + ⚠️ **Superseded 2026-09-10.** This warning used to read that + `managedKb.newDefault` had *no reader anywhere in `backend/src`*, so setting it + was a no-op that read like a behaviour change. That was true when written and is + now false: PR #1027 gave it a reader plus the app-api environment thread it was + also missing. It now has a real effect — an agent's knowledge base provisions on + the managed backend at its **first document upload**. See + `born-managed-provision-then-ingest.md` and the rollout ladder in §1. --- @@ -980,25 +1020,36 @@ answer than legacy** on a question it retrieves *better*. Both are open. ### Do these first +The build is done. What remains is proving it against prod's constraints and then +turning flags on. In order: + | | | |---|---| -| **Merge #900** | Engine exclusivity, both halves. Triggers `backend.yml` (rebuilds rag-ingestion **and** kb-sync — the content hash moves because `kb_backend` was added to both images' `SOURCE_DIRS`) and `platform.yml` (the two new IAM grants). Wait for the platform deploy before testing a deletion on a promoted assistant, or the delete fails on IAM and the `DOC#` row is deliberately kept | -| **Then re-add a document in dev** | `DOC#DOC-dc8b65658e29` is parked at `failed` from §5.31 and is not retried retroactively — there is no reprocess endpoint (task 14.4). Re-upload; that path works | -| **Then drive a migration from the *deployed* dispatcher, not the local driver** | §5.31 is the proof that the local driver cannot see IAM defects: its SSO identity is broader than either Lambda role. Every remaining unknown in this feature is of that class | +| **1. Task 15.1 — the live SDK probe** | The only item that can invalidate the whole feature in prod. The *static* half passes (`boto3==1.43.68` carries `MANAGED`, the embedding members, `FLOAT32`, all four document ops, with no `AWS_DATA_PATH`), and a real create → ingest → retrieve → promote has succeeded in dev. What is missing is doing it deliberately, with the checked-in environment and **no** side-loaded service model, and recording the result. If this fails, managed knowledge bases do not work in prod at all | +| **2. Flip `CDK_MANAGED_KB_NEW_DEFAULT` in prod** | Ladder step 2. New agents are born managed on their first document; not one existing knowledge base is touched. Lowest-blast-radius rung: each agent is independent, so a problem affects that agent and no other. Deploy first, confirm the deploy, *then* set the variable — never both in one motion | +| **3. Task 15.2 — the ingestion-concurrency probe** | Gates ladder step 3 (`MIGRATION_ENABLED`), **not** step 2. The quota page lists no account-level ingestion-concurrency limit, which is not evidence there is none. Do not size a wide fleet migration before this is answered | +| **4. Task 14.5 — the admin surface** | Also gates step 3 rather than step 2. The moment existing knowledge bases start migrating you have a mixed fleet and no view of who is on which engine — and "how many are affected?" is the first question anyone asks when something goes wrong | +| **5. Task 15.3 — full matrix in the dev container** | Housekeeping; CI already covers most of it | + +⚠️ **Before flipping anything, re-read §5.41.** Managed is *worse* than legacy for +column-structured diagrams and two-dimensional tables: legacy returned nothing, +managed returns a confident wrong answer. Born-managed makes managed the default for +every new agent, so that trade stops being opt-in. The agreed mitigation is guidance, +not code — and the `kb-chunk-inspector` spec is the tooling half of that guidance. ### Open, in rough order | Group | Notes | |---|---| | ~~**§5.40** the 2,000-char cap~~ ✅ DONE (PR #997) | Engine-aware cap: managed **8,000**, legacy 2,000 (`rag_service.resolve_context_cap`, keyed on `resolve_engine_for`). Requirement 3.2 amended; validated end-to-end on the KINES advising corpus in dev; mutation-tested | -| ~~**§5.41** diagram answers~~ ✅ DONE (task 16.2) | Understood: the vision model flattens the 2-D layout, chunks carry no column coordinates, and the 16.1 cap fix does not rescue it (re-measured 2026-09-08: 14 credits vs the chart's 19 at both caps, mis-columned `ENGR 220` persists). Closed as a **product/training matter, not code** — this is a self-service platform, and users won't know to convert a diagram to text. Guidance: demo as *retrievable where previously impossible*, never precise per-column answers | +| ~~**§5.41** diagram answers~~ ✅ DONE (task 16.2) | Understood: the vision model flattens the 2-D layout, chunks carry no column coordinates, and the 16.1 cap fix does not rescue it (re-measured 2026-09-08: 14 credits vs the chart's 19 at both caps, mis-columned `ENGR 220` persists). Closed as a **product/training matter, not code** — this is a self-service platform, and users won't know to convert a diagram to text. Guidance: demo as *retrievable where previously impossible*, never precise per-column answers. **The tooling half of that guidance is specced:** `.kiro/specs/kb-chunk-inspector/` (requirements + design written, no tasks yet) makes the extraction *visible* — an owner can see for themselves that a table came out mangled or an image description is wrong, instead of finding out via a confidently wrong answer. That is the difference between "managed fails invisibly" and "managed fails visibly", which is the most this decision can buy without touching a managed parser | | ~~**§5.33** the one fail-open line~~ ✅ DONE (PR #998) | `if not doc_ids: return vectors` now returns `[]` + `METRIC_STATUS_FILTER_FAIL_CLOSED` when a non-empty batch carries no `document_id`; an empty input stays an empty result with no metric. Guard `test_filter_fails_closed_when_no_chunk_carries_a_document_id`, mutation-tested | | ~~**engine visibility**~~ ✅ DONE (task 16.4) | The facade now logs one INFO line per query naming the served engine (`engine=managed (Managed)` / `engine=s3vectors (Classic)`), read from the same KB_Record `resolve_backend` uses. The knowledge base section carries a `Managed`/`Classic` badge, fed by a new `engine` field on `UpgradeStatusResponse` (defaults `classic`), and its per-document status vocabulary is engine-aware — `uploading → processing → ready` for managed, `chunking`/`embedding` kept for legacy. Mutation-tested. Branch `feat/kb-engine-visibility` | -| **14.4** one-click document retry | Req 21.2. Ingestion is S3-event-triggered and there is no reprocess endpoint, so this needs new backend against a live pipeline. The card currently directs the user to re-upload, which works today. Close it by building the endpoint **or** by amending Req 21.2 to accept re-upload | -| **14.5** admin surface | not started. Filter by engine, stored bytes, document counts, bulk migrate, per-KB retry | -| **15.1** packaged-SDK probe | the *static* half is done and passing (`boto3==1.43.68` carries `MANAGED`, the embedding members, `FLOAT32`, all four document ops, no `AWS_DATA_PATH`). The live half has now effectively been done by hand — a real create → ingest → retrieve → promote succeeded in dev | -| **15.2** ingestion-concurrency probe | unanswered. Do not size a wide fleet migration before it | -| **15.3** full matrix | run it once the above land | +| **14.4** one-click document retry | Req 21.2. Ingestion is S3-event-triggered and there is no reprocess endpoint, so this needs new backend against a live pipeline. The card currently directs the user to re-upload, which works today. Close it by building the endpoint **or** by amending Req 21.2 to accept re-upload. Note task 16.5's document reconciler already performs the *scheduled* form of this | +| **14.5** admin surface | Not started. Req 23.9: list knowledge bases filterable by engine, with stored bytes and document counts, bulk migrate, per-KB retry. Gates ladder **step 3**, not step 2. Two gaps it does *not* close, worth knowing before someone assumes it does: it reads our own `KB#` records, so it cannot see AWS-only orphans (the reconciler's job — and those still consume the account's knowledge-base quota); and it carries no control for granting the elevated byte tier, which remains a hand-edited `elevatedByteCap` attribute with no API or UI writer anywhere | +| **15.1** packaged-SDK probe | The *static* half is done and passing (`boto3==1.43.68` carries `MANAGED`, the embedding members, `FLOAT32`, all four document ops, no `AWS_DATA_PATH`). The live half has effectively happened by hand — a real create → ingest → retrieve → promote succeeded in dev — but has not been run deliberately with the checked-in environment and recorded. **Do this one first:** it is the only remaining item that can invalidate the feature in prod | +| **15.2** ingestion-concurrency probe | Unanswered. Gates a wide fleet migration (step 3), **not** born-managed (step 2), which provisions one knowledge base at a time as agents are created | +| **15.3** full matrix | Run it once the above land | ### RESOLVED — the `document_id` / `relevance` "known unknown" was a false alarm diff --git a/.kiro/specs/managed-kb-migration/born-managed-provision-then-ingest.md b/.kiro/specs/managed-kb-migration/born-managed-provision-then-ingest.md new file mode 100644 index 000000000..49a9b7289 --- /dev/null +++ b/.kiro/specs/managed-kb-migration/born-managed-provision-then-ingest.md @@ -0,0 +1,286 @@ +# Born-managed: provision-then-ingest on first upload + +**Status:** Draft (supersedes the enroll-reuse design in `new-default-wiring.md`) · +**Part of:** `managed-kb-migration` (rollout ladder step 2) · +**Supersedes the approach in:** PR #1027 (keeps its flag reader + infra wiring) + +## Decision + +`MANAGED_KB_NEW_DEFAULT` makes new agents born managed. Provisioning is stacked +onto the **first document upload** (not agent creation, not save): + +- Prompt-only agents and abandoned drafts never provision a KB → kind to the + ~10,000-KB/account quota. +- The agent-creation page's playground (left = form, right = test) uploads docs to + the **draft**; provisioning at first upload means what you test **is** the + managed engine you ship — WYSIWYG. +- The cost — a one-time "provisioning" delay on the first doc — is surfaced as a + status, not hidden. + +This is a **provision-then-ingest** flow, NOT the upgrade/migrate reuse. Reusing +`enroll` writes `retrievalEngine=managed` only at promotion, so the first doc would +be grabbed by the legacy pipeline (verified: the legacy handler skips a doc only +when the agent is already promoted), tested on legacy, and double-ingested. We +instead mark the KB managed up front so the first doc goes straight to managed. + +## Status vocabulary (extends task 16.4) + +Add one leading DOC# status: **`provisioning`** → label **"Provisioning knowledge +base…"**. Full managed lifecycle for the first doc becomes: + +``` +provisioning → uploading → (Processing) → complete [ or → failed ] +``` + +Only the doc that triggers KB creation shows `provisioning`. Subsequent docs (KB +already active) start at `uploading` as today. Frontend `statusLabel` +(`KnowledgeBaseSectionComponent`) gains the one new case. + +## The flow + +### 1. First upload trigger — `documents/routes.py:generate_upload_url_endpoint` +When `new_default_enabled()` AND the agent has **no KB_Record** (first doc): +- Create the KB_Record with `retrievalEngine=managed` and + `provisioningState=provisioning`, atomically/idempotently + (`create_provisioning`; a concurrent first upload loses the race benignly). +- Persist the DOC# row with status `provisioning` (instead of `uploading`). +- Enqueue a **born-managed provisioning job** for the worker (a work key / + dispatcher pickup — reuse the existing sparse-index + lease machinery; do NOT + run minutes-long provisioning in the API request). +- Issue the presigned URL as normal. + +Setting `retrievalEngine=managed` up front makes the **legacy pipeline skip** the +doc (it only ingests non-promoted agents). + +### 2. The doc lands before the KB exists — managed consumer must DEFER, not fail +The S3 event fires within seconds; provisioning takes minutes. Today the managed +consumer raises `IngestionRoutingError("not provisioned")` when +`retrievalEngine=managed` but `awsKbId` is absent → 2 redeliveries → dead-letter. + +**Change:** when the record is `provisioningState=provisioning` (managed intent, +KB not yet built), the consumer returns a benign **"deferred — provisioner owns +this"** result and leaves the DOC# in `provisioning`. It neither ingests nor +dead-letters. The born-managed job (below) owns ingestion once the KB is ready. + +### 3. Born-managed provisioning job (worker) — owns the handoff +The robust place for minutes-long provisioning (leases, dispatcher retries, not +EventBridge-capped). On pickup: +- `provision_managed_kb(assistant_id, owner_user_id=...)` → KB ACTIVE, data source + created, `awsKbId`/`awsDataSourceId` set, `provisioningState=active`. +- List DOC# rows still in `provisioning` and ingest each **directly** into managed + (reuse the ingestion consumer's ingest→wait-indexed→wait-retrievable→terminal + logic; drive `provisioning → uploading → complete`). Byte cap is enforced here + via the same S3-HEAD reconcile as PR #1019 (the first doc is never request-time + capped because the KB wasn't active at upload). +- The job owns the ingestion trigger, so correctness never depends on the 2-try S3 + redelivery window. + +### 4. Subsequent uploads — plain managed path +Once `provisioningState=active` and `retrievalEngine=managed`, further uploads are +ordinary managed: legacy skips, the managed consumer ingests on the S3 event, +request-time byte cap (PR #1019) applies, statuses start at `uploading`. + +## Failure & rollback +- **Provisioning fails.** Do NOT leave the agent stranded (managed intent, no KB → + legacy skips it, managed can't serve). **Remove `retrievalEngine`** (back to + legacy-by-absence) and mark the `provisioning` doc `failed` with an actionable + message ("couldn't prepare the knowledge base; try again"). The agent then works + on legacy and a re-upload takes the legacy path — safe fallback, no dead end. +- **Job crashes mid-provision.** Idempotent: `provision_managed_kb` resumes via its + persisted `clientToken`; the dead-letter/doc reconciler (16.5) is the backstop + for a doc stuck in `provisioning` past a grace window. +- **Agent deleted mid-provision.** Provisioning/ingestion writes are conditional on + the record existing; teardown removes the KB (task 14.6 tag-scoped teardown). + +## What PR #1027 keeps vs replaces +- **Keep:** `new_default_enabled()` (flag reader); the app-api `MANAGED_KB_NEW_DEFAULT` + env wiring + infra test; the flag-read backend tests. +- **Replace:** `maybe_enroll_new_default` (enroll-reuse) and the two finalize hooks + in `assistants/routes.py` → the first-upload trigger + the provisioning job. + +## Tasks +- [x] 1. First-upload trigger in `generate_upload_url_endpoint`: managed-intent + record + `provisioning` DOC# status + enqueue the job (gated on + `new_default_enabled()` and no existing record). +- [x] 2. New `provisioning` DOC# status; frontend `statusLabel` case + ("Provisioning knowledge base…"). +- [x] 3. Managed ingestion consumer: DEFER (benign no-op, no dead-letter) when + `provisioningState=provisioning`. +- [x] 4. Born-managed provisioning worker job: provision → set active → + ingest pending `provisioning` docs → drive to terminal; idempotent, leased. +- [x] 5. Failure path: provisioning failure removes `retrievalEngine` (legacy + fallback) + marks the doc failed; grace-window reconcile backstop. +- [x] 6. Remove the enroll-reuse helper + finalize hooks from #1027. +- [x] 7. Tests: first-upload provisions managed-intent + `provisioning` status; + legacy skips a provisioning-intent doc; consumer defers (does not + dead-letter) while provisioning; job provisions + ingests to complete; + byte cap enforced on the first doc; provisioning-failure falls back to + legacy + marks doc failed. Mutation guards: dropping the consumer DEFER + dead-letters the first doc; dropping the failure-path rollback strands the + agent. +- [ ] 8. (Turn-on, later) Bedrock KB-count quota headroom before enabling the flag. + +## As built + +Decisions taken during implementation that the design above did not fix. + +**The job is a new migration state, `born_managed`.** The spec asked for "a work +key / dispatcher pickup — reuse the existing sparse-index + lease machinery", and +the cheapest way to get exactly that is a fourth work-eligible value in +`migrationState`. It borrows the column and inherits the queue, the lease, the +generation fence and the retry-until-terminal contract without a second dispatcher. +It is deliberately NOT spelled `provisioning`: that string is already the +`provisioningState` value, and two attributes carrying one word with two meanings +is how the wrong one gets read. Terminal state on success is `retain` — the +existing "ended up on managed" terminal — with `retainUntil` left unset, because +born-managed has no legacy vectors to retain. + +**The dispatcher gates each work state on its own flag.** `born_managed` answers +to `MANAGED_KB_NEW_DEFAULT`, the migration states to +`MANAGED_KB_MIGRATION_ENABLED`, and the EventBridge rule is enabled by either. Two +alternatives were rejected: gating born-managed on the migration flag would have +made ladder step 2 useless alone (the trigger would queue jobs nothing picked up, +parking every first upload on "Provisioning…" forever), and enabling all states +from either flag would have turned step 2 into a back door for step 3's blast +radius. `_work_states()` (ordering) and `_enabled_work_states()` (flags) are +separate functions because a single one doing both cannot be tested for either. + +**The first document IS byte-capped at request time.** The design implies the +first document escapes the request-time reserve because the KB is not active yet. +It must not: the ingestion reconcile *commits* the reservation (`reservedBytes -= +n`), so a document that committed without reserving drives the counter negative and +corrupts the cap permanently. The trigger runs before the reserve and the record it +writes already resolves to managed, so the existing `_reserve_managed_upload` path +covers it unchanged. The failure path returns those reservations via +`settle_once`. + +**Ingestion reuses `ingestion_consumer.handle_object` outright** rather than +reimplementing ingest → wait-indexed → wait-retrievable → terminal. That function +is where §5.37, §5.38 and §5.39 are encoded, plus the Requirement 12.3 S3-HEAD +reconcile; a second copy would be a second place to forget them. The job moves the +row `provisioning → uploading` first, then calls it, so by the time it reads the +record `provisioningState` is `active` and it takes the ordinary managed path. + +**One document per invocation.** The worker's timeout is 15 min and one document's +indexing budget is already 10.5, so a second could not finish. Anything left over +re-arms the work key. More than one pending document only happens when the author +uploaded again during the provisioning window. + +**Failure ordering is documents → engine → terminal state.** Documents are failed +while the record still says managed (the state in which they are unambiguously this +job's to resolve), then `retrievalEngine` is REMOVEd, then the work keys go. A +crash between any two leaves the work keys in place, so the dispatcher re-runs the +job and its engine check closes out the "already rolled back" case. Terminal-first +is the one ordering that would strand the agent, so it is the one ordering avoided. + +**`provisioning` is a document-reconciler candidate.** Safe because the sweep +already skips any record with no `awsKbId`, so a document is never probed against a +knowledge base that does not exist; once one does, a document still parked past the +grace window probes `NOT_FOUND` and is re-ingested from S3. + +### Known cost, not yet addressed +Pickup latency is up to one dispatcher interval (15 min) before provisioning even +starts, so a first upload can read "Provisioning knowledge base…" for that long +before the real 47–124 s create begins. The fix is a direct async worker invoke +from the API alongside the work key (which stays as the durable anchor), and it was +left out on purpose: app-api is an ECS Fargate service, so it needs a task-role +`lambda:InvokeFunction` grant and a container env var threaded through the app-api +construct — real plumbing that does not belong in the same change as the flow +itself. Worth doing before the flag is turned on for anyone who cares about the +first-upload experience. + +## Successor design: Lambda durable functions + +Recorded because the orchestration choice here was deliberate and the better option +is now available. **Nothing below is a criticism of what shipped** — it is where to +aim if the migration engine is ever rebuilt, so the next person does not +re-derive it. + +### The mechanism this feature actually needed +Strip born-managed to its essentials and it is one linear sequence with two slow +waits in the middle: + +``` +declare the record managed → create the KB → wait for ACTIVE +→ ingest the document → wait for INDEXED → wait for retrievable → complete + ↘ on any failure → roll back to legacy +``` + +Everything else in this change is scaffolding to make that sequence survive a +process that can die at any point: the `born_managed` work state, the sparse work +keys, the lease, the 15-minute dispatcher tick, the consumer's DEFER, the +one-document-per-invocation cap, and the re-arm. All of it exists because a Lambda +is killed at 15 minutes and an S3 event gets 3 delivery attempts. + +### Why durable functions fit better than Step Functions +Step Functions was considered and rejected on two grounds (see the conversation on +PR #1027): it would be the **only** state machine in the stack — a new AWS service +for every fork maintainer to learn — and it moves the saga out of Python into ASL, +which would forfeit `tests/property/test_pbt_kb_migration_convergence.py`, the +crash-at-every-step property test that already caught a double-promotion bug. + +Lambda durable functions (re:Invent 2025; Python supported) avoid both. A durable +function **is** a normal Lambda with a `DurableConfig`, so no new service enters the +stack, and the sequence stays as ordinary Python — the SDK adds `context.step()`, +`context.wait()`, `context.waitForCondition()`, `map()`, `parallel()`. Completed +steps are checkpointed; on failure or resume Lambda replays the handler from the top +and skips them. Waits suspend for up to a year and incur **no duration charge** on +on-demand functions. + +`waitForCondition()` — pause until a supplied check function passes — is a direct +replacement for all three of this feature's hand-rolled poll loops +(`_wait_for_knowledge_base_active`, `wait_until_indexed`, +`wait_until_retrievable`), and it deletes the reason the consumer currently burns +billed Lambda time asleep. + +### What it would delete +The dispatcher Lambda; `GSI7_PK`/`GSI7_SK` and the work-key invariant; +`acquire_lease`/`migrationLeaseUntil`/`LeaseLost`; `defer_verify` and +`verifyAttempts`; `dispatch_limit` and the priority ordering in `_work_states`; +`MAX_DOCUMENTS_PER_INVOCATION` and the re-arm; the consumer's DEFER branch and the +`born_managed` state itself; the 15-minute pickup latency; and most of the document +reconciler, which exists because events dead-letter and durable executions do not. +`retain` becomes a single 30-day `wait()` instead of a stored `retainUntil` plus a +nightly job to notice it. + +### What survives any rewrite +Everything that makes the *writes* safe, as opposed to the orchestration: +`resolve_engine`'s absence-means-legacy default, the conditional-write guards +(`adopt_managed_engine`, `promote_engine`'s four guards, `attach_aws_ids`), the +persisted `clientToken` that stops a retry creating a second knowledge base, and +`byte_cap.settle_once`. Durability guarantees the sequence resumes; it does not make +an individual AWS or DynamoDB call idempotent. **And the first-upload trigger order +survives unchanged**: the engine must still be declared before the object lands, or +the legacy pipeline takes the first document. + +### Costs, which is why this is not a follow-up ticket yet +- **An existing function cannot be converted.** AWS is explicit that + `DurableConfig` cannot be added to a function created without it, so this is a new + Lambda plus a cutover, not a flag flip on the worker. +- **It forces a deploy-pipeline change.** Durable functions must be invoked by a + qualified (version/alias) ARN so replays run the same code. `backend.yml` + currently pushes images onto `$LATEST` via `update-function-code --image-uri`. +- **Replay determinism is a real footgun.** The handler re-runs from the top on + every resume, so anything outside a `step()` executes again. Code that reads a + record at the top and branches on its state needs deliberate care, and the bugs + only appear after a crash. +- **Young, and a public-repo dependency.** GA'd through 2026, AWS describes the SDKs + as fast-moving, and fork maintainers inherit the SDK plus an IAM policy + (`AWSLambdaBasicDurableExecutionRole`). Confirm region availability for this + deployment's region before planning. + +### Recommended first move +Not a rewrite. A throwaway spike of **born-managed alone** as a durable function — +it is the smallest complete instance of the pattern — to find out whether replay +determinism is pleasant or nasty against this codebase's read-record-then-branch +style. That answer decides whether the engine-wide rewrite is real. If it is, the +order is: born-managed first (no legacy corpus, lowest blast radius), then +shadow→verify→promote. + +## Risks +- **KB-count quota** (~10k/account, one KB per agent) still gates turning the flag + ON — unchanged by this design; first-doc provisioning at least bounds it to + RAG-using agents, not all agents. +- **New leading status** touches the frontend status map and any status-set + invariants (e.g. the fail-closed status filter) — keep `provisioning` + non-terminal and non-retrievable so it can never serve content. diff --git a/.kiro/specs/managed-kb-migration/new-default-wiring.md b/.kiro/specs/managed-kb-migration/new-default-wiring.md new file mode 100644 index 000000000..72317a251 --- /dev/null +++ b/.kiro/specs/managed-kb-migration/new-default-wiring.md @@ -0,0 +1,75 @@ +# NEW_DEFAULT wiring — make new agents born managed + +> **SUPERSEDED** by `born-managed-provision-then-ingest.md`. The problem statement +> and the flag/infra wiring below still hold and shipped; the *approach* — reusing +> `enroll` to migrate an empty corpus — did not, and was replaced before merge. +> Reusing `enroll` writes `retrievalEngine=managed` only at promotion, so the first +> document would be grabbed by the legacy pipeline, tested on legacy, and then +> indexed a second time on managed. Provisioning is now stacked onto the first +> upload with the engine declared up front. Kept for the record of why. + +**Status:** Superseded (approach only; flag reader + infra wiring shipped) · +**Part of:** `managed-kb-migration` (rollout ladder step 2) + +## Problem +`MANAGED_KB_NEW_DEFAULT` is meant to make a newly created agent's knowledge base +**managed from birth**, skipping the Upgrade step. It did nothing: +- **Backend:** no code read `MANAGED_KB_NEW_DEFAULT`. +- **Infra:** the app-api Lambda received `MANAGED_KB_MIGRATION_ENABLED` but not + `MANAGED_KB_NEW_DEFAULT`, so even a reader would find it absent. + +## As-built design (reuse `enroll`, not a bespoke provisioner) + +The original sketch proposed a dedicated eager-provision helper. Reading the code +changed the design: `provision_managed_kb` + the migration worker already do +crash-safe, dispatcher-driven provisioning, and `catch_up` already carries across +documents uploaded mid-flight. A brand-new agent has **no corpus**, so "born +managed" is just *migrating an empty corpus*: provision → converge instantly +(`migrated == total == 0`) → promote — through the proven machinery. + +So born-managed = **call `enroll()` at agent finalize**, gated by the flag: + +- `new_default_enabled()` reads `MANAGED_KB_NEW_DEFAULT` (allow-list of affirmative + spellings, read at call time — mirrors `migration_enabled()`). +- `maybe_enroll_new_default(assistant_id, *, owner_user_id, visibility)`: + no-ops unless the flag is on; calls `enroll()`; catches `UpgradeUnavailable` + (migration worker off → nothing would finish the provision, so stay legacy) and + swallows any other error (agent creation must never fail). Idempotent via + `enroll`'s conditional writes. +- **Fire-and-forget** from the finalize path. `enroll` is two DynamoDB writes; the + slow `CreateKnowledgeBase` happens later in the dispatcher-driven worker — so no + minutes-long request-side task. +- **Call sites:** `create_assistant_endpoint` (direct COMPLETE) and + `update_assistant_endpoint` **only on the DRAFT→COMPLETE transition**. + +**Dependency (correct, not incidental):** born-managed needs the migration worker +running, so it has effect only alongside `MANAGED_KB_MIGRATION_ENABLED`. The +`UpgradeUnavailable` catch makes that graceful. + +**Failure = legacy.** Any failure leaves `retrievalEngine` unset → the agent works +on the shared legacy index exactly as before. + +## Infra +`app-api-environment.ts`: `MANAGED_KB_NEW_DEFAULT: String(config.managedKb.newDefault)` +next to the migration flag. `config.ts` already parses `newDefault`; `platform.yml` +already forwards the GitHub var. No other wiring. + +## Risk gating TURN-ON (not this build) +Managed is one Bedrock KB per assistant; the account cap is ~10,000 KBs (there is +already an 80%-of-quota alarm). `NEW_DEFAULT=true` at fleet scale marches toward +that ceiling. Turning the flag on needs quota headroom, the KBs-by-engine +inventory (task 14.5), and likely a reaper for doc-less KBs. The wiring is safe to +ship dark. + +## Tasks +- [x] 1. `new_default_enabled()` flag reader (`kb_upgrade/service.py`). +- [x] 2. `maybe_enroll_new_default()` — flag-gated, idempotent, error-swallowing, + reuses `enroll()`. +- [x] 3. Fire-and-forget call site in `create_assistant_endpoint` (COMPLETE). +- [x] 4. Fire-and-forget call site in `update_assistant_endpoint` (DRAFT→COMPLETE). +- [x] 5. Infra: thread `MANAGED_KB_NEW_DEFAULT` into the app-api env. +- [x] 6. Backend tests: flag read (off/on/call-time); enrol-when-on; + not-called-when-off (mutation guard); swallow `UpgradeUnavailable`; swallow + unexpected errors. +- [x] 7. Infra test: `MANAGED_KB_NEW_DEFAULT` threaded + explicit `'false'` when off. +- [ ] 8. (Turn-on, later) verify Bedrock KB-count quota headroom before flipping on. diff --git a/.kiro/steering/observability.md b/.kiro/steering/observability.md index 8a133b129..7a299f605 100644 --- a/.kiro/steering/observability.md +++ b/.kiro/steering/observability.md @@ -256,6 +256,7 @@ cannot be quietly reversed. | `agentcore-high-error-rate` | `UserErrors` — our requests are malformed | Recent inference-api deploy? Check payload shape and IAM. | | `agentcore-throttles` | At the TPS or session quota | Request a quota increase. Will not self-resolve. | | `agentcore-high-latency` | p99 above 120s | Genuinely hung, not merely slow — 24s is a normal maximum here. | +| `agentcore-runtime-active-sessions` | Concurrent Runtime sessions accumulating **for a sustained hour** | **Cost before capacity** — Runtime bills memory for whole session lifetime, and there is no AWS API to terminate a session. Check mean microVM life in the Runtime log group's `/ping` access lines: 20–50 min is healthy, hours means idle reaping has regressed (see #827). ⚠️ Do **not** respond by raising the threshold: measured on 7 days of prod, load-test bursts still fire it at 500 and only go quiet near 1500, which is above the ~99-sustained #338 regime the alarm exists to catch. The 12-period (1 hour) window is what separates the two — a burst under an hour should never have paged you, so if this fired, something stayed up for a full hour. | | `bedrock-tpm-quota-usage-` | **Leading** indicator | First confirm the configured quota still matches Service Quotas — it is set by hand and nothing checks it. If current, request an increase *now*, before throttling starts. | | `bedrock-invocation-throttles` | At a model's TPM/RPM quota | Users see chats that never respond. Quota increase. | | `agentcore-memory-*` | Memory hot path failing | Users experience an agent that has forgotten the conversation. | diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d1a49ff2..1612dfc1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,105 @@ All notable changes to this project are documented in this file. Format follows For narrative release notes written for operators and product owners, see [RELEASE_NOTES.md](RELEASE_NOTES.md). +## [1.21.0] - 2026-09-13 + +Global preferences get a home. **Customize** (`/customize/{tools,skills,connectors}`) replaces the composer's settings drawer, which had been presenting durable, account-wide state as "settings for this conversation" — a user who enabled a tool to get through one question had changed the `toolConfig` of every future turn, and nothing said so. Tools and skills gain full detail pages, an MCP server's sub-tools can be switched one at a time, and `/skill-name` in the composer invokes a skill for a single message the way `@agent` already did. Chat itself stops guessing: a four-tool answer now renders as **one** card instead of five, the loading indicator states what the agent is actually doing (`Running browse_web · 4s`) from a new `agent_status` event, and each finished tool batch gets a model-written summary line off a Nova Micro side-channel that never touches the cacheable prefix. On the cost side, the tool catalog moves off a full-table Scan onto a new `EntityTypeIndex` (95 items read to return 24, before), four tenant-global catalogs gain a TTL + single-flight cache, and the per-request user-profile upsert is throttled — together roughly 24 DynamoDB writes and four uncached scans removed from every SPA first load. **Requires a CDK deploy, and a backfill must be run — see the deployment notes.** ⚠️ **This release removes the only way to select a Conversation Mode**; prod uses one (Guided Learning, ~60 sessions in the first 12 days of September). + +### 🚀 Added + +- **Customize hub** — `/customize/tools`, `/customize/skills` and `/customize/connectors`: search, category chips and a responsive grid over the same root services the drawer used, so no new endpoints and no new state. Connectors moved wholesale from Settings (`/settings/connectors` stays as a redirect, declared above the `settings` route so `loadChildren` cannot swallow it) (#1072, #1076) +- **Tool detail page** — `/customize/tools/:toolId` carries what a card cannot: the full description, an MCP server's tools one by one with their own switches (with a filter box above eight), the prompts and resources the server exposes, and the catalog facts behind it (#1081) +- **Skill detail page** — `/customize/skills/:skillId` renders the SKILL.md body, supporting files, composed skills and the advisory `allowed-tools` frontmatter. New `GET /skills/{id}` and `GET /skills/{id}/resources/{filename}`, access-checked by `resolve_accessible_skill_ids` — both registered **below** every `/mine` route, because `SKILL_ID_PATTERN` matches the literal `mine` (#1085) +- **Skills consolidated into one surface** — `/customize/skills` splits by a `scope` query param into **Yours** (authored at any status, plus catalog skills you turned on) and **Discover** (granted catalog skills still off). The authoring form moved to `/customize/skills/{new,:id/edit}`; the three `/my-skills` paths stay as redirects (#1089) +- **Slash commands in the composer** — typing `/web-research` invokes that skill for that message, sibling to the `@`-mention with the same menu, keyboard and rides-one-turn semantics. Scope is skills already enabled, so the system prompt, `toolConfig` and `` block are byte-identical whether or not a command was used — the whole cost is one line appended to the turn's user message. The composer text is the binding, so a hand-typed command and a menu pick cannot disagree (#1090) +- **`agent_status` SSE event** — `{phase, cycle, toolName?, durationMs?, ok?}` from a new `AgentStatusHook` on `BeforeModelCall` / `Before`+`AfterToolCall`, drained in `stream_coordinator` like `steering_applied` so "Using list_assignments" arrives while that tool is running. Durations come from Strands' own `AfterToolCallEvent.duration`. Deliberately no `responding` phase, and durations are live-only, never persisted. Gated by `AGENT_STATUS_ENABLED` (#1034) +- **`tool_group_summary` SSE event** — a Nova Micro side-channel turns each finished tool batch into one line ("Found the Syllabus Acknowledgment assignment in BIO 101"). Structured exactly like `session_title`: its own Bedrock call on its own messages, so it never appends to the conversation and adds nothing to the cacheable prefix. Persisted as `TSUM#` rows on the existing sessions-metadata table reusing `SessionLookupIndex` (zero new infra) and replayed on `GET /messages` as `toolSummaries`. Gated by `TOOL_SUMMARIES_ENABLED` (#1034) +- **MCP prompts and resources are discovered and stored** — `prompts/list` and `resources/list` had never been called anywhere in the stack, so two thirds of what our servers offer was invisible. Each listing is attempted independently and degrades to `supports_*=False`; "offers nothing" and "we could not ask" are recorded as different facts (#1035, #1037) +- **Try an MCP prompt** — `prompts/get` is now callable from the tool detail page: a field per argument, Compose, the server's composition rendered inline with copy. `MCPPromptArgument` gained `required` and `description`, which `_prompt_entries` had been flattening away (#1087) +- **Refresh a tool's capabilities from the admin tool list** — `POST /admin/tools/{id}/capabilities/refresh` existed and nothing in the frontend called it, so a snapshot was written once by hand and never rewritten. In dev, two servers still reported `supportsPrompts=true` with zero prompts eleven hours after gaining their first (#1062) +- **Scoped tool ids on Agent tool bindings** — `binding.ref` accepts `server/tool`, so the Rubric Builder agent can carry 7 of `canvas_faculty`'s 44 instead of all of them, dropping ~13.2k of tool definitions in the cacheable prefix to ~1.8k. `can_access_tool` now base-collapses a scoped ref like its sibling `filter_requested_tools` already did (#1047) +- **Model picker refactor** — new `shortDescription` on the managed model (80-char cap on write, permissive on read), an Effort submenu driven entirely by the model's declared `effort`/`reasoning_effort` spec, a More models submenu, and vendor icons via `iconSlug` or an S3 upload under `models/{id}/icons/{digest}.{ext}` (#1066, #1077) +- **GPT-6 Astra** (`us.openai.gpt-6-astra`) registered at the Geo CRIS Short Context card — $11.00 in / $13.75 cache-write / $1.10 cache-read / $55.00 out per MTok, `maxInputTokens` 272,000, `maxOutputTokens` 128,000 (#1055) +- **Knowledge base chunk inspector** — `GET /assistants/{id}/documents/{doc}/chunks`, owner/editor only, shows what the managed backend actually extracted. Full untruncated chunk text (the 500-char citation excerpt is exactly why the citation trace cannot serve this), `equals` on `document_id` never a prefix operator, plus a post-filter on top of the backend filter (#1057) +- **Born-managed knowledge bases** — `MANAGED_KB_NEW_DEFAULT` was a no-op: no backend code read it and the app-api Lambda never received it. Now wired end to end; a newly finalized agent provisions its KB through the proven migration worker. **Ships dark** (flag default off) (#1027) +- **Fine-tuning checkpoint + resume** — `resolve_save_steps` scales the interval to the run's own step count targeting ~10 checkpoints, SageMaker mirrors via `CheckpointConfig`, and a restarted attempt resumes. Previously `save_strategy="no"` meant a spot interruption or a `MaxRuntimeInSeconds` kill produced nothing at all for the money already spent (#1024) +- **Opt-in managed spot training** — roughly a 65% discount. Refused at submit when `checkpointing=false`: simulated at a 0.15/hr hazard, a 48h job costs ~3,200 billed hours without checkpointing and ~18 with it. Off by default (#1025) +- **`agentcore-runtime-active-sessions` alarm** — Runtime bills memory for a session's whole lifetime and AWS exposes no API to terminate one, so session accumulation is the leading indicator. Tunable via `CDK_OBSERVABILITY_AGENTCORE_ACTIVE_SESSION_THRESHOLD` (#1051, #1058) +- **Time-of-day greetings** — twenty new greetings bucketed morning/afternoon/evening/night, pooled *with* the existing five rather than replacing them, read off the viewer's own clock once at resolve time. Rebrandable via optional `timeOfDayGreetings` / `timeOfDayFallbackGreetings` (#1096) +- **Cycling discovery hints in the empty composer** — `@` and `/` are the two shortcuts nothing on the page advertises. An `aria-hidden` overlay over a transparent-but-present native placeholder, so assistive tech reads one stable string; stops after three passes, settles on the first keystroke, and honours `prefers-reduced-motion` by not rotating at all (#1096) +- **Agent governance on the assistant indicator** — a lock glyph and a menu listing what the bound Agent fixes, closing with "Your own choices in Customize don't apply in this conversation" (#1075) +- **Side nav restructured** to New Session → Agents → Artifacts, all painting at once. Assistants comes out (the route and explainer stay); the "New" badge comes off Agents; Artifacts takes the freed slot (#1063) +- **`backfill_tool_catalog_index.py`** — stamps `GSI5PK`/`GSI5SK` on tool rows written before they existed. Dry-run by default, idempotent, guarded by `attribute_not_exists(GSI5PK)` (#1069) +- **Release gate on pending backfills** — `scripts/release/check-pending-backfills.mjs` plus `.github/workflows/pending-backfills.yml` fail any PR into `main` that adds a `backend/scripts/backfill_*.py` not named in `RELEASE_NOTES.md` (#1070) + +### ✨ Improved + +- **A multi-tool answer is one card, not five.** `AssistantMessageComponent` takes a *run* of messages and flattens it into one block stream; a tool group spans message boundaries and only text or reasoning breaks it. Bedrock's tool results arrive as USER-role messages carrying nothing but `toolResult` blocks — they render at zero height but sat between every pair of assistant messages, and are now guarded by `isProtocolScaffolding`. The rail also stays *collapsed* while tools run, where it used to force itself open on any pending call (#1034) +- **The loading indicator says what is true.** Twenty invented phrases ("Pondering", "Consulting the archives") are replaced by two states read from `agent_status` — `Thinking 12s` and `Running browse_web 4s` — with the tool's own identifier as the label, distinct in colour from the amber `model_retry` notice (#1040) +- **Tools drawer drill-down, search and category grouping**, plus OAuth connection state on each tool, before the drawer was retired (#1030, #1032, #1033) +- **Readable tool descriptions** — a tool's summary with its docstring on demand, rather than the raw identifier (#1038) +- **Connector edits no longer require a credential rotation.** `update_provider` treated any non-None `oauth_discovery_url` as a discovery change, and the edit form round-trips it on every save — so changing scopes, display name, icon or enabled state was rejected with "Discovery config can only be updated together with a credential rotation", which the admin could not comply with because the client secret is never readable back. The guard now compares against the stored record (#1029) +- **Admin cost surface** — cache-read is now a per-model input rather than a hardcoded `input * 0.1`; Fable 5.1 reads at 0.025× and Grok 4.6 at 0.25×, both live on Bedrock today. Rate provenance corrected: per-model AWS model cards are authoritative for Claude, Nova and the OpenAI/Mantle family, the Price List API for xAI, Google and AgentCore (#1049) +- **Buttons settle on one radius.** `rounded-2xl` app-wide — 45 off-token primary buttons, 3 more where radius and fill were not adjacent in the class string, 6 dialog secondary buttons and the shared `ConfirmationDialog` pair (#1080, #1091) + +### 🐛 Fixed + +- **Loading a conversation page logged blocked mixed content.** `GET /api/agents/` produced Starlette's own `redirect_slashes` 307 to `http://api./agents` — the internal ALB hostname, over plain HTTP, with `/api` stripped. The browser blocks it, the caller silently gets nothing, the ALB hostname leaks into a page the user can read, and a client that followed it would land where the `__Host-` BFF cookies are not sent. CloudFront now sets `x-forwarded-prefix: /api` unconditionally and `ProxiedRedirectMiddleware` puts the public URL back. Not specific to `/agents` — `/models/`, `/files/` and `/auth/login/` did the same (#1074) +- **The agent designer's preview silently dropped tool calls.** Measured on dev: one requested `upload_course_file` produced ~316 attempts and **zero** invocations of the backing Lambda. The cause was a divergent SSE consumer, not a divergent dispatch path — `PreviewChatService` implemented 9 of ~27 events and every callback is invoked with `?.`, so an unimplemented handler dropped its event in total silence, including `oauth_required` and `tool_approval_required`, the two that gate dispatch. The fork is deleted; both preview surfaces now run on the main chat's services (#1060) +- **Conversation Mode silently stopped applying after a reload.** The session page hydrates the active mode twice on load, and the provisional call claimed the session id, so the clobber guard rejected the real hydration — while the stored preference still said the mode was on. The provisional call now passes `claim:false` (#1079) +- **Fine-tuning inference had never worked from the deployed app.** Batch Transform is two APIs and the task role was granted the job actions but not `sagemaker:CreateModel`, so every inference job died at step one with `AccessDeniedException`. Invisible from a developer machine: CloudTrail shows every successful `CreateModel` in dev was a human's SSO credentials running app-api locally. The ARN is `model/model--*` — the literal `model-` sits ahead of the prefix, so a pattern written to match the training-job ARNs looks correct and denies every call (#1023) +- **Three of five VLM catalog models could not train on their own defaults.** SmolVLM-Instruct spends 1377 tokens on a single image against a `context_length` of 1024, so truncation cut the image placeholder run to 891 and the processor rejected the batch. The trainer now renders a sample untruncated, raises the context length to fit, and logs the adjustment; defaults are raised too, so measurement stays a safety net (#1015) +- **A fresh deployment would have listed zero tools.** `seed_bootstrap_data.py` hand-builds its tool item instead of calling `ToolDefinition.to_dynamo_item`, so it wrote no `GSI5PK` — and once the catalog read moved to a sparse index, a newly bootstrapped fork or environment would have listed nothing, with no error anywhere (#1070) +- **`/users/me/settings` was read twice per load**, ~280ms apart and sequentially, so a single-flight guard would not have caught it. `getSettings()` memoizes the promise; a rejected read is deliberately not cached (#1067) +- **Nightly Build & Test failed seven consecutive nights.** Under `--coverage`, @angular/build bundles each spec into a flattened chunk emitted at the **project root**, so `import.meta.url` moves with it and every `resolve(SPEC_DIR, '..')` silently changed meaning. New `src/testing/project-root.ts` walks up from `process.cwd()` to the directory holding `angular.json`; the coverage build now also runs on PR CI, where it had been invisible (#1048) +- **The `agentcore-runtime-active-sessions` alarm fired on every load test.** Validated against 7 days of prod data it would have fired three times in three nights, all planned load tests peaking at 241, 608 and 1404 — and no threshold separates them, since load tests still trip it at 500 while the #338 reaper regression this alarm exists to catch sustained only ~99. Duration does: 75 over 60 minutes fires on neither the load tests nor a burst (#1058) +- The tool summary no longer eats the closing quote (#1039); the tool row's hover highlight runs the full drawer width (#1042); the tool rail's hover is scoped to the rail (#1043); the prose margin under the tool batch summary is dropped (#1044) + +### ⚠️ Changed + +- **Breaking (UX): the composer settings drawer is deleted, and with it the only way to select a Conversation Mode.** Skills and Tools moved to Customize, model and inference params moved to the composer's model picker, and the Conversation Mode picker that replaced the drawer's control was **parked before release** (#1088) pending user feedback on its placement. Prod carries one enabled mode — Guided Learning, a Socratic tutoring prompt — with use accelerating: 1 session in July, 20 in August, 60 in the first 12 days of September. The backend is untouched: `SystemPromptsService`, `GET /system-prompts/`, `selected_prompt_id` on `SessionPreferences` and the admin CRUD all still work, so restoring the control is a revert rather than a rebuild. **Take this regression knowingly or restore `ConversationModePickerComponent` before deploying** (#1073, #1079, #1088) +- **Breaking (route):** `/settings/connectors` and the three `/my-skills` paths are now redirects, not pages. Bookmarks and the schedules page's deep link continue to work (#1076, #1089) +- **`last_login_at` is accurate to within a window rather than to the last request** — default 5 minutes, tunable via `USER_SYNC_THROTTLE_SECONDS`. A brand-new user is unaffected: with no entry recorded the first request always claims the sync (#1065) +- **`dark:text-primary-400` and `bg-primary-50|100|200` are banned as text and tint-fill tokens.** The `primary` scale is generated from #0033a0 by lightness offset alone and keeps full chroma, so `primary-50` resolves to rgb(118,179,255) — a saturated mid-blue, not a wash. Use `text-primary-accessible dark:text-primary-accessible-dark` and neutral surfaces. The `state-*` scales *are* real tints, which is why the pattern looked safe by analogy (#1082, #1091) +- `ProxiedRedirectMiddleware` now rewrites app-generated redirects on the public URL; a fork terminating TLS elsewhere should confirm its edge sets `x-forwarded-prefix` (#1074) + +### 🔒 Security + +- **An Agent could no longer smuggle a whole MCP server through a narrow binding.** Binding `canvas_faculty` loaded all 44 of its tools including `grade_submission` and `delete_rubric`, held back only by wording in the agent's system prompt — a fence for ordinary use and none at all against a determined one. Scoped binding refs make the restriction structural (#1047) +- **The chunk inspector's document filter is `equals`, never a prefix operator** — a prefix match for `DOC-1` also admits `DOC-10`, so the operator choice *is* the isolation boundary between two owners' documents, not a query-tuning detail. A post-filter backs it up (#1057) + +### ⚡ Performance + +- **The tool catalog is listed by Query, not Scan.** The `app-roles` table is shared — tools, skills, roles, role grants, JWT mappings and one preferences row **per user** — so the read's cost grew with enrollment rather than with the number of tools: 95 items read to return 24 on dev, 17 of them per-user rows. The Query reads 24 to return 24 and stays flat as the campus grows. Two fallbacks, because an empty catalog means every user loses every tool: a missing index falls back to Scan, and a *zero* result is treated as suspect and re-read via Scan, logging an ERROR that names the backfill script (#1069, #1071) +- **Four tenant-global catalogs are TTL-cached with single flight** — models, tools, system prompts and connectors, three of which were full table scans, none cached. Single flight is the half that matters: with a cold cache 300 simultaneous sign-ins would otherwise issue 300 concurrent scans, exactly when the burst lands. The scans also move onto `asyncio.to_thread`; they were blocking boto3 calls made from `async def`, stalling the whole event loop. 60s TTL via `CONFIG_CACHE_TTL_SECONDS`. Entries hold **raw items** and callers re-parse, deliberately — `hydrate_model_roles` mutates what it is handed, and caching parsed objects would let an admin write display-only `allowedAppRoles` onto instances then served to every user (#1068) +- **The per-request user-profile upsert is throttled.** `get_current_user_from_session` fired `sync_user_from_jwt` on every authenticated request — a GetItem plus a PutItem rewriting the whole row and its GSI projections — so one SPA first load of 12 API calls issued 24 DynamoDB operations against one item, all writing identical values except `last_login_at`. The claim is recorded *before* the sync runs, because those 12 requests overlap. Also fixes a latent GC hazard: the dispatched task was unreferenced, and the loop holds only a weak reference to a bare `create_task` (#1065) + +### 🏗️ Infrastructure + +- **New `EntityTypeIndex` on the existing `{prefix}-app-roles` table** (`GSI5PK=ENTITY#{type}`, `GSI5SK=` the item's own PK, `ProjectionType.ALL`). Deliberately generic rather than TOOL-only: `list_roles` and the skills catalog scan the same table for the same reason, and one GSI per `UpdateTable` means a second entity type would otherwise need its own release. Sparse by construction — adding it changes nothing until rows are stamped. **Exactly one GSI operation on one existing table; `gsi-inventory.json` reflects it and the release guard passes** (#1069) +- **`sagemaker:CreateModel`** granted to the app-api task role under a new `SageMakerModelManagement` statement scoped to `model/model-{prefix}-*` — its own statement, because a pattern matching the training-job ARNs denies every call (#1023) +- **CloudFront sets `x-forwarded-prefix: /api`** unconditionally on the `/api/*` behaviour, so a viewer-supplied header is overwritten rather than passed through (#1074) +- **New `agentcore-runtime-active-sessions` alarm** on `ActiveSessionCount`/`AgentCore.Runtime`, threshold 75 over 12 evaluation periods (60 minutes). Net +1 alarm for a fork that configures nothing (#1051, #1058) +- **`MANAGED_KB_NEW_DEFAULT`** threaded into the app-api environment as an explicit `'false'` rather than omitted (#1027) + +### 🔧 CI/CD + +- New `pending-backfills.yml` — fails any PR into `main` adding a `backfill_*.py` script not named in `RELEASE_NOTES.md` (#1070) +- `tests.yml` gains a `run_frontend_coverage` input and a `Test frontend (coverage build)` job; `ci.yml` sets it true, so the coverage build that had been failing only in the nightly is now visible on every PR (#1048) +- New job-level `CDK_OBSERVABILITY_AGENTCORE_ACTIVE_SESSION_THRESHOLD` in `platform.yml`, plumbed through `load-env.sh` (#1051) + +### 📦 Dependencies + +- Backend: new `[tool.uv] constraint-dependencies = ["mcp<2"]`. `mcp` reaches us only through `strands-agents`, which declares `mcp>=1.23.0,<2.2` — the only thing holding us on 1.x today is that `mcp` 2.x needs `idna>=3.18` while we pin `idna==3.15` for an unrelated Dependabot alert, so a routine security bump would quietly unblock a major crossing nobody chose. The crossing lands under `integrations/mcp_apps.py`, which reaches through a Strands internal into an MCP SDK class, and the failure there is silent — App frame headers degrade to a generic glyph rather than raising. The constraint bounds the version without adding `mcp` to our dependency metadata (#1050) + +### 📚 Docs + +- `docs/specs/customize-surface.md` — the epic's spec, closed out (#1072, #1080) +- `docs/specs/skill-slash-commands.md` (#1090) +- `docs/specs/canvas-rubric-agent.md` — design plus four recorded upstream blockers, chief among them that `create_rubric` drops rating `long_description`, which is exactly where a descriptor lives: a rubric would land in Canvas structurally correct and completely empty while appearing to succeed (#1028, #1041) +- Weekly kaizen research and review-prep for 2026-09-11; the verified blockers on the AgentCore S3 Files mount; conversation-branching Phase-1 spike findings; the GPT-6 Astra context-tier decision; a `probe_bedrock_cache_point_support.py` script and the answer it produced (#1045, #1046, #1052, #1053, #1054) +- `src/branding/README.md` and the Tailwind skill: stale Tailwind-blue guidance replaced with the brand tokens (#1084) + ## [1.20.0] - 2026-09-09 Two new capabilities and one measurement that changes how the platform should be scaled. `browse_web` drives a real Chrome browser in the AgentCore Browser sandbox with a live view the user can watch, and fine-tuning gains a fourth task type — **generative VLMs**, LoRA-adapted over a 4-bit base, so a model can *write* an answer about an image instead of only classifying it. Alongside them, a load-testing harness establishes that the campus-scale ceiling is the **Bedrock TPM quota, not compute**: a representative production turn costs ~26,700 quota-counted tokens against the ~1,920 a naive load profile assumes, so a single 300-student class exceeds the default 6,000,000 TPM quota. MCP Apps get the end of a three-part chain that made every button in an embedded App fail silently after a reload, including the discovery that **AgentCore Runtime rewrites any non-2xx container response to a generic 424 and discards the body**. A managed-KB dead-letter reconciler closes the case where a document is indexed and retrievable in Bedrock but can never be cited. **Requires a CDK deploy**, and three defaults change for a fork that configures nothing — app-api Fargate sizing, the Bedrock quota alarm, and a new nightly Lambda. diff --git a/CLAUDE.MD b/CLAUDE.MD index a5d6d8033..9f1a4181e 100644 --- a/CLAUDE.MD +++ b/CLAUDE.MD @@ -39,7 +39,7 @@ npx cdk deploy {prefix}-PlatformStack - **Admin endpoints** go under `/admin//`, user-facing under `//` - **Errors stream as assistant messages** via SSE (not HTTP error codes) - **Signal-based state** throughout frontend (`signal()`, `computed()`) -- **Prompt-cache stability is a contract.** Bedrock prompt caching is exact-prefix-match: any list that reaches the system prompt or `toolConfig` (skills, tools, models, MCP tool listings) must be deterministically ordered at its source, and restored conversation history must be byte-stable between compaction-state changes (see the truncation anchor in `TurnBasedSessionManager`). An order flip or history mutation between turns silently re-writes a 30k–150k-token prefix at the cache-write premium, which is **1.25× the model's own base input rate** (2× at 1h TTL; a cache read is 0.1×) — there is no flat per-MTok figure, so price it against the model actually in play: $1.375/MTok on our default Haiku 4.5, $4.125 on Sonnet 4.6. Our `us.*` ids are **Regional (CRIS)** inference profiles and price ~10% above `global.*` for the same model; rates live in `curated-models.ts` and come from the AWS Price List API, not the pricing page. `PROMPT_CACHE_OBSERVABILITY_ENABLED=false` disables the observability layer (fingerprint hook, cacheStatus derivation, EMF metrics) — the caching itself stays on +- **Prompt-cache stability is a contract.** Bedrock prompt caching is exact-prefix-match: any list that reaches the system prompt or `toolConfig` (skills, tools, models, MCP tool listings) must be deterministically ordered at its source, and restored conversation history must be byte-stable between compaction-state changes (see the truncation anchor in `TurnBasedSessionManager`). An order flip or history mutation between turns silently re-writes a 30k–150k-token prefix at the cache-write premium, which is **1.25× the model's own base input rate** (2× at 1h TTL; a cache read is 0.1×) — there is no flat per-MTok figure, so price it against the model actually in play: $1.375/MTok on our default Haiku 4.5, $4.125 on Sonnet 4.6. Our `us.*` ids are **Regional (CRIS)** inference profiles and price ~10% above `global.*` for the same model; rates live in `curated-models.ts`, and which source is authoritative depends on the vendor: per-model AWS **model cards** are authoritative for Claude, Nova and the OpenAI/Mantle family; the **Price List API** is authoritative for xAI, Google and AgentCore. Do not re-derive Claude rates from the Price List API — a full enumeration of the `AmazonBedrock` offer file returns 10 Claude SKUs, **none newer than Claude 3**, and no us-west-2 SKU at all for Haiku 4.5, Sonnet 4.6, Fable 5.1, GPT-5.4 or Nova Micro (verified twice, a week apart). The 1.100× Regional premium *does* reproduce there, on the vendors the API does carry. `PROMPT_CACHE_OBSERVABILITY_ENABLED=false` disables the observability layer (fingerprint hook, cacheStatus derivation, EMF metrics) — the caching itself stays on - **Token cost effectiveness is a design tenet — engineer against waste, not against context.** Before merging a change that touches the model call path, answer: what does this add to the prompt, on every turn, for the life of every session? (1) Anything in the cacheable prefix (system prompt, `toolConfig`, restored history) must be deterministic and append-only — see the prompt-cache contract above. (2) Per-turn payloads (tool results, MCP responses, retrieved documents) should be bounded or offloaded, never unbounded pass-through. (3) Don't guess — verify: `cacheStatus` + fingerprint hashes on the session's `C#` rows, `GET /admin/costs/sessions/{id}/calls`, and the `AgentCoreStack/PromptCache` EMF metrics exist to prove a change's cost impact. The balance: "cost effective" means eliminating waste (avoidable cache re-writes, duplicated context, oversized payloads) — it never means stripping context the model needs for a quality answer. When cost and answer quality genuinely conflict, quality wins; look for the cheaper path to the *same* quality, not a cheaper answer - **One session can be served by more than one agent — never cache session state on an agent instance.** The agent cache keys on *configuration* (system prompt, tools, model, skills), so an `@`-mention turn builds a second `Agent`, and each `Agent` builds its own `TurnBasedSessionManager`. Both write the same DynamoDB session row and neither knows the other exists, while `initialize()` never re-runs on a cache hit — so anything a manager loads once and holds goes stale silently. This has bitten twice: conversation history (#741, fixed by aliasing the message list in `_adopt_session_conversation`) and compaction state (#751, fixed by re-reading it every turn in `_adopt_persisted_compaction_state`). Per-session state must be aliased across instances or re-read per turn, and must never move backwards — a clobbered checkpoint or truncation anchor is a prompt-cache **cost** bug before it is a correctness one - **All dependencies use exact version pins** — no `^`, `~`, or `>=` @@ -55,6 +55,8 @@ npx cdk deploy {prefix}-PlatformStack | `tool_use` / `tool_result` | Tool invocation and result | | `ui_resource` | MCP App UI for a tool (SEP-1865) — payload `{type, toolUseId, resourceUri, html, mimeType, csp, permissions, sandboxOrigin, serverName, icon, toolName}`. Emitted at the tool's `content_block_start` (early frame mount, so the App's bridge is live *while* the model streams the tool's arguments — see `ui_tool_input_partial`); falls back to right after the correlated `tool_result` if the name wasn't known at block start. At `content_block_start` a **header-only shell** (`html: ""`, full `serverName`/`icon`/`resourceUri` but no `resources/read`) is emitted *first* so the App frame's header (icon + server + tool + shimmer) replaces the plain tool rail with no flash; the full html-bearing event follows after the read and, last-write-wins, mounts the iframe (the SPA gates the iframe on a non-empty `html`). Deduped per `toolUseId` (the header shell on its own set, so it never blocks the full emit). HTML fetched server-side via `resources/read` and inlined; `sandboxOrigin` is the proxy.html origin the SPA frames it in (empty unless the mcp-sandbox stack is deployed — inference-api consumes its SSM origin only when `CDK_MCP_SANDBOX_ENABLED=true`; an empty origin means the SPA cannot frame the App). `serverName`/`icon`/`toolName` drive the App frame's connected header (Claude parity): `serverName`/`icon` resolve from the MCP `initialize` `serverInfo` (`title`→`name`, plus its `icons`); `serverName` falls back to the title-cased `ui://` authority, and `icon` falls back to the server's served MCPB `manifest.json` icon — fetched server-side from `/manifest.json` (same-origin only, cached per origin, base64-inlined as a `data:` URI; the runtime MCP protocol carries no icon, so this mirrors what Claude inlines from the installed bundle), else empty (→ generic glyph). A large auto-fetched icon is NOT persisted (size-gated against the 400KB DynamoDB item limit), so it shows live but reloads to the glyph; `toolName` is the agent-facing tool name, carried on the event so the header's name + running shimmer appear atomically with the frame's promotion (not gated on the separately-streamed message content). All three persist with the resource so the header survives reload. Gated by `AGENTCORE_MCP_APPS_HOST_ENABLED` (default true since PR #7; set `=false` to opt an environment out) | | `ui_tool_input_partial` | Streamed partial tool input for a UI tool (SEP-1865 `ui/notifications/tool-input-partial`) — payload `{type, toolUseId, arguments}`. Emitted repeatedly while the model is still streaming a UI tool's arguments (after the early `ui_resource` mount); `arguments` is the streamed prefix server-side "healed" into a valid object (`apis/shared/mcp_apps/partial_json.py`). The SPA relays each to the App via `ui/notifications/tool-input-partial` so a progressively-rendering App (e.g. Excalidraw's guided camera tour) animates as args arrive; the complete `tool-input` follows once the input is final. Same gating as `ui_resource` | +| `agent_status` | What the agent is doing right now — payload `{type, sessionId, phase, cycle, toolName?, toolUseId?, durationMs?, ok?}`. Emitted from `AgentStatusHook` (`BeforeModelCall` / `Before`+`AfterToolCall`) and drained in `stream_coordinator` exactly like `steering_applied`, before the event it precedes, so "Using list_assignments" reaches the client while that tool is running rather than after its result. Phases: `thinking` (one per event-loop cycle — a three-tool turn reports it four times, and `cycle` distinguishes them), `tool_start`, `tool_end` (carries Strands' own measured `durationMs`, and `ok=false` for BOTH a raised exception and a result with `status: "error"`). There is deliberately **no "responding" phase** — the SPA already knows text is streaming from the deltas, and a backend-derived duplicate of a fact the client holds first-hand would only disagree at the edges. Durations are live-only and deliberately NOT persisted: a reloaded conversation shows summaries without timings, where a client-invented number would be one the user could not trust. Costs nothing against the model — nothing it produces reaches the prompt, so the cacheable prefix is untouched. Gated by `AGENT_STATUS_ENABLED` (default on with a kill switch); while off the hook is registered but every callback returns immediately and the SPA stays on its cycling phrases | +| `tool_group_summary` | Model-generated one-line summary of a finished tool batch — payload `{type, sessionId, batchId, toolUseIds, summary}`, e.g. "Found the Syllabus Acknowledgment assignment in BIO 101". Produced by a Nova Micro **side-channel** task (`apis/shared/tool_summaries/summarizer.py`) structured exactly like `session_title`: its own Bedrock call on its own messages, concurrent with the agent stream, so it **never appends to the conversation** and adds nothing to the cacheable prefix. Spend is one bounded call per batch (inputs/results truncated at capture in the hook AND again in the summarizer). Lands mid-turn, out of band with the content stream, so it can arrive after the rail that shows it has rendered; the SPA keys it by `toolUseId` (not batch — client-side grouping need not match backend batches, and a group spanning two batches shows the first batch's line). Persisted as `TSUM#` rows in sessions-metadata, reusing the `SessionLookupIndex` GSI — zero new infra — and replayed on `GET /messages` as `toolSummaries`, because the event never re-streams. Deliberately NOT written onto the message content blocks: that is the Converse payload and the cacheable prefix, so a display string there would be paid at model rates every subsequent turn. Gated by `TOOL_SUMMARIES_ENABLED` (default on with a kill switch); while off the SPA's deterministic client-side formatter still renders ("Listed 4 assignments"), so absence is a downgrade in specificity, never a blank | | `session_title` | Server-generated conversation title on a session's FIRST turn — payload `{type, sessionId, title}`. Title generation (Nova Micro) runs as an asyncio task concurrent with the agent stream; the finished title is interleaved between agent events (non-blocking done-check in `stream_with_quota_warning`), so the sidebar/top-nav rename while the response is still pending. Emitted at most once per stream, possibly after `done` (the SPA parser allowlists it past Completed-state gating); never carries the "New Conversation" placeholder. Best-effort: a stream that finishes before generation emits nothing — the SPA's post-close metadata refresh (`refreshTitleFromServer`) is the fallback, reading the title the task also persisted via `update_session_title` | | `quota_session_notice` | **This conversation** has reached the tier's session-notice share of the monthly limit — payload `{type, sessionId, sessionCost, quotaLimit, sessionPercentageOfLimit, thresholdPercentage, message}`. Emitted at the head of the stream right after `quota_warning`, and re-emitted every turn while over the share (dismissal is client-side, same contract as `quota_warning`). `sessionCost` is the session's **lifetime** cost — the `totalCost` aggregate on its metadata row — deliberately not period-scoped: a conversation that opened last month and is spending this month's budget is exactly the one worth surfacing. Share is tier-configurable (`sessionNoticePercentage`, default 25%, 0 disables); the whole runway rides the `QUOTA_RUNWAY_ENABLED` kill switch (default on), which also gates the 50%/75% `quota_warning` rungs. The SPA scopes it to the conversation it names — never shown above another thread's composer | | `model_retry` | Backend is retrying a failed model call instead of surfacing it — payload `{type, attempt, delaySeconds}`. Emitted from Strands' `EventLoopThrottleEvent`; `attempt` is 1-based and counted per turn in `stream_processor` (the raw event carries only the delay). **Timing caveat:** Strands sleeps the backoff *inside* its hook and yields the event afterwards, so it lands as the next attempt BEGINS, not when the wait starts — `delaySeconds` describes the gap just endured, it is not a countdown. It also cannot cover the failing model call itself, which is indistinguishable from a slow healthy one. The SPA swaps the loading indicator's cycling phrases for a fixed amber notice, cleared on `message_start`/`done` | diff --git a/README.md b/README.md index 17c9fc933..87588267f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ **An open-source, production-ready Generative AI platform for institutions** *Built by Boise State University, designed for everyone.* -[![Release](https://img.shields.io/badge/Release-v1.20.0-6366f1?style=flat&logo=github&logoColor=white)](RELEASE_NOTES.md) +[![Release](https://img.shields.io/badge/Release-v1.21.0-6366f1?style=flat&logo=github&logoColor=white)](RELEASE_NOTES.md) [![Nightly](https://github.com/Boise-State-Development/agentcore-public-stack/actions/workflows/nightly.yml/badge.svg)](https://github.com/Boise-State-Development/agentcore-public-stack/actions/workflows/nightly.yml) ![Python](https://img.shields.io/badge/Python-3.13+-3776AB?style=flat&logo=python&logoColor=white) @@ -296,7 +296,7 @@ agentcore-public-stack/ See [RELEASE_NOTES.md](RELEASE_NOTES.md) for the full changelog, including new features, bug fixes, platform upgrades, and deployment notes for each release. -**Current release:** v1.20.0 +**Current release:** v1.21.0 --- diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 59dbf558b..c93ec87e2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,402 @@ +# Release Notes — v1.21.0 + +**Release Date:** September 13, 2026 +**Previous Release:** v1.20.0 (September 9, 2026) + +--- + +> 🏗️ **CDK deploy required.** One new GSI (`EntityTypeIndex`) on the **existing** `{prefix}-app-roles` table, `sagemaker:CreateModel` on the app-api task role, a CloudFront `x-forwarded-prefix` header on the `/api/*` behaviour, one new CloudWatch alarm, and a new app-api task-definition revision. **Exactly one GSI operation on one existing table** — the release guard passes — but the index must reach `ACTIVE` before the catalog read is trustworthy. See Deployment notes. +> +> 🗂️ **A backfill must be run, in every environment, as part of this deploy.** `backend/scripts/backfill_tool_catalog_index.py` stamps `EntityTypeIndex` keys on tool rows written before they existed. The tool catalog's read moves onto that sparse index in this same release, and a sparse index answers *"nothing matched"*, not *"something is wrong"* — an unrun backfill looks like a short tool list, not an outage. There is a Scan fallback that covers the zero-result case, but it logs an ERROR and costs an extra read per cache fill; it is a safety net, not the plan. Exact command in Deployment notes. +> +> ⚠️ **This release removes the only way to select a Conversation Mode.** The composer's settings drawer is deleted, and the picker meant to replace its Mode control was parked before release pending feedback on its placement. Prod carries one enabled mode — **Guided Learning**, a Socratic tutoring prompt — and its use is accelerating: 1 session in July, 20 in August, **60 in the first 12 days of September**. The backend is entirely untouched, so restoring the control is a revert rather than a rebuild. **This is a decision to take knowingly before deploying**, not a detail to discover afterwards. See Breaking changes. +> +> 🎨 **Two brand tokens are now banned in new code.** `dark:text-primary-400` for colored text and `bg-primary-50|100|200` as a tint fill. The `primary` scale is generated from #0033a0 by lightness offset alone and keeps full chroma at every step, so `primary-50` is a saturated mid-blue, not the pale wash its name implies — and both fail WCAG AA. Existing occurrences are swept in this release. + +--- + +## Highlights + +Global preferences finally have a home. **Customize** — `/customize/tools`, `/customize/skills`, `/customize/connectors` — replaces the composer's settings drawer, which had been quietly lying about scope: tool and skill enablement is durable, account-wide state, and it lived in a container that reads as "settings for this conversation." A user who switched on a tool to get through one question had changed the `toolConfig` of every future turn, and nothing in the UI said so. Tools and skills now have real detail pages, an MCP server's sub-tools can be enabled one at a time, and typing **`/skill-name`** in the composer invokes a skill for a single message the way `@agent` already did. + +Chat stops guessing about itself. A four-tool answer used to render as five separate assistant cards, each paying full chrome, so the actual answer was pushed below the fold; it is now **one card**. The loading indicator's twenty invented phrases ("Pondering", "Consulting the archives") are replaced by two states that are literally true — `Thinking 12s`, `Running browse_web 4s` — fed by a new **`agent_status`** SSE event, and each finished tool batch gets a model-written summary line ("Found the Syllabus Acknowledgment assignment in BIO 101") from a **`tool_group_summary`** side-channel that never touches the conversation or the cacheable prefix. + +Three cost reductions land on the same request path. The tool catalog moves off a full-table Scan onto a new sparse index — the `app-roles` table is shared with one preferences row **per user**, so that read's cost had been growing with *enrollment* rather than with the number of tools. Four tenant-global catalogs gain a TTL + single-flight cache, so 300 students signing in together no longer issue 300 concurrent scans. And the per-request user-profile upsert is throttled: one SPA first load had been issuing **24 DynamoDB operations against a single item**, all writing identical values except `last_login_at`. + +Two fixes are worth reading even if you don't use what they sit under. Loading any conversation page on a CloudFront deployment logged **blocked mixed content** — Starlette's own trailing-slash redirect, rendered from the internal ALB hostname over plain HTTP, leaking that hostname into a page the user can read while the caller silently got nothing. And the agent designer's preview pane was **silently dropping tool calls**: one requested `upload_course_file` produced ~316 attempts and zero invocations of the backing Lambda, because the preview ran a second SSE parser that implemented 9 of ~27 events and dropped the rest without so much as a parse error. + +--- + +## Customize — a home for global preferences + +Tool and skill enablement is per-user, durable and account-wide. Until this release it lived in a ~320px drawer hanging off the composer, which is the wrong container for it in two distinct ways: it presented global state as conversational, and per-tool MCP enablement means a single server can exceed the drawer's comfortable length on its own. + +### Frontend + +- `customize/tools/customize-tools.page.ts`, `customize/skills/customize-skills.page.ts` — search, category chips and a responsive grid, borrowing the browse idiom from `agents/discover`. Both read the existing root services, so a toggle here and (while it lived) the drawer stayed in sync with **no new endpoints and no new state**. +- `customize/tools/customize-tool-detail.page.ts` — `/customize/tools/:toolId`. The full description, an MCP server's tools one by one with their own switches, the prompts and resources the server exposes, and the catalog facts behind it. The drawer's tab strip did not come along: tabs existed because the pane was 320px, and on a page Tools / Prompts / Resources / About are stacked sections that find-in-page can reach. The problem tabs were solving — a 48-tool server — is solved directly, with a filter box above eight sub-tools. +- `customize/skills/customize-skill-detail.page.ts` — the SKILL.md body rendered expanded (sanitized markdown; it is text the user's own turns already load), supporting files, composed skills, and the advisory `allowed-tools` frontmatter labelled as advisory. +- `customize/connectors/customize-connectors.page.ts` — moved wholesale from Settings via `git mv`, so history follows. Connecting an account and enabling the tools that need it are one user intent; the Tools tab marks a tool "connect", and the place to act on that had lived under a different top-level page. +- `customize/components/customize-card.component.ts`, `customize-tabs.component.ts` — the shared card (with a `detailLink` input) and tab strip. + +A deliberate hazard the pages are written around: **the Agent binding lock is conversation-scoped state held on root singletons that `session.page.ts` never releases on teardown**, so a user arriving from an agent-bound chat carries it. A global preference page must not consult it. These pages read `tools()` / `skills()` and `isEnabled` rather than the `visible*` / `isShownEnabled` display shims, and write with `respectAgentLock: false`. Without that, the catalog would show the Agent's bound subset and every switch would silently do nothing. + +The detail page is a new component rather than a port of the drawer's `ToolDetailComponent`, for the same reason: the drawer was conversation-scoped and wrote through the Agent lock, and reusing it would reopen the exact seam this epic exists to close. + +### Skills, consolidated + +`/my-skills` was a top-level route reachable only by a link-out from `/customize/skills`, so the same noun lived in two places with two different answers to "what skills do I have?" — one page listed what you authored, the other what you could turn on, and neither showed the whole set. Both now live under `/customize/skills`, split by a `scope` query param: **Yours** (authored at any status, plus catalog skills you turned on) and **Discover** (granted catalog skills still off). + +No backend change was needed for the merge — it combines `GET /skills/` (accessible + ACTIVE, with the preference) and `GET /skills/mine` (the authored tier at every status). That merge is what keeps a DRAFT skill visible to its author: widening `GET /skills/` to carry drafts would surface them in the composer picker, which the runtime refuses to activate, so a draft would get a dead toggle. It has no toggle at all instead. + +### Backend + +- `apis/app_api/skills/routes.py` — new `GET /skills/{id}` and `GET /skills/{id}/resources/{filename}`, access-checked by `resolve_accessible_skill_ids`, the same resolution that builds the picker. The only per-skill read that existed was owner-scoped, so a catalog skill granted to you 404'd there. The response omits `ownerId` and `allowedAppRoles` on purpose. +- Both register **below** every `/mine` route. `SKILL_ID_PATTERN` matches the literal string `mine`, so the reverse order turns `GET /skills/mine` into a lookup for a skill named "mine". + +### Routing traps, each covered by a test + +`/settings/connectors` stays as a redirect rather than a deletion — it is in bookmarks and the schedules page linked users straight to it — and must be declared **before** the `settings` route, whose `loadChildren` would otherwise swallow the path and land the user on the settings shell with no matching child. Likewise `customize/skills/new` must stay above `customize/skills/:skillId`. Both failures are silent, which is why both are asserted. + +--- + +## Slash commands — invoking a skill for one message + +Typing `/web-research` in the composer invokes that skill for that message: the sibling of the `@`-mention, with the same menu shape, keyboard handling and rides-one-turn semantics. The menu's last row is "Browse skills →". + +**Scope is what makes it cheap.** Only skills the user already has enabled can be invoked, so the invoked skill is already in `enabled_skills` and the system prompt, `toolConfig` and `` block are **byte-identical** whether or not a command was used. The cacheable prefix is untouched; the entire cost is one line appended to the turn's user message — which is exactly what the prompt-cache contract in `CLAUDE.md` asks of anything added to the model call path. + +**The composer text is the binding.** Unlike the `@` menu there is no remembered pick: the invoked set is derived from the text, so a hand-typed command works like a menu pick and the chip cannot disagree with what is sent. The chip's ✕ edits the text, because that is where the binding lives. + +`/` is ordinary punctuation, so a command must start a word **and** not be followed by another `/`. That second clause is what keeps `/usr/bin/env` prose — an absolute path starts a word exactly like a command does. The rule is implemented three times (composer token, `findSkillCommands`, thread renderer) and all three must agree. + +### Backend + +`GET /skills/` now serves the runtime activation `slug` rather than letting the SPA re-derive it. `invoked_skills` on the invocation request is intersected against the turn's **effective** set — re-run after Agent bindings can replace it — and becomes a directive appended last, riding `original_message` so the thread shows only what the user typed. + +Spec: `docs/specs/skill-slash-commands.md`. + +--- + +## Chat tells you what it is doing + +### One card per assistant run + +A four-tool answer rendered as five separate assistant cards, each paying full chrome — card padding, a copy button, a metadata row and a 1.5rem gap — so a single response became a column of near-empty boxes with the answer pushed below the fold. + +The cause was message boundaries, not the tool rail. The agent loop starts a new Bedrock message at every tool round trip and the SPA rendered one card per message; worse, the existing tool-grouping pass could never fire, because two consecutive tool calls were always in different messages. `AssistantMessageComponent` now takes a **run** of messages and flattens it into one block stream, and `message-list` splits each turn into segments. A tool group deliberately spans message boundaries — only text and reasoning break it, because those are where the agent actually said something. + +The trap this has to handle: **Bedrock returns tool results as USER-role messages carrying nothing but `toolResult` blocks.** They render at zero height — invisible — but sit between every pair of assistant messages, so treating them as real user messages breaks the run at every single tool call. They were also already starting a spurious turn group per tool call, which moved the last group's scroll reserve out from under a streaming response. Both are now guarded by `isProtocolScaffolding`. A mid-turn steer is a real user message with real text and still breaks the run, which is correct. + +The rail itself now stays **collapsed** while tools run — it used to force itself open on any pending call, so it was at its tallest exactly while the user was watching — and leads with one line that does not grow with the group. + +### `agent_status` + +A multi-tool turn spends most of its wall-clock time in places the content stream says nothing about. A new `AgentStatusHook` records the boundaries the event loop already crosses (`BeforeModelCall`, `Before`/`AfterToolCall`) and `stream_coordinator` drains them exactly like `steering_applied`, before the event they precede — so "Using list_assignments" reaches the client **while** that tool is running rather than after its result. + +Phases are `thinking` (one per event-loop cycle, with `cycle` distinguishing them), `tool_start` and `tool_end`. Durations come from Strands' own `AfterToolCallEvent.duration` rather than being inferred from stream arrival times on the client, and `ok=false` covers both a raised exception and a result with `status: "error"`. + +There is deliberately **no `responding` phase**: the SPA already knows text is streaming from the deltas, and a backend-derived duplicate of a fact the client holds first-hand would only disagree at the edges. Durations are live-only and deliberately not persisted — a reloaded conversation shows summaries without timings, where a client-invented number would be one the user could not trust. + +It costs nothing against the model: nothing it produces reaches the prompt, so the cacheable prefix is untouched. Gated by `AGENT_STATUS_ENABLED` (default on, kill switch); while off the hook is registered but every callback returns immediately and the SPA stays on its cycling phrases. + +### `tool_group_summary` + +A Nova Micro summarizer (`apis/shared/tool_summaries/summarizer.py`) turns each finished tool batch into one line naming what was actually found. It is a **side-channel**, structured exactly like `session_title`: its own Bedrock call on its own messages, run concurrently with the agent stream. It never appends to the conversation, so it adds nothing to the cacheable prefix and cannot cause a cache re-write. Spend is one bounded call per batch, with inputs and results truncated at capture in the hook **and** again in the summarizer. + +Summaries persist as `TSUM#` rows in the existing sessions-metadata table — reusing the `SessionLookupIndex` GSI, so **zero new infra** — and replay on `GET /messages` as `toolSummaries`, because the live event is emitted once and never re-streams. They are deliberately **not** written onto the message content blocks: that is the Bedrock Converse payload and the cacheable prefix, so a display string there would be paid at model rates on every subsequent turn. + +Gated by `TOOL_SUMMARIES_ENABLED`, separately from `AGENT_STATUS_ENABLED`, so a deployment can take one without the other. While off, the SPA's deterministic client-side formatter still renders ("Listed 4 assignments") — absence is a downgrade in specificity, never a blank. That formatter reads the shape of the tool name (`verb_subject`, near-universal across MCP) plus result cardinality, because almost every tool here arrives at runtime from an MCP server and a per-tool table would leave the ones users actually see rendering as raw identifiers. + +### The loading indicator + +Twenty invented phrases, typed out character by character, were charming and were fiction — identical whether the model was generating, waiting nine seconds on a Canvas round trip, or hung. It now says only what we know: `● Thinking 12s` and `● Running browse_web 4s`. "Running" shows only while a tool really is executing, and the tool's own identifier is the label — the same name the tool rail and the admin catalog use, so humanising it would invent a second name for one thing. It renders in the routine colour, distinct from the amber `model_retry` notice. + +--- + +## MCP prompts and resources + +An MCP server exposes three listings — tools, prompts and resources — and this stack had only ever called `tools/list`. Grepping the backend for `prompts/list` or `resources/list` returned nothing, so two thirds of what our servers offer was invisible to every surface in the product. + +### Backend + +- Capability discovery attempts each listing **independently**, degrading to `supports_*=False` on failure. A server that implements tools but not prompts answers `prompts/list` with a JSON-RPC "method not found" — that is normal, and it must not cost us the resources listing or the whole snapshot. "Offers nothing" and "we could not ask" are recorded as **different facts**, because the UI has to say different things about each. +- `prompts/get` is now called for the first time anywhere in the stack. `MCPPromptArgument` carries `required` and `description` alongside the name — `_prompt_entries` had been flattening arguments to names alone, which is enough to describe a prompt and not enough to fill one in. `from_dict` accepts a bare string so snapshots taken before this rehydrate rather than break. +- `POST /admin/tools/{id}/capabilities/refresh` existed and **nothing in the frontend called it**, with no cron or sync job either, so a snapshot was written once by hand and never rewritten. In dev, `canvas_faculty` and `student_myboisestate` were probed at 05:02 UTC and gained their first prompts at 16:01 the same day; both snapshots still read `supportsPrompts=true` with zero prompts, rendering a supported-but-empty state indistinguishable from a server that genuinely offers nothing. `gmail_employee` had never been probed at all. + +### Frontend + +On a tool's detail page each prompt becomes **Try it** → a field per argument → **Compose** → the server's composition rendered inline, with copy. Prompts and resources stay a read of the stored capability snapshot rather than a live probe — probing opens an MCP session per server, and a 3LO server needs a consent token the browser does not hold — and the snapshot is fetched only for `mcp_external` tools, since nothing else has a server that could be asked. The admin tool list gains a refresh action. + +--- + +## Scoped tool bindings on Agents + +An Agent's `binding.ref` accepted only a bare catalog id, so binding an MCP server loaded **every** one of its tools. The live Rubric Builder agent needs 7 of `canvas_faculty`'s 44 and was carrying `grade_submission`, `delete_rubric` and the rest — held back only by wording in its system prompt, which is a fence for ordinary use and none at all against a determined one. It also put ~13.2k of tool definitions in the cacheable prefix every turn where ~1.8k would do. + +The scoping machinery already existed and the runtime already honoured it (`scoped_ids.py`, `collect_tool_name_filters`, `load_external_tools`); user-facing tool selection has used it all along. Three things blocked the bindings axis, and the first is the one worth noting: `AppRoleService.can_access_tool` exact-matched the id, so a scoped ref was denied **even for a user holding the whole server**, while its sibling `filter_requested_tools` base-collapsed correctly on the `enabled_tools` axis. The two now agree, which a test asserts directly. + +--- + +## Model picker + +### Short descriptions + +New `shortDescription` on the managed model (create/update/read plus the DynamoDB create path), surfaced in the admin form and rendered under the model name in the picker, falling back to the provider name when unset — so an uncurated catalog looks exactly as it did. The 10 curated catalog templates are seeded with purpose-written picker copy, deliberately shorter than the catalog card's `tagline`, which is a sentence where this is a fragment. + +The 80-char cap is on the create/update models but **not** on the read model: a stored value longer than the cap would otherwise fail validation and take the whole `/models` listing down with it. Bound the input, stay permissive about what is already persisted. + +### Effort selection + +Surfaces the existing per-model inference-param system rather than inventing a parallel one. The Effort submenu appears only when an admin marked the param supported, left it unlocked, and enumerated `allowed` levels — mirroring `_merge_inference_params`, whose enum branch keeps an override only if it is a member of that set and pins locked params to the admin default. Offering a level the backend would silently discard would show the user a choice that does nothing. Selections write through the same per-model override store every other inference param uses. + +### Icons + +Every row leads with a left-aligned vendor avatar, set two deliberately different ways. `iconSlug` points at a logo the SPA already ships (anthropic, openai, amazon, meta) — a string on the record, no storage, no round trip, and a crisp theme-aware vector at any size; for a vendor we ship, that is the better answer and not the fallback. An upload covers the rest (an in-house fine-tune, a vendor we ship no logo for): bytes to S3 under `models/{id}/icons/{digest}.{ext}`, with the record carrying only the key — the same 400 KB item-limit lesson agent icons learned. An upload wins over a slug, being the more deliberate act. With neither set the client matches on `providerName`, so every existing model keeps an icon. + +### GPT-6 Astra + +`us.openai.gpt-6-astra` is registered at the Geo CRIS **Short Context** rate card — $11.00 in / $13.75 cache-write / $1.10 cache-read / $55.00 out per MTok — with `maxInputTokens` inherited at 272,000. + +**The cap is load-bearing.** The context price tier is selected by the *actual* token count of the request, so nothing stops us being billed on the Long Context card except staying under the boundary. Crossing it bills input at 2×, output at 1.5× and both cache buckets at 2×, while a `CuratedModel` holds one flat rate per bucket — so raising the cap would silently **under-charge** every long turn. Two card-backed deviations from the GPT-5.6 family defaults: `maxOutputTokens: 128_000` and `knowledgeCutoffDate: '2026-04-30'`, both published where the sibling cards say N/A. `supportedParams` is deliberately still absent — a declared spec flips the guard from permissive to restrictive, and a wrong entry would block a parameter the model accepts. + +--- + +## Knowledge bases + +### Chunk inspector + +`GET /assistants/{id}/documents/{doc}/chunks` — owner/editor only, read-only, one bounded Retrieve, no new chunking or ingestion path. + +This is the tooling half of a decision that was otherwise "guidance, not code": the managed backend's vision step flattens a column-structured flowchart or a 2-D table at ingestion, so a per-column question gets a **confident wrong answer with no trace**, and fixing a managed parser is not on the table. But guidance nobody can verify is not guidance — an owner cannot act on "convert your flowchart to a text table" without first seeing that their flowchart came out wrong. This is where they look. + +Three things are deliberate, each mutation-guarded by a test: + +1. The document filter is `equals` on `document_id`, **never a prefix operator** — a prefix match for `DOC-1` also admits `DOC-10`, so the operator choice *is* the isolation boundary, not a query-tuning detail. +2. A post-filter drops any chunk whose `document_id` is not the requested one, on top of the backend filter. Belt and braces, because the failure mode is silent and its blast radius is one owner reading another's document. +3. **Full chunk text, untruncated.** The existing citation trace caps excerpts at 500 chars, which is exactly why it cannot serve this purpose: a flattened table's damage is usually past the cut, so a truncated excerpt of a mangled table reads like a fine excerpt of a fine table. + +### Born-managed knowledge bases + +`MANAGED_KB_NEW_DEFAULT` (rollout ladder step 2) was a **no-op**: no backend code read it, and the app-api Lambda never received it. Both are now wired. `maybe_enroll_new_default()` makes a newly finalized agent born managed by reusing `enroll()` — a new agent has no corpus, so migrating an empty one provisions the KB, converges instantly, and promotes through the proven, crash-safe, dispatcher-driven worker, with `catch_up` covering documents uploaded mid-flight. Flag-gated, idempotent, and error-swallowing so it can never fail agent creation. + +**Ships dark** (flag default off). Turning it on at fleet scale is gated by the ~10k Bedrock KB per-account quota, since managed is one KB per assistant. + +### Byte cap at upload time + +The interactive upload path was the last byte-adding path still uncapped (migration was already covered). A request-time pre-check reserves the client-declared size against `min(per_owner_cap, per_kb_ceiling)` **before** creating the `DOC#` row or issuing a presigned URL, returning HTTP 413 with the numbers. The reservation is provisional; an authoritative reconcile at ingestion takes the true size from an S3 HEAD and commits, releases the difference, or fails the document and deletes the orphaned S3 object. `settle_once()` makes commit/release exactly-once across EventBridge redeliveries and racing failure paths. Managed KBs only — legacy S3-Vectors KBs stay uncapped. + +--- + +## Fine-tuning + +### Checkpoint and resume + +Two things restart a training job: a spot interruption, and SageMaker killing it at `MaxRuntimeInSeconds` — which the dollar-quota clamp makes routine, since a $14 balance buys 3.7h on the 34B instance while a real run needs far longer. Until now `save_strategy="no"` meant the adapter was written only after `trainer.train()` returned, so **either restart produced nothing at all for the money already spent**. + +The interval is computed, not fixed. `save_steps` counts *optimizer* steps, and a 34B VLM trains at batch 1 with 16-step accumulation — a 90-sample epoch is about 6 steps. Against a hardcoded `save_steps=50` the longest, most interruption-exposed job in the catalog would never checkpoint, while a text classifier with thousands of steps would checkpoint constantly. `resolve_save_steps` scales to the run's own step count targeting ~10 checkpoints, so an interruption costs at most about a tenth of the run. `save_total_limit=1` keeps the mirror bounded. Checkpoints go to a `checkpoints/` prefix rather than inside the job's output prefix, where SageMaker writes the finished `model.tar.gz`. + +### Managed spot + +Roughly a 65% discount for a longer queue and the risk of interruption. It is only safe now that checkpointing landed: simulated at a 0.15/hr hazard, a 48h job costs **~3,200 billed hours without checkpointing and ~18 with it**. The two are enforced together — asking for spot with `checkpointing=false` is refused at submit rather than sold. + +Off by default. Measured on-demand capacity waits for these GPU families ran 28–58 minutes in us-west-2, and spot draws from the surplus of the same constrained pools, so a researcher who needs a result this afternoon should be able to pay for certainty. `MaxWaitTimeInSeconds` covers waiting for capacity *and* training and must exceed `MaxRuntimeInSeconds`, so it is the runtime plus a four-hour queue allowance; waiting is not billed. `MaxRuntime` itself is untouched — spot must not quietly extend the budget clamp. Cost accounting needs no spot branch: AWS expresses the discount by shrinking `BillableTimeInSeconds` against the same on-demand rate. + +### VLM context length + +Three of five VLM catalog models could not train on their own defaults. SmolVLM-Instruct spends **1377 tokens on a single image** against a default `context_length` of 1024, so truncation cut the image placeholder run to 891 and the processor rejected the batch — the image alone did not fit in the budget, let alone the prompt. LLaVA-1.6's AnyRes tiling and Qwen2.5-VL's dynamic resolution both go well past the 2048 those entries defaulted to. + +A fixed default cannot be right, because the token cost depends on the checkpoint's tiling **and** on the resolution of the images the user uploaded. The trainer now renders a sample of records untruncated, raises the context length to fit, and logs the adjustment, failing only when a single record genuinely exceeds the model's own maximum. Defaults are raised too (2048 for SmolVLM and LLaVA-1.5, 4096 for the three AnyRes/dynamic-resolution models) so the measured path stays a safety net rather than the norm; headroom is close to free because the collator pads to the longest item in the batch, not to `max_length`. + +--- + +## 🐛 Bug fixes + +**Loading a conversation page logged blocked mixed content, twice.** `https://dev.boisestate.ai/s/` requested `http://api.dev.boisestate.ai/agents` — a URL built nowhere in the SPA. It was Starlette's own `redirect_slashes` answer to `GET /api/agents/`, rendered from the only things app-api can see behind CloudFront: the ALB's hostname (the `/api/*` behaviour uses `ALL_VIEWER_EXCEPT_HOST_HEADER`), plain HTTP (the ALB terminates TLS), and a path with `/api` already stripped. Every part of that `Location` is wrong — the browser blocks it so the caller silently gets nothing, the internal ALB hostname leaks into a page the user can read, and a client that did follow it would land on a different origin where the `__Host-` BFF cookies are not sent and the request 401s. CloudFront now sets `x-forwarded-prefix: /api` and `ProxiedRedirectMiddleware` restores the public URL. Not specific to `/agents`: `/models/`, `/files/` and `/auth/login/` produced the same thing. (#1074) + +**The agent designer's preview silently dropped tool calls.** Measured on dev against the `canvas_faculty` MCP server: one requested `upload_course_file` produced ~316 attempts and **zero** invocations of the backing Lambda; `import_course_package` 647 attempts, zero invocations. The identical calls in the full chat succeeded first try, and nothing was surfaced to the user. The root cause was not a divergent dispatch path but a divergent SSE *consumer* — `PreviewChatService` implemented 9 of the ~27 events `processStreamEvent` dispatches, and every callback in that parser is invoked with `?.`, so an unimplemented handler drops its event in total silence: no error, not even `onParseError`. Among the dropped events were `oauth_required` and `tool_approval_required`, **the two that gate dispatch** — so the tool paused server-side waiting for an answer the preview had no way to ask for. The fork is deleted; both preview surfaces run on `ChatRequestService` / `ChatHttpService` / `StreamParserService`. (#1060) + +**Conversation Mode silently stopped applying after a reload.** The session page hydrates the active mode twice on load — provisionally, before metadata arrives, then for real — and the provisional call claimed the session id, so the clobber guard rejected the real hydration. Because chat-request sends `selected_prompt_id` from `activePromptId()`, the mode stopped being applied to every turn after a reload while the stored preference still said it was on. The provisional call now passes `claim:false`; a deliberate "None" still claims, so stale metadata cannot undo it. (#1079) + +**Fine-tuning inference had never worked from the deployed app.** Batch Transform is a two-step API — `CreateModel` registers the trained artifact, then `CreateTransformJob` runs against it — and the task role was granted the job actions but not `CreateModel`, so every inference job died at step one with `AccessDeniedException`. It went unnoticed because the failure is invisible from a developer machine: CloudTrail shows every successful `CreateModel` in dev was called by a human's SSO credentials running app-api locally against dev data. The ARN is the subtle part — `sagemaker_service` names the model `model-{job_name}` and `job_name` already starts with the project prefix, so the resource is `model/model--*` with the literal `model-` **ahead of** the prefix; a pattern written to match the training-job and transform-job ARNs looks correct and denies every call. (#1023) + +**Connector edits were rejected with an instruction the admin could not follow.** The discovery guard in `update_provider` treated any non-None `oauth_discovery_url` as a discovery change, and a discovery change requires a credential rotation — but the edit form round-trips the discovery URL on every save, so changing scopes, display name, icon or enabled state failed with "Discovery config can only be updated together with a credential rotation." The admin could not comply: the client secret is never readable back. The guard now compares against the stored record, so it means what its error message says. (#1029) + +**A fresh deployment would have listed zero tools.** `seed_bootstrap_data.py` hand-builds its tool item instead of calling `ToolDefinition.to_dynamo_item`, so it wrote `GSI1PK`/`GSI1SK` by hand and had no `GSI5PK`. Once the catalog read moved to that sparse index, a freshly bootstrapped deployment — a fork, a new environment, a rebuilt dev — would have listed **nothing**, with no error to notice, and a backfill would not even be the obvious remedy because nothing about that install is legacy. A test now derives the expected key set from the model and asserts the seeder writes all of it. (#1070) + +**`/users/me/settings` was read twice on every first load.** `settingsResource` fetched eagerly the moment `UserSettingsService` is injected; `ModelService.findUserDefaultModel` fetched again once `/models` had landed. Measured on dev they land ~280ms apart (t=863ms, t=1146ms) — **sequential, not concurrent**, so a single-flight guard would not have caught the second. `getSettings()` memoizes the promise instead. A rejected read is deliberately not cached, and `updateSettings` drops the memo before reloading. (#1067) + +**Nightly Build & Test failed seven consecutive nights** (2026-09-05 → 09-11) after being green the five before. Six filesystem-reading specs failed only under `--coverage`, which the nightly passes and PR CI did not. Under `--coverage`, @angular/build's unit-test builder bundles each spec into a flattened chunk emitted at the **project root** rather than handing Vitest the spec at its own source path, so `import.meta.url` and the `__dirname` shim move with it. That is an absolute relocation, not a fixed-depth shift, so every `resolve(SPEC_DIR, '..')` silently changed meaning: a doc-presence spec read the wrong README, a parity spec ENOENT'd at collection, and two hygiene guards walked the whole package — including generated Tailwind CSS, full of literal hex — and so "found" violations. New `src/testing/project-root.ts` walks up from `process.cwd()` to the directory holding `angular.json`, the same way the Angular CLI locates the workspace. The coverage build now also runs on PR CI. (#1048) + +**The new `agentcore-runtime-active-sessions` alarm fired on every load test.** Validated against 7 days of real prod `ActiveSessionCount` it would have fired three times in three nights, every one a planned load test, peaking at 241, 608 and 1404. No threshold fixes that: load tests still trip it at 500, and it only goes quiet around 1500, which is *above* the ~99 sustained by the `/ping` reaper regression the alarm exists to catch. Duration does separate them — the regression sustained ~99 for three months, the load tests ran 15–30 minutes. At threshold 75 over a 60-minute window, simulated firings drop to zero on that week's data while the regression would still be caught. (#1058) + +Also: the tool summary no longer eats its closing quote (#1039); the tool row's hover highlight runs the full drawer width (#1042); the tool rail's hover is scoped to the rail (#1043); the prose margin under the tool batch summary is dropped (#1044). + +--- + +## 🔒 Security + +**Scoped bindings make an Agent's tool restriction structural.** A bound MCP server previously loaded all of its tools, with the agent's system prompt as the only thing standing between a user and `grade_submission` or `delete_rubric` — prompt wording is a fence for ordinary use and none at all against a determined one. (#1047) + +**The chunk inspector's document filter is the isolation boundary.** `equals` on `document_id`, never a prefix operator, because a prefix match for `DOC-1` also admits `DOC-10`; a post-filter backs it up, because the failure mode is silent and its blast radius is one owner reading another owner's document. (#1057) + +**WCAG AA contrast sweep.** `dark:text-primary-400` resolves to #1e53c1 and fails AA in dark mode; the repo already generates `--color-primary-accessible-dark` (#437cee) for exactly this. Six legacy occurrences swept across four files, measured with `getComputedStyle` against each site's real composited backdrop — e.g. the agent-form "+Add" text moved from 2.23:1 to 3.90:1, the create-training-job check icon from 2.59:1 to 4.53:1 against a 3.0 threshold. Separately, `bg-primary-50|100|200` as a tint fill measured 4.13:1, 3.52:1, 2.23:1 and 2.63:1 across four real sites, all failing. (#1082, #1091) + +--- + +## ⚡ Performance + +### The tool catalog is Queried, not Scanned + +The `{prefix}-app-roles` table is shared: tools, skills, roles, role grants, JWT mappings **and one tool-preferences row per user** all live in it. `list_tools` scanned it and filtered, and Scan cost tracks table size rather than result size — so that read's cost grew with **enrollment**, not with the number of tools. Measured on dev: 95 items read to return 24 tools, 17 of them per-user rows. In prod that is thousands of preference rows read to return ~24 tools. The Query reads 24 to return 24, and stays flat as the campus grows. + +**The fallbacks are why this is safe to ship, not incidental hardening.** An empty tool catalog is not a degraded experience — every user loses every tool — and both ways this index can fail to answer produce exactly that: + +- **The index is absent.** `platform.yml` and `backend.yml` are ordered by nothing, a GSI is still CREATING after CloudFormation reports success, and a rolled-back stack ships its images anyway. Falls back to the Scan. It deliberately does *not* use `dynamo_errors.log_missing_index`: that helper is written for surfaces that degrade to empty and its message says so, which would be a lie here. +- **The index is present but unpopulated.** This raises nothing at all — the keys are sparse, so a catalog whose backfill has not run indexes nothing and the Query succeeds with **zero rows**. A zero result is therefore treated as suspect and re-read via Scan: if the table really holds tools we serve them and log an ERROR naming the backfill script; if it is genuinely empty (a fresh install before seeding) both agree, at the cost of one extra read per cache fill. + +### Tenant-global catalogs are cached + +Every SPA first load read four catalogs identical for every user on the deployment — models, tools, system prompts and connectors — none cached, three of the four full table scans. New `apis.shared.caching.config_cache` adds a TTL (60s, via `CONFIG_CACHE_TTL_SECONDS`) plus **single flight**, which is the half that matters under the case this exists for: with a cold cache, 300 simultaneous requests would otherwise issue 300 concurrent scans, and that stampede lands at exactly the moment the burst does. The scans also move onto `asyncio.to_thread` — they were blocking boto3 calls made from `async def`, so each stalled the whole event loop rather than just its own request. + +**Entries hold raw DynamoDB items and callers re-parse on every read**, deliberately. Callers mutate what these lists produce: `hydrate_model_roles` documents itself as "mutated in place and returned" and writes `allowed_app_roles` onto each model. Caching parsed objects would hand every caller one shared instance, so an admin opening the models page would write derived, display-only role fields onto the objects then served to every user — and `allowedAppRoles` is precisely the field the RBAC contract says must never be mistaken for a grant. Re-parsing is microseconds of CPU against a 50–100ms round trip. + +### The user-profile upsert is throttled + +`get_current_user_from_session` fired `sync_user_from_jwt` on every authenticated request, and that upsert is a GetItem followed by a PutItem that rewrites the whole profile row **and its GSI projections**. One SPA first load is 12 API calls, so a single page load issued **24 DynamoDB operations against one item**, writing identical values except `last_login_at`. A classroom signing in together multiplied that by the class size, each student's writes landing on their own hot partition. + +Throttling is safe because this was never the authoritative write — the BFF callback's `_sync_user_from_id_token` syncs on every login off the ID token (the only place the full claim set exists), and `POST /users/me/sync` writes through the repository directly. What remained here is a periodic refresh, so it only needs to run periodically: default 5 minutes via `USER_SYNC_THROTTLE_SECONDS`. The claim is recorded **before** the sync runs rather than on completion, because the 12 requests of a page load overlap and a marker written on completion would let most of them through before the first one landed. A brand-new user is unaffected: with no entry recorded, the first request always claims the sync. + +This also fixes a latent bug at the same site — the dispatched task was not referenced anywhere, and the event loop holds only a weak reference to a bare `asyncio.create_task(...)`, so the GC could collect it mid-await. + +--- + +## ⚠️ Breaking changes + +### The composer settings drawer is gone, and with it the only way to select a Conversation Mode + +This is the one item in the release that needs a decision rather than a note. + +The Customize epic removed the drawer in stages: Skills and Tools left for `/customize` because they are global; the model picker and inference-param form moved to the composer; agent-lock state moved to the assistant indicator. **Conversation Mode could not follow Skills and Tools** — it applies to *this* conversation, and putting it on a global page would have recreated the exact scope lie the epic exists to fix. So it went the other way, into the composer beside the model and effort controls, replacing the old passive chip that could display an active mode but never select one. + +That picker was then **parked before release** (#1088). The placement was verified end to end on dev and works; the reasoning for pulling it is that a permanent composer slot is a bigger commitment than the current evidence supports, and it should come back on user feedback about where it belongs. + +The consequence is that this release is the first to carry both the drawer's deletion and no replacement picker. **Prod selects Guided Learning — a Socratic tutoring prompt — through that drawer today**, and use is accelerating: 1 session in July, 20 in August, 60 in the first 12 days of September. After this deploy there is no UI anywhere to select a mode. + +Only the control was removed. `SystemPromptsService`, `GET /system-prompts/`, `selected_prompt_id` on `SessionPreferences` and the admin CRUD are all untouched, and sessions that already carry a mode keep applying it. Restoring `ConversationModePickerComponent` is a revert, not a rebuild. + +**Three options, in order of preference:** restore the picker before merging; land a replacement placement; or take the regression knowingly with a plan for when the control returns. + +### Route redirects + +`/settings/connectors` and the three `/my-skills` paths are now redirects rather than pages. Bookmarks and the schedules page's deep link continue to work, and the ordering that keeps them working is asserted by tests — but a fork that has customised either route should re-check it. + +### Token bans + +`dark:text-primary-400` (colored text and icons) and `bg-primary-50|100|200` (tint fills) must not be used in new code. The `primary` scale is generated from #0033a0 by lightness offset alone and keeps full chroma at every step, so `primary-50` resolves to rgb(118, 179, 255) — a saturated mid-blue, not a wash. Used as a chip, badge, icon tile or selected-row fill it reads as a blue blob behind small text and fails AA. The `state-*` scales **are** real tints (`state-success-50` = rgb(240, 253, 244)), which is exactly why the pattern looked safe by analogy and wasn't. Use `text-primary-accessible dark:text-primary-accessible-dark` and neutral surfaces with the brand blue in the text. + +### `last_login_at` granularity + +Accurate to within `USER_SYNC_THROTTLE_SECONDS` (default 300) rather than to the last request. Anything reading it as a precise last-seen timestamp should be adjusted or the throttle lowered. + +--- + +## 🏗️ Infrastructure + +- **`EntityTypeIndex` on the existing `{prefix}-app-roles` table** — `GSI5PK=ENTITY#{type}`, `GSI5SK=` the item's own PK, `ProjectionType.ALL`. Full projection because the catalog is rebuilt from these rows; an INCLUDE projection would force a base-table read per tool and give back the amplification the index exists to remove. Deliberately generic rather than TOOL-only: `list_roles` and the skills catalog scan the same table for the same reason, and DynamoDB permits only **one GSI creation per `UpdateTable`** — so giving a second entity type its own index later would need its own release, and two accumulating into one release rolls the whole stack back (the 1.12.0 lesson of 2026-08-01). One partition per entity type costs nothing now and leaves that door open. Sparse by construction, so adding it changes nothing until rows are stamped. `infrastructure/gsi-inventory.json` shows exactly one index added to one existing table. +- **`sagemaker:CreateModel`** on the app-api task role, as its own `SageMakerModelManagement` statement scoped to `arn:aws:sagemaker:::model/model--*`. Its own statement because the literal `model-` sits ahead of the project prefix, so a pattern that matches the training-job and transform-job ARNs denies every call. Grants only `CreateModel` — `create_model` is the sole model API the service calls. Models are left behind after each transform job, which is a tidiness follow-up, not a reason to grant `DeleteModel` here. +- **CloudFront `x-forwarded-prefix: /api`** on the `/api/*` behaviour, set unconditionally rather than only on the stripping branches, so a viewer-supplied header is always overwritten rather than passed through to the origin. +- **`agentcore-runtime-active-sessions` alarm** on `ActiveSessionCount` / `AgentCore.Runtime`, threshold `CDK_OBSERVABILITY_AGENTCORE_ACTIVE_SESSION_THRESHOLD` (default **75**) over 12 evaluation periods — a **60-minute** window, which is what separates a real regression from a load test. Runtime bills memory for a session's whole lifetime and AWS still exposes no API to list or force-terminate one, so session accumulation is the leading indicator; the `/ping` reaper bug ran undetected for three months at 73% of the platform bill. The threshold is deliberately **not** a fraction of the 5,000-session account quota — quota exhaustion is already owned by `agentcore-throttles`. +- **`MANAGED_KB_NEW_DEFAULT`** threaded into the app-api environment as an explicit `'false'` rather than omitted, for the same visibility reason as its siblings. + +--- + +## 🔧 CI/CD + +- **`pending-backfills.yml`** — a new gate on every PR into `main`. `scripts/release/check-pending-backfills.mjs` diffs `backend/scripts/backfill_*.py` against `origin/main` and fails when a script added in the range is not named in `RELEASE_NOTES.md`. There are now six backfill scripts and no step that surfaced "this release needs one run"; they reached the changelog only when whoever wrote it remembered. It cannot verify a backfill was actually *run* — no CI job can — but it guarantees the instruction reaches the person who can. +- **`tests.yml`** gains a `run_frontend_coverage` input and a `Test frontend (coverage build)` job, which `ci.yml` sets true. The coverage build had been running only in the nightly, which is why six specs could break for seven consecutive nights without a single PR going red. +- **`platform.yml`** gains job-level `CDK_OBSERVABILITY_AGENTCORE_ACTIVE_SESSION_THRESHOLD`, plumbed through `load-env.sh` like every other observability threshold. + +--- + +## 📦 Dependencies + +| Component | Change | From | To | +|---|---|---|---| +| Backend (uv constraint) | `mcp` — transitive, via `strands-agents` | unbounded within `>=1.23.0,<2.2` | `<2` | + +`mcp` reaches us only through `strands-agents`; we do not depend on it directly. `strands-agents==1.55.0` declares `mcp>=1.23.0,<2.2`, so the resolver was free to cross into 2.x at any time. The only thing holding us on the 1.x line was **incidental**: `mcp` 2.x pulls `httpx2>=2.5.0`, which needs `idna>=3.18`, and we pin `idna==3.15` for an unrelated Dependabot alert. A routine security bump of `idna` would have quietly unblocked the major crossing — nobody would be choosing it, and nothing in that diff would mention MCP. + +The crossing lands under `integrations/mcp_apps.py`, which substitutes `strands.tools.mcp.mcp_client.ClientSession` to advertise the MCP Apps UI extension on `initialize`. That seam reaches through a Strands internal into an MCP SDK class and has never been exercised against 2.x, where the transport dependency changed (httpx → httpx2) and the types moved to a separate distribution. The failure is silent: App frame headers degrade to a generic glyph rather than raising. `[tool.uv] constraint-dependencies` bounds the version **if** `mcp` is in the resolution graph, without adding it to our dependency metadata. + +--- + +## 🧪 Test coverage + +Roughly 5,000 lines of new backend and frontend tests. Notable scopes: the Customize pages and both detail views (~1,100 lines of specs); `test_tool_catalog_index_read.py` (240 lines) covering both index-absent and index-unpopulated fallbacks; `test_backfill_tool_catalog_index.py` (202 lines) including the idempotency and never-resurrect-a-deleted-row guards; route-ordering assertions for the four redirect/parameterised-route traps, all of which fail silently; `pending-backfills.test.ts` and the infra tests for the new IAM statement and alarm; and a mutation-verified seeder test that derives the expected GSI key set from `ToolDefinition` so the next index added to tools cannot be forgotten in the one place that mirrors the model by hand. + +--- + +## 🚀 Deployment notes + +**Order: `platform.yml` (CDK) → wait for the index → run the backfill → `backend.yml` → `frontend-deploy.yml`.** + +### 1. Deploy the platform stack + +One GSI operation on one existing table. The release guard confirms it: + +```bash +node scripts/release/check-gsi-update-limit.mjs +``` + +### 2. Wait for `EntityTypeIndex` to report `ACTIVE` + +CloudFormation reporting `UPDATE_COMPLETE` is **not** the same as the index being usable: + +```bash +aws dynamodb describe-table --table-name -app-roles \ + --query 'Table.GlobalSecondaryIndexes[?IndexName==`EntityTypeIndex`].{Name:IndexName,Status:IndexStatus}' +``` + +### 3. Run the tool-catalog backfill — required, in every environment + +Populates `GSI5PK`/`GSI5SK` on tool rows written before the keys existed. **Dry-run by default; idempotent; guarded by `attribute_not_exists(GSI5PK)` and `attribute_exists(SK)`, so it never resurrects a deleted row and never overwrites one the writer has since stamped.** It touches only tool metadata rows — `CAPABILITIES` snapshots, skills, roles and user preferences are left alone. + +```bash +AWS_PROFILE= python backend/scripts/backfill_tool_catalog_index.py \ + --table -app-roles --region us-west-2 +``` + +Then, once the dry run looks right: + +```bash +AWS_PROFILE= python backend/scripts/backfill_tool_catalog_index.py \ + --table -app-roles --region us-west-2 --apply +``` + +**Verify `skipped=0 failed=0` and that the index item count matches the tool count before considering the deploy complete.** Run against dev first, then prod. If the backfill is skipped, the Query returns zero rows, the zero-result fallback re-reads via Scan and logs an ERROR naming this script — so the catalog still serves, but at Scan cost and with a standing error in the logs. + +### 4. Decide on the Conversation Mode regression + +See Breaking changes. There is no operational workaround after the deploy: sessions that already carry a mode keep applying it, but no user can select or change one. Restore `ConversationModePickerComponent`, land a replacement placement, or accept the regression deliberately. + +### 5. Optional configuration + +| Variable | Default | Effect | +|---|---|---| +| `AGENT_STATUS_ENABLED` | on | `=false` stops `agent_status` emission; the SPA falls back to cycling phrases | +| `TOOL_SUMMARIES_ENABLED` | on | `=false` stops the Nova Micro side-channel; the SPA's deterministic formatter still renders | +| `CONFIG_CACHE_TTL_SECONDS` | `60` | Lifetime of the tenant-global catalog cache | +| `USER_SYNC_THROTTLE_SECONDS` | `300` | Minimum interval between per-request user-profile upserts | +| `CDK_OBSERVABILITY_AGENTCORE_ACTIVE_SESSION_THRESHOLD` | `75` | Concurrent-Runtime-session alarm threshold, over a 60-minute window | +| `MANAGED_KB_NEW_DEFAULT` | `false` | Born-managed knowledge bases. Requires the migration worker running; gated at fleet scale by the ~10k Bedrock KB account quota | + +### 6. No action required + +No new table. No seeder run is required for this release. No agent, session or artifact data migration. The `mcp<2` constraint takes effect on the next `uv sync` with no code change. + +--- + # Release Notes — v1.20.0 **Release Date:** September 9, 2026 diff --git a/VERSION b/VERSION index 398935591..3500250a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.20.0 +1.21.0 diff --git a/backend/Dockerfile.kb-sync b/backend/Dockerfile.kb-sync index 3bfb4549d..8038e36ae 100644 --- a/backend/Dockerfile.kb-sync +++ b/backend/Dockerfile.kb-sync @@ -34,6 +34,12 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt COPY backend/src/apis/shared/__init__.py ${LAMBDA_TASK_ROOT}/apis/shared/__init__.py COPY backend/src/apis/shared/timestamps.py ${LAMBDA_TASK_ROOT}/apis/shared/timestamps.py COPY backend/src/apis/shared/dynamo_errors.py ${LAMBDA_TASK_ROOT}/apis/shared/dynamo_errors.py +# feature_flags.py + caching/ — oauth/provider_repository.py memoizes its +# provider list in the process-wide config cache, which reads the +# CONFIG_CACHE_ENABLED kill switch. Both are module-level imports, so a miss +# here is a cold-start ModuleNotFoundError, not a degraded path. +COPY backend/src/apis/shared/feature_flags.py ${LAMBDA_TASK_ROOT}/apis/shared/feature_flags.py +COPY backend/src/apis/shared/caching/ ${LAMBDA_TASK_ROOT}/apis/shared/caching/ COPY backend/src/apis/shared/sync_policies/ ${LAMBDA_TASK_ROOT}/apis/shared/sync_policies/ COPY backend/src/apis/shared/oauth/ ${LAMBDA_TASK_ROOT}/apis/shared/oauth/ COPY backend/src/apis/shared/embeddings/ ${LAMBDA_TASK_ROOT}/apis/shared/embeddings/ @@ -43,6 +49,11 @@ COPY backend/src/apis/shared/embeddings/ ${LAMBDA_TASK_ROOT}/apis/shared/embeddi # that package are stdlib only (test_kb_backend_boundary.py), so this adds # files, not dependencies. COPY backend/src/apis/shared/kb_backend/ ${LAMBDA_TASK_ROOT}/apis/shared/kb_backend/ +# observability/ — kb_backend/metrics.py (reached via byte_cap, which +# document_service uses to release a managed-KB byte reservation on a stale +# or failed upload) emits through observability/emf.py. Stdlib-only closure +# (test_lambda_image_imports.py enforces it); this adds files, not deps. +COPY backend/src/apis/shared/observability/ ${LAMBDA_TASK_ROOT}/apis/shared/observability/ # assistants/ — document_service verifies assistant ownership via # get_assistant before soft-deleting; the worker's miss-eviction path # (_delete_missing_web_document) goes through it. diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f13c76cb2..7b838145c 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agentcore-stack" -version = "1.20.0" +version = "1.21.0" requires-python = ">=3.10" description = "Multi-agent conversational AI system with AWS Bedrock AgentCore" readme = "README.md" @@ -106,6 +106,34 @@ all = [ "agentcore-stack[agentcore,bidi,dev]", ] +[tool.uv] +# Guard rail on a TRANSITIVE dependency — this is not a loosened pin. +# +# `mcp` arrives only via `strands-agents`; we do not depend on it directly and +# deliberately do not promote it to a direct dependency. `strands-agents==1.55.0` +# — the version pinned above, today — already declares `mcp>=1.23.0,<2.2`, so +# the resolver is free to cross into `mcp` 2.x whenever it gets the chance. +# +# The ONLY thing holding us on the 1.x line right now is incidental: `mcp` 2.x +# pulls `httpx2>=2.5.0`, which needs `idna>=3.18`, and we pin `idna==3.15` for +# an unrelated Dependabot alert. A routine security bump of `idna` would quietly +# unblock the major crossing — nobody would be choosing it, and nothing in the +# diff would say "MCP". +# +# That crossing lands under `src/agents/main_agent/integrations/mcp_apps.py`, +# which substitutes `strands.tools.mcp.mcp_client.ClientSession` with +# `_UIExtensionClientSession` to advertise the MCP Apps UI extension (SEP-1865) +# on `initialize`. That seam reaches through a Strands internal into an MCP SDK +# class and has never been exercised against 2.x — where the transport +# dependency itself changed (httpx -> httpx2) and the types moved to a separate +# `mcp-types` distribution. The failure mode is silent: MCP App frame headers +# degrade to a generic glyph rather than raising an ImportError. +# +# A constraint, not a dependency: it bounds the version IF `mcp` is in the +# resolution graph, without adding it to our own dependency metadata. Lift it +# deliberately, together with a test pass over that `ClientSession` patch. +constraint-dependencies = ["mcp<2"] + [tool.setuptools] package-dir = {"" = "src"} diff --git a/backend/scripts/backfill_tool_catalog_index.py b/backend/scripts/backfill_tool_catalog_index.py new file mode 100644 index 000000000..6963c2d9c --- /dev/null +++ b/backend/scripts/backfill_tool_catalog_index.py @@ -0,0 +1,217 @@ +"""Backfill: stamp GSI5PK/GSI5SK on tool-catalog rows written before they existed. + +``ToolDefinition.to_dynamo_item`` began stamping EntityTypeIndex keys in +``perf(tools): index the tool catalog so listing it stops scanning`` — every +tool row written before that carries neither attribute: + + tool row : PK=TOOL#{tool_id} SK=METADATA + + GSI5PK=ENTITY#TOOL + + GSI5SK=TOOL#{tool_id} + +``EntityTypeIndex`` is **sparse**: DynamoDB indexes a row only if it carries +the index's key attributes. A row missing them is not "stale" in the index, it +is *absent from it forever* — and the omission is silent, no error anywhere. +Switching ``list_tools`` to that index without this backfill would serve an +empty (or partial) tool catalog to every user on the deployment, which reads +as "all my tools disappeared", not as an outage anyone gets paged for. + +WHY THIS EXISTS AT ALL +---------------------- +The app-roles table is shared. Tools, skills, roles, role grants, JWT mappings +and one tool-preferences row PER USER live in it, so listing tools by Scan +reads the whole table and filters. Scan cost tracks table size, not result +size — measured on dev, 95 items read to return 24 tools, 17 of them per-user +rows. That read's cost therefore grows with enrollment rather than with the +number of tools. The index turns it into a Query on one partition. + +RUN THIS BEFORE THE INDEX IS CREATED, IF YOU CAN +------------------------------------------------ +DynamoDB backfills a new GSI at creation time from rows that already carry its +keys, so stamping first means the index is complete the moment it reports +ACTIVE, with no partially-populated window. The attributes are inert until an +index consumes them, so running early costs nothing. + +Running *after* creation is also fine and is the expected order here, because +the CDK change and this script ship in the same PR: the index simply picks up +each row as this script stamps it. Either way the rule that matters is the +same — **do not switch the read to the index until this reports +``skipped=0 failed=0`` and the index item count matches the tool count.** + +SAFETY +------ +* **Dry-run by default.** Pass ``--apply`` to write. +* **Idempotent.** Guarded by ``attribute_not_exists(GSI5PK)``, so a second run + finds nothing and a row the writer has since stamped is left alone. +* **Never resurrects a deleted row.** ``attribute_exists(SK)`` on every update. +* **Touches only tool metadata rows.** ``TOOL#``/``CAPABILITIES`` snapshots, + skills, roles and user preferences are left alone — they are not tools, and + the tool partition of the index must contain exactly the tool catalog. +* **Invents nothing.** Both key values derive from the row's own PK, so a + stamped row is byte-identical to what the writer would have written. + +Run against dev first, then prod:: + + AWS_PROFILE=dev-ai python backend/scripts/backfill_tool_catalog_index.py \\ + --table dev-boisestateai-v2-app-roles --region us-west-2 + AWS_PROFILE=dev-ai python backend/scripts/backfill_tool_catalog_index.py \\ + --table dev-boisestateai-v2-app-roles --region us-west-2 --apply +""" + +from __future__ import annotations + +import argparse +import logging +from typing import Any, Dict, Iterator, List + +import boto3 +from botocore.exceptions import ClientError + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" +) +logger = logging.getLogger("backfill_tool_catalog_index") + +# Kept as a literal rather than imported from apis.shared.tools.models so the +# script runs standalone against a deployed table without importing the app. +# The value is asserted against the model in +# tests/test_backfill_tool_catalog_index.py, so the two cannot drift. +ENTITY_TYPE_TOOL = "ENTITY#TOOL" + + +def iter_tool_rows(table: Any) -> Iterator[Dict[str, Any]]: + """Every tool METADATA row in the table. + + A Scan, not a Query — this migration exists precisely because there is no + index to query yet. ``FilterExpression`` runs server-side to cut payload; + DynamoDB still reads every item either way, so it saves bandwidth, not + capacity. + """ + kwargs: Dict[str, Any] = { + "FilterExpression": "begins_with(PK, :p) AND SK = :sk", + "ExpressionAttributeValues": {":p": "TOOL#", ":sk": "METADATA"}, + } + while True: + resp = table.scan(**kwargs) + for item in resp.get("Items", []): + yield item + last = resp.get("LastEvaluatedKey") + if not last: + return + kwargs["ExclusiveStartKey"] = last + + +def plan_row(item: Dict[str, Any]) -> Dict[str, Any] | None: + """What this row needs, or None if it needs nothing. + + Returns the computed keys, or ``{"skip": reason}`` for a row that cannot + be stamped safely. + """ + if "GSI5PK" in item: + return None # already stamped — writer or a previous run + + pk = str(item.get("PK", "")) + if not pk.startswith("TOOL#"): + return {"skip": f"unexpected PK {pk!r}"} + if pk == "TOOL#": + return {"skip": "empty tool id in PK"} + + # GSI5SK is the row's own PK. Deriving both keys from the PK rather than + # from the `toolId` attribute means a row with a missing or disagreeing + # `toolId` still lands in the index under the identity the base table + # already uses — there is no way for this to invent a different one. + return {"gsi5pk": ENTITY_TYPE_TOOL, "gsi5sk": pk} + + +def backfill(table: Any, apply: bool) -> Dict[str, int]: + stats = {"tool_rows": 0, "already": 0, "stamped": 0, "skipped": 0, "failed": 0} + skipped: List[str] = [] + + for item in iter_tool_rows(table): + stats["tool_rows"] += 1 + plan = plan_row(item) + + if plan is None: + stats["already"] += 1 + continue + + pk = str(item.get("PK", "")) + if "skip" in plan: + stats["skipped"] += 1 + skipped.append(f"{pk}: {plan['skip']}") + continue + + logger.info("stamp %s -> GSI5PK=%s GSI5SK=%s", pk, plan["gsi5pk"], plan["gsi5sk"]) + if not apply: + stats["stamped"] += 1 + continue + + try: + table.update_item( + Key={"PK": item["PK"], "SK": item["SK"]}, + UpdateExpression="SET GSI5PK = :pk, GSI5SK = :sk", + ExpressionAttributeValues={ + ":pk": plan["gsi5pk"], + ":sk": plan["gsi5sk"], + }, + # attribute_exists(SK): never resurrect a row the delete path + # removed between the scan and this write. + # attribute_not_exists(GSI5PK): idempotent, and yields to the + # writer if it stamped the row in the meantime. + ConditionExpression=( + "attribute_exists(SK) AND attribute_not_exists(GSI5PK)" + ), + ) + stats["stamped"] += 1 + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code == "ConditionalCheckFailedException": + # Deleted, or stamped by the writer, while we scanned. + stats["already"] += 1 + continue + stats["failed"] += 1 + logger.error("failed to stamp %s: %s", pk, code, exc_info=True) + + if skipped: + logger.warning("%s row(s) could not be stamped:", len(skipped)) + for line in skipped: + logger.warning(" %s", line) + + return stats + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--table", required=True, help="app-roles table name") + parser.add_argument("--region", default="us-west-2") + parser.add_argument( + "--apply", + action="store_true", + help="actually write (default is a dry run)", + ) + args = parser.parse_args() + + table = boto3.resource("dynamodb", region_name=args.region).Table(args.table) + + if not args.apply: + logger.info("DRY RUN — no writes. Pass --apply to commit.") + + stats = backfill(table, args.apply) + + logger.info( + "tool rows=%s already-stamped=%s stamped=%s skipped=%s failed=%s", + stats["tool_rows"], + stats["already"], + stats["stamped"], + stats["skipped"], + stats["failed"], + ) + if stats["skipped"] or stats["failed"]: + logger.warning( + "Index will be INCOMPLETE for the rows above. Resolve them before " + "switching list_tools to EntityTypeIndex — a sparse index drops " + "them silently, and the catalog just looks short." + ) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/probe_bedrock_cache_point_support.py b/backend/scripts/probe_bedrock_cache_point_support.py new file mode 100644 index 000000000..83ede82f4 --- /dev/null +++ b/backend/scripts/probe_bedrock_cache_point_support.py @@ -0,0 +1,277 @@ +"""Which Bedrock model families actually accept an explicit cachePoint? + +`ModelConfig.bedrock_cache_points_supported()` answers "Anthropic only", because +it mirrors what *Strands* recognizes (`BedrockModel._cache_strategy`). That is +not the same question as what *Bedrock* accepts, and the two have already +diverged — Nova Micro honors a system cachePoint that our predicate refuses. +More families will gain prompt caching, and nothing in CI can notice: the +predicate is a string test, so it keeps returning the same answer as the +platform moves underneath it. + +This is the measurement that closes that gap. It reports two independent +layers, and the interesting rows are where they disagree: + + 1. OFFLINE — what the pinned strands-agents emits for a model id. No AWS + calls. Answers "does upstream place a tools point / auto-inject a system + point", and "does upstream strip a hand-placed system point" (it does not, + which is the whole reason our gate exists). + 2. LIVE — whether Bedrock accepts a request carrying an explicit system + cachePoint, and whether it actually caches. A model that fails answers + `AccessDeniedException` ("You invoked an unsupported model or your request + did not allow prompt caching"), NOT the ValidationException our comments + claimed until 2026-09-11. + +Read-only apart from the model invocations themselves. Nothing is written to +DynamoDB, the catalog, or the agent loop; each probe is its own boto3 +`converse` call with `maxTokens: 5`. + +⚠️ Real spend, but small: two calls per model at ~7.9k input tokens (the prefix +has to clear the largest cache minimum — see _SYSTEM_TEXT). The script prints an +estimate and totals actual tokens at the end. A model the account cannot invoke +at all is reported as SKIPPED, distinct from a model that refuses the cache +point. + +Usage: + + cd backend + AWS_PROFILE=dev-ai uv run python scripts/probe_bedrock_cache_point_support.py + + # offline layer only — no AWS calls, no spend + uv run python scripts/probe_bedrock_cache_point_support.py --offline-only + + # check a newly announced family + AWS_PROFILE=dev-ai uv run python scripts/probe_bedrock_cache_point_support.py \ + --model-id us.amazon.nova-premier-v1:0 --model-id us.writer.palmyra-x5-v1:0 + +Baseline, us-west-2, 2026-09-11, strands-agents 1.55.0: + + us.anthropic.claude-haiku-4-5 strands=anthropic ACCEPTS+CACHES + us.amazon.nova-micro-v1:0 strands=None ACCEPTS+CACHES <- divergence + us.meta.llama3-3-70b strands=None REFUSED (AccessDenied) + mistral.mistral-large-2407 strands=None REFUSED (AccessDenied) + us.deepseek.r1-v1:0 strands=None REFUSED (AccessDenied) +""" + +import argparse +import sys +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +import boto3 +import botocore.exceptions + +# Default sweep: one model known to cache (the control), one known divergence, +# and three known refusals. Add families here as they gain caching. +DEFAULT_MODEL_IDS = [ + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "us.amazon.nova-micro-v1:0", + "us.meta.llama3-3-70b-instruct-v1:0", + "mistral.mistral-large-2407-v1:0", + "us.deepseek.r1-v1:0", +] + +# Sized well over every model's cache minimum, because BELOW that minimum +# Bedrock silently ignores the cache point — no error, no cache buckets, a +# result indistinguishable from "unsupported". Measured on Haiku 4.5 in +# us-west-2 (2026-09-11): 2,351 and 3,911 tokens both wrote NOTHING, 5,202 +# wrote in full, bracketing the real floor at 4,096 — twice the 2,048 the +# first-party Anthropic docs give for Haiku. Do not shrink this to save +# pennies; an undersized prefix turns every row into a false negative. +_FILLER = "You are a helpful assistant operating inside a controlled test harness. " +_SYSTEM_TEXT = _FILLER * 600 +_APPROX_PROMPT_TOKENS = 7_900 +_MESSAGES = [{"role": "user", "content": [{"text": "Say OK."}]}] + + +@dataclass +class ProbeResult: + """One model's answer from both layers.""" + + model_id: str + strands_strategy: Optional[str] = None + strands_emits_tools_point: bool = False + strands_autoinjects_system_point: bool = False + strands_strips_placed_point: bool = False + live_status: str = "not run" + live_detail: str = "" + cache_write_tokens: int = 0 + cache_read_tokens: int = 0 + input_tokens_uncached: int = 0 + input_tokens_cached: int = 0 + tokens_billed: int = 0 + notes: List[str] = field(default_factory=list) + + +def probe_offline(model_id: str, result: ProbeResult) -> None: + """Ask the pinned SDK what it would put on the wire. No AWS calls.""" + from strands.models import BedrockModel, CacheConfig + + tool_specs = [{"name": "t", "description": "d", "inputSchema": {"json": {"type": "object"}}}] + model = BedrockModel( + model_id=model_id, + cache_config=CacheConfig(strategy="auto", system_prompt_ttl=True, tools_ttl=True), + region_name="us-west-2", + ) + result.strands_strategy = model._cache_strategy + + plain = model.format_request(_MESSAGES, tool_specs, system_prompt_content=[{"text": "SYSTEM"}]) + result.strands_emits_tools_point = any("cachePoint" in t for t in plain["toolConfig"]["tools"]) + result.strands_autoinjects_system_point = any("cachePoint" in b for b in plain["system"]) + + # The load-bearing question: does upstream filter a point WE placed? + placed = model.format_request( + _MESSAGES, + tool_specs, + system_prompt_content=[{"text": "SYSTEM"}, {"cachePoint": {"type": "default"}}], + ) + survivors = [b for b in placed["system"] if "cachePoint" in b] + result.strands_strips_placed_point = not survivors + if len(survivors) > 1: + result.notes.append(f"DOUBLED system cachePoint ({len(survivors)}) — invariant broken") + + +def probe_live(client: Any, model_id: str, result: ProbeResult) -> None: + """Send a real request with, and without, an explicit system cachePoint.""" + try: + control = client.converse( + modelId=model_id, + system=[{"text": _SYSTEM_TEXT}], + messages=_MESSAGES, + inferenceConfig={"maxTokens": 5}, + ) + except botocore.exceptions.ClientError as exc: + result.live_status = "SKIPPED" + result.live_detail = f"control call failed: {exc.response['Error']['Code']}" + return + + result.input_tokens_uncached = control["usage"]["inputTokens"] + result.tokens_billed += control["usage"]["totalTokens"] + + try: + cached = client.converse( + modelId=model_id, + system=[{"text": _SYSTEM_TEXT}, {"cachePoint": {"type": "default"}}], + messages=_MESSAGES, + inferenceConfig={"maxTokens": 5}, + ) + except botocore.exceptions.ClientError as exc: + error = exc.response["Error"] + result.live_status = "REFUSED" + result.live_detail = f"{error['Code']}: {error['Message'][:110]}" + return + + usage = cached["usage"] + result.tokens_billed += usage["totalTokens"] + result.input_tokens_cached = usage["inputTokens"] + result.cache_write_tokens = usage.get("cacheWriteInputTokens", 0) + result.cache_read_tokens = usage.get("cacheReadInputTokens", 0) + + if result.cache_write_tokens or result.cache_read_tokens: + result.live_status = "ACCEPTS+CACHES" + else: + # Accepted the block but reported no cache buckets. Two different + # worlds, and the API does not distinguish them: either the model + # ignores cache points, or this prefix is under that model's minimum. + # Rule the second out before believing the first. + result.live_status = "ACCEPTS (no cache)" + result.notes.append( + f"accepted the cache point but cached nothing at {result.input_tokens_cached:,} " + "tokens — re-run with a larger prefix before concluding it is unsupported" + ) + + +def print_report(results: List[ProbeResult], offline_only: bool) -> None: + """Render both layers and call out every divergence.""" + print("\n" + "=" * 96) + print("OFFLINE — what strands-agents puts on the wire") + print("=" * 96) + print(f"{'model_id':<46}{'_cache_strategy':<18}{'tools pt':<11}{'our sys pt survives':<20}") + for r in results: + survives = "no (stripped)" if r.strands_strips_placed_point else "YES" + print( + f"{r.model_id:<46}{str(r.strands_strategy):<18}" + f"{('yes' if r.strands_emits_tools_point else 'no'):<11}{survives:<20}" + ) + + if offline_only: + return + + print("\n" + "=" * 96) + print("LIVE — what Bedrock does with an explicit system cachePoint") + print("=" * 96) + # Both buckets, because a repeat run READS the entry the previous run wrote + # — a zero in cacheWrite next to a non-zero cacheRead is a hit, not a miss. + print(f"{'model_id':<46}{'verdict':<20}{'input (ctl->cached)':<22}{'cacheWrite':<12}{'cacheRead':<11}") + for r in results: + movement = f"{r.input_tokens_uncached} -> {r.input_tokens_cached}" if r.input_tokens_cached else "-" + print( + f"{r.model_id:<46}{r.live_status:<20}{movement:<22}" + f"{str(r.cache_write_tokens or '-'):<12}{str(r.cache_read_tokens or '-'):<11}" + ) + if r.live_detail: + print(f"{'':<46}{r.live_detail}") + + print("\n" + "=" * 96) + print("DIVERGENCE — Bedrock caches it, our predicate refuses it") + print("=" * 96) + divergent = [ + r for r in results if r.live_status == "ACCEPTS+CACHES" and r.strands_strategy != "anthropic" + ] + if not divergent: + print(" none — upstream's 'anthropic only' test still matches what Bedrock accepts.") + else: + for r in divergent: + saved = r.input_tokens_uncached - r.input_tokens_cached + print( + f" {r.model_id}: caches {r.cache_write_tokens} tokens " + f"({saved} fewer billed as fresh input), but _cache_strategy is " + f"{r.strands_strategy!r} so neither we nor Strands will place a point." + ) + print( + "\n Widening bedrock_cache_points_supported() to cover these means widening\n" + " past upstream. Re-read its docstring before changing it." + ) + + for r in results: + for note in r.notes: + print(f"\n⚠️ {r.model_id}: {note}") + + total = sum(r.tokens_billed for r in results) + print(f"\nTokens billed by this run: {total:,}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model-id", + action="append", + dest="model_ids", + help="Model id to probe; repeatable. Replaces the default sweep.", + ) + parser.add_argument("--region", default="us-west-2") + parser.add_argument( + "--offline-only", + action="store_true", + help="Only ask the pinned SDK what it emits. No AWS calls, no spend.", + ) + args = parser.parse_args() + + model_ids = args.model_ids or DEFAULT_MODEL_IDS + results = [ProbeResult(model_id=m) for m in model_ids] + + for result in results: + probe_offline(result.model_id, result) + + if not args.offline_only: + estimate = len(model_ids) * 2 * _APPROX_PROMPT_TOKENS + print(f"Probing {len(model_ids)} models in {args.region} (~{estimate:,} input tokens).") + client = boto3.client("bedrock-runtime", region_name=args.region) + for result in results: + probe_live(client, result.model_id, result) + + print_report(results, args.offline_only) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/seed_bootstrap_data.py b/backend/scripts/seed_bootstrap_data.py index f22c191d9..c2f471ae0 100644 --- a/backend/scripts/seed_bootstrap_data.py +++ b/backend/scripts/seed_bootstrap_data.py @@ -806,6 +806,14 @@ def seed_default_tools( "SK": sk, "GSI1PK": f"CATEGORY#{tool_def['category']}", "GSI1SK": pk, + # EntityTypeIndex (GSI5) — mirrors ToolDefinition.to_dynamo_item. + # This seeder hand-builds the item rather than going through the + # model, so a new index key has to be added in BOTH places. Miss it + # here and a freshly bootstrapped deployment lists zero tools once + # the catalog read moves to the index — with no error, because a + # sparse index answers "nothing matched", not "something is wrong". + "GSI5PK": "ENTITY#TOOL", + "GSI5SK": pk, "toolId": tool_id, "displayName": tool_def["displayName"], "description": tool_def["description"], diff --git a/backend/src/.env.example b/backend/src/.env.example index 24a71394a..087c29bf5 100644 --- a/backend/src/.env.example +++ b/backend/src/.env.example @@ -863,7 +863,13 @@ DYNAMODB_USER_SETTINGS_TABLE_NAME= # "Concise", etc.) that users can opt into per-conversation. App API # performs full CRUD; Inference API only reads (GetItem) at invocation # time to compose the active prompt onto the base system prompt. +# Local Development: Leave empty to disable Conversation Modes. +# ⚠️ Unset does NOT error. SystemPromptsRepository disables itself and +# `GET /system-prompts/` returns `{"prompts": [], "total": 0}`, so the +# composer's Mode picker never appears — it renders only when a mode exists. +# If you are debugging a missing Mode picker locally, check this first. # CDK Deployment: Created by AppApiStack +# Example: -system-prompts DYNAMODB_SYSTEM_PROMPTS_TABLE_NAME= # S3 vector store bucket name (REQUIRED for RAG) diff --git a/backend/src/agents/main_agent/base_agent.py b/backend/src/agents/main_agent/base_agent.py index 15a29fe0b..79043d3fe 100644 --- a/backend/src/agents/main_agent/base_agent.py +++ b/backend/src/agents/main_agent/base_agent.py @@ -13,6 +13,7 @@ from agents.main_agent.core import ModelConfig, SystemPromptBuilder, AgentFactory from agents.main_agent.session import SessionFactory from agents.main_agent.session.hooks import ( + AgentStatusHook, DisplayTextHook, SteeringHook, StopHook, @@ -282,6 +283,8 @@ def _create_hooks(self) -> List: what the UI renders for an interrupted turn - OAuthConsentHook: Pauses the agent (Strands interrupt) when an OAuth-gated MCP tool is about to run without a cached token + - AgentStatusHook: Records model/tool boundaries so the UI can say what + the agent is doing while the turn streams - Approval hooks: Gate dangerous operations for user confirmation Returns: @@ -328,6 +331,15 @@ def _create_hooks(self) -> List: # final metadata SSE event. hooks.append(ContextAttributionHook()) + # Live narration of what the agent is doing (model call / tool call + # boundaries) plus Strands-measured per-tool durations. Held on the + # wrapper so the stream coordinator can drain the transitions into + # `agent_status` SSE events, and the closed tool batches into the + # tool-summary side-channel. Registered unconditionally; the callbacks + # return immediately when AGENT_STATUS_ENABLED=false. + self.agent_status_hook = AgentStatusHook() + hooks.append(self.agent_status_hook) + # Per-model-call prompt-cache prefix fingerprints (toolConfig / # system prompt / history hashes). Best-effort; the stream # coordinator persists them on each call's metadata row so avoidable diff --git a/backend/src/agents/main_agent/core/agent_factory.py b/backend/src/agents/main_agent/core/agent_factory.py index 0d1f05366..5a0b04e67 100644 --- a/backend/src/agents/main_agent/core/agent_factory.py +++ b/backend/src/agents/main_agent/core/agent_factory.py @@ -259,14 +259,49 @@ def create_agent( # Bedrock prompt caching: give the system prompt its own cachePoint by # passing it as a SystemContentBlock list with a trailing cachePoint - # (the cache_prompt model-config key is deprecated). Together with - # cache_tools (set in to_bedrock_config) this keeps the stable - # system+tools prefix readable from cache even when the auto-placed - # message-level cache point misses — see the cachePoint budget comment - # in ModelConfig.to_bedrock_config. Strands' auto strategy strips only - # message-level cachePoints, never system ones. Agent.system_prompt - # remains the plain string (split_system_prompt concatenates the text - # blocks), so hashing/attribution/voice consumers are unaffected. + # (the cache_prompt model-config key is deprecated). Together with the + # tools cachePoint (CacheConfig(tools_ttl=...) in to_bedrock_config; + # the model-level cache_tools key it replaces is deprecated as of + # strands-agents 1.55.0) this keeps the stable system+tools prefix + # readable from cache even when the auto-placed message-level cache + # point misses — see the cachePoint budget comment in + # ModelConfig.to_bedrock_config. + # + # INVARIANT, as of strands-agents 1.55.0 (the pinned version): this + # hand-placed system cachePoint is honored, never doubled. 1.55 does + # place a system cachePoint of its own — _should_cache_system(), with + # CacheConfig.system_prompt_ttl defaulting to True — but it arms on two + # conditions that together can never catch a block list this branch + # skipped. (1) It returns early unless _cache_strategy == "anthropic", + # i.e. "claude"/"anthropic" in the model id — the same test inside + # bedrock_cache_points_supported(), so on any model where upstream + # would place one, the list below already carries ours. (2) Its final + # guard is `not any("cachePoint" in block for block in system_blocks)`, + # which sees that block and stands down. Upstream's own CacheConfig + # docstring says the same thing ("A hand-placed system cache point is + # honored rather than doubled"). The older claim that auto strategy + # "strips only message-level cachePoints, never system ones" was a + # 1.51-era fact and is NOT the reason this is safe — do not restore it. + # + # Measured on the pinned 1.55.0 (2026-09-11) rather than read off the + # source: formatting a request with this block present yields exactly + # ONE system cachePoint, and with it absent upstream injects exactly one + # of its own. The same probe shows the converse, which is why the + # bedrock_cache_points_supported() gate below cannot be dropped — + # on a NON-Anthropic model this block is passed through untouched and + # Bedrock rejects the call with AccessDeniedException. + # + # RE-VERIFY BEFORE ANY BUMP PAST 1.55.0. This is a statement about + # upstream internals and it has already rotted once. Re-check + # _should_cache_system's guard, CacheConfig.system_prompt_ttl's + # default, and that tools_ttl=True still emits a bare + # {"cachePoint": {"type": "default"}} while cache_config.ttl is unset — + # the tools point is the tail of the cached prefix, so a stray ttl key + # there is a fleet-wide prefix re-write. + # + # Agent.system_prompt remains the plain string (split_system_prompt + # concatenates the text blocks), so hashing/attribution/voice consumers + # are unaffected. agent_system_prompt: Any = system_prompt if system_prompt and model_config.bedrock_cache_points_supported(): agent_system_prompt = [ diff --git a/backend/src/agents/main_agent/core/model_config.py b/backend/src/agents/main_agent/core/model_config.py index 83fcdf01c..8ec486b89 100644 --- a/backend/src/agents/main_agent/core/model_config.py +++ b/backend/src/agents/main_agent/core/model_config.py @@ -343,18 +343,45 @@ def get_provider(self) -> ModelProvider: return self.provider def bedrock_cache_points_supported(self) -> bool: - """Whether explicit Bedrock cachePoints (tools/system) may be sent. - - Mirrors Strands' ``BedrockModel._cache_strategy`` predicate: Anthropic - models are the only Bedrock family with prompt-cache support. Auto - (message-level) caching no-ops safely on other models, but explicit - tools/system cachePoints would be sent verbatim and rejected with a - ValidationException — so both are gated here, alongside - ``caching_enabled``, on the Bedrock provider path. + """Whether a hand-placed Bedrock system cachePoint may be sent. + + Mirrors Strands' ``BedrockModel._cache_strategy`` predicate. What it is + load-bearing FOR is the system cachePoint that + ``AgentFactory.create_agent`` places — the one explicit point in this + codebase upstream will not filter for us. ``format_request`` copies + ``system_prompt_content`` verbatim (bedrock.py:376) and + ``_apply_system_cache_ttl`` only ever rewrites a TTL, never removes a + point, so a point placed on a model that cannot cache reaches Bedrock + and the call fails with **AccessDeniedException** — "You invoked an + unsupported model or your request did not allow prompt caching." + Measured live in us-west-2 (2026-09-11) on llama3-3-70b, + mistral-large-2407 and deepseek-r1; the identical request without the + point succeeds on all three. + + It is also passed as ``tools_ttl``, where since strands-agents 1.55.0 + it is belt-and-braces rather than load-bearing: + ``_build_tools_cache_point`` applies the same + ``_cache_strategy != "anthropic"`` test itself (bedrock.py:579), so + True and False emit byte-identical requests on a non-Anthropic model. + Kept as the single predicate both points read, so the two can never + disagree about which models get an explicit point. + + ⚠️ This is deliberately NARROWER than "which Bedrock models support + prompt caching" — it tracks what *Strands* recognizes, not what + *Bedrock* accepts, and the two have already diverged. Nova Micro + accepts a system cachePoint and honors it (7,203 input tokens -> 2, + with 7,201 cache-written, same live probe) and this predicate denies + it. Widening it means widening past upstream's ``_cache_strategy``, so + re-measure with ``scripts/probe_bedrock_cache_point_support.py`` first + and widen the tools point in the same change. """ model_lower = self.model_id.lower() return ( self.caching_enabled + # Redundant with the family test below rather than an independent + # condition: get_provider() returns BEDROCK for any claude/anthropic + # id regardless of the configured provider. Kept as documentation of + # the surface this gates — the Converse path, not Mantle/Responses. and self.get_provider() == ModelProvider.BEDROCK and ("claude" in model_lower or "anthropic" in model_lower) ) @@ -392,8 +419,13 @@ def to_bedrock_config(self) -> Dict[str, Any]: # not touch the system/tools points. # # The tools+system points make a message-level lookup miss cost a - # cache READ of the stable prefix instead of a full re-write at - # $2.5/MTok (write premium). One proven miss mode is structural: + # cache READ of the stable prefix instead of a full re-write at the + # cache-write premium. There is no flat per-MTok figure for that + # premium: it is 1.25x the model's OWN base input rate, so price it + # against the model in play ($1.375/MTok on our default Haiku 4.5, + # $4.125 on Sonnet 4.6) — see the prompt-cache contract in CLAUDE.md, + # which is the single place that rule is maintained. + # One proven miss mode is structural: # Anthropic's cache lookback checks only ~20 content blocks behind # the breakpoint, so a wide parallel tool fan-out (e.g. 18 parallel # calls = ~38 new blocks) pushes the previous checkpoint out of range @@ -402,10 +434,12 @@ def to_bedrock_config(self) -> Dict[str, Any]: # ~28k-token static prefix still reads from cache on those turns. # # For a model whose id Strands doesn't recognize as cache-capable, - # auto strategy logs a warning and no-ops — but the tools and system - # cachePoints are sent unconditionally once configured, so both are - # gated on bedrock_cache_points_supported() (the same predicate - # Strands' auto mode uses). Requires strands-agents>=1.48.0: a + # auto strategy logs a warning and no-ops. The tools point is filtered + # by upstream on the same test as of 1.55.0, but a hand-placed SYSTEM + # point is passed through verbatim and Bedrock answers + # AccessDeniedException — so both read + # bedrock_cache_points_supported(), which is where that asymmetry and + # its live measurement are documented. Requires strands-agents>=1.48.0: a # cachePoint trailing a non-PDF `document` attachment is rejected by # Bedrock's Anthropic adapter with "ValidationException ... # content.N.type: Field required" (agent force-stop on any turn with a diff --git a/backend/src/agents/main_agent/session/hooks/__init__.py b/backend/src/agents/main_agent/session/hooks/__init__.py index 0251a49a7..88efadd8e 100644 --- a/backend/src/agents/main_agent/session/hooks/__init__.py +++ b/backend/src/agents/main_agent/session/hooks/__init__.py @@ -1,5 +1,6 @@ """Hooks for Main Agent""" +from agents.main_agent.session.hooks.agent_status import AgentStatusHook from agents.main_agent.session.hooks.context_attribution import ContextAttributionHook from agents.main_agent.session.hooks.display_text import DisplayTextHook from agents.main_agent.session.hooks.oauth_consent import OAuthConsentHook @@ -9,6 +10,7 @@ from agents.main_agent.session.hooks.tool_approval import MCPExternalApprovalHook __all__ = [ + "AgentStatusHook", "ContextAttributionHook", "DisplayTextHook", "OAuthConsentHook", diff --git a/backend/src/agents/main_agent/session/hooks/agent_status.py b/backend/src/agents/main_agent/session/hooks/agent_status.py new file mode 100644 index 000000000..73824625f --- /dev/null +++ b/backend/src/agents/main_agent/session/hooks/agent_status.py @@ -0,0 +1,324 @@ +"""Narrate what the agent is doing while a turn is still streaming. + +A multi-tool turn spends most of its wall-clock time in places the content +stream says nothing about: the model deciding, a Lambda-backed MCP tool doing +a round trip, a batch of parallel calls finishing at different times. The SPA +covered that silence with cycling phrases ("Pondering...", "Cross-referencing") +which are honest about *nothing* — they look identical whether the agent is +thinking, waiting on Canvas, or hung. + +This hook watches the boundaries the event loop already crosses and records a +small status transition at each one. The stream coordinator drains them into +``agent_status`` SSE events (see ``_drain_agent_status_events``), and the SPA +turns them into a live line — "Using list_assignments..." — plus a real +duration on every finished tool row. + +WHAT IT OBSERVES +---------------- +``BeforeInvocationEvent`` + Turn boundary. Resets per-turn state so a queue left behind by an aborted + or interrupted turn can never leak into the next one. + +``BeforeModelCallEvent`` + The model is generating. Emitted once per event-loop cycle, so a turn that + calls three tools in series reports "thinking" four times — that is the + real shape of the turn and the SPA renders each as a fresh cycle. + +``BeforeToolCallEvent`` / ``AfterToolCallEvent`` + One tool starting and finishing. ``AfterToolCallEvent`` carries Strands' + own ``duration``, so the per-tool timing the rail shows is measured by the + event loop rather than guessed from stream arrival times on the client. + +``AfterToolsEvent`` + The batch is done. Closes the batch and parks a **bounded** record of it + (tool names, truncated inputs, truncated results) for the tool-summary + side-channel to consume. Fires from a ``finally``, so it also fires on the + cancel / error / interrupt paths — a batch closed there is still a true + record of tools that ran, which is exactly what a summary should describe. + +WHAT IT DELIBERATELY DOES NOT OBSERVE +------------------------------------- +"The agent is writing its answer." The SPA already knows that: text deltas are +arriving. Deriving it here would mean a second source of truth for a fact the +client holds first-hand, and the two would disagree at the edges. + +COST +---- +Nothing reaches the model. The hook writes to two in-process lists and the +coordinator drains them between stream events; nothing it produces is appended +to the conversation, so the cacheable prefix is untouched (CLAUDE.md +prompt-cache contract). Both lists are capped — a pathological 500-tool turn +costs a bounded amount of memory and drops the overflow rather than growing +without limit. + +Best-effort in every direction: every callback is wrapped, and any failure is +swallowed. A status line is a nicety; it must never be able to break a turn. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, Dict, List, Optional + +from strands.hooks import ( + AfterToolCallEvent, + AfterToolsEvent, + BeforeInvocationEvent, + BeforeModelCallEvent, + BeforeToolCallEvent, + HookProvider, + HookRegistry, +) + +logger = logging.getLogger(__name__) + +# Queue caps. A normal turn produces a handful of transitions; these exist so a +# runaway agent loop degrades to "stops narrating" instead of "grows until the +# container is killed". +_MAX_QUEUED_STATUSES = 400 +_MAX_QUEUED_BATCHES = 40 +# Per-tool payload caps for the summarizer record. The summarizer truncates +# again before it builds its prompt; this is the first bound, applied at +# capture time so an 8MB tool result never sits in memory for the turn. +_MAX_INPUT_CHARS = 600 +_MAX_RESULT_CHARS = 1200 +# A batch wider than this is summarized from its first N calls. Beyond that the +# marginal call adds prompt cost without changing the one-line summary. +_MAX_CALLS_PER_BATCH = 12 + + +def _truncate(text: str, limit: int) -> str: + """Clip ``text`` to ``limit`` characters with a visible ellipsis.""" + if len(text) <= limit: + return text + return text[:limit] + "..." + + +def _stringify(value: Any, limit: int) -> str: + """Render a tool input/result as bounded text for the summarizer. + + JSON where possible (the summarizer reads structure better than a repr), + ``str()`` otherwise. Never raises: a value that will not serialize is worth + less than the turn it would break. + """ + if value is None: + return "" + if isinstance(value, str): + return _truncate(value, limit) + try: + return _truncate(json.dumps(value, default=str), limit) + except Exception: # noqa: BLE001 - defensive, see docstring + return _truncate(str(value), limit) + + +def _duration_ms(raw: Any) -> Optional[int]: + """Normalize Strands' ``AfterToolCallEvent.duration`` to whole ms. + + The SDK has expressed this as both a ``float`` of seconds and a + ``timedelta`` across versions, and the attribute is absent on older ones. + All three degrade to ``None`` rather than to a wrong number, because a + wrong duration on a tool row is worse than no duration. + """ + if raw is None: + return None + total_seconds = getattr(raw, "total_seconds", None) + if callable(total_seconds): + try: + return max(0, int(total_seconds() * 1000)) + except Exception: # noqa: BLE001 + return None + if isinstance(raw, (int, float)): + return max(0, int(float(raw) * 1000)) + return None + + +def _tool_use_fields(event: Any) -> tuple[Optional[str], Optional[str], Any]: + """Pull ``(tool_use_id, tool_name, input)`` off a tool hook event.""" + tool_use = getattr(event, "tool_use", None) or {} + if not isinstance(tool_use, dict): + return None, None, None + tool_use_id = tool_use.get("toolUseId") or tool_use.get("tool_use_id") + name = tool_use.get("name") + return ( + str(tool_use_id) if tool_use_id else None, + str(name) if name else None, + tool_use.get("input"), + ) + + +class AgentStatusHook(HookProvider): + """Record model-call and tool-call transitions for the live status line. + + Holds only per-turn state, reset at ``BeforeInvocationEvent`` and drained + within the same turn. Per the CLAUDE.md rule that per-session state is + never cached on an agent instance: there is no session state here to go + stale — an ``@``-mention turn builds a second ``Agent`` with its own hook, + and each narrates its own turn. + """ + + def __init__(self) -> None: + # Status transitions awaiting a drain by the stream coordinator. + self._statuses: List[Dict[str, Any]] = [] + # Closed tool batches awaiting a summarizer task. + self._batches: List[Dict[str, Any]] = [] + # Calls completed in the batch currently in flight. + self._open_batch: List[Dict[str, Any]] = [] + # Event-loop cycle counter, 1-based. Lets the SPA tell "thinking again + # after a tool" apart from a stutter in the same cycle. + self._cycle = 0 + # Monotonic batch counter, used to build a stable batch id. + self._batch_seq = 0 + + def register_hooks(self, registry: HookRegistry, **kwargs: Any) -> None: + registry.add_callback(BeforeInvocationEvent, self._on_turn_start) + registry.add_callback(BeforeModelCallEvent, self._on_before_model_call) + registry.add_callback(BeforeToolCallEvent, self._on_before_tool_call) + registry.add_callback(AfterToolCallEvent, self._on_after_tool_call) + registry.add_callback(AfterToolsEvent, self._on_after_tools) + + # -- drains (called by the stream coordinator) ------------------------- + + def drain_statuses(self) -> List[Dict[str, Any]]: + """Take the status transitions recorded since the last drain.""" + statuses, self._statuses = self._statuses, [] + return statuses + + def drain_batches(self) -> List[Dict[str, Any]]: + """Take the tool batches closed since the last drain.""" + batches, self._batches = self._batches, [] + return batches + + # -- callbacks --------------------------------------------------------- + + def _on_turn_start(self, event: BeforeInvocationEvent) -> None: + """Drop anything a previous turn left behind. + + The interrupt path (OAuth consent, tool approval) unwinds the event + loop without draining, so without this reset the next turn's first + drain would replay stale transitions and the SPA would narrate tools + that are not running. + """ + self._statuses = [] + self._batches = [] + self._open_batch = [] + self._cycle = 0 + self._batch_seq = 0 + + def _on_before_model_call(self, event: BeforeModelCallEvent) -> None: + if not self._enabled(): + return + try: + self._cycle += 1 + self._push({"phase": "thinking", "cycle": self._cycle}) + except Exception: # noqa: BLE001 - narration must never break a turn + logger.debug("Agent status (before model call) skipped", exc_info=True) + + def _on_before_tool_call(self, event: BeforeToolCallEvent) -> None: + if not self._enabled(): + return + try: + tool_use_id, name, _ = _tool_use_fields(event) + if not name: + return + self._push( + { + "phase": "tool_start", + "cycle": self._cycle, + "toolName": name, + "toolUseId": tool_use_id, + } + ) + except Exception: # noqa: BLE001 + logger.debug("Agent status (before tool call) skipped", exc_info=True) + + def _on_after_tool_call(self, event: AfterToolCallEvent) -> None: + """Record the finished call: a status transition AND a batch entry. + + The two are gated separately on purpose. The status transition is the + live line and rides ``AGENT_STATUS_ENABLED``; the batch entry is what + the tool-summary side-channel consumes and rides + ``TOOL_SUMMARIES_ENABLED`` (checked downstream, at the summarizer). A + deployment that wants summaries without a live status line — or the + reverse — gets exactly that, instead of one flag silently disabling + the other feature. + """ + try: + tool_use_id, name, tool_input = _tool_use_fields(event) + if not name: + return + duration_ms = _duration_ms(getattr(event, "duration", None)) + ok = getattr(event, "exception", None) is None + result = getattr(event, "result", None) + if ok and isinstance(result, dict): + # A tool can fail *inside* a successful invocation; Strands + # reports that as a result with status "error", not as a raised + # exception. The rail's red dot depends on catching both. + ok = result.get("status") != "error" + + if self._enabled(): + self._push( + { + "phase": "tool_end", + "cycle": self._cycle, + "toolName": name, + "toolUseId": tool_use_id, + "durationMs": duration_ms, + "ok": ok, + } + ) + + if len(self._open_batch) < _MAX_CALLS_PER_BATCH: + self._open_batch.append( + { + "toolUseId": tool_use_id, + "toolName": name, + "input": _stringify(tool_input, _MAX_INPUT_CHARS), + "result": _stringify(result, _MAX_RESULT_CHARS), + "ok": ok, + "durationMs": duration_ms, + } + ) + except Exception: # noqa: BLE001 + logger.debug("Agent status (after tool call) skipped", exc_info=True) + + def _on_after_tools(self, event: AfterToolsEvent) -> None: + """Close the in-flight batch and park it for the summarizer. + + Runs even with the status flag off, because the tool-summary + side-channel is gated separately — a deployment can want summaries + without the live status line, and vice versa. + """ + try: + calls, self._open_batch = self._open_batch, [] + if not calls: + return + self._batch_seq += 1 + batch_id = calls[0].get("toolUseId") or f"batch-{self._batch_seq}" + if len(self._batches) < _MAX_QUEUED_BATCHES: + self._batches.append( + { + "batchId": str(batch_id), + "cycle": self._cycle, + "toolUseIds": [ + c["toolUseId"] for c in calls if c.get("toolUseId") + ], + "calls": calls, + } + ) + except Exception: # noqa: BLE001 + logger.debug("Agent status (after tools) skipped", exc_info=True) + + # -- internals --------------------------------------------------------- + + @staticmethod + def _enabled() -> bool: + from apis.shared.feature_flags import agent_status_enabled + + return agent_status_enabled() + + def _push(self, status: Dict[str, Any]) -> None: + """Queue a transition, dropping it if the turn has run away.""" + if len(self._statuses) >= _MAX_QUEUED_STATUSES: + return + self._statuses.append(status) diff --git a/backend/src/agents/main_agent/session/turn_based_session_manager.py b/backend/src/agents/main_agent/session/turn_based_session_manager.py index 5f8227d73..025e598fb 100644 --- a/backend/src/agents/main_agent/session/turn_based_session_manager.py +++ b/backend/src/agents/main_agent/session/turn_based_session_manager.py @@ -16,8 +16,10 @@ stored history must produce byte-identical ``agent.messages``. Bedrock prompt caching requires an exact prefix match, so any per-restore mutation of older turns (e.g. a sliding truncation window) breaks the cached prefix and forces a -full re-write (~$2.5/MTok on a 35k–150k prefix) nearly every turn — far more -expensive than the read tokens truncation saves. +full re-write of a 35k–150k prefix nearly every turn, at the cache-write +premium — 1.25x the model's own base input rate, not a flat per-MTok figure; +see the prompt-cache contract in CLAUDE.md — far more expensive than the read +tokens truncation saves. Based on: https://github.com/aws-samples/sample-strands-agent-with-agentcore """ diff --git a/backend/src/agents/main_agent/streaming/stream_coordinator.py b/backend/src/agents/main_agent/streaming/stream_coordinator.py index 6b954c042..0ffd89b3b 100644 --- a/backend/src/agents/main_agent/streaming/stream_coordinator.py +++ b/backend/src/agents/main_agent/streaming/stream_coordinator.py @@ -314,6 +314,14 @@ async def stream_response( ui_block_index_to_tool_use_id: Dict[int, str] = {} ui_partial_input_acc: Dict[str, str] = {} + # Tool-batch summary tasks in flight for this turn. Each is a Nova + # Micro side-channel call started when a tool batch closed; the emit + # loop harvests whichever have finished, and `_collect_tool_summary_ + # events` rewrites this list in place as it drains. Per-turn only — + # never state that outlives the turn, so the CLAUDE.md rule about + # caching session state on an agent instance does not bite here. + tool_summary_tasks: List[Any] = [] + # Accumulate metadata from stream accumulated_metadata: Dict[str, Any] = {"usage": {}, "metrics": {}} @@ -586,6 +594,13 @@ async def stream_response( # interrupt flavor, so any extractor's resume path can rebuild # the agent shape after a refresh / cache eviction. if event.get("type") == "done": + # Last chance for a summary still in flight to reach the + # live view. Bounded wait; a straggler past it is left to + # its own persistence and shows up on reload instead. + for summary_sse in await self._collect_tool_summary_events( + tool_summary_tasks, drain_all=True + ): + yield summary_sse await self._persist_paused_turn_snapshot( agent, session_id=session_id, @@ -977,6 +992,29 @@ async def stream_response( ): yield steering_sse + # Live narration: the status hook records model-call and + # tool-call boundaries from inside Strands' event loop, which + # has no route to the SSE stream. Drained here, before the + # event it precedes is yielded, so "Using list_assignments" + # reaches the client while that tool is actually running + # rather than after its result. + for status_sse in self._drain_agent_status_events( + main_agent_wrapper, session_id + ): + yield status_sse + + # Tool-batch summaries: start a Nova Micro side-channel task + # for each batch that just closed, then harvest whichever + # earlier tasks have finished. Non-blocking in both + # directions — the agent stream never waits on Nova. + self._spawn_tool_summary_tasks( + main_agent_wrapper, session_id, user_id, tool_summary_tasks + ) + for summary_sse in await self._collect_tool_summary_events( + tool_summary_tasks + ): + yield summary_sse + # Format as SSE event and yield (including done event after metadata) sse_event = self._format_sse_event(event) yield sse_event @@ -2237,6 +2275,170 @@ def _drain_steering_events( ) return events + def _drain_agent_status_events( + self, main_agent_wrapper: Any, session_id: str + ) -> List[str]: + """Emit one `agent_status` SSE per transition the status hook recorded. + + Drained rather than pushed for the same reason as steering: the hook + runs inside Strands' event loop, which has no route to the SSE stream. + Draining on every iteration of the emit loop keeps the transitions + roughly interleaved with the content they describe — "thinking" lands + before the text it precedes, "tool_start" before that tool's result. + + Best-effort: a wrapper without the hook (voice, tests) and any failure + both yield nothing, leaving the SPA on its cycling phrases. + """ + hook = getattr(main_agent_wrapper, "agent_status_hook", None) + if hook is None: + return [] + try: + statuses = hook.drain_statuses() + except Exception: # noqa: BLE001 - never break the stream on narration + logger.warning("Agent status drain failed", exc_info=True) + return [] + + events = [] + for status in statuses: + payload = {"type": "agent_status", "sessionId": session_id, **status} + events.append(f"event: agent_status\ndata: {json.dumps(payload)}\n\n") + return events + + def _spawn_tool_summary_tasks( + self, + main_agent_wrapper: Any, + session_id: str, + user_id: str, + tasks: List[Any], + ) -> None: + """Kick off a Nova Micro summary for each tool batch that just closed. + + Concurrent with the agent stream, exactly like conversation-title + generation: the batch is finished, so nothing downstream waits on this, + and the agent's next model call is already in flight while Nova runs. + + Each task persists its own result before returning it, so reload + survival does not depend on the emit loop still being alive when the + summary lands — a turn that ends (or is cancelled) between the call and + its completion still leaves the row behind for `GET /messages`. + """ + hook = getattr(main_agent_wrapper, "agent_status_hook", None) + if hook is None: + return + try: + batches = hook.drain_batches() + except Exception: # noqa: BLE001 + logger.warning("Tool batch drain failed", exc_info=True) + return + if not batches: + return + + from apis.shared.feature_flags import tool_summaries_enabled + + if not tool_summaries_enabled(): + return + + for batch in batches: + tasks.append( + asyncio.create_task( + self._summarize_and_persist_batch(batch, session_id, user_id) + ) + ) + + @staticmethod + async def _summarize_and_persist_batch( + batch: Dict[str, Any], session_id: str, user_id: str + ) -> Optional[Dict[str, Any]]: + """Summarize one batch, persist it, and return the SSE payload. + + Returns None whenever there is nothing worth showing — the SPA keeps + the deterministic formatter line it is already displaying, which is a + good enough answer that no failure here is worth surfacing. + """ + try: + from apis.shared.tool_summaries import ( + get_tool_summary_store, + summarize_tool_batch, + ) + + summary = await summarize_tool_batch(batch.get("calls") or []) + if not summary: + return None + + batch_id = str(batch.get("batchId") or "") + tool_use_ids = [str(t) for t in (batch.get("toolUseIds") or [])] + + # Persist before returning: the emit loop may never get to this + # payload (cancelled turn, dropped connection), but the row is + # what makes the summary survive a reload either way. + await asyncio.to_thread( + get_tool_summary_store().store, + user_id=user_id, + session_id=session_id, + batch_id=batch_id, + tool_use_ids=tool_use_ids, + summary=summary, + ) + return { + "type": "tool_group_summary", + "sessionId": session_id, + "batchId": batch_id, + "toolUseIds": tool_use_ids, + "summary": summary, + } + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - a summary is never worth an error + logger.debug("Tool batch summary task failed", exc_info=True) + return None + + @staticmethod + async def _collect_tool_summary_events( + tasks: List[Any], *, drain_all: bool = False, timeout: float = 3.0 + ) -> List[str]: + """Harvest finished summary tasks into `tool_group_summary` SSEs. + + Non-blocking by default — only tasks that are already done are + collected, so the agent stream is never held up waiting on Nova. + + `drain_all` is used once, just before the turn's final metadata and + `done`: it waits up to `timeout` for stragglers so a summary that lands + late still reaches the live view instead of only appearing on reload. + A task that misses even that window is abandoned here but NOT + cancelled — it has already persisted (or is about to), and the reload + path will show it. + """ + if not tasks: + return [] + + if drain_all: + pending = [t for t in tasks if not t.done()] + if pending: + try: + await asyncio.wait(pending, timeout=timeout) + except Exception: # noqa: BLE001 + pass + + events: List[str] = [] + still_running: List[Any] = [] + for task in tasks: + if not task.done(): + still_running.append(task) + continue + try: + payload = task.result() + except asyncio.CancelledError: + continue + except Exception: # noqa: BLE001 + logger.debug("Tool summary task raised", exc_info=True) + continue + if payload: + events.append( + f"event: tool_group_summary\ndata: {json.dumps(payload)}\n\n" + ) + tasks[:] = still_running + return events + def _format_sse_event(self, event: Dict[str, Any]) -> str: """ Format processed event as SSE (Server-Sent Event) diff --git a/backend/src/apis/app_api/admin/models.py b/backend/src/apis/app_api/admin/models.py index caf6fd20d..29cca61b6 100644 --- a/backend/src/apis/app_api/admin/models.py +++ b/backend/src/apis/app_api/admin/models.py @@ -138,3 +138,21 @@ class ManagedModelsListResponse(BaseModel): models: List[ManagedModel] total_count: int = Field(..., alias="totalCount") + + +class ManagedModelIconResponse(BaseModel): + """The result of uploading or clearing a managed model's icon. + + Both fields are ``None`` after a remove, which is the signal the SPA needs to + fall back to the model's ``iconSlug`` (or its provider-name match) without + re-reading the whole catalog. + """ + model_config = ConfigDict(populate_by_name=True) + + model_id: str = Field(..., alias="id", description="Managed model record id") + icon_key: Optional[str] = Field( + None, alias="iconKey", description="S3 object key now on the record; None after a remove" + ) + icon_url: Optional[str] = Field( + None, alias="iconUrl", description="Where to render it from; None → the iconSlug fallback" + ) diff --git a/backend/src/apis/app_api/admin/oauth/routes.py b/backend/src/apis/app_api/admin/oauth/routes.py index 4f5ed0524..cf06e5d51 100644 --- a/backend/src/apis/app_api/admin/oauth/routes.py +++ b/backend/src/apis/app_api/admin/oauth/routes.py @@ -285,13 +285,32 @@ async def update_provider( ) rotating_credentials = bool(updates.client_id and updates.client_secret) + # Only a *different* value counts as a discovery change. Clients that + # round-trip the whole record (the connector edit form does) resend the + # unchanged discovery URL on every save; treating any non-None value as + # a change would make metadata-only edits — scopes, display name, icon, + # enabled — impossible for a discovery-URL provider, because the admin + # cannot satisfy the rotation requirement (AgentCore never echoes the + # client secret back, so there is nothing to re-enter). changing_discovery = ( updates.oauth_discovery_url is not None - or updates.authorization_server_metadata is not None + and updates.oauth_discovery_url != existing.oauth_discovery_url + ) or ( + updates.authorization_server_metadata is not None + and updates.authorization_server_metadata + != existing.authorization_server_metadata ) credential_info: CredentialProviderInfo | None = None if rotating_credentials or changing_discovery: + if not rotating_credentials: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Discovery config can only be updated together with a " + "credential rotation (client_id + client_secret)." + ), + ) discovery_url = ( updates.oauth_discovery_url if updates.oauth_discovery_url is not None @@ -302,14 +321,6 @@ async def update_provider( if updates.authorization_server_metadata is not None else existing.authorization_server_metadata ) - if not rotating_credentials: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "Discovery config can only be updated together with a " - "credential rotation (client_id + client_secret)." - ), - ) try: credential_info = registrar.update_credential_provider( provider_id=provider_id, diff --git a/backend/src/apis/app_api/admin/routes.py b/backend/src/apis/app_api/admin/routes.py index 470202c93..2e4956931 100644 --- a/backend/src/apis/app_api/admin/routes.py +++ b/backend/src/apis/app_api/admin/routes.py @@ -4,7 +4,7 @@ Requires admin role (Admin or SuperAdmin) via JWT token. """ -from fastapi import APIRouter, HTTPException, Depends, Query, status +from fastapi import APIRouter, File, HTTPException, Depends, Query, UploadFile, status from typing import List, Literal, Optional import logging import os @@ -21,6 +21,7 @@ OpenAIModelSummary, MantleModelsResponse, MantleModelSummary, + ManagedModelIconResponse, ManagedModelsListResponse, ) from apis.shared.models.models import ( @@ -38,6 +39,11 @@ update_managed_model, delete_managed_model, ) +from .services.model_icons import ( + ModelIconError, + remove_model_icon, + upload_model_icon, +) from .services.model_roles import get_model_role_service logger = logging.getLogger(__name__) @@ -803,6 +809,58 @@ async def delete_managed_model_endpoint( ) +# ---------------------------------------------------------------- model icons +# Writing an icon is editing the catalog, so it rides the admin.models scope. +# Reading is deliberately NOT here: every signed-in user renders these in the +# chat model picker, so the serve route lives on the user-facing /models router. +@router.post("/managed-models/{model_id}/icon", response_model=ManagedModelIconResponse) +async def upload_managed_model_icon( + model_id: str, + file: UploadFile = File(...), + admin_user: User = Depends(require_models_admin), +): + """Upload a custom icon for a model (admin only). + + Square PNG or JPEG, at least 256×256 and at most 400 KB; stored re-encoded at + 512×512, which is also what strips EXIF. Prefer setting ``iconSlug`` when we + ship a logo for the vendor — it stays a crisp, theme-aware vector. Rejections + carry the limit and the supplied value, since "invalid image" sends an admin + back to the file picker with nothing to change. + """ + content = await file.read() + try: + icon_key, icon_url = await upload_model_icon(model_id, content) + except ModelIconError as e: + raise HTTPException(status_code=e.status_code, detail=e.message) + except Exception as e: + logger.error("Unexpected error uploading model icon", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to upload model icon: {str(e)}") + + return ManagedModelIconResponse(model_id=model_id, icon_key=icon_key, icon_url=icon_url) + + +@router.delete("/managed-models/{model_id}/icon", response_model=ManagedModelIconResponse) +async def delete_managed_model_icon( + model_id: str, + admin_user: User = Depends(require_models_admin), +): + """Remove the uploaded icon, falling back to the model's ``iconSlug`` (admin only). + + Separate from clearing ``iconSlug`` through the model form on purpose: the two + are independent, and an admin who uploaded the wrong file should get their + built-in logo back rather than a blank tile. + """ + try: + icon_key, icon_url = await remove_model_icon(model_id) + except ModelIconError as e: + raise HTTPException(status_code=e.status_code, detail=e.message) + except Exception as e: + logger.error("Unexpected error removing model icon", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to remove model icon: {str(e)}") + + return ManagedModelIconResponse(model_id=model_id, icon_key=icon_key, icon_url=icon_url) + + @router.get("/managed-models/{model_id}/roles", response_model=List[ModelRoleAssignment]) async def get_managed_model_roles( model_id: str, diff --git a/backend/src/apis/app_api/admin/services/model_icons.py b/backend/src/apis/app_api/admin/services/model_icons.py new file mode 100644 index 000000000..f253ee4d6 --- /dev/null +++ b/backend/src/apis/app_api/admin/services/model_icons.py @@ -0,0 +1,118 @@ +"""Upload / remove a managed model's icon. + +The one place the three storage concerns meet: validation +(:mod:`apis.shared.images.icons`), the object write (S3), and the ``iconKey`` +attribute write (``write_model_icon_key``). + +Writing is admin-only — it rides the ``admin.models`` scope like every other +mutation on the model catalog. *Reading* is not: the icon renders in every +signed-in user's model picker, which is why the serve route lives on the +user-facing ``/models`` router instead (``apis.app_api.models.routes``). +""" + +from __future__ import annotations + +import logging +from typing import Optional, Tuple + +from apis.shared.models.managed_models import get_managed_model, write_model_icon_key +from apis.shared.models.model_icons import ( + IconError, + IconStoreError, + get_model_icon_store, + model_icon_url, + model_icon_version, + normalize_icon, +) + +logger = logging.getLogger(__name__) + + +class ModelIconError(Exception): + """An icon operation we cannot complete, with a message written for the admin. + + ``status_code`` maps to the HTTP response: 404 missing model or missing icon, + 400 an image that fails the limits, 503 storage unconfigured. + """ + + def __init__(self, message: str, status_code: int = 400) -> None: + super().__init__(message) + self.message = message + self.status_code = status_code + + +async def _require_model(model_id: str): + model = await get_managed_model(model_id) + if not model: + raise ModelIconError(f"Model not found: {model_id}", status_code=404) + return model + + +async def upload_model_icon(model_id: str, content: bytes) -> Tuple[Optional[str], Optional[str]]: + """Validate, store and record a new icon; return ``(icon_key, icon_url)``. + + The old object is deleted only *after* the record points at the new one, so a + failure in the middle leaves the model with its previous icon rather than + none. The key is content-addressed, which makes re-uploading the same image + idempotent — and makes the delete a no-op in exactly that case, which is why + it is skipped when the key is unchanged. + """ + model = await _require_model(model_id) + + try: + data, ext, content_type = normalize_icon(content) + except IconError as e: + raise ModelIconError(str(e), status_code=400) from e + + store = get_model_icon_store() + try: + key = store.put(model_id=model_id, content=data, ext=ext, content_type=content_type) + except IconStoreError as e: + logger.error(f"Icon storage unavailable for model {model_id}: {e}") + raise ModelIconError("Icon storage is unavailable.", status_code=503) from e + + previous = model.icon_key + await write_model_icon_key(model_id, key) + if previous and previous != key: + store.delete(previous) + + logger.info(f"🖼️ model-icons: uploaded icon for model {model_id}") + return key, model_icon_url(model_id, key) + + +async def remove_model_icon(model_id: str) -> Tuple[Optional[str], Optional[str]]: + """Clear the uploaded icon, returning the model to its ``iconSlug`` (or the + client-side provider fallback when it has none).""" + model = await _require_model(model_id) + previous = model.icon_key + + await write_model_icon_key(model_id, None) + if previous: + get_model_icon_store().delete(previous) + + logger.info(f"🖼️ model-icons: removed icon for model {model_id}") + return None, None + + +async def read_model_icon(model_id: str) -> Tuple[bytes, str, str]: + """Return ``(bytes, content_type, version)`` for a model's uploaded icon. + + No access check beyond being signed in: the catalog already hands every user + this model's name, provider and pricing, so its logo is no new disclosure — + and gating it on the per-role model grant would render a broken tile in the + admin form for a model the admin's own roles happen not to grant. + """ + model = await _require_model(model_id) + if not model.icon_key: + raise ModelIconError("This model has no icon.", status_code=404) + + try: + data, content_type = get_model_icon_store().get(model.icon_key) + except IconStoreError as e: + # A key that outlived its object: 404 rather than 500, so the SPA's + # error path falls through to the slug/provider fallback instead of + # showing a broken tile. + logger.warning(f"Icon object missing for model {model_id}: {e}") + raise ModelIconError("This model has no icon.", status_code=404) from e + + return data, content_type, model_icon_version(model.icon_key) or "" diff --git a/backend/src/apis/app_api/admin/tools/routes.py b/backend/src/apis/app_api/admin/tools/routes.py index 7cf4f245c..0f4743720 100644 --- a/backend/src/apis/app_api/admin/tools/routes.py +++ b/backend/src/apis/app_api/admin/tools/routes.py @@ -20,7 +20,10 @@ get_provider_repository, ) from apis.app_api.tools.service import get_tool_catalog_service -from apis.app_api.tools.discovery import discover_tools_for_saved_tool +from apis.app_api.tools.discovery import ( + discover_capabilities_for_saved_tool, + discover_tools_for_saved_tool, +) from apis.shared.tools.gateway_target_service import ( GatewayTargetConflictError, GatewayTargetNotFoundError, @@ -40,6 +43,7 @@ DiscoveredMCPTool, GatewayTargetStatusResponse, MCPAuthType, + ToolCapabilitySnapshot, ) logger = logging.getLogger(__name__) @@ -734,3 +738,94 @@ async def remove_roles_from_tool( raise HTTPException(status_code=400, detail=str(e)) +@router.post( + "/{tool_id}/capabilities/refresh", response_model=ToolCapabilitySnapshot +) +async def admin_refresh_tool_capabilities( + tool_id: str, + admin: User = Depends(require_tools_admin), + provider_repo: OAuthProviderRepository = Depends(get_provider_repository), +): + """Ask a saved MCP server what prompts and resources it exposes, and store it. + + An admin action rather than a user one: probing opens a live MCP session, so + doing it per user per page view would hammer every server in the catalog. + Users read the stored snapshot. + + A 3LO server is probed with the **admin's own** vaulted token, exactly as + ``POST /admin/tools/discover`` does. The snapshot therefore reflects what the + admin's connection can see — for providers that scope-filter by grant, a user + with narrower scopes may see less. That is the same caveat the tool listing + has always carried, and it is far better than the alternative: six servers in + prod currently expose nothing at all because discovery runs without a token. + + A failed probe is **not** written over a good snapshot. Losing a working + listing because a server was briefly down would be a worse outcome than + showing a slightly stale one, so the previous snapshot is returned with the + error attached. + """ + service = get_tool_catalog_service() + tool = await service.repository.get_tool(tool_id) + if tool is None: + raise HTTPException(status_code=404, detail="Tool not found") + + oauth_token: Optional[str] = None + if tool.requires_oauth_provider: + provider = await provider_repo.get_provider(tool.requires_oauth_provider) + if provider is None: + raise HTTPException( + status_code=400, + detail=f"Unknown OAuth provider '{tool.requires_oauth_provider}'.", + ) + identity = get_agentcore_identity_client() + try: + result = await identity.get_token_for_user( + provider_name=provider.provider_id, + scopes=provider.scopes, + user_id=admin.user_id, + # customParameters are part of the AgentCore vault key — omitting + # them would falsely report consent-required for a vaulted token. + custom_parameters=custom_parameters_for(provider.custom_parameters), + ) + except (WorkloadTokenUnavailableError, CallbackUrlUnavailableError) as err: + logger.warning("Capability discovery token unavailable: %s", err) + raise HTTPException(status_code=503, detail=str(err)) + if result.requires_consent or not result.access_token: + raise HTTPException( + status_code=409, + detail=f"You haven't connected '{provider.display_name}' yet. " + "Connect it in your connector settings, then retry.", + ) + oauth_token = result.access_token + elif getattr(tool, "forward_auth_token", False): + if not admin.raw_token: + raise HTTPException( + status_code=400, + detail="Forward-auth discovery needs your session token, which " + "this request didn't carry.", + ) + oauth_token = admin.raw_token + + snapshot = await discover_capabilities_for_saved_tool( + tool, oauth_token=oauth_token, discovered_by=admin.user_id + ) + + if snapshot.error: + previous = await service.repository.get_capabilities(tool_id) + if previous is not None: + previous.error = snapshot.error + return previous + return snapshot + + return await service.repository.put_capabilities(snapshot) + + +@router.get("/{tool_id}/capabilities", response_model=ToolCapabilitySnapshot) +async def admin_get_tool_capabilities( + tool_id: str, + admin: User = Depends(require_tools_admin), +): + """The stored snapshot, without probing. Empty when never discovered.""" + service = get_tool_catalog_service() + snapshot = await service.repository.get_capabilities(tool_id) + return snapshot or ToolCapabilitySnapshot(tool_id=tool_id) diff --git a/backend/src/apis/app_api/agent_designer/services/binding_validation.py b/backend/src/apis/app_api/agent_designer/services/binding_validation.py index 88ae6e1fa..3eed9eb29 100644 --- a/backend/src/apis/app_api/agent_designer/services/binding_validation.py +++ b/backend/src/apis/app_api/agent_designer/services/binding_validation.py @@ -8,8 +8,11 @@ - ``model`` → must exist + author passes ``ModelAccessService.can_access_model``. - ``tool`` → author must have the tool in the ``/agents/bindable`` palette (``ToolCatalogService.get_user_accessible_tools``, the SAME source the picker fetches, so - "if the palette offers it, the write accepts it" — cf. the model check). Run-time then - re-resolves each bound tool against the *invoker* (``AppRoleService.can_access_tool``, D5). + "if the palette offers it, the write accepts it" — cf. the model check). A ref may be + *scoped* (``toolId::mcpToolName``) to bind a subset of an MCP server's tools: the base + id is checked against the palette and the tool name against that server's discovered + ``serverTools`` list. Run-time then re-resolves each bound tool against the *invoker* + (``AppRoleService.can_access_tool``, D5). - ``skill`` → feature-flagged; author must have the skill in the ``/agents/bindable`` palette (``resolve_accessible_skill_ids``, the SAME source the picker fetches). Run-time then re-resolves each bound skill against the *invoker* (``resolve_invocable_skill_ids``, D5). @@ -28,6 +31,7 @@ from apis.shared.memory.service import MemorySpaceService from apis.shared.models.managed_models import list_all_managed_models from apis.shared.skills.access import resolve_accessible_skill_ids +from apis.shared.tools.scoped_ids import SCOPE_DELIMITER, parse_scoped_tool_id from apis.app_api.admin.services.model_access import ModelAccessService from apis.app_api.tools.service import ToolCatalogService @@ -73,15 +77,22 @@ async def validate_agent_write( # kind is present, then validate each binding synchronously against those sets — the # same sources the palette uses, so the picker and the write agree (cf. the model # check). Each is fetched lazily (only when that kind is actually bound). - accessible_tool_ids: Optional[set] = None + # Tools map catalog id -> the server's discovered tool names (empty for a + # non-MCP tool, or an MCP server with no discovery snapshot), because a scoped + # ref has to be checked on both axes: base id in the palette, tool name exposed + # by that server. + accessible_tools: Optional[dict] = None if any(b.kind == "tool" for b in bindings): svc = tool_service or ToolCatalogService() - accessible_tool_ids = {t.tool_id for t in await svc.get_user_accessible_tools(user)} + accessible_tools = { + t.tool_id: {st.name for st in t.server_tools} + for t in await svc.get_user_accessible_tools(user) + } accessible_skill_ids: Optional[set] = None if skills_enabled() and any(b.kind == "skill" for b in bindings): accessible_skill_ids = set(await resolve_accessible_skill_ids(user)) for binding in bindings: - _validate_binding(user, binding, mem, accessible_tool_ids, accessible_skill_ids) + _validate_binding(user, binding, mem, accessible_tools, accessible_skill_ids) async def _validate_model(user: User, cfg: AgentModelConfig, svc: ModelAccessService) -> None: @@ -156,7 +167,7 @@ def _validate_binding( user: User, binding: AgentBinding, mem: MemorySpaceService, - accessible_tool_ids: Optional[set] = None, + accessible_tools: Optional[dict] = None, accessible_skill_ids: Optional[set] = None, ) -> None: kind = binding.kind @@ -170,7 +181,7 @@ def _validate_binding( ) if kind == "tool": - _validate_tool(binding, accessible_tool_ids or set()) + _validate_tool(binding, accessible_tools or {}) return if kind == "skill": @@ -181,17 +192,49 @@ def _validate_binding( _validate_memory_space(user, binding, mem) -def _validate_tool(binding: AgentBinding, accessible_tool_ids: set) -> None: +def _validate_tool(binding: AgentBinding, accessible_tools: dict) -> None: + """Validate one ``tool`` binding, bare (whole server) or scoped (one tool of it). + + Mirrors ``ToolCatalogService.save_user_preferences``, which validates the same + ``base::tool`` shape on the user-preference axis: the **base** id must be in the + author's palette (``get_user_accessible_tools`` — the exact source the Designer + picker fetches, so a bindable tool is always writable), and the tool name must be + one the server actually exposes. + + A server with an empty ``serverTools`` list has simply never been discovered + ("Discover from server" on the admin tool page populates it); the name check is + skipped there rather than rejecting every scoped ref, exactly as the preference + path does. That cached list is a UI/validation aid only — it is never consulted at + run time, so it can gate what an author may *write* but is not the enforcement + point. Enforcement is the scoped id itself reaching ``collect_tool_name_filters``. + """ ref = (binding.ref or "").strip() if not ref: raise BindingValidationError("tool binding requires a non-empty 'ref'.", status_code=400) - # The tool id must be in the author's palette (get_user_accessible_tools) — the exact - # source the Designer picker fetches, so a bindable tool is always writable. Run-time - # re-resolves against the invoker via AppRoleService.can_access_tool (D5). - if ref not in accessible_tool_ids: + + base, tool_name = parse_scoped_tool_id(ref) + if tool_name is None and SCOPE_DELIMITER in ref: + # "canvas_faculty::" parses to a bare ref, so it would silently store as a + # whole-server binding — the opposite of what the author meant to narrow. raise BindingValidationError( - f"You do not have access to tool '{ref}'.", status_code=403 + f"tool binding ref '{ref}' is missing a tool name after " + f"'{SCOPE_DELIMITER}'.", + status_code=400, ) + # Run-time re-resolves against the invoker via AppRoleService.can_access_tool (D5), + # which likewise admits a scoped ref whose base server is granted. + if base not in accessible_tools: + raise BindingValidationError( + f"You do not have access to tool '{base}'.", status_code=403 + ) + + if tool_name is not None: + server_names = accessible_tools[base] + if server_names and tool_name not in server_names: + raise BindingValidationError( + f"Tool '{base}' does not expose a tool named '{tool_name}'.", + status_code=400, + ) def _validate_skill(binding: AgentBinding, accessible_skill_ids: set) -> None: diff --git a/backend/src/apis/app_api/assistants/routes.py b/backend/src/apis/app_api/assistants/routes.py index ac28211b9..ca7b90aaa 100644 --- a/backend/src/apis/app_api/assistants/routes.py +++ b/backend/src/apis/app_api/assistants/routes.py @@ -156,6 +156,7 @@ async def create_assistant_endpoint(request: CreateAssistantRequest, current_use # Convert to response model (excludes owner_id for privacy) assistant_dict = assistant.model_dump(by_alias=True, exclude={"ownerId"}) + return AssistantResponse.model_validate(assistant_dict) except Exception as e: diff --git a/backend/src/apis/app_api/documents/models.py b/backend/src/apis/app_api/documents/models.py index eb3339eea..bb29eef61 100644 --- a/backend/src/apis/app_api/documents/models.py +++ b/backend/src/apis/app_api/documents/models.py @@ -6,7 +6,15 @@ from pydantic import BaseModel, ConfigDict, Field # Type alias for document processing status -DocumentStatus = Literal["uploading", "chunking", "embedding", "complete", "failed", "deleting"] +# +# 'provisioning' is the leading status a born-managed first upload carries while +# its Bedrock knowledge base is being created (MANAGED_KB_NEW_DEFAULT). It is +# non-terminal and non-retrievable — the retrieval facade serves only 'complete' — +# so it can never answer a question from a knowledge base that does not exist yet. +# 'chunking'/'embedding' are written only by the legacy S3-Vectors pipeline. +DocumentStatus = Literal[ + "provisioning", "uploading", "chunking", "embedding", "complete", "failed", "deleting" +] @dataclass(frozen=True) @@ -177,3 +185,46 @@ class ImportDocumentsResponse(BaseModel): model_config = ConfigDict(populate_by_name=True) documents: List[DocumentResponse] = Field(..., description="Created document records, each in 'uploading' state") + + +class ExtractedChunkResponse(BaseModel): + """One passage as the knowledge base actually holds it. + + ``text`` is the FULL extracted text, deliberately not truncated. The existing + per-answer citation trace caps excerpts at 500 characters, which is exactly why it + cannot serve this purpose: a flattened table's damage is usually past the cut, so a + truncated excerpt of a mangled table reads like a fine excerpt of a fine table. + """ + + model_config = ConfigDict(populate_by_name=True) + + text: str = Field(..., description="Full extracted text of this chunk, untruncated") + order: int = Field(..., description="Position in the returned set — NOT the document's own order") + score: Optional[float] = Field(None, description="Relevance as the backend reported it; higher is better") + page: Optional[int] = Field(None, description="Page number when the backend supplied one; never inferred") + + +class ExtractedChunksResponse(BaseModel): + """What the knowledge base extracted from one document. + + ``available`` is false, with a ``reason``, for a document the inspector cannot + show — a classic knowledge base cannot scope a retrieval to a single document. The + shape is identical either way so the client never branches on the engine. + + ``capReached`` is honesty rather than a paging hint. Bedrock exposes no + chunk-enumeration API, so a complete set is never guaranteed and the UI must say + "up to N" instead of implying the document has exactly N chunks. + """ + + model_config = ConfigDict(populate_by_name=True) + + documentId: str = Field(..., description="Document identifier", alias="documentId") + fileName: str = Field(..., description="Original filename", alias="fileName") + engine: str = Field(..., description="Engine serving this knowledge base: 'managed' or 's3vectors'") + available: bool = Field(..., description="False when this engine cannot show a single document's chunks") + reason: Optional[str] = Field(None, description="Owner-facing explanation when available is false") + chunks: List[ExtractedChunkResponse] = Field(default_factory=list, description="The chunks returned, unordered") + returned: int = Field(0, description="How many chunks are in this response") + capReached: bool = Field( + False, description="True when the per-call ceiling was hit, so more chunks may exist", alias="capReached" + ) diff --git a/backend/src/apis/app_api/documents/routes.py b/backend/src/apis/app_api/documents/routes.py index 18bf44321..831d0cfdd 100644 --- a/backend/src/apis/app_api/documents/routes.py +++ b/backend/src/apis/app_api/documents/routes.py @@ -14,12 +14,18 @@ DocumentResponse, DocumentsListResponse, DownloadUrlResponse, + ExtractedChunkResponse, + ExtractedChunksResponse, ImportDocumentsRequest, ImportDocumentsResponse, ReportUploadFailureRequest, UploadUrlResponse, ) -from apis.app_api.documents.services.document_service import _generate_document_id, create_document, list_assistant_documents, update_document_status +from apis.app_api.documents.services.chunk_inspector import ( + DocumentNotInspectable, + inspect_document_chunks, +) +from apis.app_api.documents.services.document_service import _generate_document_id, create_document, list_assistant_documents, update_document_status, release_reservation_if_managed from apis.app_api.documents.services.document_service import get_document as get_document_service from apis.app_api.documents.services.import_service import run_import from apis.app_api.documents.services.storage_service import ( @@ -29,12 +35,14 @@ generate_upload_url, ) from apis.app_api.file_sources.service import require_file_source_token, resolve_file_source +from apis.app_api.kb_upgrade.born_managed import STATUS_PROVISIONING, begin_born_managed from apis.shared.auth import User, get_current_user_from_session from apis.shared.oauth.provider_repository import ( OAuthProviderRepository, get_provider_repository, ) from apis.shared.rbac.service import AppRoleService, get_app_role_service +from apis.shared.kb_backend.byte_cap import ByteCapExceeded logger = logging.getLogger(__name__) @@ -60,6 +68,73 @@ async def _require_edit_permission(assistant_id: str, current_user: User) -> str return assistant.owner_id +async def _resolve_managed_kb(assistant_id: str) -> tuple[bool, bool]: + """Return ``(is_managed, elevated)`` for this assistant's knowledge base. + + Reads the KB_Record once. A legacy knowledge base — an absent record, or any + record whose engine is not the exact managed literal — returns + ``(False, False)`` so the caller skips all cap logic: legacy S3-Vectors + knowledge bases stay uncapped (Requirement 12.11 scopes the cap to managed + KBs). The elevated tier is READ from the existing ``elevatedByteCap`` flag, + never written here — granting it is a separate feature. + """ + from apis.shared.kb_backend.records import ENGINE_MANAGED, get_kb_record, resolve_engine + + # app_kb_id == assistant_id this phase. get_kb_record is a blocking boto3 call. + record = await asyncio.to_thread(get_kb_record, assistant_id, assistant_id) + if resolve_engine(record) != ENGINE_MANAGED: + return False, False + return True, bool((record or {}).get("elevatedByteCap")) + + +async def _reserve_managed_upload(assistant_id: str, size_bytes: int) -> int: + """Provisionally reserve a declared upload size against the managed-KB cap. + + Returns the number of bytes reserved — ``0`` for a legacy knowledge base, + which is uncapped and never touched. Raises + :class:`~apis.shared.kb_backend.byte_cap.ByteCapExceeded` if the reservation + would breach the binding cap; the endpoint turns that into HTTP 413 with the + numbers (Requirement 12.12). + + The reservation is PROVISIONAL. ``size_bytes`` is the client's own declaration + and a client that under-reports its size would defeat the cap, so this only + buys fast, friendly feedback before a presigned URL is issued — the + authoritative gate is the S3-HEAD reconcile at ingestion (Requirement 12.3). + ``effective_cap`` folds the per-owner allowance and the per-KB ceiling into one + atomic conditional write (Requirement 12.1/12.5). + """ + from apis.shared.kb_backend import byte_cap + + is_managed, elevated = await _resolve_managed_kb(assistant_id) + if not is_managed: + return 0 + cap = byte_cap.effective_cap(elevated) + await asyncio.to_thread(byte_cap.reserve, assistant_id, assistant_id, size_bytes, cap) + return size_bytes + + +async def _release_managed_reservation(assistant_id: str, size_bytes: int) -> None: + """Return a managed-KB reservation taken earlier in THIS request. + + Used only to unwind the request-time reservation when a later step of the same + upload-URL request fails (document-row create, presigned-URL generation) before + the client is ever handed a URL. No ``settle_once`` guard here: the reservation + was taken microseconds ago by this same request, the document is not yet + visible to any other settlement path, and the ``DOC#`` row may not even exist — + stamping a marker on it would conjure a partial row. The abandoned-after-URL + cases (client upload failure, stale sweep) settle through their own guarded + paths (Requirement 12.6). + """ + from apis.shared.kb_backend import byte_cap + + if size_bytes <= 0: + return + is_managed, _ = await _resolve_managed_kb(assistant_id) + if not is_managed: + return + await asyncio.to_thread(byte_cap.release, assistant_id, assistant_id, size_bytes) + + @router.post("/upload-url", response_model=UploadUrlResponse, status_code=status.HTTP_200_OK) async def generate_upload_url_endpoint( assistant_id: str, @@ -78,7 +153,33 @@ async def generate_upload_url_endpoint( """ try: # 1. Resolve permission — owner or editor may upload documents - await _require_edit_permission(assistant_id, current_user) + assistant_owner_id = await _require_edit_permission(assistant_id, current_user) + + # 1b. Born-managed (MANAGED_KB_NEW_DEFAULT): the FIRST document is what + # triggers knowledge-base provisioning, so a prompt-only agent or an + # abandoned draft never spends a Bedrock knowledge base. This declares + # the engine managed up front — which is what makes the legacy pipeline + # skip the object about to land — and queues the provisioning job for + # the worker. It never raises: a failure leaves the agent on legacy. + # Returns True only while the knowledge base is still being built. + provisioning = await begin_born_managed( + assistant_id, owner_user_id=assistant_owner_id + ) + + # 1c. Managed KBs are byte-capped on EVERY path that adds bytes + # (Requirement 12.11); the interactive upload path is enforced here. + # Reserve the client-declared size BEFORE creating the DOC# row or + # issuing a presigned URL, so an over-cap upload is refused with a 413 + # the client can act on rather than after the bytes are already staged. + # Legacy KBs return 0 and are never checked. + # + # This runs AFTER the born-managed trigger deliberately: the record it + # creates already resolves to managed, so a born-managed first document + # is capped like every other one. Skipping the reserve for it would be + # the subtle bug — the reconcile at ingestion COMMITS the reservation + # (`reservedBytes -= n`), so a document that committed without reserving + # would drive the counter negative and corrupt the cap permanently. + reserved_bytes = await _reserve_managed_upload(assistant_id, request.size_bytes) # 2. Generate document_id and S3 key from apis.app_api.documents.services.storage_service import _get_s3_key, _sanitize_filename @@ -88,24 +189,48 @@ async def generate_upload_url_endpoint( sanitized_filename = _sanitize_filename(request.filename) s3_key = _get_s3_key(assistant_id, document_id, sanitized_filename) - # 3. Create document record in DynamoDB (status='uploading') - _ = await create_document( - assistant_id=assistant_id, - filename=request.filename, - content_type=request.content_type, - size_bytes=request.size_bytes, - s3_key=s3_key, - document_id=document_id, - ) + try: + # 3. Create document record in DynamoDB (status='uploading', or + # 'provisioning' when this upload is what triggered the knowledge + # base being built). The declared size is persisted as sizeBytes — + # the value the ingestion step reconciles the true S3 size against + # (Requirement 12.3). + _ = await create_document( + assistant_id=assistant_id, + filename=request.filename, + content_type=request.content_type, + size_bytes=request.size_bytes, + s3_key=s3_key, + document_id=document_id, + status=STATUS_PROVISIONING if provisioning else "uploading", + ) - # 4. Generate presigned S3 URL - presigned_url, _ = await generate_upload_url( - assistant_id=assistant_id, document_id=document_id, filename=request.filename, content_type=request.content_type, expires_in=3600 - ) + # 4. Generate presigned S3 URL + presigned_url, _ = await generate_upload_url( + assistant_id=assistant_id, document_id=document_id, filename=request.filename, content_type=request.content_type, expires_in=3600 + ) + except Exception: + # A step after the reservation failed and the client never received a + # URL, so this upload can never settle the bytes. Return the + # reservation now rather than leak it (Requirement 12.6). + await _release_managed_reservation(assistant_id, reserved_bytes) + raise # 5. Return response return UploadUrlResponse(documentId=document_id, uploadUrl=presigned_url, expiresIn=3600) + except ByteCapExceeded as e: + # Requirement 12.12: the numbers make the error actionable — the user can + # see how far over they are and request an elevated tier. + used = "" if e.already_used is None else f" (currently using {e.already_used} bytes)" + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=( + f"This file ({e.requested} bytes) would put the assistant over its " + f"{e.cap}-byte knowledge-base limit{used}. Delete unused documents " + f"or request an elevated storage tier." + ), + ) except HTTPException: raise except Exception as e: @@ -240,6 +365,12 @@ async def report_upload_failure( if not updated: raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to update document status") + # The client's upload to S3 never landed, so the bytes reserved at + # request time will never be settled by the ingestion consumer. Return + # them now (Requirement 12.6). settle_once makes this idempotent against a + # concurrent stale-document sweep marking the same document failed. + await release_reservation_if_managed(document) + return DocumentResponse.model_validate(updated.model_dump(by_alias=True)) except HTTPException: @@ -340,6 +471,76 @@ async def get_download_url( raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to generate download URL: {str(e)}") +@router.get( + "/{document_id}/chunks", + response_model=ExtractedChunksResponse, + status_code=status.HTTP_200_OK, +) +async def get_document_chunks( + assistant_id: str, document_id: str, current_user: User = Depends(get_current_user_from_session) +) -> ExtractedChunksResponse: + """Show what the knowledge base actually extracted from this document. + + The tooling half of the task-16.2 decision on managed-kb-migration §5.41. The + managed backend flattens a column-structured diagram or a 2-D table at ingestion, + so a per-column question gets a *confident wrong answer with no trace*. We do not + fix the parser — it is a managed service and this is a self-service platform — so + instead the extraction is made visible and the owner can decide to reformat their + source. Guidance nobody can verify is not guidance. + + Owners and editors only, the same gate as every other document endpoint. Read-only: + no chunking, parsing or ingestion path is touched (Requirement 5). + """ + try: + assistant_owner_id = await _require_edit_permission(assistant_id, current_user) + document = await get_document_service(assistant_id, document_id, assistant_owner_id) + + if not document or document.status == "deleting": + # A soft-deleted document is being removed on purpose; surfacing its + # content would resurrect it in the one place the user is told it is gone. + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"Document not found: {document_id}" + ) + + result = await inspect_document_chunks( + assistant_id, + document_id, + file_name=document.filename, + status=document.status, + ) + + return ExtractedChunksResponse( + documentId=result.document_id, + fileName=result.file_name, + engine=result.engine, + available=result.available, + reason=result.reason, + chunks=[ + ExtractedChunkResponse( + text=chunk.text, order=chunk.order, score=chunk.score, page=chunk.page + ) + for chunk in result.chunks + ], + returned=result.returned, + capReached=result.cap_reached, + ) + + except DocumentNotInspectable as e: + # 409 rather than 404: the document exists, it simply has no content in the + # knowledge base yet. The carried copy is written for the owner, not an + # operator — `provisioning` in particular gets its own sentence, because a + # born-managed first upload is waiting on the knowledge base itself. + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=e.reason) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error inspecting document chunks: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to read extracted content: {str(e)}", + ) + + @router.delete("/{document_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_document( assistant_id: str, document_id: str, current_user: User = Depends(get_current_user_from_session) diff --git a/backend/src/apis/app_api/documents/services/chunk_inspector.py b/backend/src/apis/app_api/documents/services/chunk_inspector.py new file mode 100644 index 000000000..8fa164e18 --- /dev/null +++ b/backend/src/apis/app_api/documents/services/chunk_inspector.py @@ -0,0 +1,277 @@ +"""Show an owner the content their knowledge base actually extracted from a document. + +Feature: `kb-chunk-inspector`. The tooling half of the task-16.2 decision on +`managed-kb-migration` §5.41. + +Why this exists +--------------- +The two backends fail differently, and the managed one fails worse: + +* **Legacy** failed **silently** — a column-structured flowchart produced 0 chunks, + so a question about it got no answer. Bad, but visible. +* **Managed** fails **invisibly** — the vision/parse step flattens a 2-D layout at + ingestion, so a question about "semester 4" or a per-column total gets a + *confident wrong answer* with no trace of why. + +Task 16.2 decided not to fix the parser: it is a managed service, and this is a +self-service platform where owners upload their own documents. The mitigation is +guidance. But guidance is useless if the user cannot see the problem — nobody can act +on "reformat your flowchart as a text table" without first believing their flowchart +came out wrong. This module is what lets them look. + +Read-only. No new chunking, parsing or ingestion path (Requirement 5). + +Managed-only, and that is a correction to the design +---------------------------------------------------- +The design document assumed ``backend.search(..., retrieval_filter=...)`` was part of +the protocol and that the inspector could therefore be engine-agnostic. It is not: +``retrieval_filter`` exists **only** on ``ManagedKbBackend.search``. The legacy +adapter's signature is ``search(kb_ref, query, top_k)``, it accepts no filter, and it +ignores ``top_k`` — it always asks its index for a fixed five results across the +*whole* knowledge base. + +So on legacy there is no way to scope a retrieval to one document. Running it anyway +would return the top five chunks of the entire knowledge base, most or all of them +belonging to **other documents** — the user opens "view extracted content" on their +syllabus and reads somebody else's handbook. That is a cross-document leak, which is +precisely what Requirement 4 and ``ISOLATION_SAFE_FILTER_OPERATORS`` exist to +prevent; it is not a cosmetic problem to be tidied up later. + +Teaching the legacy adapter to filter was rejected: Requirement 5 says the inspector +must not depend on the legacy pipeline continuing to exist, and that pipeline is being +deprecated. Building new capability into it would be work with a negative lifespan. + +So a legacy document returns ``available=False`` with a reason, as a **200 rather than +an error**: the owner asked a reasonable question and "your knowledge base is on the +classic engine, which cannot show this" is an answer, not a fault. The response shape +is identical either way, so the UI never branches on the engine — which is what +Requirement 2 was actually protecting. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + +#: Chunks requested in one `Retrieve`. Bedrock documents 100 as the ceiling for +#: `numberOfResults`, and the inspector wants as complete a picture as one bounded +#: call can give — unlike a real query, where 5 is the answer-quality choice. +#: +#: One call, not a pagination loop (Requirement 6). `Retrieve` is query-ranked with no +#: cursor, so "page 2" is not a thing it offers; repeating the call with a different +#: query would return an overlapping arbitrary subset and cost another 662–695 ms for +#: no guarantee of new content. +INSPECT_TOP_K = 100 + +#: Statuses that can be inspected. Only `complete` — the retrieval facade serves only +#: `complete`, so anything else has no chunks in the knowledge base to show. +INSPECTABLE_STATUS = "complete" + +#: Why a document cannot be inspected yet, in the owner's language. Keyed on the +#: `DOC#` status. `provisioning` is born-managed's leading status (the knowledge base +#: itself is still being created), and it gets its own sentence because "not found" +#: would be a lie and "still processing" would understate the wait. +_NOT_READY_REASONS = { + "provisioning": ( + "This assistant's knowledge base is still being created. The extracted " + "content will be available once the first document has finished processing." + ), + "uploading": "This document is still being processed. Check back shortly.", + "chunking": "This document is still being processed. Check back shortly.", + "embedding": "This document is still being processed. Check back shortly.", + "failed": ( + "This document could not be processed, so the knowledge base holds no " + "content for it. Upload it again, or convert it to a different format." + ), +} + +_NOT_READY_FALLBACK = "This document is not ready to inspect yet." + +LEGACY_UNAVAILABLE_REASON = ( + "This assistant uses the classic knowledge base, which cannot list the " + "extracted content for a single document. Upgrading the assistant's knowledge " + "base makes this view available." +) + + +class DocumentNotInspectable(Exception): + """The document exists but has no content to show yet. Carries owner-safe copy.""" + + def __init__(self, reason: str, status: str) -> None: + super().__init__(reason) + self.reason = reason + self.status = status + + +@dataclass +class InspectedChunk: + """One passage as the knowledge base holds it. + + ``text`` is deliberately **not** truncated. The existing citation trace caps + excerpts at 500 characters, which is the whole reason it cannot serve this + purpose: a flattened table's damage is frequently past the cut, and a truncated + excerpt of a mangled table looks like a fine excerpt of a fine table. + """ + + text: str + order: int + score: Optional[float] = None + page: Optional[int] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class InspectionResult: + document_id: str + file_name: str + engine: str + available: bool + chunks: List[InspectedChunk] = field(default_factory=list) + returned: int = 0 + cap_reached: bool = False + reason: Optional[str] = None + + +def document_filter(document_id: str) -> Dict[str, Any]: + """The retrieval filter scoping results to exactly one document. + + ``equals``, never a prefix or substring operator. This is the same filter the + ingestion consumer's retrievability probe and the document reconciler use, and it + is in ``ISOLATION_SAFE_FILTER_OPERATORS`` for a reason worth restating: a prefix + match for ``DOC-1`` also admits ``DOC-10``, ``DOC-11`` and so on, so the operator + choice is the isolation boundary rather than a query-tuning detail. + """ + return {"equals": {"key": "document_id", "value": document_id}} + + +def _page_of(metadata: Dict[str, Any]) -> Optional[int]: + """Page number when the backend supplied one, else ``None``. + + Never invented. A fabricated page number would be indistinguishable from a real + one and would make an unordered result set look authoritatively ordered. + """ + for key in ("page", "pageNumber", "page_number", "x-amz-bedrock-kb-document-page-number"): + raw = metadata.get(key) + if raw is None: + continue + try: + return int(float(raw)) + except (TypeError, ValueError): + continue + return None + + +def _dedupe(chunks: List[Any]) -> List[Any]: + """Drop repeated passages, preserving first-seen order. + + ``Retrieve`` is query-ranked rather than a cursor over a set, so the same passage + can legitimately come back more than once. Showing an owner the same mangled + table three times would make them think it was ingested three times. + """ + seen = set() + unique = [] + for chunk in chunks: + fingerprint = hash(chunk.text) + if fingerprint in seen: + continue + seen.add(fingerprint) + unique.append(chunk) + return unique + + +async def inspect_document_chunks( + assistant_id: str, + document_id: str, + *, + file_name: str, + status: str, +) -> InspectionResult: + """The content the knowledge base holds for one document. + + Raises :class:`DocumentNotInspectable` when the document exists but has nothing + to show; the caller turns that into a 409 with the carried copy. + """ + from apis.shared.kb_backend import resolver + from apis.shared.kb_backend.query_guard import clamp_query + from apis.shared.kb_backend.records import ENGINE_MANAGED + + if status != INSPECTABLE_STATUS: + raise DocumentNotInspectable( + _NOT_READY_REASONS.get(status, _NOT_READY_FALLBACK), status + ) + + record = resolver.load_record(assistant_id) + engine = resolver.resolve_engine_for(assistant_id, record=record) + + if engine != ENGINE_MANAGED: + # See the module docstring: legacy cannot scope a retrieval to one document, + # and returning the whole knowledge base's top chunks would leak other + # documents' content into this view. + return InspectionResult( + document_id=document_id, + file_name=file_name, + engine=engine, + available=False, + reason=LEGACY_UNAVAILABLE_REASON, + ) + + backend = resolver.resolve_backend(assistant_id, record=record) + + # The filter is what scopes the result; this text exists only because `Retrieve` + # requires a query. The filename is the most document-anchored string available + # without reading the object, and clamp_query is reused rather than reinvented so + # an absurd filename cannot become an absurd query. + query, _ = clamp_query(file_name or document_id) + + chunks = await backend.search( + assistant_id, + query, + INSPECT_TOP_K, + retrieval_filter=document_filter(document_id), + ) + + # Defence in depth. The filter should make this a no-op, and if it ever is not, + # the failure must not be "the owner reads another document's content". + scoped = [chunk for chunk in chunks if chunk.document_id == document_id] + if len(scoped) != len(chunks): + logger.error( + f"chunk inspector: {len(chunks) - len(scoped)} chunk(s) for a document " + f"other than {document_id} came back through a document-scoped filter; " + f"dropped them" + ) + + unique = _dedupe(scoped) + + inspected = [ + InspectedChunk( + text=chunk.text, + order=index, + score=chunk.relevance, + page=_page_of(chunk.metadata or {}), + metadata={}, + ) + for index, chunk in enumerate(unique) + ] + + # `capReached` is honesty, not a paging hint (Requirement 3). Bedrock exposes no + # chunk-enumeration API, so a full result set is never guaranteed and the UI must + # say "up to N" rather than implying the document has exactly N chunks. + cap_reached = len(chunks) >= INSPECT_TOP_K + + logger.info( + f"chunk inspector: assistant={assistant_id} document={document_id} " + f"engine={engine} returned={len(inspected)} capReached={cap_reached}" + ) + + return InspectionResult( + document_id=document_id, + file_name=file_name, + engine=engine, + available=True, + chunks=inspected, + returned=len(inspected), + cap_reached=cap_reached, + ) diff --git a/backend/src/apis/app_api/documents/services/document_service.py b/backend/src/apis/app_api/documents/services/document_service.py index 046bed8bc..94b01bb28 100644 --- a/backend/src/apis/app_api/documents/services/document_service.py +++ b/backend/src/apis/app_api/documents/services/document_service.py @@ -82,9 +82,49 @@ async def _auto_fail_stale_document(document: Document) -> Document: error_message='Processing timed out. The document may need to be re-uploaded.', error_details=f'Document was stuck in "{document.status}" state since {document.updated_at}', ) + # A document that timed out in a processing state never reached the ingestion + # consumer's terminal reconcile, so its request-time byte reservation (managed + # KBs only) would leak. Return it (Requirement 12.6); settle_once keeps this + # idempotent against a concurrent client-reported failure on the same document. + await release_reservation_if_managed(document) return updated if updated else document +async def release_reservation_if_managed(document: Document) -> None: + """Return a managed-KB byte reservation for a document abandoned before its + bytes were settled by the ingestion consumer (Requirement 12.6). + + Exactly-once via ``byte_cap.settle_once``: the request-time reservation is + released by whichever terminal path the document reaches first — a + client-reported upload failure, this stale sweep, or the ingestion consumer's + own failure path — and the guard stops two of them double-crediting the + allowance. A no-op for legacy knowledge bases (uncapped) and for zero-size + rows (nothing was reserved). + + ``app_kb_id == assistant_id`` this phase. boto3 is called synchronously here, + matching the rest of this module. + """ + from apis.shared.kb_backend import byte_cap + from apis.shared.kb_backend.records import ENGINE_MANAGED, get_kb_record, resolve_engine + + size_bytes = int(document.size_bytes or 0) + if size_bytes <= 0: + return + assistant_id = document.assistant_id + try: + record = get_kb_record(assistant_id, assistant_id) + if resolve_engine(record) != ENGINE_MANAGED: + return + if not byte_cap.settle_once(assistant_id, document.document_id): + return + byte_cap.release(assistant_id, assistant_id, size_bytes) + except Exception as e: # noqa: BLE001 - a bookkeeping failure must not break the status write + logger.error( + f"Failed to release byte reservation for document {document.document_id}: {e}", + exc_info=True, + ) + + async def create_document( assistant_id: str, filename: str, @@ -92,7 +132,8 @@ async def create_document( size_bytes: int, s3_key: str, document_id: Optional[str] = None, - provenance: Optional[DocumentProvenance] = None + provenance: Optional[DocumentProvenance] = None, + status: DocumentStatus = 'uploading' ) -> Document: """ Create a new document record in DynamoDB @@ -110,9 +151,16 @@ async def create_document( provenance: Optional file-source origin metadata. Set only for documents imported from an external connector; None for device uploads. + status: Initial status. Defaults to 'uploading' — the only caller that + overrides it is the born-managed first upload, which uses + 'provisioning' because the knowledge base its document is bound for + is still being created (MANAGED_KB_NEW_DEFAULT). Passing it in rather + than patching the row afterwards keeps the document from ever being + visible as 'uploading' to a poller, which is what would make the + managed ingestion consumer's defer look like a stuck upload. Returns: - Document object with status='uploading' + Document object with the requested initial status """ try: import boto3 @@ -136,7 +184,7 @@ async def create_document( content_type=content_type, size_bytes=size_bytes, s3_key=s3_key, - status='uploading', + status=status, created_at=now, updated_at=now, source_connector_id=provenance.source_connector_id if provenance else None, diff --git a/backend/src/apis/app_api/fine_tuning/job_models.py b/backend/src/apis/app_api/fine_tuning/job_models.py index 97a693de7..dffde30f2 100644 --- a/backend/src/apis/app_api/fine_tuning/job_models.py +++ b/backend/src/apis/app_api/fine_tuning/job_models.py @@ -275,7 +275,7 @@ def _hyperparameters(task_type: str, **overrides: str) -> Dict[str, str]: description="7.6B parameter LLaVA-NeXT on Mistral, higher-resolution tiling than 1.5 and correspondingly slower per record", task_type=_VLM, default_instance_type="ml.g6e.xlarge", - default_hyperparameters=_hyperparameters(_VLM, context_length="2048"), + default_hyperparameters=_hyperparameters(_VLM, context_length="4096"), ), AvailableModel( model_id="qwen25-vl-7b-instruct", @@ -284,7 +284,7 @@ def _hyperparameters(task_type: str, **overrides: str) -> Dict[str, str]: description="8.3B parameter vision-language model from Alibaba with dynamic resolution, strong on documents, charts and OCR-heavy images", task_type=_VLM, default_instance_type="ml.g6e.xlarge", - default_hyperparameters=_hyperparameters(_VLM, context_length="2048"), + default_hyperparameters=_hyperparameters(_VLM, context_length="4096"), ), AvailableModel( model_id="llava-1.6-34b", @@ -299,7 +299,7 @@ def _hyperparameters(task_type: str, **overrides: str) -> Dict[str, str]: default_instance_type="ml.g6e.4xlarge", default_hyperparameters=_hyperparameters( _VLM, - context_length="2048", + context_length="4096", gradient_accumulation_steps="16", lora_r="8", lora_alpha="16", @@ -343,6 +343,11 @@ class CreateJobRequest(BaseModel): hyperparameters: Optional[Dict[str, str]] = None max_runtime_seconds: int = Field(default=86400, le=432000, gt=0) custom_huggingface_model_id: Optional[str] = None + #: Run on managed spot capacity. Opt-in, not default: spot trades a large + #: discount for a longer queue, and measured on-demand waits for these GPU + #: families already ran 28-58 minutes — a researcher who needs a result + #: this afternoon should be able to pay for certainty. + use_spot: bool = False class JobResponse(BaseModel): @@ -369,6 +374,7 @@ class JobResponse(BaseModel): error_message: Optional[str] = None max_runtime_seconds: int = 86400 training_progress: Optional[float] = None + use_spot: bool = False class JobListResponse(BaseModel): diff --git a/backend/src/apis/app_api/fine_tuning/job_repository.py b/backend/src/apis/app_api/fine_tuning/job_repository.py index adf6011f6..275ea2431 100644 --- a/backend/src/apis/app_api/fine_tuning/job_repository.py +++ b/backend/src/apis/app_api/fine_tuning/job_repository.py @@ -67,6 +67,9 @@ def _item_to_dict(self, item: dict) -> dict: "updated_at": item["updatedAt"], "error_message": item.get("error_message"), "max_runtime_seconds": int(item.get("max_runtime_seconds", 86400)), + # Absent on every row written before spot existed, and those all + # ran on-demand. + "use_spot": bool(item.get("use_spot", False)), "training_progress": round(float(item["training_progress"]) * 100, 1) if item.get("training_progress") is not None else None, } return result @@ -84,6 +87,7 @@ def create_job( sagemaker_job_name: str, output_s3_prefix: str, max_runtime_seconds: int = 86400, + use_spot: bool = False, task_type: str = task_types.DEFAULT_TASK_TYPE, ) -> dict: """Create a new training job record.""" @@ -105,6 +109,7 @@ def create_job( "instance_count": 1, "sagemaker_job_name": sagemaker_job_name, "max_runtime_seconds": max_runtime_seconds, + "use_spot": use_spot, "createdAt": now, "updatedAt": now, } diff --git a/backend/src/apis/app_api/fine_tuning/pricing.py b/backend/src/apis/app_api/fine_tuning/pricing.py index efece7068..bd464d7c5 100644 --- a/backend/src/apis/app_api/fine_tuning/pricing.py +++ b/backend/src/apis/app_api/fine_tuning/pricing.py @@ -130,16 +130,31 @@ def transform_rate(instance_type: str) -> Optional[float]: def calculate_cost( - instance_type: str, billable_seconds: int, *, transform: bool = False + instance_type: str, + billable_seconds: int, + *, + transform: bool = False, + instance_count: int = 1, ) -> float: """Cost in USD for ``billable_seconds`` on ``instance_type``. + ``BillableTimeInSeconds`` is per instance — AWS documents multiplying it + by the instance count to get the total compute time billed. Harmless + while every job runs on one instance, but silent under-billing the moment + a multi-instance job lands, so the multiply is here rather than waiting to + be discovered. + + This is also correct for **managed spot**, with no special casing: AWS + expresses the spot discount by *shrinking* BillableTimeInSeconds against + the same on-demand rate, which is why the documented savings formula is + ``(1 - BillableTimeInSeconds / TrainingTimeInSeconds) * 100``. + Returns 0.0 for an instance we have no rate for. Callers must not rely on that to mean "free" — validate the instance up front instead; a silent 0.0 is exactly the blind spot that lets unpriced GPU time go unbilled. """ rate = transform_rate(instance_type) if transform else training_rate(instance_type) - return round((rate or 0.0) * (billable_seconds / 3600), 4) + return round((rate or 0.0) * (billable_seconds / 3600) * max(1, instance_count), 4) def estimate_max_cost( diff --git a/backend/src/apis/app_api/fine_tuning/routes.py b/backend/src/apis/app_api/fine_tuning/routes.py index 14605f300..4e94bc903 100644 --- a/backend/src/apis/app_api/fine_tuning/routes.py +++ b/backend/src/apis/app_api/fine_tuning/routes.py @@ -573,6 +573,22 @@ async def create_job( if request.hyperparameters: hyperparameters.update(request.hyperparameters) + + # Spot restarts the job from the last checkpoint. With checkpointing off + # it restarts from zero, and a run longer than the mean time between + # interruptions then almost never completes — it just re-bills the same + # early steps forever. Refuse the combination rather than sell it. + if request.use_spot and not task_types.str2bool( + hyperparameters.get("checkpointing", "true") + ): + raise HTTPException( + status_code=400, + detail=( + "Managed spot requires checkpointing. Without it an " + "interrupted job restarts from the beginning, so a long run " + "may never finish while still being billed for every attempt." + ), + ) hyperparameters["model_name_or_path"] = huggingface_id hyperparameters["task_type"] = spec.task_type @@ -596,6 +612,7 @@ async def create_job( # S3 paths output_s3_prefix = s3_service.get_output_s3_prefix(user.user_id, job_id) output_s3_uri = s3_service.get_output_s3_uri(user.user_id, job_id) + checkpoint_s3_uri = s3_service.get_checkpoint_s3_uri(user.user_id, job_id) input_s3_uri = f"s3://{s3_service.bucket_name}/{request.dataset_s3_key}" # Create DynamoDB job record @@ -612,6 +629,7 @@ async def create_job( sagemaker_job_name=sagemaker_job_name, output_s3_prefix=output_s3_prefix, max_runtime_seconds=max_runtime_seconds, + use_spot=request.use_spot, ) # Start SageMaker training job @@ -625,6 +643,8 @@ async def create_job( max_runtime=max_runtime_seconds, source_dir_s3_uri=scripts_s3_uri, task_type=spec.task_type, + checkpoint_s3_uri=checkpoint_s3_uri, + use_spot=request.use_spot, ) job = jobs_repo.update_job_status(user.user_id, job_id, "TRAINING") except Exception as e: @@ -767,6 +787,11 @@ def _estimate_training_progress(sm_status: dict, job: dict) -> Optional[float]: "DownloadingTrainingImage": 6.0, "Downloading": 8.0, "Uploading": 92.0, + # Spot only: the job is stalled waiting to be rescheduled, not + # progressing. Without this the elapsed-time curve below keeps + # climbing and the bar lies about a job that is standing still. + "Interrupted": 10.0, + "MaxWaitTimeExceeded": 0.0, } if secondary in phase_progress: return phase_progress[secondary] diff --git a/backend/src/apis/app_api/fine_tuning/s3_service.py b/backend/src/apis/app_api/fine_tuning/s3_service.py index 35de98986..3acb4d4eb 100644 --- a/backend/src/apis/app_api/fine_tuning/s3_service.py +++ b/backend/src/apis/app_api/fine_tuning/s3_service.py @@ -95,6 +95,16 @@ def get_output_s3_uri(self, user_id: str, job_id: str) -> str: prefix = self.get_output_s3_prefix(user_id, job_id) return f"s3://{self.bucket_name}/{prefix}" + def get_checkpoint_s3_uri(self, user_id: str, job_id: str) -> str: + """Return the s3:// URI SageMaker mirrors the checkpoint dir to. + + Kept beside the job's output rather than inside it: SageMaker writes + the finished ``model.tar.gz`` under the output prefix, and mixing a + live-mirrored directory into the same prefix makes it ambiguous which + objects belong to the completed artifact. + """ + return f"s3://{self.bucket_name}/checkpoints/{user_id}/{job_id}" + # ===================================================================== # Inference (Batch Transform) S3 methods # ===================================================================== diff --git a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_common.py b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_common.py index c11306f10..89830245d 100644 --- a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_common.py +++ b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_common.py @@ -20,6 +20,7 @@ """ import logging +import math import os import shutil import zipfile @@ -57,6 +58,17 @@ #: Image files an archive-based dataset may reference. SUPPORTED_IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png", ".bmp", ".gif", ".webp", ".tiff") +#: Where Trainer writes checkpoints. SageMaker mirrors this directory to S3 +#: continuously when CheckpointConfig names the same LocalPath, and restores it +#: before the script runs again — which is what makes a restart resumable. +CHECKPOINT_DIR = "/opt/ml/checkpoints" + +#: Roughly how many checkpoints a run should produce. Fewer means more lost +#: work per interruption; more means more S3 traffic. Ten is a compromise that +#: costs at most ~10% of a run. +TARGET_CHECKPOINTS = 10 + + #: Directory the training script unpacks an uploaded archive into. Sits #: outside the input channel so the extracted tree is never mistaken for #: another dataset file on a re-scan. @@ -287,6 +299,72 @@ def split_frame(frame, split_ratio, seed): return dataset["train"].shuffle(seed=seed), dataset["test"].shuffle(seed=seed) +# ========================================================================= +# Checkpointing +# ========================================================================= + +def resolve_save_steps(num_examples, batch_size, gradient_accumulation_steps, + epochs, target=TARGET_CHECKPOINTS): + """Optimizer-step interval that yields roughly ``target`` checkpoints. + + A fixed interval cannot work across these jobs. ``save_steps`` counts + *optimizer* steps, not samples, and a generative VLM trains at batch 1 with + heavy gradient accumulation — a 90-sample epoch is about 5 steps. Against + a hardcoded ``save_steps=50`` the longest, most interruption-exposed job in + the catalog would never checkpoint at all, while a fast text classifier + with thousands of steps would checkpoint constantly. + + Scaling to the run's own step count fixes both ends. Returns at least 1. + """ + per_epoch_batches = max(1, math.ceil(num_examples / max(1, batch_size))) + steps_per_epoch = max(1, math.ceil( + per_epoch_batches / max(1, gradient_accumulation_steps) + )) + total_steps = max(1, int(steps_per_epoch * max(1, epochs))) + return max(1, total_steps // max(1, target)) + + +def checkpoint_arguments(save_steps, enabled=True): + """TrainingArguments kwargs for periodic checkpointing. + + ``save_total_limit=1`` keeps only the newest checkpoint: resume needs the + latest and nothing else, and an unbounded set would grow the mirrored S3 + prefix for the whole run. + """ + if not enabled or not save_steps or save_steps <= 0: + return {"save_strategy": "no"} + return { + "save_strategy": "steps", + "save_steps": int(save_steps), + "save_total_limit": 1, + } + + +def latest_checkpoint(checkpoint_dir=CHECKPOINT_DIR): + """Path to the checkpoint SageMaker restored, or None for a fresh start. + + On a spot interruption or any other restart SageMaker repopulates + ``CHECKPOINT_DIR`` from S3 before re-running the script, so the presence of + a checkpoint here is how a resumed attempt distinguishes itself from a + first one. Never raises: a malformed checkpoint should cost the run its + progress, not fail the job outright. + """ + if not os.path.isdir(checkpoint_dir): + return None + + try: + from transformers.trainer_utils import get_last_checkpoint + + found = get_last_checkpoint(checkpoint_dir) + except Exception as error: # pragma: no cover - defensive + logger.warning(f"Could not inspect {checkpoint_dir}: {error}") + return None + + if found: + logger.info(f"Resuming from checkpoint {found}") + return found + + # ========================================================================= # Model helpers # ========================================================================= diff --git a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_classification.py b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_classification.py index 3fc026218..9e56175bd 100644 --- a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_classification.py +++ b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_classification.py @@ -103,19 +103,32 @@ def train(args, spec): frame, args.split_ratio, args.seed ) + # Checkpoint often enough that an interruption costs a fraction of the + # run, and rarely enough that the S3 mirror stays cheap. + save_steps = task_common.resolve_save_steps( + len(train_dataset), args.per_device_train_batch_size, 1, args.epochs + ) + checkpointing = task_common.checkpoint_arguments( + save_steps, enabled=args.checkpointing + ) + logger.info( + f"Checkpointing: {checkpointing.get('save_strategy')} " + f"every {checkpointing.get('save_steps', 'n/a')} step(s)" + ) + training_args = task_common.build_training_arguments( - output_dir="/opt/ml/checkpoints", + output_dir=task_common.CHECKPOINT_DIR, learning_rate=args.learning_rate, num_train_epochs=args.epochs, per_device_train_batch_size=args.per_device_train_batch_size, weight_decay=args.weight_decay, eval_strategy="epoch", - save_strategy="no", logging_dir="/opt/ml/output/tensorboard", # The collator returns pixel tensors, not model-signature columns; # Trainer's default column pruning would strip the image paths it # needs before the collator ever sees them. remove_unused_columns=False, + **checkpointing, ) trainer = Trainer( @@ -134,7 +147,7 @@ def train(args, spec): f"batch_size={args.per_device_train_batch_size}, " f"image_size={args.image_size}" ) - trainer.train() + trainer.train(resume_from_checkpoint=task_common.latest_checkpoint()) metrics = trainer.evaluate() logger.info(f"Final evaluation: accuracy={metrics.get('eval_accuracy', 'N/A')}") diff --git a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_text_classification.py b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_text_classification.py index c53b76e0f..291a5dd8b 100644 --- a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_text_classification.py +++ b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_text_classification.py @@ -248,17 +248,30 @@ def train(args, spec): frame, args.split_ratio, args.seed ) + # Checkpoint often enough that an interruption costs a fraction of the + # run, and rarely enough that the S3 mirror stays cheap. + save_steps = task_common.resolve_save_steps( + len(train_dataset), args.per_device_train_batch_size, 1, args.epochs + ) + checkpointing = task_common.checkpoint_arguments( + save_steps, enabled=args.checkpointing + ) + logger.info( + f"Checkpointing: {checkpointing.get('save_strategy')} " + f"every {checkpointing.get('save_steps', 'n/a')} step(s)" + ) + training_args = task_common.build_training_arguments( - output_dir="/opt/ml/checkpoints", + output_dir=task_common.CHECKPOINT_DIR, learning_rate=args.learning_rate, num_train_epochs=args.epochs, per_device_train_batch_size=args.per_device_train_batch_size, weight_decay=args.weight_decay, eval_strategy="epoch", - save_strategy="no", logging_dir="/opt/ml/output/tensorboard", remove_unused_columns=False, label_names=["labels"], + **checkpointing, ) trainer = Trainer( @@ -276,7 +289,7 @@ def train(args, spec): f"model={args.model_name_or_path}, epochs={args.epochs}, " f"batch_size={args.per_device_train_batch_size}" ) - trainer.train() + trainer.train(resume_from_checkpoint=task_common.latest_checkpoint()) metrics = trainer.evaluate() logger.info(f"Final evaluation: accuracy={metrics.get('eval_accuracy', 'N/A')}") diff --git a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_text_to_text.py b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_text_to_text.py index e09249a43..b1a51a773 100644 --- a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_text_to_text.py +++ b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_image_text_to_text.py @@ -282,14 +282,28 @@ def train(args, spec): model = load_base_model(args.model_name_or_path, args.load_in_4bit) max_ctx = task_common.resolve_max_context_length(model.config, tokenizer) - effective_context = ( - min(args.context_length, max_ctx) if max_ctx else args.context_length + measured = measure_required_context(processor, spec, frame.to_dict("records")) + effective_context, raised = resolve_effective_context( + args.context_length, measured, max_ctx ) logger.info( f"Context length: requested={args.context_length}, " - f"effective={effective_context}" - f"{' (capped)' if max_ctx and args.context_length > max_ctx else ''}" + f"measured={measured}, effective={effective_context}" ) + if raised: + logger.warning( + f"Raised context length from {args.context_length} to " + f"{effective_context}: this model spends {measured} tokens on a " + f"record, mostly on the image. Truncating to the requested length " + f"would have cut into the image tokens and failed the job." + ) + if measured and max_ctx and measured > max_ctx: + raise ValueError( + f"A single record needs {measured} tokens but " + f"{args.model_name_or_path} supports at most {max_ctx}. The image " + f"alone does not fit. Use smaller images, or a model that tiles " + f"less aggressively." + ) if args.load_in_4bit: model = prepare_model_for_kbit_training( @@ -336,15 +350,27 @@ def train(args, spec): image_token_ids = resolve_image_token_ids(processor, model.config) logger.info(f"Masking image placeholder token ids: {sorted(image_token_ids)}") + # Checkpoint often enough that an interruption costs a fraction of the + # run, and rarely enough that the S3 mirror stays cheap. + save_steps = task_common.resolve_save_steps( + len(train_dataset), args.per_device_train_batch_size, args.gradient_accumulation_steps, args.epochs + ) + checkpointing = task_common.checkpoint_arguments( + save_steps, enabled=args.checkpointing + ) + logger.info( + f"Checkpointing: {checkpointing.get('save_strategy')} " + f"every {checkpointing.get('save_steps', 'n/a')} step(s)" + ) + training_args = task_common.build_training_arguments( - output_dir="/opt/ml/checkpoints", + output_dir=task_common.CHECKPOINT_DIR, learning_rate=args.learning_rate, num_train_epochs=args.epochs, per_device_train_batch_size=args.per_device_train_batch_size, gradient_accumulation_steps=args.gradient_accumulation_steps, weight_decay=args.weight_decay, eval_strategy="epoch", - save_strategy="no", logging_dir="/opt/ml/output/tensorboard", remove_unused_columns=False, label_names=["labels"], @@ -354,6 +380,7 @@ def train(args, spec): # sequence causes; it is a bitsandbytes optimiser, so it is only # available on the quantised path. optim="paged_adamw_8bit" if args.load_in_4bit else "adamw_torch", + **checkpointing, ) collator = build_collator(processor, spec, effective_context, image_token_ids) @@ -374,7 +401,7 @@ def train(args, spec): f"batch_size={args.per_device_train_batch_size} x " f"{args.gradient_accumulation_steps} accumulation" ) - trainer.train() + trainer.train(resume_from_checkpoint=task_common.latest_checkpoint()) metrics = trainer.evaluate() logger.info(f"Final evaluation: loss={metrics.get('eval_loss', 'N/A')}") @@ -389,6 +416,74 @@ def train(args, spec): return metrics +#: Tokens left for the prompt and response after the image is accounted for, +#: when a context length has to be raised to fit. Generous on purpose: the +#: cost of overshooting is a little wasted padding, the cost of undershooting +#: is a failed job on a billed GPU. +TEXT_TOKEN_HEADROOM = 256 + + +def measure_required_context(processor, spec, dataset, sample_size=4): + """Longest untruncated rendering across a sample of records. + + A vision-language model spends most of its sequence on the image, and how + much is not knowable in advance: it depends on the checkpoint's tiling + strategy *and* on the resolution of the images the user uploaded. + SmolVLM-Instruct spends 1377 tokens on a single image; LLaVA-1.6's AnyRes + tiling can spend more than twice that. + + That is why a fixed default context length cannot be right for every + model, and why guessing one and letting truncation cut into the image + placeholder run is a job that fails minutes into a billed GPU. Measuring + is cheap — a handful of CPU-side processor calls — so measure. + + Returns None when the sample cannot be processed at all; the caller keeps + the requested length and lets :func:`check_collation` produce the error. + """ + try: + from . import task_image_classification + except ImportError: # pragma: no cover - flat sourcedir + import task_image_classification # type: ignore + + longest = 0 + for index in range(min(sample_size, len(dataset))): + record = dataset[index] + try: + image = task_image_classification.load_image(record[spec.image_column]) + text = render_chat( + processor, + build_messages(record[spec.text_column], record[spec.response_column]), + add_generation_prompt=False, + ) + # No truncation: the point is to find out how long it really is. + batch = processor(images=[image], text=[text], return_tensors="pt") + longest = max(longest, int(batch["input_ids"].shape[1])) + except Exception as error: # pragma: no cover - defer to check_collation + logger.warning(f"Could not measure record {index}: {error}") + return None + + return longest or None + + +def resolve_effective_context(requested, measured, model_max): + """Context length to actually use. + + Raises the requested length to fit the measured records, then clamps to + what the model supports. Returns ``(effective, raised)``. + """ + effective = int(requested) + raised = False + + if measured and measured > effective: + effective = measured + TEXT_TOKEN_HEADROOM + raised = True + + if model_max: + effective = min(effective, int(model_max)) + + return effective, raised + + def check_collation(collator, dataset, sample_size=2): """Collate a couple of records before the Trainer starts. diff --git a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_text_classification.py b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_text_classification.py index 39be5f504..3fbd67815 100644 --- a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_text_classification.py +++ b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/task_text_classification.py @@ -92,15 +92,28 @@ def tokenize_function(examples): train_dataset = train_dataset.map(tokenize_function, batched=True) eval_dataset = eval_dataset.map(tokenize_function, batched=True) + # Checkpoint often enough that an interruption costs a fraction of the + # run, and rarely enough that the S3 mirror stays cheap. + save_steps = task_common.resolve_save_steps( + len(train_dataset), args.per_device_train_batch_size, 1, args.epochs + ) + checkpointing = task_common.checkpoint_arguments( + save_steps, enabled=args.checkpointing + ) + logger.info( + f"Checkpointing: {checkpointing.get('save_strategy')} " + f"every {checkpointing.get('save_steps', 'n/a')} step(s)" + ) + training_args = task_common.build_training_arguments( - output_dir="/opt/ml/checkpoints", + output_dir=task_common.CHECKPOINT_DIR, learning_rate=args.learning_rate, num_train_epochs=args.epochs, per_device_train_batch_size=args.per_device_train_batch_size, weight_decay=args.weight_decay, eval_strategy="epoch", - save_strategy="no", logging_dir="/opt/ml/output/tensorboard", + **checkpointing, ) trainer = Trainer( @@ -117,7 +130,7 @@ def tokenize_function(examples): f"model={args.model_name_or_path}, epochs={args.epochs}, " f"batch_size={args.per_device_train_batch_size}" ) - trainer.train() + trainer.train(resume_from_checkpoint=task_common.latest_checkpoint()) metrics = trainer.evaluate() logger.info(f"Final evaluation: accuracy={metrics.get('eval_accuracy', 'N/A')}") diff --git a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/train.py b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/train.py index 578b62251..18d64fca2 100644 --- a/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/train.py +++ b/backend/src/apis/app_api/fine_tuning/sagemaker_scripts/train.py @@ -54,15 +54,10 @@ } -def str2bool(value): - """Parse a boolean hyperparameter. - - SageMaker passes every hyperparameter as a string, so ``bool("false")`` — - which is True — is the trap this exists to avoid. - """ - if isinstance(value, bool): - return value - return str(value).strip().lower() in ("1", "true", "yes", "on") +#: Re-exported so the container script and app-api parse boolean +#: hyperparameters identically — a disagreement here would let app-api admit a +#: job the trainer then runs with different settings. +str2bool = task_types.str2bool def resolve_task_module(task_type): @@ -103,6 +98,10 @@ def parse_args(argv=None): parser.add_argument("--context_length", type=int, default=512) parser.add_argument("--image_size", type=int, default=224) parser.add_argument("--gradient_accumulation_steps", type=int, default=1) + # Kill switch. Checkpointing is what makes a restart — a spot interruption, + # or a job killed at the budget-clamped MaxRuntime — resume instead of + # starting over, so it is on unless deliberately turned off. + parser.add_argument("--checkpointing", type=str2bool, default=True) # Generative VLM (LoRA) hyperparameters. Ignored by the classification # tasks, which train every weight of a much smaller model. diff --git a/backend/src/apis/app_api/fine_tuning/sagemaker_service.py b/backend/src/apis/app_api/fine_tuning/sagemaker_service.py index 236e5c6b1..a3033f169 100644 --- a/backend/src/apis/app_api/fine_tuning/sagemaker_service.py +++ b/backend/src/apis/app_api/fine_tuning/sagemaker_service.py @@ -48,6 +48,22 @@ # The AWS-owned account that publishes Deep Learning Containers. Same in every # region we support. +#: Must match ``task_common.CHECKPOINT_DIR`` — SageMaker only mirrors the +#: directory it is told about, and the trainer only writes to the one it knows. +CHECKPOINT_LOCAL_PATH = "/opt/ml/checkpoints" + +#: Head-room added to MaxRuntimeInSeconds to get MaxWaitTimeInSeconds for a +#: spot job. MaxWaitTime covers capacity waiting *and* training, and the API +#: requires it to exceed MaxRuntime, so it cannot simply equal it. +#: +#: Four hours because measured on-demand capacity waits for the GPU families +#: this feature uses ran 28-58 minutes in us-west-2; spot draws from the +#: surplus of those same constrained pools, so its queue is longer, not +#: shorter. Waiting is not billed — only running is — so a generous allowance +#: costs nothing but patience, while a tight one fails the job outright with +#: MaxWaitTimeExceeded after it has already queued. +SPOT_CAPACITY_WAIT_ALLOWANCE_SECONDS = 4 * 60 * 60 + _DLC_REGISTRY_ACCOUNT = "763104351884" _SUPPORTED_DLC_REGIONS = ( @@ -126,6 +142,8 @@ def create_training_job( instance_type: str, instance_count: int = 1, max_runtime: int = 86400, + checkpoint_s3_uri: Optional[str] = None, + use_spot: bool = False, source_dir_s3_uri: str = "", task_type: Optional[str] = None, volume_size_gb: Optional[int] = None, @@ -190,6 +208,30 @@ def create_training_job( "HyperParameters": hyperparameters, } + # Mirror the container's checkpoint directory to S3 while the job runs, + # and restore it before a restarted attempt runs again. Without this + # the trainer still writes checkpoints, but they die with the instance + # — so a job stopped at MaxRuntime, or interrupted, yields nothing. + if checkpoint_s3_uri: + params["CheckpointConfig"] = { + "S3Uri": checkpoint_s3_uri, + "LocalPath": CHECKPOINT_LOCAL_PATH, + } + + if use_spot: + # MaxWaitTimeInSeconds must be strictly larger than + # MaxRuntimeInSeconds, and it covers waiting for capacity *plus* + # training — so it is the runtime plus an allowance for the queue, + # not the runtime itself. + params["EnableManagedSpotTraining"] = True + params["StoppingCondition"]["MaxWaitTimeInSeconds"] = ( + max_runtime + SPOT_CAPACITY_WAIT_ALLOWANCE_SECONDS + ) + logger.info( + f"Managed spot enabled for {job_name}: MaxWait=" + f"{params['StoppingCondition']['MaxWaitTimeInSeconds']}s" + ) + if subnets and security_groups: params["VpcConfig"] = { "SecurityGroupIds": security_groups, diff --git a/backend/src/apis/app_api/fine_tuning/task_types.py b/backend/src/apis/app_api/fine_tuning/task_types.py index bb001687d..1f62d40db 100644 --- a/backend/src/apis/app_api/fine_tuning/task_types.py +++ b/backend/src/apis/app_api/fine_tuning/task_types.py @@ -298,7 +298,13 @@ def supports_inference_extension(self, filename: str) -> bool: # literal batch of 16 OOMs on any instance we offer. "per_device_train_batch_size": "1", "gradient_accumulation_steps": "8", - "context_length": "1024", + # Generous on purpose. A VLM spends most of its sequence on the + # image — SmolVLM-Instruct measures 1377 tokens for one image, so + # the old 1024 default could not fit the image, let alone the + # prompt. The collator pads to the longest item in the batch, not + # to this value, so headroom here costs nothing; too little is a + # failed job on a billed GPU. + "context_length": "2048", "load_in_4bit": "true", "lora_r": "16", "lora_alpha": "32", @@ -329,6 +335,22 @@ def supports_inference_extension(self, filename: str) -> bool: ) +def str2bool(value) -> bool: + """Parse a boolean hyperparameter. + + Lives here because both sides need it and this is the only module they + share: app-api validates a submission before billing a GPU, and the + training script parses the same value inside the container. + + ``bool("false")`` is True, which is the trap this exists to avoid — + SageMaker passes every hyperparameter as a string, and JSON-parses some of + them back into Python-style ``"False"`` on the command line. + """ + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("1", "true", "yes", "on") + + def get_task_spec(task_type: Optional[str]) -> TaskSpec: """Return the spec for ``task_type``. diff --git a/backend/src/apis/app_api/kb_migration/dispatcher.py b/backend/src/apis/app_api/kb_migration/dispatcher.py index fa0de5690..cba38fed8 100644 --- a/backend/src/apis/app_api/kb_migration/dispatcher.py +++ b/backend/src/apis/app_api/kb_migration/dispatcher.py @@ -42,9 +42,20 @@ logger.setLevel(logging.INFO) #: The migration flag. Absent, empty, or anything outside the truthy set means the -#: dispatcher invokes nothing. +#: dispatcher invokes nothing for the shadow→verify→promote states. FLAG_MIGRATION_ENABLED = "MANAGED_KB_MIGRATION_ENABLED" +#: Born-managed (rollout ladder step 2). Gates the ``born_managed`` state ONLY. +#: +#: Two flags rather than one because the ladder's steps are meant to be +#: independent: step 2 makes new agents born managed, step 3 starts migrating the +#: existing fleet, and a deployment sitting on step 2 must not have its fleet +#: quietly migrated. Gating this dispatcher on ``MANAGED_KB_MIGRATION_ENABLED`` +#: alone would have made step 2 useless on its own — the trigger would queue a +#: provisioning job that nothing ever picked up, leaving the author's first +#: document parked at "Provisioning knowledge base…" forever. +FLAG_NEW_DEFAULT = "MANAGED_KB_NEW_DEFAULT" + #: Recognised affirmative spellings, matching the reconciler's. An allow-list #: rather than a truthiness test, because the failure being designed around is a #: value that is present but empty: ``bool("")`` is correct by luck, @@ -68,7 +79,7 @@ def migration_enabled() -> bool: - """Whether the dispatcher may invoke the worker at all. + """Whether the dispatcher may invoke the worker for MIGRATION work at all. Read at call time. Bound as a module constant it would be captured at import and a test overriding the variable would silently get the production value — @@ -77,6 +88,22 @@ def migration_enabled() -> bool: return (os.environ.get(FLAG_MIGRATION_ENABLED) or "").strip().lower() in _TRUTHY +def new_default_enabled() -> bool: + """Whether the dispatcher may invoke the worker for BORN-MANAGED provisioning. + + Read at call time, same reason as :func:`migration_enabled`. Mirrors + ``kb_upgrade.service.new_default_enabled`` — the flag is read on both sides of + the handoff because each side is useless without the other: the trigger queues + the job, this dispatcher is the only thing that picks it up. + """ + return (os.environ.get(FLAG_NEW_DEFAULT) or "").strip().lower() in _TRUTHY + + +def dispatcher_enabled() -> bool: + """Whether this tick has any reason to run at all.""" + return migration_enabled() or new_default_enabled() + + def dispatch_limit() -> int: """Records taken per tick, bounded above by :data:`DISPATCH_LIMIT_CEILING`.""" raw = os.environ.get("KB_MIGRATION_DISPATCH_LIMIT") @@ -106,22 +133,35 @@ def _now_iso() -> str: def _work_states() -> List[str]: - """Every work-eligible state, drained-first. + """Every work-eligible state, drained-first. Ordering only — no flag gating. Derived from ``WORK_ELIGIBLE_STATES`` rather than restated, with an explicit - priority order laid over it. A record in ``promote`` is one conditional write - from being finished, so serving it ahead of new ``shadow`` work drains the queue - instead of accumulating half-migrated knowledge bases. + priority order laid over it. ``born_managed`` is served first because somebody + is watching an upload spinner for it, and a record in ``promote`` next because + it is one conditional write from being finished — serving those ahead of new + ``shadow`` work drains the queue instead of accumulating half-migrated + knowledge bases. Anything work-eligible but absent from the priority list is appended rather than dropped. A state added to the records module and forgotten here then migrates slowly, which is a scheduling nuisance; dropped, it would stall forever with its work keys written and nothing ever reading them — invisible, because the record still looks queued. + + Which of these a given tick may actually sweep is :func:`_enabled_work_states`. + The two are separate on purpose: this one answers "what work exists and in what + order", that one answers "what is switched on", and a single function doing both + cannot be tested for either. """ - from apis.shared.kb_backend.records import PROMOTE, SHADOW, VERIFY, WORK_ELIGIBLE_STATES + from apis.shared.kb_backend.records import ( + BORN_MANAGED, + PROMOTE, + SHADOW, + VERIFY, + WORK_ELIGIBLE_STATES, + ) - priority = (PROMOTE, VERIFY, SHADOW) + priority = (BORN_MANAGED, PROMOTE, VERIFY, SHADOW) ordered = [state for state in priority if state in WORK_ELIGIBLE_STATES] remainder = sorted(set(WORK_ELIGIBLE_STATES) - set(priority)) if remainder: @@ -132,6 +172,30 @@ def _work_states() -> List[str]: return ordered + remainder +def _enabled_work_states() -> List[str]: + """The states THIS tick may sweep, each gated on its own flag. + + This is what keeps the rollout ladder's rungs independent. ``born_managed`` + answers to ``MANAGED_KB_NEW_DEFAULT`` (step 2) and every migration state to + ``MANAGED_KB_MIGRATION_ENABLED`` (step 3), so a deployment sitting on step 2 + has new agents born managed and NOT ONE existing knowledge base touched. + + Gating them together — either flag enabling all of them — would turn step 2 + into a back door for step 3's blast radius; gating born-managed on the + migration flag would make step 2 useless alone, queueing provisioning jobs that + nothing ever picks up and parking every first upload on "Provisioning…". + """ + from apis.shared.kb_backend.records import BORN_MANAGED + + allowed_migration = migration_enabled() + allowed_born = new_default_enabled() + return [ + state + for state in _work_states() + if (allowed_born if state == BORN_MANAGED else allowed_migration) + ] + + def _invoke_worker(payload: Dict[str, Any]) -> None: """Async-invoke the migration worker. Same shape as the sync dispatcher's.""" import boto3 @@ -170,7 +234,7 @@ async def _due_records(limit: int, now_iso: str) -> List[Dict[str, Any]]: from apis.shared.kb_backend.records import query_due_work collected: List[Dict[str, Any]] = [] - for state in _work_states(): + for state in _enabled_work_states(): if len(collected) >= limit: break remaining = limit - len(collected) @@ -187,8 +251,11 @@ async def dispatch_once() -> Dict[str, int]: """One dispatcher tick. Returns the metric counts (also emitted).""" counts: Dict[str, int] = {"Due": 0, "Dispatched": 0, "Failed": 0} - if not migration_enabled(): - logger.info(f"{FLAG_MIGRATION_ENABLED} is not truthy; dispatcher tick is a no-op") + if not dispatcher_enabled(): + logger.info( + f"neither {FLAG_MIGRATION_ENABLED} nor {FLAG_NEW_DEFAULT} is truthy; " + f"dispatcher tick is a no-op" + ) return counts limit = dispatch_limit() diff --git a/backend/src/apis/app_api/kb_migration/document_reconciler.py b/backend/src/apis/app_api/kb_migration/document_reconciler.py index ec7f5c745..0893c53a0 100644 --- a/backend/src/apis/app_api/kb_migration/document_reconciler.py +++ b/backend/src/apis/app_api/kb_migration/document_reconciler.py @@ -102,7 +102,16 @@ # minus its terminal members. ``deleting`` is deliberately NOT here: a # soft-deleted document is being removed on purpose and must never be resurrected # to ``complete``. -NON_TERMINAL_STATUSES = frozenset({"uploading", "chunking", "embedding"}) +# +# ``provisioning`` (born-managed, MANAGED_KB_NEW_DEFAULT) is here as the long- +# horizon backstop for a first document whose provisioning job was killed between +# building the knowledge base and handing the document over. It is safe to include +# precisely because the sweep already skips any record with no ``awsKbId`` — while +# the knowledge base does not exist there is nothing to probe, so such a document is +# never even considered. Once it does exist, a document still parked here past the +# grace window probes ``NOT_FOUND`` and is re-ingested from S3, which is exactly the +# correction that is owed. +NON_TERMINAL_STATUSES = frozenset({"provisioning", "uploading", "chunking", "embedding"}) # ── Tunables, resolved at call time ────────────────────────────────────────── # diff --git a/backend/src/apis/app_api/kb_migration/ingestion_consumer.py b/backend/src/apis/app_api/kb_migration/ingestion_consumer.py index 5af29c385..5da298593 100644 --- a/backend/src/apis/app_api/kb_migration/ingestion_consumer.py +++ b/backend/src/apis/app_api/kb_migration/ingestion_consumer.py @@ -68,6 +68,22 @@ STATUS_COMPLETE = "complete" STATUS_FAILED = "failed" +#: The leading status of a born-managed first upload (``MANAGED_KB_NEW_DEFAULT``): +#: the document is uploaded and its knowledge base is still being created, so the +#: managed lifecycle is ``provisioning → uploading → complete``. +#: +#: Defined HERE, in the module that owns the managed document lifecycle, rather +#: than beside the API-side trigger, because the migration Lambda image carries +#: only ``apis/shared/kb_backend`` and ``apis/app_api/kb_migration`` — importing +#: the trigger's package from the provisioning job would fail on a cold start. +#: ``kb_upgrade.born_managed`` re-exports this name for the API side, so the two +#: halves cannot drift apart. +#: +#: Non-terminal and non-retrievable: the retrieval facade serves only +#: ``complete``, so a document in this status can never answer a question out of a +#: knowledge base that does not exist yet. +STATUS_PROVISIONING = "provisioning" + #: How long to wait for a document to become genuinely retrievable after Bedrock #: reports it INDEXED. The observed gap is 0.75-1.03 s; the margin is wide because #: the cost of waiting is a few seconds of Lambda time and the cost of not waiting @@ -467,9 +483,127 @@ def wait_until_retrievable( return None +def _get_doc_row(assistant_id: str, document_id: str) -> Optional[Dict[str, Any]]: + """The ``DOC#`` row, or ``None``. Read once per invocation. + + Carries the declared ``sizeBytes`` reconciled against the true S3 size, and the + ``byteCapSettled`` marker that makes settlement idempotent across redeliveries. + """ + response = _table().get_item( + Key={"PK": f"AST#{assistant_id}", "SK": f"DOC#{document_id}"} + ) + return response.get("Item") + + +def _declared_bytes(doc_row: Optional[Dict[str, Any]]) -> int: + """The size the client declared at request time, or 0 if the row has none. + + 0 is the correct default for a document that reserved nothing at request time + (an imported file, whose row is created with ``sizeBytes=0`` and whose true + size is only known here) — the reconcile then reserves the whole real size. + """ + if not doc_row: + return 0 + try: + return int(doc_row.get("sizeBytes") or 0) + except (TypeError, ValueError): + return 0 + + +def _delete_s3_object(bucket: str, key: str) -> None: + """Best-effort delete of an orphaned source object that overshot the cap. + + A failure to delete must not turn a cap rejection into an unhandled error: the + document is already being failed, and a stranded object is a smaller problem + than a stuck ingestion. boto3 is imported here to keep this module's + module-level imports stdlib-only (image-size discipline). + """ + try: + import boto3 + + boto3.client("s3").delete_object(Bucket=bucket, Key=key) + except Exception as exc: # noqa: BLE001 - cleanup is best-effort + logger.warning(f"failed to delete orphaned object s3://{bucket}/{key}: {exc}") + + +def _release_reservation(assistant_id: str, document_id: str, declared: int) -> None: + """Return the request-time reservation on a terminal ingestion FAILURE. + + Exactly-once via ``byte_cap.settle_once`` (Requirement 12.6). Called only from + genuinely terminal failure paths — never from the "leave it non-terminal for + redelivery" paths, where the bytes must stay reserved for the delivery that + eventually completes the document. + """ + from apis.shared.kb_backend import byte_cap + + if declared <= 0: + return + if not byte_cap.settle_once(assistant_id, document_id): + return + byte_cap.release(assistant_id, assistant_id, declared) + + +def _reconcile_bytes_on_complete( + assistant_id: str, + document_id: str, + bucket: str, + key: str, + record: Optional[Dict[str, Any]], + declared: int, +) -> None: + """Settle the byte reservation against the AUTHORITATIVE S3 size (Req 12.3/12.4). + + The request-time reservation used the client-declared size, which is an input, + not a measurement. Here — the document is indexed and its bytes are really in + S3 — the true size is taken from an S3 HEAD and the reservation is reconciled: + + * ``real == declared`` — commit the reservation as-is. + * ``real < declared`` — the client over-reported; commit the real size and + release the difference so the owner is not charged for bytes never stored. + * ``real > declared`` — the client UNDER-reported, which is the case that could + defeat the cap. Reserve the shortfall against the binding cap. If that + breaches it, the document overshoots: raise :class:`ByteCapExceeded` so the + caller fails it — but first return the original reservation and delete the + orphaned S3 object, leaving no bytes charged and no stranded source. + + Exactly-once via ``byte_cap.settle_once``: a redelivery that re-examines an + already-settled document does nothing, which is what stops a second commit + driving ``reservedBytes`` negative. + """ + from apis.shared.kb_backend import byte_cap + + if not byte_cap.settle_once(assistant_id, document_id): + # A prior delivery already settled this document's bytes. + return + + real = byte_cap.object_size_bytes(bucket, key) + + if real == declared: + byte_cap.commit(assistant_id, assistant_id, real) + return + if real < declared: + byte_cap.commit(assistant_id, assistant_id, real) + byte_cap.release(assistant_id, assistant_id, declared - real) + return + + # real > declared: reserve the shortfall the client did not declare. + elevated = bool((record or {}).get("elevatedByteCap")) + cap = byte_cap.effective_cap(elevated) + try: + byte_cap.reserve(assistant_id, assistant_id, real - declared, cap) + except byte_cap.ByteCapExceeded: + # Overshoots the cap. Undo everything this document reserved and remove the + # source object, then let the caller mark it failed. + if declared > 0: + byte_cap.release(assistant_id, assistant_id, declared) + _delete_s3_object(bucket, key) + raise + byte_cap.commit(assistant_id, assistant_id, real) + + def handle_object(bucket: str, key: str) -> Dict[str, Any]: """Route one uploaded object. Returns a summary for logging and tests.""" - from apis.shared.kb_backend.records import ENGINE_MANAGED + from apis.shared.kb_backend.records import BORN_MANAGED, ENGINE_MANAGED assistant_id, document_id, filename = parse_object_key(key) engine, record = resolve_engine_for(assistant_id) @@ -486,9 +620,38 @@ def handle_object(bucket: str, key: str) -> Dict[str, Any]: aws_kb_id = (record or {}).get("awsKbId") data_source_id = (record or {}).get("awsDataSourceId") if not aws_kb_id or not data_source_id: - # Managed engine but no identifiers means provisioning has not finished. - # Failing loudly is correct: silently falling back to legacy would create - # exactly the dual-index this function exists to prevent. + # Managed engine but no identifiers. Two very different situations share + # this shape, and telling them apart is the difference between a working + # born-managed first upload and a dead-lettered one. + if (record or {}).get("migrationState") == BORN_MANAGED: + # Born-managed (MANAGED_KB_NEW_DEFAULT): the knowledge base is being + # created right now and this is the document that triggered it. The S3 + # event fires within seconds; provisioning takes minutes. Lambda's + # asynchronous retry is capped at 2 attempts — a hard service limit — + # so waiting here or raising for redelivery both end in the DLQ well + # before the knowledge base exists. + # + # So this defers instead: a benign no-op that ingests nothing, writes + # nothing, and does NOT fail. The provisioning job owns the handoff and + # ingests this document itself once the knowledge base is ACTIVE + # (kb_migration/provisioner.py), which is why correctness here does not + # depend on the redelivery window at all. The DOC# row is left in + # `provisioning`, which is what the user sees. + logger.info( + f"document {document_id} belongs to a knowledge base still being " + f"provisioned (born-managed); deferring to the provisioning job, " + f"which owns its ingestion" + ) + return { + "routed": "managed", + "ingested": False, + "document_id": document_id, + "note": "deferred-provisioning", + } + + # Not born-managed: managed intent with no provisioner behind it. Failing + # loudly is correct — silently falling back to legacy would create exactly + # the dual-index this function exists to prevent. raise IngestionRoutingError( f"assistant {assistant_id} resolves to the managed engine but its " f"knowledge base is not provisioned (awsKbId={aws_kb_id!r}, " @@ -500,6 +663,30 @@ def handle_object(bucket: str, key: str) -> Dict[str, Any]: from apis.shared.kb_backend.managed_backend import ManagedKbBackend from apis.shared.kb_backend.protocol import DocumentSource + # The DOC# row carries the size declared and reserved at request time + # (sizeBytes) and the byteCapSettled marker. Read it once. If a PRIOR delivery + # already drove this document terminal AND settled its bytes, this is a + # redelivery and re-running the byte accounting would double-count — a second + # commit drives reservedBytes negative, a second release over-credits the cap. + # Return without touching anything (Requirement 12.4/12.5). + doc_row = _get_doc_row(assistant_id, document_id) + if ( + doc_row + and doc_row.get("byteCapSettled") + and doc_row.get("status") in (STATUS_COMPLETE, STATUS_FAILED) + ): + logger.info( + f"document {document_id} is already {doc_row.get('status')} and its " + f"bytes are settled; skipping to avoid double-counting" + ) + return { + "routed": "managed", + "ingested": False, + "document_id": document_id, + "status": doc_row.get("status"), + "note": "already-settled", + } + declared = _declared_bytes(doc_row) # The backend takes the App_KB_Id and resolves the AWS identifiers itself on # every operation. Threading them in from here would defeat that: a # dormancy/rehydration cycle replaces them, and a caller holding a stale pair @@ -527,6 +714,7 @@ def handle_object(bucket: str, key: str) -> Dict[str, Any]: assistant_id, document_id, STATUS_FAILED, error=f"the knowledge base reports this document as {status}", ) + _release_reservation(assistant_id, document_id, declared) return {"routed": "managed", "ingested": False, "document_id": document_id, "status": status} @@ -537,6 +725,7 @@ def handle_object(bucket: str, key: str) -> Dict[str, Any]: except Exception as exc: logger.error(f"direct ingestion of {document_id} failed: {exc}", exc_info=True) set_document_terminal(assistant_id, document_id, STATUS_FAILED, error=str(exc)) + _release_reservation(assistant_id, document_id, declared) raise else: # Already submitted — a redelivery, or a document still being worked on. @@ -560,6 +749,7 @@ def handle_object(bucket: str, key: str) -> Dict[str, Any]: assistant_id, document_id, STATUS_FAILED, error=f"the knowledge base reports this document as {status}", ) + _release_reservation(assistant_id, document_id, declared) return {"routed": "managed", "ingested": True, "document_id": document_id, "status": status} @@ -594,6 +784,40 @@ def handle_object(bucket: str, key: str) -> Dict[str, Any]: f"poll window; leaving it for redelivery" ) + # INDEXED and retrievable. Settle the byte reservation against the AUTHORITATIVE + # S3 size before declaring success (Requirement 12.3): the request-time reserve + # used the client's declared size, which is not trustworthy. This commits the + # true bytes, or — if the client under-reported and the real size overshoots the + # cap — fails the document and removes the orphaned object. + from apis.shared.kb_backend.byte_cap import ByteCapExceeded + + try: + _reconcile_bytes_on_complete( + assistant_id, document_id, bucket, key, record, declared + ) + except ByteCapExceeded: + # The reservation has been returned and the source object deleted inside the + # reconcile. Fail the document with an actionable message (Requirement + # 12.12) rather than reporting a success that breaches the cap. + logger.warning( + f"document {document_id} indexed but its true size overshoots the byte " + f"cap; marking failed and removing the orphaned object" + ) + set_document_terminal( + assistant_id, document_id, STATUS_FAILED, + error=( + "this document exceeds the knowledge base's storage limit; delete " + "unused documents or request an elevated storage tier" + ), + ) + return { + "routed": "managed", + "ingested": True, + "document_id": document_id, + "status": STATUS_FAILED, + "note": "byte-cap-exceeded", + } + set_document_terminal( assistant_id, document_id, diff --git a/backend/src/apis/app_api/kb_migration/provisioner.py b/backend/src/apis/app_api/kb_migration/provisioner.py new file mode 100644 index 000000000..18e521fc1 --- /dev/null +++ b/backend/src/apis/app_api/kb_migration/provisioner.py @@ -0,0 +1,456 @@ +"""Born-managed provisioning job: build the knowledge base, then own the handoff. + +The worker-side half of ``MANAGED_KB_NEW_DEFAULT``. The API-side half +(:mod:`apis.app_api.kb_upgrade.born_managed`) does three conditional writes on the +first document upload — create the record, declare the engine managed, queue this +job — and returns in milliseconds. Everything slow happens here. + +Why the ingestion trigger lives in this job and not in the S3 event +------------------------------------------------------------------- +The S3 ``ObjectCreated`` event for the first document fires within seconds of the +client's upload. ``CreateKnowledgeBase`` takes 47–124 s to ``ACTIVE`` and the whole +provision takes minutes. Lambda's asynchronous retry is capped at **2** attempts — +a hard service limit, not a setting — so the event is exhausted and dead-lettered +long before there is a knowledge base to ingest into. That is precisely the §5.37 +dead-letter failure that leaves a document permanently invisible, and designing a +new feature that walks into it on its very first document would be a choice. + +So the managed ingestion consumer **defers** while a record is born-managed and +unprovisioned (a benign no-op, no dead-letter), and this job ingests the pending +documents itself once the knowledge base exists. Correctness then rests on the +dispatcher's work-key queue and the worker lease — retried until terminal, leased +so two workers cannot both provision — rather than on a two-try event window. + +Failure means legacy, never limbo +--------------------------------- +Managed intent with no knowledge base is a dead end in both directions: the legacy +pipeline skips the agent's uploads because the record says managed, and the managed +one cannot serve because there is nothing to serve from. So a provisioning failure +does not merely stop — it **removes** ``retrievalEngine``, returning the agent to +legacy-by-absence, and fails the waiting documents with a message that says to try +again. A re-upload then takes the legacy path and works. See :func:`_fall_back_to_legacy`. + +Import boundary +--------------- +Module-level imports are stdlib only, like every other module in this package: this +code ships in the size-constrained migration Lambda image. Everything heavy — +boto3, the provisioner, the ingestion consumer — is imported inside the functions +that need it. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + +METRIC_BORN_MANAGED_PROVISIONED = "KbBornManagedProvisioned" +METRIC_BORN_MANAGED_FAILED = "KbBornManagedFailed" +METRIC_BORN_MANAGED_DOCUMENTS = "KbBornManagedDocuments" + +#: Documents ingested per invocation. **One**, deliberately. +#: +#: The worker's Lambda timeout is 15 minutes and one document's ingestion budget is +#: already 10.5 (``INDEXED_POLL_TIMEOUT_SECONDS`` + the retrievable poll), because +#: image-heavy PDFs run the vision model per page. Two documents in one invocation +#: could not both finish, and being killed mid-wait is the one outcome worth +#: engineering away: it costs a whole dispatcher interval and teaches nothing. +#: +#: More than one pending document only happens when the author uploaded again +#: during the provisioning window, so the common case is exactly one. Anything left +#: over re-arms the work key and is picked up on the next tick. +MAX_DOCUMENTS_PER_INVOCATION = 1 + +#: The user-facing copy for a document orphaned by a failed provision. Written for +#: the person looking at the upload, not for an operator reading a log. +PROVISIONING_FAILED_MESSAGE = ( + "We could not prepare this assistant's knowledge base, so this document was " + "not added. Please upload it again." +) + + +def _now_iso() -> str: + from apis.shared.timestamps import utc_now_iso + + return utc_now_iso() + + +def _documents_bucket() -> str: + bucket = os.environ.get("S3_ASSISTANTS_DOCUMENTS_BUCKET_NAME") + if not bucket: + raise RuntimeError("S3_ASSISTANTS_DOCUMENTS_BUCKET_NAME is not set") + return bucket + + +def pending_documents(assistant_id: str) -> List[Dict[str, Any]]: + """``DOC#`` rows still waiting on provisioning, oldest first. + + Reads the raw table rather than ``list_assistant_documents`` for the import + reason in the module docstring, and because that function auto-fails stale rows + as a side effect of reading them — a write triggered by a query is not something + this job wants happening underneath it. + """ + import boto3 + from boto3.dynamodb.conditions import Key + + from apis.app_api.kb_migration.ingestion_consumer import STATUS_PROVISIONING + + table = boto3.resource("dynamodb").Table(os.environ["DYNAMODB_ASSISTANTS_TABLE_NAME"]) + items: List[Dict[str, Any]] = [] + kwargs: Dict[str, Any] = { + "KeyConditionExpression": Key("PK").eq(f"AST#{assistant_id}") + & Key("SK").begins_with("DOC#"), + } + while True: + response = table.query(**kwargs) + items.extend(response.get("Items") or []) + last = response.get("LastEvaluatedKey") + if not last: + break + kwargs["ExclusiveStartKey"] = last + + waiting = [ + item for item in items if str(item.get("status") or "") == STATUS_PROVISIONING + ] + waiting.sort(key=lambda item: str(item.get("createdAt") or "")) + return waiting + + +async def run_born_managed( + assistant_id: str, + app_kb_id: str, + record: Dict[str, Any], +): + """Provision the knowledge base and ingest the documents waiting on it. + + Returns the worker's :class:`~apis.app_api.kb_migration.worker.StepResult`. + Called under the worker's lease, from :func:`worker.run_step`, for a record in + ``migrationState = born_managed``. + + Idempotent at every step, because the dispatcher will bring this record back + until it is terminal and a 15-minute Lambda can be killed at any point: + + * ``provision_managed_kb`` resumes from the record's persisted ``clientToken``, + so a retry adopts the half-created knowledge base instead of making a second. + * ``handle_object`` probes Bedrock before ingesting, so a re-run of a document + already submitted waits on it rather than re-submitting it. + * The terminal transition is the last write, so anything killed before it is + simply re-run. + """ + from apis.shared.kb_backend import records as r + from apis.shared.kb_backend.metrics import emit_count + from apis.app_api.kb_migration.worker import StepResult + + generation = int(record.get("migrationGeneration") or 0) + + if r.resolve_engine(record) != r.ENGINE_MANAGED: + # A previous attempt already fell back to legacy but was killed before it + # could clear the work keys. Finish that, rather than provisioning a + # knowledge base for a record that has stopped pointing at one. + logger.info( + f"kb {app_kb_id}: born-managed record is back on legacy; closing out " + f"the abandoned provisioning job" + ) + await _finish(assistant_id, app_kb_id, generation, r.MIGRATION_FAILED, + reason="provisioning already rolled back to legacy") + return StepResult( + assistant_id, app_kb_id, r.BORN_MANAGED, r.MIGRATION_FAILED, + detail="already rolled back to legacy", + ) + + provisioned = await _provision(assistant_id, app_kb_id, record) + if provisioned is None: + # Another worker holds the provisioning. Ours re-arms and steps aside + # rather than creating a second knowledge base. + await _rearm(assistant_id, app_kb_id, generation) + return StepResult( + assistant_id, app_kb_id, r.BORN_MANAGED, r.BORN_MANAGED, + detail="provisioning already in progress elsewhere; re-queued", + ) + if provisioned is False: + emit_count(METRIC_BORN_MANAGED_FAILED) + return StepResult( + assistant_id, app_kb_id, r.BORN_MANAGED, r.MIGRATION_FAILED, + detail="provisioning failed; the agent is back on legacy", + ) + + emit_count(METRIC_BORN_MANAGED_PROVISIONED) + + try: + return await _hand_off(assistant_id, app_kb_id, generation) + except Exception as exc: # noqa: BLE001 + # The knowledge base exists but the handoff broke in a way this job did not + # anticipate. Falling back is still the right move and this clause is what + # guarantees it: letting the exception reach `worker.run_step` would send the + # record to `failed` with `retrievalEngine` still set — managed intent, no + # usable pipeline, and no work keys left to fix it. That is the one outcome + # this design exists to make impossible. + logger.error( + f"kb {app_kb_id}: born-managed handoff failed after provisioning: {exc}", + exc_info=True, + ) + await _fall_back_to_legacy(assistant_id, app_kb_id, record, reason=str(exc)) + emit_count(METRIC_BORN_MANAGED_FAILED) + return StepResult( + assistant_id, app_kb_id, r.BORN_MANAGED, r.MIGRATION_FAILED, + detail="handoff failed after provisioning; the agent is back on legacy", + ) + + +async def _hand_off(assistant_id: str, app_kb_id: str, generation: int): + """Ingest the documents that were deferred while the knowledge base was built.""" + from apis.shared.kb_backend import records as r + from apis.shared.kb_backend.metrics import emit_count + from apis.app_api.kb_migration.worker import StepResult + + waiting = await _to_thread(pending_documents, assistant_id) + if not waiting: + # Nothing is waiting: the knowledge base is built and the ordinary managed + # path (S3 event → ingestion consumer) owns every upload from here. + await _finish(assistant_id, app_kb_id, generation, r.RETAIN) + return StepResult( + assistant_id, app_kb_id, r.BORN_MANAGED, r.RETAIN, + converged=True, detail="knowledge base ready; no documents waiting", + ) + + ingested = await _ingest_waiting(assistant_id, waiting[:MAX_DOCUMENTS_PER_INVOCATION]) + remaining = len(waiting) - MAX_DOCUMENTS_PER_INVOCATION + + if remaining > 0: + await _rearm(assistant_id, app_kb_id, generation) + return StepResult( + assistant_id, app_kb_id, r.BORN_MANAGED, r.BORN_MANAGED, + documents_migrated=ingested, + detail=f"{remaining} document(s) still waiting; re-queued", + ) + + await _finish(assistant_id, app_kb_id, generation, r.RETAIN) + if ingested: + emit_count(METRIC_BORN_MANAGED_DOCUMENTS, ingested) + return StepResult( + assistant_id, app_kb_id, r.BORN_MANAGED, r.RETAIN, + documents_migrated=ingested, converged=True, + detail="knowledge base ready and the first document ingested", + ) + + +async def _provision(assistant_id: str, app_kb_id: str, record: Dict[str, Any]): + """Build the knowledge base. ``True`` built, ``None`` someone else's, ``False`` failed. + + A three-valued answer rather than an exception pair because the three outcomes + need three different things from the caller: carry on, step aside, or fall back + to legacy. Collapsing "someone else is provisioning" into a failure would roll + an agent back to legacy while a perfectly good knowledge base was being built + for it. + """ + from apis.shared.kb_backend.provisioning import ( + ProvisioningInProgress, + provision_managed_kb, + ) + + try: + await provision_managed_kb( + assistant_id, + app_kb_id, + owner_user_id=str(record.get("ownerUserId") or ""), + ) + return True + except ProvisioningInProgress as exc: + logger.info(f"kb {app_kb_id}: {exc}") + return None + except Exception as exc: # noqa: BLE001 — every other failure means legacy + logger.error( + f"kb {app_kb_id}: born-managed provisioning failed: {exc}", exc_info=True + ) + await _fall_back_to_legacy(assistant_id, app_kb_id, record, reason=str(exc)) + return False + + +async def _ingest_waiting(assistant_id: str, waiting: List[Dict[str, Any]]) -> int: + """Ingest documents whose S3 event was deferred. Returns how many finished. + + Reuses the managed ingestion consumer's ``handle_object`` outright rather than + reimplementing ingest → wait-indexed → wait-retrievable → terminal. That + function is where §5.37, §5.38 and §5.39 are encoded — Bedrock reports + ``INDEXED`` up to a second before a document is retrievable, ``TEXT_INDEXED`` + is not in the SDK's enum, a document already in flight must not be + re-submitted — plus the authoritative S3-HEAD byte-cap reconcile from + Requirement 12.3. A second implementation of that would be a second place for + those lessons to be forgotten. + + The row is moved ``provisioning → uploading`` first, so the author watching the + upload sees it leave "Provisioning knowledge base…" the moment the wait becomes + an ordinary indexing wait — and so the stale-document sweep's clock starts from + the point the document really entered the normal pipeline. + """ + from apis.app_api.kb_migration import ingestion_consumer as ic + + bucket = _documents_bucket() + done = 0 + + for document in waiting: + document_id = str(document.get("documentId") or "") + s3_key = str(document.get("s3Key") or "") + if not document_id or not s3_key: + logger.warning( + f"skipping malformed waiting DOC# row {document.get('SK')!r}: " + f"documentId or s3Key is missing" + ) + continue + + await _to_thread( + ic.set_document_terminal, assistant_id, document_id, "uploading" + ) + try: + await _to_thread(ic.handle_object, bucket, s3_key) + done += 1 + except Exception as exc: # noqa: BLE001 + # handle_object has already marked the document failed for anything + # genuinely terminal; what reaches here is its "leave it for + # redelivery" signal, and there is no redelivery for a deferred event. + # Leaving the row at `uploading` is correct: this job re-arms, and the + # document reconciler (16.5) is the longer-horizon backstop. + logger.warning( + f"document {document_id} did not finish indexing in this " + f"invocation; it will be retried: {exc}" + ) + + return done + + +async def _fall_back_to_legacy( + assistant_id: str, + app_kb_id: str, + record: Dict[str, Any], + *, + reason: str, +) -> None: + """Undo the managed intent so the agent works on legacy again. + + Order matters and is the opposite of intuition — documents first, engine + second, terminal state last: + + 1. **Fail the waiting documents** (and return their byte reservations). Done + while the record still says managed, because that is the state in which + these rows are unambiguously this job's to resolve. + 2. **Remove ``retrievalEngine``** (``rollback_engine``). The agent is legacy by + absence again, byte-identical to one that never tried, so the next upload + takes the legacy pipeline and simply works. + 3. **Go terminal**, which removes the work keys. + + A crash between 1 and 2, or 2 and 3, leaves the work keys in place, so the + dispatcher brings the record back and :func:`run_born_managed` re-runs from + whichever point it reached — its engine check handles the "already rolled back" + case. A crash the other way round, terminal-state-first, is the one shape that + would strand the agent: managed intent, no knowledge base, and nothing left in + the queue to fix it. + """ + from apis.shared.kb_backend import records as r + + generation = int(record.get("migrationGeneration") or 0) + + try: + await _fail_waiting_documents(assistant_id) + except Exception as exc: # noqa: BLE001 — the rollback matters more + logger.error( + f"kb {app_kb_id}: could not fail the documents waiting on provisioning; " + f"rolling back to legacy anyway: {exc}" + ) + + try: + await _to_thread(r.rollback_engine, assistant_id, app_kb_id, _now_iso()) + logger.info(f"kb {app_kb_id}: rolled back to legacy after a failed provision") + except r.TransitionLost: + # Already not managed. Nothing to undo. + logger.info(f"kb {app_kb_id}: engine was already legacy at rollback") + + await _finish(assistant_id, app_kb_id, generation, r.MIGRATION_FAILED, reason=reason) + + +async def _fail_waiting_documents(assistant_id: str) -> None: + """Mark every document waiting on provisioning failed, and return its bytes.""" + from apis.app_api.kb_migration import ingestion_consumer as ic + + for document in await _to_thread(pending_documents, assistant_id): + document_id = str(document.get("documentId") or "") + if not document_id: + continue + await _to_thread( + ic.set_document_terminal, + assistant_id, + document_id, + ic.STATUS_FAILED, + None, + None, + PROVISIONING_FAILED_MESSAGE, + ) + # The request-time reservation would otherwise leak: nothing else will ever + # settle a document whose ingestion never happened. settle_once keeps this + # exactly-once against the stale sweep reaching the same row. + await _to_thread( + ic._release_reservation, + assistant_id, + document_id, + ic._declared_bytes(document), + ) + + +async def _rearm(assistant_id: str, app_kb_id: str, generation: int) -> None: + """Keep the job queued, due now, without changing state.""" + from apis.shared.kb_backend import records as r + + try: + await _to_thread( + r.set_migration_state, + assistant_id, + app_kb_id, + r.BORN_MANAGED, + generation, + _now_iso(), + ) + except r.TransitionLost: + logger.info(f"kb {app_kb_id}: could not re-arm the provisioning job; a newer generation owns it") + + +async def _finish( + assistant_id: str, + app_kb_id: str, + generation: int, + state: str, + *, + reason: Optional[str] = None, +) -> None: + """Move to a terminal state, which is what removes the work keys. + + ``retain`` is the terminal state for a knowledge base that ends up on managed. + For born-managed it carries no retention obligation — there are no legacy + vectors to keep, so ``retainUntil`` is deliberately not set and a future + reclaim pass finds nothing to reclaim. + """ + from apis.shared.kb_backend import records as r + + try: + await _to_thread( + r.set_migration_state, + assistant_id, + app_kb_id, + state, + generation, + None, + None, + reason[:1000] if reason else None, + ) + except Exception as exc: # noqa: BLE001 + # The work keys survive, so the dispatcher brings this record back and the + # step re-runs. That is the safe direction: every step here is idempotent, + # whereas a record wrongly taken out of the queue is never looked at again. + logger.error(f"kb {app_kb_id}: could not record the terminal state {state}: {exc}") + + +async def _to_thread(fn, *args): + """Run a blocking boto3 call off the event loop (Requirement 20.7).""" + import asyncio + + return await asyncio.to_thread(fn, *args) diff --git a/backend/src/apis/app_api/kb_migration/worker.py b/backend/src/apis/app_api/kb_migration/worker.py index 49da7230a..5fea6e735 100644 --- a/backend/src/apis/app_api/kb_migration/worker.py +++ b/backend/src/apis/app_api/kb_migration/worker.py @@ -909,7 +909,15 @@ async def run_step( # the clause would be unreachable, which is how a guard becomes decoration. await take_lease(assistant_id, app_kb_id) - if state == r.SHADOW: + if state == r.BORN_MANAGED: + # Born-managed provisioning (MANAGED_KB_NEW_DEFAULT). Not part of the + # shadow→verify→promote migration — there is no legacy corpus to carry + # across — but it runs here to inherit the lease and the work-key + # queue, which is what makes a minutes-long provision survive a crash. + from apis.app_api.kb_migration.provisioner import run_born_managed + + result = await run_born_managed(assistant_id, app_kb_id, record) + elif state == r.SHADOW: result = await run_shadow(assistant_id, app_kb_id, record, backend) elif state == r.VERIFY: result = await run_verify(assistant_id, app_kb_id, record, backend) diff --git a/backend/src/apis/app_api/kb_upgrade/born_managed.py b/backend/src/apis/app_api/kb_upgrade/born_managed.py new file mode 100644 index 000000000..beedb1226 --- /dev/null +++ b/backend/src/apis/app_api/kb_upgrade/born_managed.py @@ -0,0 +1,251 @@ +"""Born-managed: stack knowledge-base provisioning onto the FIRST document upload. + +``MANAGED_KB_NEW_DEFAULT`` (rollout ladder step 2) makes a new agent's knowledge +base managed from the start. This module is the API-side half — the trigger — and +:mod:`apis.app_api.kb_migration.provisioner` is the worker-side half that does the +minutes-long work. + +Why the first upload, and not agent creation +-------------------------------------------- +Three reasons, in order of how much they cost if ignored: + +1. **Quota.** Managed is one Bedrock knowledge base per agent against a ~10,000 + per-account ceiling. Provisioning at agent creation spends that budget on + prompt-only agents and abandoned drafts — agents that will never hold a + document. Provisioning at first upload bounds it to agents that actually use + RAG. +2. **WYSIWYG.** The agent-creation page uploads documents to the *draft* and lets + the author test against them on the spot. Provisioning at first upload means + the engine they test on is the engine they ship. +3. **Nothing to migrate.** A brand-new agent has no corpus, so there is no + shadow-index-and-verify to run. Trying to reuse the migration for this — which + an earlier revision of this feature did — writes ``retrievalEngine=managed`` + only at *promotion*, so the very first document is grabbed by the legacy + pipeline, tested on legacy, and then indexed a second time on managed. + +Which is why this declares the engine **up front**: the legacy handler skips a +document only when its record already resolves to ``managed`` +(``documents/ingestion/handler._resolve_engine``). Declaring first is what makes +the first document go straight to the managed pipeline instead of both. + +What this function must never do +-------------------------------- +Fail an upload. Provisioning is an optimisation of *which engine* serves a +knowledge base, never a precondition for storing a document, so every failure +path here is logged and swallowed and leaves the agent on legacy — exactly where +it would have been without this feature. The one thing the caller learns is +whether to label this document ``provisioning``. + +It also never blocks. The AWS ``CreateKnowledgeBase`` call is 47–124 s to +``ACTIVE`` and takes minutes end to end; all this does is three conditional +DynamoDB writes and hand the job to the dispatcher's queue. +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, Optional + +from apis.app_api.kb_migration.ingestion_consumer import ( + STATUS_PROVISIONING as _STATUS_PROVISIONING, +) +from apis.app_api.kb_upgrade.service import new_default_enabled + +logger = logging.getLogger(__name__) + +#: The leading ``DOC#`` status a born-managed first upload carries, extending the +#: managed lifecycle to ``provisioning → uploading → complete``. +#: +#: Re-exported from the managed ingestion consumer rather than declared here: the +#: consumer and the provisioning job live in a Lambda image that carries only +#: ``kb_backend`` and ``kb_migration``, so the constant has to be defined on that +#: side of the boundary. Both halves must agree on this string — the trigger writes +#: it, the consumer defers on it, the provisioner clears it. +STATUS_PROVISIONING = _STATUS_PROVISIONING + + +def is_provisioning_managed(record: Optional[Dict[str, Any]]) -> bool: + """Whether this record is a managed knowledge base that is not built yet. + + The single definition of "defer to the provisioner", read by the trigger here + and by the managed ingestion consumer. Both halves must agree: if the consumer + thought a record was ready while the trigger thought it was provisioning, the + consumer would dead-letter the first document. + + Tests the engine AND the absence of the AWS identifiers rather than + ``provisioningState`` alone, because the identifiers are what ingestion + actually needs. ``provisioningState`` flips to ``active`` in the same write + that attaches them (``records.attach_aws_ids``), so the two cannot disagree — + but reading the thing that is used keeps it that way. + """ + from apis.shared.kb_backend.records import ENGINE_MANAGED, resolve_engine + + if resolve_engine(record) != ENGINE_MANAGED: + return False + record = record or {} + return not (record.get("awsKbId") and record.get("awsDataSourceId")) + + +async def begin_born_managed( + assistant_id: str, + *, + owner_user_id: str, + visibility: str = "PRIVATE", +) -> bool: + """Make sure this agent is heading for a managed knowledge base. + + Returns ``True`` when the document being uploaded should be recorded as + :data:`STATUS_PROVISIONING` — that is, when the knowledge base is being built + and this document is waiting on it. ``False`` means "carry on exactly as + before": either the flag is off, or the knowledge base is already built, or + something went wrong and the agent stays on legacy. + + Idempotent and safe under concurrency. Two simultaneous first uploads produce + one knowledge base and one provisioning job; the loser of each conditional + write falls through to the same answer the winner got. + """ + if not new_default_enabled(): + return False + + try: + import asyncio + + from apis.shared.kb_backend import records as r + + # app_kb_id == assistant_id this phase. + record = await asyncio.to_thread(r.get_kb_record, assistant_id, assistant_id) + if record is None: + return await _start(assistant_id, owner_user_id, visibility) + return await _join(assistant_id, record) + except Exception as exc: # noqa: BLE001 — born-managed must never fail an upload + logger.warning( + f"kb {assistant_id}: born-managed provisioning could not be started, " + f"so this upload takes the legacy path: {exc}", + exc_info=True, + ) + return False + + +async def _start(assistant_id: str, owner_user_id: str, visibility: str) -> bool: + """First upload for an agent that has no KB_Record at all. + + Three conditional writes, in this order and for these reasons: + + 1. ``create_provisioning`` — the record must exist before anything claims it, + and ``attribute_not_exists(PK)`` makes it the arbiter of the concurrent + first-upload race. + 2. ``adopt_managed_engine`` — declare the engine, so the legacy pipeline skips + the document that is about to land. + 3. ``set_migration_state(BORN_MANAGED)`` — write the sparse work keys, which is + what puts the job in the dispatcher's queue. **Last**, because a job picked + up before step 2 would provision a knowledge base nothing routes to. + + A crash between any two of these is recoverable rather than stranding: see + :func:`_join`, which is what the next upload runs into. + """ + import asyncio + + from apis.shared.kb_backend import records as r + from apis.shared.kb_backend.provisioning import ( + build_client_token, + new_managed_kb_record, + ) + from apis.shared.timestamps import utc_now_iso + + fresh = new_managed_kb_record( + assistant_id, + owner_user_id, + # The same deterministic token ``provision_managed_kb`` would build for + # itself, persisted now so its resume path adopts this record rather than + # inventing a second token for the same knowledge base. + client_token=build_client_token(assistant_id, "knowledge-base"), + visibility=visibility, + ) + try: + await asyncio.to_thread(r.create_provisioning, assistant_id, fresh) + except r.TransitionLost: + # A concurrent first upload created it between our read and our write. + # Not an error — re-read and answer from whatever it actually says. + logger.info( + f"kb {assistant_id}: record created concurrently during the first upload" + ) + record = await asyncio.to_thread(r.get_kb_record, assistant_id, assistant_id) + return await _join(assistant_id, record) + + now = utc_now_iso() + try: + await asyncio.to_thread(r.adopt_managed_engine, assistant_id, assistant_id, now) + except r.TransitionLost: + # Somebody declared the engine first. Their job owns the provision; this + # document just waits on it. + logger.info(f"kb {assistant_id}: engine already declared managed by a concurrent upload") + + await _enqueue(assistant_id, generation=0) + logger.info( + f"kb {assistant_id}: born managed — knowledge base queued for provisioning " + f"on its first document" + ) + return True + + +async def _join(assistant_id: str, record: Optional[Dict[str, Any]]) -> bool: + """An upload arriving while a born-managed provision is (or should be) in flight. + + Covers three situations that look the same from here and must be handled the + same way: + + * a second document uploaded during the provisioning window; + * a retried request whose predecessor already did the work; + * a crash part-way through :func:`_start`, which leaves a managed-intent record + with no work keys and therefore nothing to finish it. + + The last is why the work keys are re-asserted rather than assumed. Without it a + single unlucky crash would leave the agent permanently unable to ingest: managed + intent makes the legacy pipeline skip every upload, and no knowledge base exists + for the managed one to use. + """ + if not is_provisioning_managed(record): + # Either legacy (nothing owed) or a built managed knowledge base (the + # ordinary managed upload path, byte cap and all). + return False + + generation = int((record or {}).get("migrationGeneration") or 0) + state = str((record or {}).get("migrationState") or "") + + from apis.shared.kb_backend import records as r + + if state != r.BORN_MANAGED: + logger.warning( + f"kb {assistant_id}: managed intent with no provisioning job " + f"(migrationState={state!r}); re-queueing it" + ) + await _enqueue(assistant_id, generation=generation) + return True + + +async def _enqueue(assistant_id: str, *, generation: int) -> None: + """Put the provisioning job in the dispatcher's queue, due immediately. + + ``due_at`` is now rather than later: a person is watching an upload spinner, + and the dispatcher is rate-bounded anyway. Losing the conditional write means a + concurrent request queued the same job, which is the desired end state, so it + is logged at info and not raised. + """ + import asyncio + + from apis.shared.kb_backend import records as r + from apis.shared.timestamps import utc_now_iso + + try: + await asyncio.to_thread( + r.set_migration_state, + assistant_id, + assistant_id, + r.BORN_MANAGED, + generation, + utc_now_iso(), + ) + except r.TransitionLost: + logger.info( + f"kb {assistant_id}: provisioning job already queued by a concurrent request" + ) diff --git a/backend/src/apis/app_api/kb_upgrade/service.py b/backend/src/apis/app_api/kb_upgrade/service.py index 253b1ab62..eb102f56c 100644 --- a/backend/src/apis/app_api/kb_upgrade/service.py +++ b/backend/src/apis/app_api/kb_upgrade/service.py @@ -45,6 +45,19 @@ class UpgradeUnavailable(Exception): #: action is available, and "available" has to mean actionable. FLAG_MIGRATION_ENABLED = "MANAGED_KB_MIGRATION_ENABLED" +#: Born-managed: when set, an agent's knowledge base is created on the managed +#: backend the moment its FIRST document is uploaded, skipping the user-facing +#: Upgrade step entirely (rollout ladder step 2). Read here so the flag has one +#: definition, and acted on in ``kb_upgrade.born_managed`` (the upload trigger) and +#: ``kb_migration.provisioner`` (the job that builds the knowledge base). +#: +#: Independent of ``MANAGED_KB_MIGRATION_ENABLED`` by design: born-managed has no +#: corpus to carry across and does not go through shadow/verify/promote, so a +#: deployment can sit on step 2 — new agents managed, existing fleet untouched — +#: for as long as it likes. The migration dispatcher reads BOTH flags and gates +#: each work state on its own. +FLAG_NEW_DEFAULT = "MANAGED_KB_NEW_DEFAULT" + #: Affirmative spellings, matching ``dispatcher._TRUTHY`` exactly. An allow-list #: rather than truthiness, because the value being designed around is present but #: empty: ``bool("")`` is right by luck and ``bool("false")`` is not. @@ -67,6 +80,17 @@ def migration_enabled() -> bool: return (os.environ.get(FLAG_MIGRATION_ENABLED) or "").strip().lower() in _TRUTHY +def new_default_enabled() -> bool: + """Whether a new agent's knowledge base should be born on the managed backend. + + Read at call time, never bound as a default argument — same reason as + :func:`migration_enabled`. Sufficient on its own: born-managed provisioning is + served by the migration dispatcher under this flag alone, so it does not also + require ``MANAGED_KB_MIGRATION_ENABLED``. + """ + return (os.environ.get(FLAG_NEW_DEFAULT) or "").strip().lower() in _TRUTHY + + def _now() -> datetime: return datetime.now(timezone.utc) diff --git a/backend/src/apis/app_api/main.py b/backend/src/apis/app_api/main.py index 324b4613a..2755a4876 100644 --- a/backend/src/apis/app_api/main.py +++ b/backend/src/apis/app_api/main.py @@ -171,6 +171,17 @@ async def lifespan(app: FastAPI): app.add_middleware(SessionRefreshMiddleware) logger.info("Added BFF session-refresh + CSRF middlewares (dormant until cookie present)") +# Outermost middleware: repair `Location` headers on redirects this app +# generates for itself (Starlette's `redirect_slashes`, chiefly). Behind +# CloudFront those come out as `http://api./` — the internal +# ALB host, over plain HTTP, without the stripped `/api` prefix — which a +# browser blocks as mixed content. Added last so it wraps every other +# middleware and sees the final response headers. +from apis.shared.middleware.proxied_redirect import ProxiedRedirectMiddleware + +app.add_middleware(ProxiedRedirectMiddleware) +logger.info("Added proxied-redirect middleware") + # Import routers from apis.app_api.health import router as health_router diff --git a/backend/src/apis/app_api/models/routes.py b/backend/src/apis/app_api/models/routes.py index 3f1058c5c..f451d37a3 100644 --- a/backend/src/apis/app_api/models/routes.py +++ b/backend/src/apis/app_api/models/routes.py @@ -4,12 +4,13 @@ Supports both AppRole-based access (preferred) and legacy JWT role-based access. """ -from fastapi import APIRouter, HTTPException, Depends, status +from fastapi import APIRouter, HTTPException, Depends, Request, Response, status import logging from apis.app_api.admin.models import ManagedModelsListResponse from apis.shared.auth import User, get_current_user_from_session from apis.shared.models.managed_models import list_all_managed_models +from apis.app_api.admin.services.model_icons import ModelIconError, read_model_icon from apis.app_api.admin.services.model_access import ( ModelAccessService, get_model_access_service, @@ -88,3 +89,48 @@ async def list_models_for_user( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error listing models: {str(e)}" ) + + +@router.get("/{model_id}/icon") +async def get_model_icon( + model_id: str, + request: Request, + current_user: User = Depends(get_current_user_from_session), +): + """Serve a model's uploaded icon bytes. + + User-facing, not admin-facing: this renders in the chat model picker for + everyone. Signed-in is the whole check — the catalog already hands every user + this model's name, provider and pricing, so its logo discloses nothing new. + + The object is immutable — its key *is* its content digest — so this answers + with a one-year ``immutable`` directive and the digest as the ETag. A + replacement changes ``iconUrl``'s ``?v=``, which is what busts the cache; the + ``If-None-Match`` 304 below is for the same URL being asked for twice. + + A missing icon is a 404, so the SPA falls through to the model's ``iconSlug`` + (or its provider-name match) rather than rendering a broken tile. + """ + try: + data, content_type, version = await read_model_icon(model_id) + except ModelIconError as e: + raise HTTPException(status_code=e.status_code, detail=e.message) + except Exception as e: + logger.error(f"Error reading model icon: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to read model icon: {str(e)}") + + etag = f'"{version}"' + # `immutable` is only true of the VERSIONED url. `?v=` names one + # specific object and can never mean anything else, so a year is right. The + # bare path tracks whatever the record points at now — promising a year for + # that pins a replaced or removed icon in every cache that saw it, and the + # removal simply never becomes visible. Revalidating costs a 304 against the + # ETag below, which is the same round trip the versioned url avoids anyway. + if request.query_params.get("v") == version: + cache_control = "public, max-age=31536000, immutable" + else: + cache_control = "no-cache" + headers = {"Cache-Control": cache_control, "ETag": etag} + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers=headers) + return Response(content=data, media_type=content_type, headers=headers) diff --git a/backend/src/apis/app_api/sessions/services/metadata.py b/backend/src/apis/app_api/sessions/services/metadata.py deleted file mode 100644 index 59aa2cfaa..000000000 --- a/backend/src/apis/app_api/sessions/services/metadata.py +++ /dev/null @@ -1,1060 +0,0 @@ -"""Metadata storage service for messages and conversations - -This service handles storing message metadata (token usage, latency) after -streaming completes. It uses DynamoDB for all storage operations. - -Architecture: -- Cloud: Stores metadata in DynamoDB table specified by DYNAMODB_SESSIONS_METADATA_TABLE_NAME -""" - -import logging -import json -import os -import base64 -from typing import Optional, Tuple, Any, Dict, Union -from decimal import Decimal - -from apis.shared.sessions.models import MessageMetadata, SessionMetadata - -logger = logging.getLogger(__name__) - - -def _convert_floats_to_decimal(obj: Any) -> Any: - """ - Recursively convert floats to Decimal for DynamoDB - - DynamoDB doesn't support float type, requires Decimal instead. - """ - if isinstance(obj, float): - return Decimal(str(obj)) - elif isinstance(obj, dict): - return {k: _convert_floats_to_decimal(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [_convert_floats_to_decimal(item) for item in obj] - else: - return obj - - -def _convert_decimal_to_float(obj: Any) -> Any: - """ - Recursively convert Decimal to float for JSON serialization - - DynamoDB returns Decimal objects, which need to be converted back to float. - """ - if isinstance(obj, Decimal): - return float(obj) - elif isinstance(obj, dict): - return {k: _convert_decimal_to_float(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [_convert_decimal_to_float(item) for item in obj] - else: - return obj - - -async def store_message_metadata( - session_id: str, - user_id: str, - message_id: Union[int, str], - message_metadata: MessageMetadata -) -> None: - """ - Store message metadata after streaming completes - - This function updates DynamoDB records with message cost and usage metadata. - - Args: - session_id: Session identifier - user_id: User identifier - message_id: Message index or namespaced key (e.g. 1 for default, "voice:1" for voice) - message_metadata: MessageMetadata object to store - - Note: - This should be called AFTER the session manager flushes messages, - ensuring the message file exists before we try to update it. - """ - sessions_metadata_table = os.environ.get('DYNAMODB_SESSIONS_METADATA_TABLE_NAME') - if not sessions_metadata_table: - raise RuntimeError("DYNAMODB_SESSIONS_METADATA_TABLE_NAME environment variable is required") - - await _store_message_metadata_cloud( - session_id=session_id, - user_id=user_id, - message_id=message_id, - message_metadata=message_metadata, - table_name=sessions_metadata_table - ) - - -async def _store_message_metadata_cloud( - session_id: str, - user_id: str, - message_id: Union[int, str], - message_metadata: MessageMetadata, - table_name: str -) -> None: - """ - Store message metadata (cost record) in DynamoDB and update cost summary - - This stores cost/usage data as a separate record with C# prefix SK pattern. - Cost records are independent of session records and persist even when sessions - are deleted (for audit trail and billing accuracy). - - Args: - session_id: Session identifier - user_id: User identifier - message_id: Message number (stored as attribute, not in SK) - message_metadata: MessageMetadata to store - table_name: DynamoDB table name from DYNAMODB_SESSIONS_METADATA_TABLE_NAME env var - - Schema: - PK: USER#{user_id} - SK: C#{timestamp}#{uuid} - - GSI1: UserTimestampIndex (time-range queries by user) - GSI1PK: USER#{user_id} - GSI1SK: {timestamp} - - GSI2: SessionLookupIndex (per-session cost queries) - GSI_PK: SESSION#{session_id} - GSI_SK: C#{timestamp} - - Benefits: - - Clean separation from session records (S# prefix) - - Time-ordered by default - - Unique SK via UUID prevents collisions - - Per-session cost queries via SessionLookupIndex GSI - - Time-range queries via UserTimestampIndex GSI - - TTL only affects cost records (sessions don't have ttl) - """ - try: - import boto3 - import uuid as uuid_lib - from datetime import datetime, timezone, timedelta - - dynamodb = boto3.resource('dynamodb') - table = dynamodb.Table(table_name) - - # Prepare item for DynamoDB - metadata_dict = message_metadata.model_dump(by_alias=True, exclude_none=True) - - # Convert floats to Decimal for DynamoDB compatibility - metadata_decimal = _convert_floats_to_decimal(metadata_dict) - - # Extract timestamp for SK and GSI - timestamp = metadata_dict.get("attribution", {}).get("timestamp", datetime.now(timezone.utc).isoformat()) - - # Generate unique ID for SK to prevent collisions - unique_id = str(uuid_lib.uuid4()) - - # Calculate TTL (365 days from now, matching AgentCore Memory retention) - # Only cost records have TTL - sessions persist until soft-deleted - ttl = int((datetime.now(timezone.utc) + timedelta(days=365)).timestamp()) - - # Build item with new SK pattern - item = { - # Primary key with C# prefix for cost records - "PK": f"USER#{user_id}", - "SK": f"C#{timestamp}#{unique_id}", - - # GSI1 keys for UserTimestampIndex - enables time-range queries across all user messages - "GSI1PK": f"USER#{user_id}", - "GSI1SK": timestamp, - - # GSI keys for SessionLookupIndex - enables per-session cost queries - "GSI_PK": f"SESSION#{session_id}", - "GSI_SK": f"C#{timestamp}", - - # Session reference (for linking back to session) - "sessionId": session_id, - "messageId": message_id, - - # Attribution - "userId": user_id, - "timestamp": timestamp, - - # TTL - only cost records have this attribute - "ttl": ttl, - - # Cost and usage metadata - **metadata_decimal - } - - # Store in DynamoDB - table.put_item(Item=item) - - logger.info(f"💾 Stored cost record in DynamoDB table {table_name}") - logger.info(f" Session: {session_id}, Message: {message_id}, SK: C#{timestamp}#{unique_id[:8]}...") - - # Update pre-aggregated cost summary for fast quota checks - # This is done asynchronously and non-blocking - failures don't affect the main flow - await _update_cost_summary_async( - user_id=user_id, - timestamp=timestamp, - message_metadata=message_metadata - ) - - except Exception as e: - logger.error(f"Failed to store message metadata in DynamoDB: {e}", exc_info=True) - # Don't raise - metadata storage failures shouldn't break the app - - -async def _update_cost_summary_async( - user_id: str, - timestamp: str, - message_metadata: MessageMetadata -) -> None: - """ - Update pre-aggregated cost summary (async, non-blocking) - - This atomically increments the user's cost summary in DynamoDB for <10ms quota checks. - Uses atomic ADD operations for concurrent safety. - Also updates per-model breakdown and calculates cache savings. - - Additionally triggers system-wide rollup updates (async, fire-and-forget) for: - - Daily rollups (ROLLUP#DAILY) - - Monthly rollups (ROLLUP#MONTHLY) - - Per-model rollups (ROLLUP#MODEL) - - Args: - user_id: User identifier - timestamp: ISO timestamp of the message - message_metadata: MessageMetadata containing cost, usage, and model info - """ - try: - import asyncio - from datetime import datetime - - # Extract cost from metadata — may be a float (legacy) or a breakdown dict - raw_cost = message_metadata.cost - if isinstance(raw_cost, dict): - cost = raw_cost.get("total", 0.0) - else: - cost = raw_cost or 0.0 - token_usage = message_metadata.token_usage - - usage_delta = {} - cache_read_tokens = 0 - if token_usage: - cache_read_tokens = token_usage.cache_read_input_tokens or 0 - usage_delta = { - "inputTokens": token_usage.input_tokens or 0, - "outputTokens": token_usage.output_tokens or 0, - "cacheReadInputTokens": cache_read_tokens, - "cacheWriteInputTokens": token_usage.cache_write_input_tokens or 0, - } - - # Extract model info for per-model breakdown - model_id = None - model_name = None - provider = None - if message_metadata.model_info: - model_id = message_metadata.model_info.model_id - model_name = message_metadata.model_info.model_name - provider = message_metadata.model_info.provider - - # Calculate cache savings from pricing snapshot - # Savings = (cache_read_tokens * input_price) - (cache_read_tokens * cache_read_price) - cache_savings = 0.0 - if cache_read_tokens > 0: - logger.debug(f"🔍 Cache savings calculation: cache_read_tokens={cache_read_tokens}") - if message_metadata.model_info: - pricing = message_metadata.model_info.pricing_snapshot - logger.debug(f"🔍 Pricing snapshot: {pricing}") - if pricing: - # Get pricing values (handle both dict and Pydantic model) - if hasattr(pricing, 'model_dump'): - pricing_dict = pricing.model_dump(by_alias=True) - else: - pricing_dict = pricing - - logger.debug(f"🔍 Pricing dict: {pricing_dict}") - - input_price = pricing_dict.get("inputPricePerMtok", 0) - cache_read_price = pricing_dict.get("cacheReadPricePerMtok", 0) - - # Calculate savings: what we would have paid vs what we actually paid - standard_cost = (cache_read_tokens / 1_000_000) * input_price - actual_cache_cost = (cache_read_tokens / 1_000_000) * cache_read_price - cache_savings = standard_cost - actual_cache_cost - - logger.info( - f"💰 Cache savings: ${cache_savings:.6f} " - f"({cache_read_tokens:,} tokens @ input=${input_price}/Mtok vs cache_read=${cache_read_price}/Mtok, " - f"standard_cost=${standard_cost:.6f}, actual_cache_cost=${actual_cache_cost:.6f})" - ) - else: - logger.warning(f"⚠️ No pricing snapshot available for cache savings calculation") - else: - logger.warning(f"⚠️ No model_info available for cache savings calculation") - - # Determine period key from timestamp (YYYY-MM format) and date (YYYY-MM-DD) - try: - dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) - period = dt.strftime('%Y-%m') - date = dt.strftime('%Y-%m-%d') - except (ValueError, AttributeError): - # Fallback to current month/day if timestamp parsing fails - from datetime import timezone - now = datetime.now(timezone.utc) - period = now.strftime('%Y-%m') - date = now.strftime('%Y-%m-%d') - - # Use storage abstraction for the atomic update - from apis.shared.storage import get_metadata_storage - storage = get_metadata_storage() - - await storage.update_user_cost_summary( - user_id=user_id, - period=period, - cost_delta=cost, - usage_delta=usage_delta, - timestamp=timestamp, - model_id=model_id, - model_name=model_name, - cache_savings_delta=cache_savings, - provider=provider - ) - - model_info_str = f", model={model_id}" if model_id else "" - savings_str = f", savings=${cache_savings:.6f}" if cache_savings > 0 else "" - logger.info(f"📊 Updated cost summary: user={user_id}, period={period}, cost=${cost:.6f}{model_info_str}{savings_str}") - - # Fire-and-forget: Update system-wide rollups asynchronously - # These updates don't block the main request flow - asyncio.create_task( - _update_system_rollups_async( - user_id=user_id, - period=period, - date=date, - cost=cost, - usage_delta=usage_delta, - cache_savings=cache_savings, - model_id=model_id, - model_name=model_name, - provider=provider - ) - ) - - except Exception as e: - # Log but don't raise - cost summary updates shouldn't break the app - logger.error(f"Failed to update cost summary: {e}", exc_info=True) - - -async def _update_system_rollups_async( - user_id: str, - period: str, - date: str, - cost: float, - usage_delta: dict, - cache_savings: float, - model_id: str | None, - model_name: str | None, - provider: str | None -) -> None: - """ - Update system-wide rollups for admin dashboard (async, fire-and-forget) - - This updates: - - Daily rollup (ROLLUP#DAILY, SK: YYYY-MM-DD) - - Monthly rollup (ROLLUP#MONTHLY, SK: YYYY-MM) - - Per-model rollup (ROLLUP#MODEL, SK: YYYY-MM#model_id) - - These updates are non-blocking and failures don't affect the main request flow. - The rollups support the admin cost dashboard with pre-aggregated system-wide metrics. - - Args: - user_id: User identifier (for tracking unique active users) - period: Monthly period (YYYY-MM) - date: Daily date (YYYY-MM-DD) - cost: Cost delta to add - usage_delta: Token usage delta - cache_savings: Cache savings delta - model_id: Model identifier - model_name: Human-readable model name - provider: LLM provider - """ - try: - # Check if we're using DynamoDB storage (rollups only make sense in cloud mode) - system_rollup_table = os.environ.get("DYNAMODB_SYSTEM_ROLLUP_TABLE_NAME") - if not system_rollup_table: - logger.debug("System rollup table not configured, skipping rollup updates") - return - - from apis.shared.storage.dynamodb_storage import DynamoDBStorage - storage = DynamoDBStorage() - - # Track active users using conditional writes - # Returns (is_new_today, is_new_this_month) - True if first request for that period - is_new_today, is_new_this_month = await storage.track_active_user( - user_id=user_id, - period=period, - date=date - ) - - # Update daily rollup - await storage.update_daily_rollup( - date=date, - cost_delta=cost, - usage_delta=usage_delta, - is_new_user=is_new_today, - model_id=model_id - ) - - # Update monthly rollup - await storage.update_monthly_rollup( - period=period, - cost_delta=cost, - usage_delta=usage_delta, - cache_savings_delta=cache_savings, - is_new_user=is_new_this_month, - model_id=model_id - ) - - # Update per-model rollup if model info is available - if model_id and model_name and provider: - # Track active users per model separately (user may use multiple models) - is_new_user_for_model = await storage.track_active_user_for_model( - user_id=user_id, - period=period, - model_id=model_id - ) - - await storage.update_model_rollup( - period=period, - model_id=model_id, - model_name=model_name, - provider=provider, - cost_delta=cost, - usage_delta=usage_delta, - is_new_user_for_model=is_new_user_for_model - ) - - logger.debug(f"📈 Updated system rollups: date={date}, period={period}, new_today={is_new_today}, new_month={is_new_this_month}") - - except Exception as e: - # Log but don't raise - rollup updates are supplementary - logger.error(f"Failed to update system rollups: {e}", exc_info=True) - - -async def store_session_metadata( - session_id: str, - user_id: str, - session_metadata: SessionMetadata -) -> None: - """ - Store or update session metadata - - This function creates or updates session metadata in DynamoDB. - - Args: - session_id: Session identifier - user_id: User identifier - session_metadata: SessionMetadata object to store - - Note: - This performs a deep merge - existing fields are preserved unless - explicitly overwritten by new values. - """ - sessions_metadata_table = os.environ.get('DYNAMODB_SESSIONS_METADATA_TABLE_NAME') - if not sessions_metadata_table: - raise RuntimeError("DYNAMODB_SESSIONS_METADATA_TABLE_NAME environment variable is required") - - await _store_session_metadata_cloud( - session_id=session_id, - user_id=user_id, - session_metadata=session_metadata, - table_name=sessions_metadata_table - ) - - -async def _store_session_metadata_cloud( - session_id: str, - user_id: str, - session_metadata: SessionMetadata, - table_name: str -) -> None: - """ - Store session metadata in DynamoDB with new SK pattern - - This creates or updates the session record in DynamoDB. - For updates where last_message_at changes, the record is moved (delete old, put new) - because the SK contains the timestamp. - - Args: - session_id: Session identifier - user_id: User identifier - session_metadata: SessionMetadata to store - table_name: DynamoDB table name from DYNAMODB_SESSIONS_METADATA_TABLE_NAME env var - - Schema: - PK: USER#{user_id} - SK: S#ACTIVE#{last_message_at}#{session_id} (active sessions) - S#DELETED#{deleted_at}#{session_id} (deleted sessions) - - GSI: SessionLookupIndex - GSI_PK: SESSION#{session_id} - GSI_SK: META - - This allows: - - Querying all active sessions: begins_with(SK, 'S#ACTIVE#') - - Sessions sorted by timestamp in SK (no in-memory sorting needed) - - Direct session lookup via GSI - """ - try: - import boto3 - from botocore.exceptions import ClientError - from datetime import datetime, timezone - - dynamodb = boto3.resource('dynamodb') - table = dynamodb.Table(table_name) - - # First, check if session exists via GSI to get current SK - existing_session = await _get_session_by_gsi(session_id, user_id, table) - - # Prepare item for DynamoDB - item = session_metadata.model_dump(by_alias=True, exclude_none=True) - - # Convert floats to Decimal for DynamoDB compatibility - item = _convert_floats_to_decimal(item) - - # Determine SK based on session status - last_message_at = session_metadata.last_message_at or datetime.now(timezone.utc).isoformat() - - if session_metadata.deleted: - deleted_at = session_metadata.deleted_at or datetime.now(timezone.utc).isoformat() - new_sk = f"S#DELETED#{deleted_at}#{session_id}" - else: - new_sk = f"S#ACTIVE#{last_message_at}#{session_id}" - - # Build primary key - pk = f'USER#{user_id}' - - # Add GSI keys for direct lookup - item['GSI_PK'] = f'SESSION#{session_id}' - item['GSI_SK'] = 'META' - - if existing_session: - # Session exists - check if SK needs to change - old_sk = existing_session.get('SK') - - if old_sk and old_sk != new_sk: - # SK changed (timestamp updated) - need transactional move - # Deep merge existing with new data - merged_item = _deep_merge( - {k: v for k, v in existing_session.items() if k not in ['PK', 'SK']}, - item - ) - merged_item['PK'] = pk - merged_item['SK'] = new_sk - - # Move session: put new SK first, then delete old SK - # Using high-level Table API (put_item + delete_item) instead of - # transact_write_items to avoid low-level serialization issues - logger.debug(f"🔄 Moving session: old_sk={old_sk[:50]}..., new_sk={new_sk[:50]}...") - try: - # Convert floats to Decimal for DynamoDB compatibility - decimal_item = _convert_floats_to_decimal(merged_item) - - # Put new item first — if this fails, original is untouched - table.put_item(Item=decimal_item) - # Delete old item - table.delete_item(Key={'PK': pk, 'SK': old_sk}) - logger.info(f"💾 Moved session metadata in DynamoDB (SK changed)") - except Exception as move_error: - logger.error(f"Session move failed - PK={pk}, old_SK={old_sk}, new_SK={new_sk}") - logger.error(f"Move error: {move_error}") - raise - else: - # SK unchanged - simple update with deep merge - # Build update expression for partial update - update_expression_parts = [] - expression_attribute_names = {} - expression_attribute_values = {} - - for key_name, value in item.items(): - # Skip keys that are part of the primary key or GSI - if key_name in ['sessionId', 'userId', 'PK', 'SK']: - continue - - placeholder_name = f"#{key_name}" - placeholder_value = f":{key_name}" - - update_expression_parts.append(f"{placeholder_name} = {placeholder_value}") - expression_attribute_names[placeholder_name] = key_name - expression_attribute_values[placeholder_value] = value - - if update_expression_parts: - update_expression = "SET " + ", ".join(update_expression_parts) - table.update_item( - Key={'PK': pk, 'SK': old_sk}, - UpdateExpression=update_expression, - ExpressionAttributeNames=expression_attribute_names, - ExpressionAttributeValues=expression_attribute_values - ) - logger.info(f"💾 Updated session metadata in DynamoDB table {table_name}") - else: - # New session - create with put_item - item['PK'] = pk - item['SK'] = new_sk - table.put_item(Item=item) - logger.info(f"💾 Created session metadata in DynamoDB table {table_name}") - - logger.info(f" Session: {session_id}, User: {user_id}") - - except Exception as e: - logger.error(f"Failed to store session metadata in DynamoDB: {e}", exc_info=True) - # Don't raise - metadata storage failures shouldn't break the app - - -async def _get_session_by_gsi(session_id: str, user_id: str, table) -> Optional[dict]: - """ - Get session record using GSI (SessionLookupIndex) - - This allows looking up a session by ID without knowing its SK (which contains timestamp). - - Args: - session_id: Session identifier - user_id: User identifier (for ownership verification) - table: DynamoDB table resource - - Returns: - Raw DynamoDB item dict if found, None otherwise - """ - try: - from boto3.dynamodb.conditions import Key - - response = table.query( - IndexName='SessionLookupIndex', - KeyConditionExpression=Key('GSI_PK').eq(f'SESSION#{session_id}') & Key('GSI_SK').eq('META') - ) - - items = response.get('Items', []) - if not items: - return None - - item = items[0] - - # Verify user ownership - if item.get('userId') != user_id: - logger.warning(f"Session {session_id} belongs to different user") - return None - - return _convert_decimal_to_float(item) - - except Exception as e: - # GSI might not exist yet - fall back to None - logger.debug(f"GSI lookup failed (may not exist yet): {e}") - return None - - - -async def get_session_metadata(session_id: str, user_id: str) -> Optional[SessionMetadata]: - """ - Retrieve session metadata - - Args: - session_id: Session identifier - user_id: User identifier - - Returns: - SessionMetadata object if found, None otherwise - """ - sessions_metadata_table = os.environ.get('DYNAMODB_SESSIONS_METADATA_TABLE_NAME') - if not sessions_metadata_table: - raise RuntimeError("DYNAMODB_SESSIONS_METADATA_TABLE_NAME environment variable is required") - - return await _get_session_metadata_cloud( - session_id=session_id, - user_id=user_id, - table_name=sessions_metadata_table - ) - - -async def get_all_message_metadata(session_id: str, user_id: str) -> Dict[str, Any]: - """ - Retrieve all message metadata for a session. - - Queries the DynamoDB table for all records matching the session_id prefix - in the sort key. - - Returns: - Dictionary mapping message_id (str) to metadata dict - """ - sessions_metadata_table = os.environ.get('DYNAMODB_SESSIONS_METADATA_TABLE_NAME') - if not sessions_metadata_table: - raise RuntimeError("DYNAMODB_SESSIONS_METADATA_TABLE_NAME environment variable is required") - - return await _get_all_message_metadata_cloud(session_id, user_id, sessions_metadata_table) - - -async def _get_all_message_metadata_cloud(session_id: str, user_id: str, table_name: str) -> Dict[str, Any]: - """ - Retrieve all message metadata (cost records + display text) for a session from DynamoDB - - Uses the SessionLookupIndex GSI to query records by session ID. - Cost records have SK pattern: C#{timestamp}#{uuid}, GSI_SK: C#{timestamp} - Display text records have SK pattern: D#{session_id}#{message_id}, GSI_SK: D#{message_id} - - Args: - session_id: Session identifier - user_id: User identifier - table_name: DynamoDB table name - - Returns: - Dictionary mapping message_id (str) to metadata dict - """ - try: - import boto3 - from boto3.dynamodb.conditions import Key - - dynamodb = boto3.resource('dynamodb') - table = dynamodb.Table(table_name) - - logger.info(f"🔍 Querying cost records via GSI for session {session_id}") - - # Query cost records (C#) and display text records (D#) in parallel - cost_response = table.query( - IndexName='SessionLookupIndex', - KeyConditionExpression=Key('GSI_PK').eq(f'SESSION#{session_id}') & Key('GSI_SK').begins_with('C#') - ) - display_response = table.query( - IndexName='SessionLookupIndex', - KeyConditionExpression=Key('GSI_PK').eq(f'SESSION#{session_id}') & Key('GSI_SK').begins_with('D#') - ) - - items = cost_response.get("Items", []) - display_items = display_response.get("Items", []) - metadata_index = {} - - logger.info(f"📦 DynamoDB returned {len(items)} cost record items, {len(display_items)} display text items") - - for item in items: - # Verify user ownership - if item.get('userId') != user_id: - logger.warning(f"Cost record belongs to different user, skipping") - continue - - # Convert Decimal to float - item_float = _convert_decimal_to_float(item) - - # Extract message_id as integer (DynamoDB returns Decimal, convert to int then str) - # Must convert to int first to avoid "0.0" -> "0" mismatch - message_id_raw = item_float.get("messageId") - message_id = str(int(message_id_raw)) if isinstance(message_id_raw, (int, float)) else str(message_id_raw) - - logger.debug(f"Processing cost record for message_id={message_id}, SK={item_float.get('SK')}") - - # Remove DynamoDB-specific keys and top-level fields not needed in metadata dict - for key in ["PK", "SK", "GSI_PK", "GSI_SK", "ttl", "userId", "sessionId", "messageId", "timestamp"]: - item_float.pop(key, None) - - metadata_index[message_id] = item_float - - logger.info(f"📂 Retrieved {len(metadata_index)} cost records from DynamoDB") - - # Merge displayText from D# records into metadata index - for item in display_items: - if item.get('userId') != user_id: - continue - item_float = _convert_decimal_to_float(item) - message_id_raw = item_float.get("messageId") - message_id = str(int(message_id_raw)) if isinstance(message_id_raw, (int, float)) else str(message_id_raw) - display_text = item_float.get("displayText") - if display_text: - if message_id in metadata_index: - metadata_index[message_id]["displayText"] = display_text - else: - metadata_index[message_id] = {"displayText": display_text} - logger.debug(f"🔗 Merged displayText for user message {message_id}") - - logger.info(f"📋 Metadata keys: {sorted(metadata_index.keys())}") - return metadata_index - - except Exception as e: - logger.error(f"Failed to query message metadata from DynamoDB: {e}", exc_info=True) - return {} - - -async def _get_session_metadata_cloud( - session_id: str, - user_id: str, - table_name: str -) -> Optional[SessionMetadata]: - """ - Retrieve session metadata from DynamoDB using GSI - - With the new SK pattern (S#ACTIVE#{last_message_at}#{session_id}), we can't - use get_item directly because we don't know the last_message_at timestamp. - Instead, we use the SessionLookupIndex GSI for direct session lookup by ID. - - Args: - session_id: Session identifier - user_id: User identifier - table_name: DynamoDB table name - - Returns: - SessionMetadata object if found, None otherwise - - Schema: - GSI: SessionLookupIndex - GSI_PK: SESSION#{session_id} - GSI_SK: META - - This allows looking up sessions by ID without knowing the timestamp. - """ - try: - import boto3 - from boto3.dynamodb.conditions import Key - - dynamodb = boto3.resource('dynamodb') - table = dynamodb.Table(table_name) - - # Use GSI for session lookup by ID - response = table.query( - IndexName='SessionLookupIndex', - KeyConditionExpression=Key('GSI_PK').eq(f'SESSION#{session_id}') & Key('GSI_SK').eq('META') - ) - - items = response.get('Items', []) - if not items: - logger.info(f"Session metadata not found in DynamoDB: {session_id}") - return None - - item = items[0] - - # Verify user ownership - if item.get('userId') != user_id: - logger.warning(f"Session {session_id} belongs to different user") - return None - - # Convert Decimal to float for JSON serialization - item = _convert_decimal_to_float(item) - - # Remove DynamoDB keys before validation - for key in ['PK', 'SK', 'GSI_PK', 'GSI_SK']: - item.pop(key, None) - - return SessionMetadata.model_validate(item) - - except Exception as e: - logger.error(f"Failed to retrieve session metadata from DynamoDB: {e}", exc_info=True) - return None - - -def _apply_pagination( - sessions: list[SessionMetadata], - limit: Optional[int] = None, - next_token: Optional[str] = None -) -> Tuple[list[SessionMetadata], Optional[str]]: - """ - Apply pagination to a list of sessions - - Args: - sessions: List of sessions (should be sorted by last_message_at descending) - limit: Maximum number of sessions to return - next_token: Pagination token (base64-encoded last_message_at timestamp to start from) - - Returns: - Tuple of (paginated sessions, next_token if more sessions exist) - """ - start_index = 0 - - # Decode next_token if provided (it's a base64-encoded last_message_at timestamp) - if next_token: - try: - decoded = base64.b64decode(next_token).decode('utf-8') - # Find the index of the first session with last_message_at < decoded timestamp - # This skips all sessions with the same timestamp as the token (to avoid duplicates) - for idx, session in enumerate(sessions): - if session.last_message_at < decoded: - start_index = idx - break - else: - # If no session found with timestamp < decoded, we've reached the end - start_index = len(sessions) - except Exception as e: - logger.warning(f"Invalid next_token: {e}, starting from beginning") - start_index = 0 - - # Apply start index - paginated_sessions = sessions[start_index:] - - # Apply limit - if limit and limit > 0: - paginated_sessions = paginated_sessions[:limit] - # Check if there are more sessions - if start_index + limit < len(sessions): - # Use the last_message_at of the last session in this page as the next token - last_session = paginated_sessions[-1] - next_token = base64.b64encode(last_session.last_message_at.encode('utf-8')).decode('utf-8') - else: - next_token = None - else: - next_token = None - - return paginated_sessions, next_token - - -async def list_user_sessions( - user_id: str, - limit: Optional[int] = None, - next_token: Optional[str] = None -) -> Tuple[list[SessionMetadata], Optional[str]]: - """ - List sessions for a user with pagination support - - Args: - user_id: User identifier - limit: Maximum number of sessions to return (optional) - next_token: Pagination token for retrieving next page (optional) - - Returns: - Tuple of (list of SessionMetadata objects, next_token if more sessions exist) - Sessions are sorted by last_message_at descending (most recent first) - """ - sessions_metadata_table = os.environ.get('DYNAMODB_SESSIONS_METADATA_TABLE_NAME') - if not sessions_metadata_table: - raise RuntimeError("DYNAMODB_SESSIONS_METADATA_TABLE_NAME environment variable is required") - - return await _list_user_sessions_cloud( - user_id=user_id, - table_name=sessions_metadata_table, - limit=limit, - next_token=next_token - ) - - -async def _list_user_sessions_cloud( - user_id: str, - table_name: str, - limit: Optional[int] = None, - next_token: Optional[str] = None -) -> Tuple[list[SessionMetadata], Optional[str]]: - """ - List active sessions for a user from DynamoDB with efficient pagination - - Args: - user_id: User identifier - table_name: DynamoDB table name - limit: Maximum number of sessions to return (optional) - next_token: Pagination token for retrieving next page (optional) - - Returns: - Tuple of (list of SessionMetadata objects, next_token if more sessions exist) - Sessions are sorted by last_message_at descending (most recent first) - - Schema: - PK: USER#{user_id} - SK: S#ACTIVE#{last_message_at}#{session_id} - - Performance improvements over old schema: - - Query only returns session records (no cost records with C# prefix) - - No in-memory filtering needed - - Sessions sorted by timestamp in SK (no in-memory sorting) - - True server-side pagination via DynamoDB's native mechanism - - O(page_size) instead of O(sessions + messages) - """ - try: - import boto3 - from boto3.dynamodb.conditions import Key - - dynamodb = boto3.resource('dynamodb') - table = dynamodb.Table(table_name) - - # Decode next_token to get ExclusiveStartKey if provided - exclusive_start_key = None - if next_token: - try: - decoded = base64.b64decode(next_token).decode('utf-8') - exclusive_start_key = json.loads(decoded) - except Exception as e: - logger.warning(f"Invalid next_token: {e}") - - # Build query parameters with new S#ACTIVE# prefix - # This cleanly separates from: - # - S#DELETED# (soft-deleted sessions) - # - C# (cost records) - query_params = { - 'KeyConditionExpression': Key('PK').eq(f'USER#{user_id}') & Key('SK').begins_with('S#ACTIVE#'), - 'ScanIndexForward': False # Descending order (most recent first) - timestamp is in SK! - } - - if exclusive_start_key: - query_params['ExclusiveStartKey'] = exclusive_start_key - - if limit: - # With new schema, we can use exact limit - no over-fetching needed - query_params['Limit'] = limit - - # Execute query - response = table.query(**query_params) - - # Parse items - no filtering needed, all items are active sessions - sessions = [] - for item in response['Items']: - try: - # Convert Decimal to float - item = _convert_decimal_to_float(item) - - # Remove DynamoDB keys - for key in ['PK', 'SK', 'GSI_PK', 'GSI_SK']: - item.pop(key, None) - - metadata = SessionMetadata.model_validate(item) - sessions.append(metadata) - except Exception as e: - logger.warning(f"Failed to parse session item: {e}") - continue - - # No in-memory sorting needed! With new SK pattern S#ACTIVE#{last_message_at}#{session_id}, - # DynamoDB returns items already sorted by timestamp (via ScanIndexForward=False) - - # No limit trimming needed either - we use exact Limit in query params - - # Generate next_token from LastEvaluatedKey if present. - # Only return a next_token when we actually filled the requested page. - # DynamoDB's Limit caps items *evaluated*, so filtered-out items - # (parse failures) can cause fewer results than the limit while still - # producing a LastEvaluatedKey. Returning that token to the client - # makes it show a "Load More" button with nothing left to load. - next_page_token = None - if 'LastEvaluatedKey' in response and (not limit or len(sessions) >= limit): - next_page_token = base64.b64encode( - json.dumps(response['LastEvaluatedKey']).encode('utf-8') - ).decode('utf-8') - - logger.info(f"Listed {len(sessions)} sessions for user {user_id} from DynamoDB") - - return sessions, next_page_token - - except Exception as e: - logger.error(f"Failed to list user sessions from DynamoDB: {e}", exc_info=True) - return [], None - - -def _deep_merge(base: dict, updates: dict) -> dict: - """ - Deep merge two dictionaries - - Args: - base: Base dictionary (existing data) - updates: Updates to apply (new data) - - Returns: - Merged dictionary - - Note: - Updates take precedence. Nested dictionaries are merged recursively. - """ - result = base.copy() - - for key, value in updates.items(): - if key in result and isinstance(result[key], dict) and isinstance(value, dict): - # Recursively merge nested dictionaries - result[key] = _deep_merge(result[key], value) - else: - # Overwrite with new value - result[key] = value - - return result - diff --git a/backend/src/apis/app_api/sessions/tests/test_cache_savings.py b/backend/src/apis/app_api/sessions/tests/test_cache_savings.py index 54c0e747a..58c4a07bc 100644 --- a/backend/src/apis/app_api/sessions/tests/test_cache_savings.py +++ b/backend/src/apis/app_api/sessions/tests/test_cache_savings.py @@ -76,7 +76,7 @@ async def test_cache_savings_calculation(self, mock_storage, sample_message_meta 'apis.shared.storage.get_metadata_storage', return_value=mock_storage ): - from apis.app_api.sessions.services.metadata import _update_cost_summary_async + from apis.shared.sessions.metadata import _update_cost_summary_async # Call the function await _update_cost_summary_async( @@ -141,7 +141,7 @@ async def test_cache_savings_zero_when_no_cache_reads(self, mock_storage): 'apis.shared.storage.get_metadata_storage', return_value=mock_storage ): - from apis.app_api.sessions.services.metadata import _update_cost_summary_async + from apis.shared.sessions.metadata import _update_cost_summary_async await _update_cost_summary_async( user_id="test_user", @@ -183,7 +183,7 @@ async def test_cache_savings_zero_when_no_pricing_snapshot(self, mock_storage): 'apis.shared.storage.get_metadata_storage', return_value=mock_storage ): - from apis.app_api.sessions.services.metadata import _update_cost_summary_async + from apis.shared.sessions.metadata import _update_cost_summary_async await _update_cost_summary_async( user_id="test_user", @@ -233,7 +233,7 @@ async def test_cache_savings_large_cache_hit(self, mock_storage): 'apis.shared.storage.get_metadata_storage', return_value=mock_storage ): - from apis.app_api.sessions.services.metadata import _update_cost_summary_async + from apis.shared.sessions.metadata import _update_cost_summary_async await _update_cost_summary_async( user_id="test_user", @@ -290,7 +290,7 @@ async def test_cache_savings_with_haiku_pricing(self, mock_storage): 'apis.shared.storage.get_metadata_storage', return_value=mock_storage ): - from apis.app_api.sessions.services.metadata import _update_cost_summary_async + from apis.shared.sessions.metadata import _update_cost_summary_async await _update_cost_summary_async( user_id="test_user", diff --git a/backend/src/apis/app_api/skills/routes.py b/backend/src/apis/app_api/skills/routes.py index 6751eee14..487c5dbb7 100644 --- a/backend/src/apis/app_api/skills/routes.py +++ b/backend/src/apis/app_api/skills/routes.py @@ -42,6 +42,7 @@ from apis.shared.auth import User, get_current_user_from_session from apis.shared.skills.access import resolve_accessible_skill_ids +from apis.shared.skills.bundle import slugify_skill_name from apis.shared.skills.models import ( SkillDefinition, SkillResourceRef, @@ -54,6 +55,7 @@ safe_download_content_type, ) +from .service import get_skill_catalog_service from .user_service import ( UserSkillError, UserSkillLimitError, @@ -75,6 +77,13 @@ class UserSkillResponse(BaseModel): category: Optional[str] = None user_enabled: Optional[bool] = Field(None, alias="userEnabled") is_enabled: bool = Field(..., alias="isEnabled") + # The runtime's activation key for this skill — the same slug the + # ``AgentSkills`` plugin injects as ``Skill.name`` and accepts on its + # ``skills`` tool. Served rather than re-derived client-side so the token + # the composer's `/` menu writes into a message is byte-identical to the + # one the model reads in ````; a slug rule that drifted + # between the two would show the user a command the model cannot resolve. + slug: str model_config = {"populate_by_name": True} @@ -127,6 +136,7 @@ async def get_user_skills( # the two must agree or the UI would show skills as active that the # turn never loads. is_enabled=preferences.get(record.skill_id, False), + slug=slugify_skill_name(record.skill_id), ) for record in records if record.status == SkillStatus.ACTIVE @@ -450,3 +460,153 @@ async def delete_my_skill_resource( raise _resource_value_error(e) return SkillResourcesResponse(skill_id=skill_id, resources=resources) + + +# ----------------------------------------------------------------------------- +# One accessible skill (read-only detail) +# +# ⚠️ REGISTRATION ORDER IS LOAD-BEARING. These routes must stay BELOW every +# ``/mine`` route in this module. Starlette matches in registration order and +# ``SKILL_ID_PATTERN`` happily matches the literal string ``mine`` — declare +# ``/{skill_id}`` first and ``GET /skills/mine`` becomes a lookup for a skill +# called "mine", which 404s for every user in the product. +# ----------------------------------------------------------------------------- + + +class SkillDetailResponse(BaseModel): + """One skill the user can reach, with everything the picker line omits. + + The fat sibling of ``UserSkillResponse``. ``GET /skills/`` stays thin on + purpose — it is a first-load payload, and putting every granted skill's + SKILL.md body on it would buy nothing for the list and cost on every load. + This is fetched once, for the one skill the user opened. + + ``instructions`` is served to anyone the skill is granted to. It is not a + secret from them: it is the text their own turns load on dispatch, so the + page is showing the user what they are already talking to. + + Deliberately absent: ``ownerId`` (``isOwned`` is the only part of it this + surface needs, and a raw owner id would leak one user's identity to + another) and ``allowedAppRoles`` (an admin-display projection — see the + RBAC note in CLAUDE.md — which would expose the role topology to any user + holding the skill). + """ + + skill_id: str = Field(..., alias="skillId") + display_name: str = Field(..., alias="displayName") + description: str + instructions: str = "" + compose: List[str] = Field(default_factory=list) + allowed_tools: List[str] = Field(default_factory=list, alias="allowedTools") + skill_metadata: Dict = Field(default_factory=dict, alias="skillMetadata") + resources: List[SkillResourceRef] = Field(default_factory=list) + status: str = SkillStatus.ACTIVE.value + category: Optional[str] = None + user_enabled: Optional[bool] = Field(None, alias="userEnabled") + is_enabled: bool = Field(..., alias="isEnabled") + #: True when the caller authored this skill — drives the SPA's "Edit in My + #: Skills" affordance. Ownership is its own grant (``access.py``), so this + #: can be true for a user with no skill-granting role at all. + is_owned: bool = Field(..., alias="isOwned") + created_at: Optional[str] = Field(None, alias="createdAt") + updated_at: Optional[str] = Field(None, alias="updatedAt") + + model_config = {"populate_by_name": True} + + +async def _require_accessible_skill(skill_id: str, user: User) -> SkillDefinition: + """Load a skill the user can reach, or raise 404. + + Access is ``resolve_accessible_skill_ids`` — the *same* resolution that + builds the picker and that the runtime uses to decide what a turn may + activate. A skill the user cannot reach and a skill that does not exist + both 404, so this never discloses the existence of a skill someone else + was granted. + + Status is checked too: ``GET /skills/`` filters to ACTIVE, so a drilled-in + DRAFT or DISABLED catalog skill would otherwise be reachable by id from a + surface that never listed it. + """ + accessible = await resolve_accessible_skill_ids(user) + if skill_id not in accessible: + raise HTTPException(status_code=404, detail=f"Skill '{skill_id}' not found") + + skill = await get_skill_catalog_service().get_skill(skill_id) + if skill is None: + raise HTTPException(status_code=404, detail=f"Skill '{skill_id}' not found") + + status = skill.status.value if hasattr(skill.status, "value") else skill.status + if status != SkillStatus.ACTIVE.value and skill.owner_id != user.user_id: + raise HTTPException(status_code=404, detail=f"Skill '{skill_id}' not found") + + return skill + + +@router.get("/{skill_id}", response_model=SkillDetailResponse) +async def get_accessible_skill( + skill_id: str, + user: User = Depends(get_current_user_from_session), +) -> SkillDetailResponse: + """Read one skill the current user can reach, catalog or self-authored.""" + logger.info(f"User {user.name} reading skill '{skill_id}'") + + skill = await _require_accessible_skill(skill_id, user) + + repo = get_skill_catalog_repository() + preferences = (await repo.get_user_preferences(user.user_id)).skill_preferences + + status = skill.status.value if hasattr(skill.status, "value") else skill.status + return SkillDetailResponse( + skill_id=skill.skill_id, + display_name=skill.display_name, + description=skill.description, + instructions=skill.instructions, + compose=list(skill.compose), + allowed_tools=list(skill.allowed_tools), + skill_metadata=dict(skill.skill_metadata), + resources=list(skill.resources), + status=str(status), + category=skill.category, + user_enabled=preferences.get(skill.skill_id), + # Skills v2 D6 opt-in: untouched is OFF. Same default as the picker — + # the two must agree or the page would contradict the list it came from. + is_enabled=preferences.get(skill.skill_id, False), + is_owned=skill.owner_id == user.user_id, + created_at=skill.created_at.isoformat() if skill.created_at else None, + updated_at=skill.updated_at.isoformat() if skill.updated_at else None, + ) + + +@router.get("/{skill_id}/resources/{filename}") +async def read_accessible_skill_resource( + skill_id: str, + filename: str, + user: User = Depends(get_current_user_from_session), +): + """Return the raw bytes of one supporting file on an accessible skill. + + The read counterpart of ``/mine/{id}/resources/{filename}``, scoped by + *access* rather than ownership so a user granted a catalog skill can open + its reference files — the level-3 half of the same progressive disclosure + whose level-2 body this page already renders. Read-only by construction: + there is no accessible-scoped upload or delete. + + Hardened identically to the owner and admin read routes: the media type is + re-derived from the filename and the body is served ``attachment`` + + ``nosniff`` + inert CSP, so a resource can never become a script-bearing + document on the SPA's origin (``apis.shared.skills.resource_types``). + """ + await _require_accessible_skill(skill_id, user) + + try: + ref, content = await get_skill_catalog_service().read_resource( + skill_id, filename + ) + except ValueError as e: + raise _resource_value_error(e) + + return Response( + content=content, + media_type=safe_download_content_type(ref.filename), + headers=resource_download_headers(ref.filename), + ) diff --git a/backend/src/apis/app_api/tools/discovery.py b/backend/src/apis/app_api/tools/discovery.py index 147bc67bb..9fec082d9 100644 --- a/backend/src/apis/app_api/tools/discovery.py +++ b/backend/src/apis/app_api/tools/discovery.py @@ -20,9 +20,24 @@ import asyncio import logging -from typing import List, Optional +from datetime import datetime, timezone +from typing import Dict, List, Optional -from apis.shared.tools.models import DiscoveredMCPTool, ToolDefinition +from apis.shared.tools.models import ( + MAX_CAPABILITY_ENTRIES, + MAX_CAPABILITY_PAGES, + MAX_RESOLVED_PROMPT_CHARS, + MAX_RESOLVED_PROMPT_MESSAGES, + DiscoveredMCPTool, + MCPPromptArgument, + MCPPromptEntry, + MCPResourceEntry, + ResolvedPrompt, + ResolvedPromptMessage, + ToolCapabilitySnapshot, + ToolDefinition, + _clip, +) logger = logging.getLogger(__name__) @@ -100,3 +115,291 @@ def _list_tools(): DiscoveredMCPTool(name=name, description=getattr(spec, "description", None)) ) return discovered + + +# ============================================================================= +# Capability discovery (prompts + resources) +# ============================================================================= + + +def _paginate(list_page, extract) -> tuple[list, bool]: + """Walk an MCP listing's cursor, capped by entries and by pages. + + Returns ``(entries, truncated)``. Two caps, because they fail differently: + a server with thousands of resources would blow the DynamoDB item, and a + server with a broken cursor would loop forever. + """ + entries: list = [] + cursor = None + truncated = False + for _ in range(MAX_CAPABILITY_PAGES): + result = list_page(cursor) + entries.extend(extract(result)) + if len(entries) >= MAX_CAPABILITY_ENTRIES: + entries = entries[:MAX_CAPABILITY_ENTRIES] + truncated = True + break + cursor = getattr(result, "nextCursor", None) + if not cursor: + break + else: + truncated = True + return entries, truncated + + +def _prompt_entries(result) -> List[MCPPromptEntry]: + out: List[MCPPromptEntry] = [] + for prompt in getattr(result, "prompts", None) or []: + name = getattr(prompt, "name", None) + if not name: + continue + out.append( + MCPPromptEntry( + name=name, + title=_clip(getattr(prompt, "title", None)), + description=_clip(getattr(prompt, "description", None)), + arguments=[ + MCPPromptArgument( + name=getattr(arg, "name", ""), + description=_clip(getattr(arg, "description", None)), + required=bool(getattr(arg, "required", False)), + ) + for arg in (getattr(prompt, "arguments", None) or []) + if getattr(arg, "name", None) + ], + ) + ) + return out + + +def _resource_entries(result, *, templates: bool) -> List[MCPResourceEntry]: + out: List[MCPResourceEntry] = [] + source = ( + getattr(result, "resourceTemplates", None) + if templates + else getattr(result, "resources", None) + ) or [] + for resource in source: + # A template carries `uriTemplate`; a concrete resource carries `uri`. + uri = getattr(resource, "uriTemplate", None) if templates else None + uri = uri or getattr(resource, "uri", None) + if not uri: + continue + out.append( + MCPResourceEntry( + uri=str(uri), + name=_clip(getattr(resource, "name", None)), + description=_clip(getattr(resource, "description", None)), + mime_type=getattr(resource, "mimeType", None), + uri_template=templates, + ) + ) + return out + + +async def discover_capabilities_for_saved_tool( + tool: ToolDefinition, + oauth_token: Optional[str] = None, + discovered_by: Optional[str] = None, +) -> ToolCapabilitySnapshot: + """Ask a saved MCP tool what prompts and resources it exposes. + + Each listing is attempted independently and its failure is swallowed into + ``supports_*=False``. A server that implements tools but not prompts answers + ``prompts/list`` with a JSON-RPC "method not found", which is normal and must + not cost us the resources listing — or the whole snapshot. + + A transport-level failure (server unreachable, auth rejected) is different: + nothing was learned, so the snapshot records ``error`` and the caller can + keep showing the previous one rather than replacing it with emptiness. + + Gateway (``mcp``) tools are not probed. The AgentCore Gateway enumerates a + target's *tools* at registration and exposes no prompt or resource surface, + so there is nothing on the other end to ask. + """ + snapshot = ToolCapabilitySnapshot( + tool_id=tool.tool_id, + discovered_at=datetime.now(timezone.utc).isoformat(), + discovered_by=discovered_by, + ) + + if tool.protocol != "mcp_external" or not tool.mcp_config: + snapshot.error = "This tool is not an external MCP server." + return snapshot + + from agents.main_agent.integrations.external_mcp_client import ( + create_external_mcp_client, + ) + + forward = bool(getattr(tool, "forward_auth_token", False)) + client = create_external_mcp_client( + config=tool.mcp_config, + tool_definition=tool, + oauth_token=oauth_token if (forward or tool.requires_oauth_provider) else None, + ) + if client is None: + snapshot.error = "Could not build a client for this server." + return snapshot + + def _probe() -> ToolCapabilitySnapshot: + # One session for both listings — a second connect would double the + # handshake cost and, for a 3LO server, the token round-trip with it. + with client: + try: + prompts, prompts_truncated = _paginate( + lambda cursor: client.list_prompts_sync(pagination_token=cursor), + _prompt_entries, + ) + snapshot.prompts = prompts + snapshot.supports_prompts = True + snapshot.truncated = snapshot.truncated or prompts_truncated + except Exception as exc: # noqa: BLE001 - unsupported is the common case + logger.debug("prompts/list unavailable for %s: %s", tool.tool_id, exc) + + resources: List[MCPResourceEntry] = [] + supports_resources = False + try: + listed, listed_truncated = _paginate( + lambda cursor: client.list_resources_sync(pagination_token=cursor), + lambda result: _resource_entries(result, templates=False), + ) + resources.extend(listed) + supports_resources = True + snapshot.truncated = snapshot.truncated or listed_truncated + except Exception as exc: # noqa: BLE001 + logger.debug("resources/list unavailable for %s: %s", tool.tool_id, exc) + + try: + templated, templated_truncated = _paginate( + lambda cursor: client.list_resource_templates_sync( + pagination_token=cursor + ), + lambda result: _resource_entries(result, templates=True), + ) + resources.extend(templated) + supports_resources = True + snapshot.truncated = snapshot.truncated or templated_truncated + except Exception as exc: # noqa: BLE001 + logger.debug( + "resources/templates/list unavailable for %s: %s", + tool.tool_id, + exc, + ) + + snapshot.resources = resources[:MAX_CAPABILITY_ENTRIES] + snapshot.supports_resources = supports_resources + return snapshot + + try: + return await asyncio.to_thread(_probe) + except Exception as exc: # noqa: BLE001 - surfaced to the caller as `error` + logger.warning("Capability discovery failed for %s: %s", tool.tool_id, exc) + snapshot.error = f"Could not reach the MCP server: {exc}" + return snapshot + + +# ============================================================================= +# Prompt resolution (prompts/get) +# ============================================================================= + + +def _message_text(content) -> tuple[str, str]: + """Flatten one ``PromptMessage`` content block to ``(kind, text)``. + + An MCP prompt message can carry text, an image, audio, a resource link or an + embedded resource. Only text survives into something a person can read and + edit, so everything else is reported by kind and its payload is dropped + rather than base64'd into the response — a preview is not the place to move + megabytes, and the caller renders the kind so nothing goes missing silently. + """ + kind = getattr(content, "type", None) or "text" + if kind == "text": + return "text", getattr(content, "text", "") or "" + if kind == "resource_link": + return kind, str(getattr(content, "uri", "") or "") + if kind == "resource": + resource = getattr(content, "resource", None) + # An embedded *text* resource is still readable; binary blobs are not. + text = getattr(resource, "text", None) + if text: + return "text", text + return kind, str(getattr(resource, "uri", "") or "") + return kind, "" + + +async def resolve_prompt_for_saved_tool( + tool: ToolDefinition, + prompt_name: str, + arguments: Dict[str, str], + oauth_token: Optional[str] = None, +) -> ResolvedPrompt: + """Ask a saved MCP server to compose one of its prompts (``prompts/get``). + + Unlike the listings, this is deliberately a *live* call and not a stored + snapshot: the result depends on the arguments the user just typed, and for a + 3LO server on the token only they hold. + + Raises: + RuntimeError: the server could not be reached or refused the prompt. + The route translates this into a 502. + """ + if tool.protocol != "mcp_external" or not tool.mcp_config: + raise RuntimeError("This tool is not an external MCP server.") + + from agents.main_agent.integrations.external_mcp_client import ( + create_external_mcp_client, + ) + + forward = bool(getattr(tool, "forward_auth_token", False)) + client = create_external_mcp_client( + config=tool.mcp_config, + tool_definition=tool, + oauth_token=oauth_token if (forward or tool.requires_oauth_provider) else None, + ) + if client is None: + raise RuntimeError("Could not build a client for this server.") + + def _get() -> ResolvedPrompt: + with client: + return _to_resolved(client.get_prompt_sync(prompt_name, arguments)) + + try: + return await asyncio.to_thread(_get) + except Exception as exc: # noqa: BLE001 - surfaced as a 502 by the route + logger.warning( + "prompts/get failed for %s/%s: %s", tool.tool_id, prompt_name, exc + ) + raise RuntimeError(f"The server could not compose that prompt: {exc}") from exc + + +def _to_resolved(result) -> ResolvedPrompt: + """Cap and flatten a ``GetPromptResult`` into the wire model.""" + messages: List[ResolvedPromptMessage] = [] + budget = MAX_RESOLVED_PROMPT_CHARS + truncated = False + + for message in (getattr(result, "messages", None) or [])[:MAX_RESOLVED_PROMPT_MESSAGES]: + kind, text = _message_text(getattr(message, "content", None)) + if len(text) > budget: + text = text[:budget] + truncated = True + budget -= len(text) + messages.append( + ResolvedPromptMessage( + role=getattr(message, "role", "user") or "user", + kind=kind, + text=text, + ) + ) + if budget <= 0: + truncated = True + break + + if len(getattr(result, "messages", None) or []) > MAX_RESOLVED_PROMPT_MESSAGES: + truncated = True + + return ResolvedPrompt( + description=_clip(getattr(result, "description", None)), + messages=messages, + truncated=truncated, + ) diff --git a/backend/src/apis/app_api/tools/routes.py b/backend/src/apis/app_api/tools/routes.py index 61ba8771e..037ec1728 100644 --- a/backend/src/apis/app_api/tools/routes.py +++ b/backend/src/apis/app_api/tools/routes.py @@ -20,11 +20,13 @@ # Import new service and models from .service import get_tool_catalog_service -from .discovery import discover_tools_for_saved_tool +from .discovery import discover_tools_for_saved_tool, resolve_prompt_for_saved_tool from apis.shared.tools.models import ( UserToolsResponse, ToolPreferencesRequest, MCPDiscoverResponse, + ResolvedPrompt, + ToolCapabilitySnapshot, ) logger = logging.getLogger(__name__) @@ -165,6 +167,79 @@ async def discover_my_tool_tools( return MCPDiscoverResponse(tools=tools) +@router.get("/{tool_id}/capabilities", response_model=ToolCapabilitySnapshot) +async def get_my_tool_capabilities( + tool_id: str, + user: User = Depends(get_current_user_from_session), +): + """The stored prompts/resources snapshot for a tool the user can access. + + Reads the persisted snapshot rather than probing the server. A detail view + has to render immediately, a 3LO server cannot be probed without the user's + consent token, and a catalogue of thirty servers opening a session each + would be unusable. Refreshing is an admin action. + + A tool that has never been discovered returns an empty snapshot with + ``discoveredAt: null`` rather than a 404 — "nobody has asked this server + yet" is a state the UI should render, not an error. + """ + service = get_tool_catalog_service() + + # RBAC: same gate as the per-tool discover route — the user must already + # have access to this catalog tool. + accessible = await service.get_user_accessible_tools(user) + if not any(t.tool_id == tool_id for t in accessible): + raise HTTPException(status_code=404, detail="Tool not found or not accessible") + + snapshot = await service.repository.get_capabilities(tool_id) + return snapshot or ToolCapabilitySnapshot(tool_id=tool_id) + + +class ResolvePromptRequest(BaseModel): + """Argument values for one ``prompts/get`` call, keyed by argument name.""" + + arguments: dict[str, str] = Field(default_factory=dict) + + +@router.post("/{tool_id}/prompts/{prompt_name}", response_model=ResolvedPrompt) +async def resolve_my_tool_prompt( + tool_id: str, + prompt_name: str, + body: ResolvePromptRequest, + user: User = Depends(get_current_user_from_session), +): + """Compose one of a server's prompts with the caller's argument values. + + Live, unlike the capability listings: the result depends on arguments typed + a moment ago, and a 3LO server can only compose it under the caller's own + token. Nothing is persisted — the composition belongs to this request. + + Lives on app-api rather than inference-api deliberately. This is user-facing + CRUD, not part of the AgentCore Runtime invocation path, and a route added to + inference-api would 404 in cloud before reaching the container. + """ + service = get_tool_catalog_service() + + # RBAC: same gate as the discover and capabilities routes. + accessible = await service.get_user_accessible_tools(user) + if not any(t.tool_id == tool_id for t in accessible): + raise HTTPException(status_code=404, detail="Tool not found or not accessible") + + tool = await service.repository.get_tool(tool_id) + if tool is None: + raise HTTPException(status_code=404, detail="Tool not found") + + try: + return await resolve_prompt_for_saved_tool( + tool, + prompt_name, + body.arguments, + oauth_token=user.raw_token, + ) + except RuntimeError as exc: + raise HTTPException(status_code=502, detail=str(exc)) + + # ============================================================================= # Legacy Public Endpoints (backward compatibility) # ============================================================================= diff --git a/backend/src/apis/app_api/tools/service.py b/backend/src/apis/app_api/tools/service.py index f5fe1a046..8254afe57 100644 --- a/backend/src/apis/app_api/tools/service.py +++ b/backend/src/apis/app_api/tools/service.py @@ -135,6 +135,11 @@ async def get_user_accessible_tools(self, user: User) -> List[UserToolAccess]: category=tool.category, protocol=tool.protocol, status=tool.status, + # UserToolAccess has always declared this field; nothing ever + # passed it, so every tool reported `requiresOauthProvider: + # null` and the SPA had no way to tell that 13 of the 31 + # tools in prod need a connection before they will work. + requires_oauth_provider=tool.requires_oauth_provider, granted_by=granted_by, enabled_by_default=tool.enabled_by_default, user_enabled=user_enabled, diff --git a/backend/src/apis/inference_api/chat/agent_binding_resolver.py b/backend/src/apis/inference_api/chat/agent_binding_resolver.py index d02fd0afd..9f644d7cb 100644 --- a/backend/src/apis/inference_api/chat/agent_binding_resolver.py +++ b/backend/src/apis/inference_api/chat/agent_binding_resolver.py @@ -20,6 +20,9 @@ override): when an Agent binds tools they *are* its toolset, re-resolved per invoker via the same ``AppRoleService.can_access_tool`` gate; a bound tool the invoker lacks blocks the turn (D5). Absent tool bindings ⇒ the request's ``enabled_tools`` drive the turn as today. + A ref may be *scoped* (``toolId::mcpToolName``) to bind a subset of an MCP server's tools + rather than all of them; the scoped id rides through to the runtime, where + ``collect_tool_name_filters`` turns it into that server's ``allowed_tool_names``. - ``skill`` bindings → the effective skill set (**replace**, same shape as tools): when an Agent binds skills they *are* the turn's skills, re-resolved per invoker via the invoke-through predicate (§6/D7, ``resolve_invocable_skill_ids``); a bound skill the @@ -42,6 +45,7 @@ from apis.shared.memory.service import MemorySpaceService from apis.shared.rbac.service import get_app_role_service from apis.shared.skills.access import resolve_invocable_skill_ids +from apis.shared.tools.scoped_ids import base_tool_id _ROLE_RANK = {"viewer": 1, "editor": 2, "owner": 3} @@ -94,6 +98,12 @@ class ResolvedTools: hand straight to the tool filter. An empty ``tool_ids`` is meaningful — the Agent deliberately runs with *no* tools — and is distinct from ``plan.tools is None`` (no tool binding, fall through to the request). + + Ids are carried **verbatim**, scoping included: a scoped ``base::tool`` id must survive + into ``enabled_tools`` for ``collect_tool_name_filters`` to fold it into that server's + ``allowed_tool_names`` and build a filtered MCP client. Collapsing one to its base here + would silently restore the whole server — the exact bug the scoping exists to prevent, + and invisible from the outside because the turn would still work. """ tool_ids: List[str] @@ -223,6 +233,11 @@ async def _resolve_tools(assistant: Assistant, invoker: User) -> Optional[Resolv (block-with-message, no silent drop — D5). Returns ``None`` when the Agent binds no tools, leaving the request's ``enabled_tools`` in force. The RBAC service is fetched lazily (only when the Agent actually binds tools) — mirroring how ``_resolve_memory`` builds its own. + + A ref may be **scoped** (``toolId::mcpToolName``) to bind a subset of an MCP server's + tools. Access is a property of the *server*, so the gate is keyed on the base id — which + is also what an administrator would grant, and so what the block message names. The + scoped id itself is preserved in the result; that is what narrows the turn. """ tool_bindings = [b for b in (assistant.bindings or []) if b.kind == "tool"] if not tool_bindings: @@ -230,13 +245,19 @@ async def _resolve_tools(assistant: Assistant, invoker: User) -> Optional[Resolv app_role_service = get_app_role_service() resolved: List[str] = [] + # Access is per server, so check each base once: an Agent binding seven tools of one + # MCP server is the normal shape here, and it should cost one gate call, not seven. + checked_bases: set = set() for binding in tool_bindings: ref = binding.ref - if not await app_role_service.can_access_tool(invoker, ref): - raise AgentBindingBlockedError( - f"This agent uses the tool **{ref}**, which isn't available to your account. " - "Ask an administrator for access, or use a different agent." - ) + base = base_tool_id(ref) + if base not in checked_bases: + if not await app_role_service.can_access_tool(invoker, ref): + raise AgentBindingBlockedError( + f"This agent uses the tool **{base}**, which isn't available to your " + "account. Ask an administrator for access, or use a different agent." + ) + checked_bases.add(base) if ref not in resolved: resolved.append(ref) diff --git a/backend/src/apis/inference_api/chat/models.py b/backend/src/apis/inference_api/chat/models.py index 221f2f2a5..b56321d85 100644 --- a/backend/src/apis/inference_api/chat/models.py +++ b/backend/src/apis/inference_api/chat/models.py @@ -185,6 +185,17 @@ class InvocationRequest(BaseModel): # — client input can narrow the set, never grant. An empty (or fully # inaccessible) list yields zero skills, so the turn is plain chat. enabled_skills: Optional[List[str]] = None + # Skills the user named with a `/` slash command in the composer, for this + # turn only. A strict subset of the turn's effective skills — it is + # intersected server-side exactly like ``enabled_skills``, so it can never + # widen the set and an id that is not already active is simply dropped. + # + # It changes nothing about what is *disclosed*: the same skills are in + # ```` either way, so the cacheable prefix is untouched. + # All it adds is a short directive on the user message telling the model to + # activate the named skill before answering, which is what makes a slash + # command deterministic rather than a hint the model may ignore. + invoked_skills: Optional[List[str]] = None # User-selected custom system prompt ("conversation mode") for this # turn. The frontend forwards the active selection on every submit so # the inference path doesn't have to round-trip session metadata to diff --git a/backend/src/apis/inference_api/chat/routes.py b/backend/src/apis/inference_api/chat/routes.py index 1f440008b..3bc00454f 100644 --- a/backend/src/apis/inference_api/chat/routes.py +++ b/backend/src/apis/inference_api/chat/routes.py @@ -43,6 +43,7 @@ ) from apis.shared.rbac.service import get_app_role_service +from apis.shared.skills.bundle import slugify_skill_name from apis.inference_api.chat.agent_binding_resolver import ( AgentBindingBlockedError, resolve_agent_invocation, @@ -1206,6 +1207,55 @@ def _apply_enabled_skills_filter( return [sid for sid in accessible_skill_ids if sid in requested] +def _resolve_invoked_skill_slugs( + effective_skill_ids: Optional[list[str]], invoked_skills: Optional[list[str]] +) -> list[str]: + """Activation slugs for the skills the user named with a `/` command. + + Intersected against the turn's **effective** set — the same narrow-never-grant + rule ``_apply_enabled_skills_filter`` applies, re-run here because the effective + set can still shrink after that call (an Agent's skill bindings replace it + wholesale). A slash command for a skill the turn does not actually disclose is + dropped rather than honoured: the directive would name a skill that is absent + from ````, and the model would burn a tool call discovering + that. + + Returns slugs, not ids, because the slug is the activation key the ``skills`` + tool takes — the id never appears in anything the model can see. + """ + if not invoked_skills or not effective_skill_ids: + return [] + requested = set(invoked_skills) + # Ordered by the effective set, not by the request: the directive is part of + # the persisted message, and a list whose order followed client input would + # differ between two turns that named the same skills. + return [slugify_skill_name(sid) for sid in effective_skill_ids if sid in requested] + + +def _build_skill_invocation_note(skill_slugs: list[str]) -> str: + """Directive appended to a turn whose user invoked skills by slash command. + + A slash command is an explicit instruction, not a hint — the user picked the + skill by name from a menu. But the only activation path is the plugin's + ``skills`` tool, which the *model* has to call, so "explicit" has to be + expressed as a directive rather than enforced by pre-loading the instructions + (doing that server-side would duplicate the plugin's response formatting and + bypass its activation-state tracking). + + Kept to one line per skill. It rides the user message, so it is paid once as + input on this turn and then again as cached history on every later turn of the + session; the disclosure block it points at is already in the prefix either way, + so this is the whole cost of the feature. + """ + named = ", ".join(f"`{slug}`" for slug in skill_slugs) + plural = "s" if len(skill_slugs) > 1 else "" + return ( + f"[The user invoked the {named} skill{plural} with a slash command. " + f"Activate {'each' if plural else 'it'} with the `skills` tool before " + "answering, and follow the loaded instructions for this message.]" + ) + + @router.post("/invocations") async def invocations(request: InvocationRequest, current_user: User = Depends(get_current_user_trusted)): """ @@ -2631,6 +2681,21 @@ def _session_title_sse() -> Optional[str]: f"{_build_interruption_note(interrupted_turn_reason)}\n\n{final_message}" ) + # Slash commands: the user named one or more skills in the + # composer. Appended LAST, after every prepended note, so the + # directive is the closest thing to the model's first token — + # an instruction about what to do with the message it follows. + # Narrowed against the effective set here rather than at parse + # time because an Agent's skill bindings can still have + # replaced that set above. + invoked_skill_slugs = _resolve_invoked_skill_slugs( + effective_skill_ids, input_data.invoked_skills + ) + if invoked_skill_slugs: + final_message = ( + f"{final_message}\n\n{_build_skill_invocation_note(invoked_skill_slugs)}" + ) + message_will_be_modified = ( final_message != input_data.message # RAG augmentation / attachment guidance / inventory or bool(files_to_send) # File attachments diff --git a/backend/src/apis/shared/assistants/icons.py b/backend/src/apis/shared/assistants/icons.py index 6fb8f6328..b0b1764e9 100644 --- a/backend/src/apis/shared/assistants/icons.py +++ b/backend/src/apis/shared/assistants/icons.py @@ -5,6 +5,11 @@ way against the 400 KB DynamoDB item limit, except here the limit would be hit by design rather than by accident, since the icon ceiling *is* 400 KB. +Validation, normalization (including the EXIF-stripping re-encode) and the S3 put/get/ +delete live in :mod:`apis.shared.images.icons` — managed models attach icons the same +way, and one validator is the point. What stays here is the part that is about Agents: +the key layout and the URL that serves it. + Storage ------- Objects land in the existing assistants asset bucket @@ -12,65 +17,28 @@ ``assistants/{agent_id}/`` prefix that assistant documents already use: assistants/{agent_id}/icons/{sha256[:16]}.{png|jpg} - -The key is **content-addressed**, which buys two things: re-uploading the same image is a -no-op rather than a new object, and the digest doubles as the cache version — the serve -route hands it out as the ETag and the read shapes hang it off ``?v=`` so an icon can be -cached ``immutable`` and still change the moment a new one is uploaded. - -Validation ----------- -Everything a caller sends is untrusted, so the ``Content-Type`` header is ignored and the -format is sniffed from the bytes. Beyond the D5 limits (PNG or JPEG, ≤ 400 KB, square), -the image is **always re-encoded** even when it already measures 512×512. That is not -redundant work: re-encoding is what strips EXIF, and an author uploading a phone photo as -an icon would otherwise publish its GPS coordinates to the whole institution. """ from __future__ import annotations -import hashlib -import io import logging -import os -from typing import Optional, Tuple - -try: # boto3 is absent in some local-dev setups - import boto3 - from botocore.exceptions import ClientError -except ImportError: # pragma: no cover - exercised only without boto3 - boto3 = None - ClientError = Exception # type: ignore[assignment, misc] +from typing import Optional + +from apis.shared.images.icons import ( # noqa: F401 - re-exported for existing importers + ICON_MAX_BYTES, + ICON_MIN_SOURCE, + ICON_SIZE, + ICON_SQUARE_TOLERANCE_PX, + IconError, + IconStore, + IconStoreError, + content_digest, + key_version, + normalize_icon, +) logger = logging.getLogger(__name__) -# D5 limits. -ICON_MAX_BYTES = 400 * 1024 -ICON_SIZE = 512 -# Below this, upscaling to 512 produces a soft tile that reads worse than the generated -# gradient it replaced — so we decline rather than accept a downgrade. -ICON_MIN_SOURCE = 256 -# A hand-cropped square is often off by a pixel; a 4:3 photo is not. -ICON_SQUARE_TOLERANCE_PX = 2 - -_FORMAT_EXT = {"PNG": "png", "JPEG": "jpg"} -_EXT_CONTENT_TYPE = {"png": "image/png", "jpg": "image/jpeg"} - -# AWS-managed (SSE-S3 / AES256) encryption, matching the bucket default. -_SSE_ALGORITHM = "AES256" - - -class IconError(ValueError): - """An icon the author cannot publish, with a message written for the author. - - Every message names the limit *and* what was actually supplied, because "invalid - image" sends someone back to a file picker with nothing to change. - """ - - -class IconStoreError(RuntimeError): - """Storage is unavailable or the object could not be read/written.""" - # ── keys and URLs ──────────────────────────────────────────────────────────────────── @@ -86,9 +54,7 @@ def icon_version(icon_key: Optional[str]) -> Optional[str]: Used as the ``?v=`` on ``iconUrl`` and as the ETag on the serve route, so a stored icon can be cached ``immutable`` while a replacement busts it immediately. """ - if not icon_key: - return None - return icon_key.rsplit("/", 1)[-1].rsplit(".", 1)[0] + return key_version(icon_key) def icon_url(agent_id: str, icon_key: Optional[str]) -> Optional[str]: @@ -113,184 +79,19 @@ def icon_url(agent_id: str, icon_key: Optional[str]) -> Optional[str]: return f"/agents/{agent_id}/icon?v={version}" -# ── validation / normalization ─────────────────────────────────────────────────────── - - -def normalize_icon(content: bytes) -> Tuple[bytes, str, str]: - """Validate and normalize an uploaded icon. - - Returns ``(bytes, ext, content_type)`` for a 512×512, metadata-free PNG or JPEG. - Raises :class:`IconError` with an author-facing message on anything it declines. - """ - from PIL import Image, UnidentifiedImageError # lazy: keeps PIL off every importer - - if not content: - raise IconError("The uploaded file is empty.") - if len(content) > ICON_MAX_BYTES: - raise IconError( - f"Icons must be {ICON_MAX_BYTES // 1024} KB or smaller " - f"(this one is {len(content) // 1024} KB)." - ) - - try: - image = Image.open(io.BytesIO(content)) - source_format = image.format - image.load() - except (UnidentifiedImageError, OSError, ValueError) as e: - raise IconError("Icons must be a PNG or JPEG image.") from e - - if source_format not in _FORMAT_EXT: - raise IconError( - f"Icons must be a PNG or JPEG image (this one is {source_format or 'an unknown format'})." - ) - - width, height = image.size - if abs(width - height) > ICON_SQUARE_TOLERANCE_PX: - raise IconError(f"Icons must be square (this one is {width}×{height}).") - if min(width, height) < ICON_MIN_SOURCE: - raise IconError( - f"Icons must be at least {ICON_MIN_SOURCE}×{ICON_MIN_SOURCE} " - f"(this one is {width}×{height})." - ) - - ext = _FORMAT_EXT[source_format] - # Re-encoding always happens — see the module docstring on EXIF. LANCZOS because a - # 28px tile is a 18× downscale of the stored icon and cheaper filters alias badly. - image = image.convert("RGBA" if ext == "png" else "RGB") - if (width, height) != (ICON_SIZE, ICON_SIZE): - image = image.resize((ICON_SIZE, ICON_SIZE), Image.Resampling.LANCZOS) - - encoded = _encode_within_limit(image, ext) - if encoded is None: - raise IconError( - f"This icon could not be stored under {ICON_MAX_BYTES // 1024} KB. " - "Try a simpler image or fewer colors." - ) - data, ext = encoded - return data, ext, _EXT_CONTENT_TYPE[ext] - - -def _encode_within_limit(image, ext: str) -> Optional[Tuple[bytes, str]]: - """Encode at 512×512 under the size ceiling, degrading in defined steps. - - A downscale to 512 almost always lands well under 400 KB; this ladder exists for the - input that arrives *already* 512×512 and near the ceiling, where re-encoding could - push it over. Each rung is deliberate rather than a retry loop: JPEG loses quality, - an opaque PNG becomes a JPEG, and a transparent PNG loses colors but keeps its alpha. - """ - from PIL import Image - - if ext == "jpg": - for quality in (92, 85, 78): - data = _save(image, "JPEG", quality=quality, optimize=True, progressive=True) - if len(data) <= ICON_MAX_BYTES: - return data, "jpg" - return None - - data = _save(image, "PNG", optimize=True) - if len(data) <= ICON_MAX_BYTES: - return data, "png" - - has_alpha = image.getchannel("A").getextrema()[0] < 255 - if not has_alpha: - for quality in (92, 85): - data = _save(image.convert("RGB"), "JPEG", quality=quality, optimize=True) - if len(data) <= ICON_MAX_BYTES: - return data, "jpg" - return None - - # FASTOCTREE, not the default MEDIANCUT: it is the only method Pillow will quantize - # an RGBA image with, and this rung exists precisely to keep the alpha. - quantized = image.quantize(colors=256, method=Image.Quantize.FASTOCTREE) - data = _save(quantized, "PNG", optimize=True) - return (data, "png") if len(data) <= ICON_MAX_BYTES else None - - -def _save(image, fmt: str, **kwargs) -> bytes: - buffer = io.BytesIO() - image.save(buffer, format=fmt, **kwargs) - return buffer.getvalue() - - -def content_digest(content: bytes) -> str: - """The 16-hex-char content address used in the key and as the cache version.""" - return hashlib.sha256(content).hexdigest()[:16] - - # ── S3 ─────────────────────────────────────────────────────────────────────────────── -class AgentIconStore: +class AgentIconStore(IconStore): """Put / get / delete agent icons in the assistants asset bucket.""" def __init__(self, bucket_name: Optional[str] = None, s3_client: Optional[object] = None) -> None: - self.bucket_name = bucket_name or os.environ.get("S3_ASSISTANTS_DOCUMENTS_BUCKET_NAME") - # Lazily constructed so importing this module never needs AWS credentials; - # tests inject a client. - self._s3 = s3_client - - @property - def enabled(self) -> bool: - return bool(self.bucket_name) and boto3 is not None - - def _client(self): - if self._s3 is None: - if boto3 is None: # pragma: no cover - import-guarded above - raise IconStoreError("icon storage unavailable: boto3 is not installed") - self._s3 = boto3.client("s3") - return self._s3 - - def _require_enabled(self) -> None: - if not self.enabled: - raise IconStoreError( - "icon storage is not configured (S3_ASSISTANTS_DOCUMENTS_BUCKET_NAME is unset)" - ) + super().__init__(label="agent-icons", bucket_name=bucket_name, s3_client=s3_client) def put(self, *, agent_id: str, content: bytes, ext: str, content_type: str) -> str: """Store normalized bytes and return the content-addressed key.""" - self._require_enabled() key = build_icon_key(agent_id, content_digest(content), ext) - try: - self._client().put_object( - Bucket=self.bucket_name, - Key=key, - Body=content, - ContentType=content_type, - ServerSideEncryption=_SSE_ALGORITHM, - # The object is immutable by construction (the key is its digest), so the - # cache directive belongs on the object as much as on the serve response. - CacheControl="public, max-age=31536000, immutable", - ) - except ClientError as e: # pragma: no cover - network/permission path - logger.error(f"agent-icons: put failed for agent={agent_id} key={key}: {e}") - raise IconStoreError(f"failed to store icon for agent '{agent_id}'") from e - - logger.info(f"🖼️ agent-icons: stored agent={agent_id} key={key} ({len(content)} bytes)") - return key - - def get(self, icon_key: str) -> Tuple[bytes, str]: - """Return ``(bytes, content_type)`` for a stored icon.""" - self._require_enabled() - try: - response = self._client().get_object(Bucket=self.bucket_name, Key=icon_key) - body = response["Body"].read() - return body, response.get("ContentType") or "application/octet-stream" - except ClientError as e: - code = e.response.get("Error", {}).get("Code", "") - if code in ("NoSuchKey", "404"): - raise IconStoreError(f"icon not found at key '{icon_key}'") from e - logger.error(f"agent-icons: get failed for key={icon_key}: {e}") - raise IconStoreError(f"failed to read icon at key '{icon_key}'") from e - - def delete(self, icon_key: str) -> None: - """Best-effort delete. Never raises: a replaced icon's old object going missing - is not a reason to fail the upload that replaced it.""" - if not self.enabled or not icon_key: - return - try: - self._client().delete_object(Bucket=self.bucket_name, Key=icon_key) - except ClientError as e: # pragma: no cover - network/permission path - logger.warning(f"agent-icons: delete failed for key={icon_key}: {e}") + return self.put_object(key=key, content=content, content_type=content_type) _store: Optional[AgentIconStore] = None diff --git a/backend/src/apis/shared/auth/dependencies.py b/backend/src/apis/shared/auth/dependencies.py index 629ff404b..b6b82be2a 100644 --- a/backend/src/apis/shared/auth/dependencies.py +++ b/backend/src/apis/shared/auth/dependencies.py @@ -4,6 +4,7 @@ import jwt import logging import os +import time from typing import Optional from fastapi import Depends, HTTPException, Request, status from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials @@ -62,6 +63,84 @@ def invalidate_user_profile_cache(user_id: str) -> None: """ _user_profile_cache.pop(user_id, None) + +# ─── Background User-Sync Throttle ───────────────────────────────────── +# `sync_user_from_jwt` is an upsert: a GetItem followed by a PutItem that +# rewrites the whole profile row and its GSI projections. Firing it on every +# authenticated request meant one SPA first load — 12 API calls — issued 24 +# DynamoDB operations against a single item, writing identical values except +# `last_login_at`. A classroom signing in together multiplied that by the +# class size, against one hot partition per student. +# +# Throttling is safe because this is not the authoritative write. The BFF +# callback's `_sync_user_from_id_token` syncs on every login off the ID token +# (which is the only place the full claim set exists), and `POST /users/me/sync` +# writes through the repository directly. What remains here is a periodic +# refresh, so it only needs to run periodically. The cost is that +# `last_login_at` is accurate to within the window rather than to the request. +# +# A brand-new user is unaffected: with no entry recorded, the first request +# always claims the sync, so row creation still happens immediately. + +_USER_SYNC_THROTTLE_SECONDS = int(os.environ.get("USER_SYNC_THROTTLE_SECONDS", "300")) +_USER_SYNC_TRACKER_MAX = 10_000 + +_user_sync_last_run: dict[str, float] = {} + +# Strong references to in-flight sync tasks. The event loop only holds a weak +# reference to a bare `asyncio.create_task(...)`, so without keeping one here +# the GC can collect the task mid-await — the same hazard +# `SessionRefreshMiddleware` guards its slide writes against. +_user_sync_tasks: set[asyncio.Task] = set() + + +def _claim_user_sync(user_id: str) -> bool: + """Return True if this request should run the background user sync. + + The claim is recorded BEFORE the sync runs, not after it completes. The + 12 requests of a single page load overlap, so a marker written on + completion would let most of them through before the first one landed — + which is the exact pileup this exists to prevent. + """ + now = time.monotonic() + last = _user_sync_last_run.get(user_id) + if last is not None and now - last < _USER_SYNC_THROTTLE_SECONDS: + return False + + _user_sync_last_run[user_id] = now + if len(_user_sync_last_run) > _USER_SYNC_TRACKER_MAX: + _prune_user_sync_tracker(now) + return True + + +def _prune_user_sync_tracker(now: float) -> None: + """Drop entries older than the throttle window. + + Bounds the dict on a long-lived container that has served many distinct + users. An entry past the window is already a no-op for `_claim_user_sync`, + so dropping it changes no behaviour. + """ + stale = [ + uid + for uid, ts in _user_sync_last_run.items() + if now - ts >= _USER_SYNC_THROTTLE_SECONDS + ] + for uid in stale: + _user_sync_last_run.pop(uid, None) + + +def reset_user_sync_throttle(user_id: Optional[str] = None) -> None: + """Let the next request re-sync `user_id` (or every user when None). + + Exposed for tests and for any caller that has just invalidated profile + state and wants the refresh to happen now rather than at the next window. + """ + if user_id is None: + _user_sync_last_run.clear() + else: + _user_sync_last_run.pop(user_id, None) + + _user_repository = None @@ -95,8 +174,6 @@ async def _enrich_user_from_store(user: User) -> None: Results are cached in-memory to avoid per-request DynamoDB lookups. """ - import time - from apis.shared.rbac.version import get_roles_version current_version = get_roles_version() @@ -145,6 +222,24 @@ async def _sync_user_background(sync_service, user: User) -> None: # Log but don't fail - sync should never break authentication logger.warning(f"Failed to sync user {user.user_id}: {e}") + +def _schedule_user_sync(user: User) -> None: + """Dispatch the throttled background profile sync for `user`. + + No-op when sync is unconfigured or the user was already synced inside the + throttle window. Never raises: a sync problem must not break auth. + """ + sync_service = _get_user_sync_service() + if not (sync_service and sync_service.enabled): + return + if not _claim_user_sync(user.user_id): + return + + task = asyncio.create_task(_sync_user_background(sync_service, user)) + _user_sync_tasks.add(task) + task.add_done_callback(_user_sync_tasks.discard) + + # Cognito JWT validator for tokens minted by the BFF confidential client. # The SPA-public PKCE client was decommissioned in Phase 7; all browser-facing # auth now flows through the BFF, which issues tokens carrying @@ -277,9 +372,7 @@ async def get_current_user_from_session(request: Request) -> User: user.raw_token = record.cognito_access_token await _enrich_user_from_store(user) - sync_service = _get_user_sync_service() - if sync_service and sync_service.enabled: - asyncio.create_task(_sync_user_background(sync_service, user)) + _schedule_user_sync(user) return user @@ -380,9 +473,7 @@ async def get_current_user_trusted( await _enrich_user_from_store(user) # Fire-and-forget sync to Users table - sync_service = _get_user_sync_service() - if sync_service and sync_service.enabled: - asyncio.create_task(_sync_user_background(sync_service, user)) + _schedule_user_sync(user) return user diff --git a/backend/src/apis/shared/caching/__init__.py b/backend/src/apis/shared/caching/__init__.py new file mode 100644 index 000000000..1aee3a99b --- /dev/null +++ b/backend/src/apis/shared/caching/__init__.py @@ -0,0 +1,27 @@ +"""Shared caching primitives.""" + +from .config_cache import ( + MANAGED_MODELS, + OAUTH_PROVIDERS_ALL, + OAUTH_PROVIDERS_ENABLED, + SYSTEM_PROMPTS, + TOOL_CATALOG, + ConfigListCache, + get_config_cache, + get_or_load, + invalidate, + invalidate_oauth_providers, +) + +__all__ = [ + "MANAGED_MODELS", + "OAUTH_PROVIDERS_ALL", + "OAUTH_PROVIDERS_ENABLED", + "SYSTEM_PROMPTS", + "TOOL_CATALOG", + "ConfigListCache", + "get_config_cache", + "get_or_load", + "invalidate", + "invalidate_oauth_providers", +] diff --git a/backend/src/apis/shared/caching/config_cache.py b/backend/src/apis/shared/caching/config_cache.py new file mode 100644 index 000000000..e53b4b6b5 --- /dev/null +++ b/backend/src/apis/shared/caching/config_cache.py @@ -0,0 +1,215 @@ +"""TTL + single-flight cache for tenant-global config item lists. + +The SPA's first load reads several catalogs that are the *same for everyone* +on the deployment — the model catalog, the tool catalog, the system-prompt +list, the connector list. Each read was an uncached DynamoDB scan (or, for +connectors, a GSI query) that also re-parsed every row. A classroom signing in +together ran one of each per student. + +Two properties matter here, and the second is the one that actually shows up +under a burst: + +**TTL.** A catalog changes when an admin edits it, which is rare, so a short +window collapses ~all of the reads. + +**Single flight.** A cold cache with 300 simultaneous requests would otherwise +produce 300 concurrent scans — the stampede lands at exactly the moment the +burst does, which is the case this exists to prevent. One loader runs per key; +everyone else awaits it. + +## What is cached: raw items, not parsed objects + +Entries hold the **raw DynamoDB items**, and callers re-parse on every read. + +That is deliberate, and it is not a missed optimization. Callers mutate the +objects these lists produce: ``hydrate_model_roles`` documents itself as +"models to hydrate (mutated in place and returned)" and writes +``allowed_app_roles`` onto each model, and ``list_tools_with_roles`` does the +same to ``ToolDefinition``. Caching parsed objects would hand every caller a +reference to one shared instance, so an admin opening the models page would +write derived, display-only role fields onto the objects subsequently served +to every user — and per CLAUDE.md ``allowedAppRoles`` is precisely the field +that must never be mistaken for a grant. + +Re-parsing costs microseconds of CPU against a network round trip of ~50-100ms +(measured medians: ``/models`` 46ms, ``/tools/`` 91ms), so the saving that +matters is kept while the shared-mutable-state class of bug is designed out +entirely — including for callers added later, who cannot be audited in advance. + +The list itself is shallow-copied on read so a caller cannot append to or sort +the cached list. The item dicts are shared; parsers treat them as read-only. + +## Scope: per process, like the rest of our caches + +This is an in-process cache, matching ``AppRoleCache`` and the roles-version +watermark. A write invalidates only the task that served it, so a second ECS +task can serve a stale catalog until its entry expires. That is why the default +TTL is short (60s) rather than the 5-10 minutes ``AppRoleCache`` uses for +authorization data: an admin edit should not appear to "not take" for minutes. +Bounding it to a minute keeps the burst win (a classroom arrives inside a few +seconds) while keeping admin feedback close to immediate. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import time +from typing import Awaitable, Callable, Dict, List, Optional + +logger = logging.getLogger(__name__) + +DEFAULT_TTL_SECONDS = 60 + +# Cache keys. Named here rather than passed as bare strings at each call site +# so a typo cannot silently create a second, never-invalidated entry. +MANAGED_MODELS = "managed_models" +TOOL_CATALOG = "tool_catalog" +SYSTEM_PROMPTS = "system_prompts" + +# Providers are read two ways — the enabled-only GSI query that `/connectors/` +# uses on first load, and the full scan the admin console uses. They are +# separate entries because they return different rows, and BOTH are dropped on +# any provider write: `enabled` is part of the GSI partition key, so flipping it +# moves a provider between the two result sets. +OAUTH_PROVIDERS_ENABLED = "oauth_providers:enabled" +OAUTH_PROVIDERS_ALL = "oauth_providers:all" + +Items = List[dict] +Loader = Callable[[], Awaitable[Items]] + + +def _ttl_seconds() -> int: + """Cache lifetime, from ``CONFIG_CACHE_TTL_SECONDS``. + + A non-numeric or negative value falls back to the default rather than + raising: a malformed env var should degrade to sane caching, not break + every catalog read on the deployment. + """ + raw = os.environ.get("CONFIG_CACHE_TTL_SECONDS", "").strip() + if not raw: + return DEFAULT_TTL_SECONDS + try: + value = int(raw) + except ValueError: + logger.warning( + "CONFIG_CACHE_TTL_SECONDS=%r is not an integer; using %ds", + raw, + DEFAULT_TTL_SECONDS, + ) + return DEFAULT_TTL_SECONDS + if value < 0: + logger.warning( + "CONFIG_CACHE_TTL_SECONDS=%d is negative; using %ds", + value, + DEFAULT_TTL_SECONDS, + ) + return DEFAULT_TTL_SECONDS + return value + + +class ConfigListCache: + """TTL cache with single-flight loading, holding raw DynamoDB items.""" + + def __init__(self) -> None: + # key -> (expires_at_monotonic, items) + self._entries: Dict[str, tuple[float, Items]] = {} + # key -> lock serialising loads for that key. Created lazily; an + # asyncio.Lock built at import time would be fine on 3.10+ but the + # per-key dict has to be lazy regardless. + self._locks: Dict[str, asyncio.Lock] = {} + + def _lock_for(self, key: str) -> asyncio.Lock: + lock = self._locks.get(key) + if lock is None: + lock = asyncio.Lock() + self._locks[key] = lock + return lock + + def _live_entry(self, key: str) -> Optional[Items]: + entry = self._entries.get(key) + if entry is None: + return None + expires_at, items = entry + if time.monotonic() >= expires_at: + return None + return items + + async def get_or_load(self, key: str, loader: Loader) -> Items: + """Return cached items for ``key``, loading via ``loader`` on a miss. + + Concurrent callers that miss together run ``loader`` exactly once; the + rest wait on the same load and read the result. + + A loader that raises propagates to every waiter and caches nothing, so + the next request retries rather than inheriting a failure. An expired + entry is not served as a fallback: these catalogs drive authorization + surfaces, and serving one the store may have since changed is worse + than surfacing the error the caller already knows how to handle. + """ + from apis.shared.feature_flags import config_cache_enabled + + if not config_cache_enabled(): + return await loader() + + cached = self._live_entry(key) + if cached is not None: + return list(cached) + + async with self._lock_for(key): + # Re-check: another coroutine may have loaded while we queued. + cached = self._live_entry(key) + if cached is not None: + return list(cached) + + items = await loader() + self._entries[key] = (time.monotonic() + _ttl_seconds(), items) + logger.debug("Config cache filled %s (%d items)", key, len(items)) + return list(items) + + def invalidate(self, key: str) -> None: + """Drop ``key`` so the next read reloads from DynamoDB. + + Called from the write paths themselves rather than from admin routes, + so a new mutation cannot forget to invalidate. + """ + if self._entries.pop(key, None) is not None: + logger.debug("Config cache invalidated %s", key) + + def clear(self) -> None: + """Drop every entry. For tests and process-wide invalidation.""" + self._entries.clear() + + +_cache: Optional[ConfigListCache] = None + + +def get_config_cache() -> ConfigListCache: + """Return the process-wide cache instance.""" + global _cache + if _cache is None: + _cache = ConfigListCache() + return _cache + + +async def get_or_load(key: str, loader: Loader) -> Items: + """Module-level convenience wrapper over the process-wide cache.""" + return await get_config_cache().get_or_load(key, loader) + + +def invalidate(key: str) -> None: + """Module-level convenience wrapper over the process-wide cache.""" + get_config_cache().invalidate(key) + + +def invalidate_oauth_providers() -> None: + """Drop both provider entries. + + Always both: ``enabled`` is part of the GSI partition key backing the + enabled-only read, so a write that flips it changes what BOTH the query and + the scan return. + """ + cache = get_config_cache() + cache.invalidate(OAUTH_PROVIDERS_ENABLED) + cache.invalidate(OAUTH_PROVIDERS_ALL) diff --git a/backend/src/apis/shared/feature_flags.py b/backend/src/apis/shared/feature_flags.py index d9e8e9748..0c2be954b 100644 --- a/backend/src/apis/shared/feature_flags.py +++ b/backend/src/apis/shared/feature_flags.py @@ -234,3 +234,72 @@ def artifact_share_inbox_enabled() -> bool: os.environ.get("ARTIFACT_SHARE_INBOX_ENABLED", "").strip().lower() != "false" ) + + +def agent_status_enabled() -> bool: + """Whether the agent narrates what it is doing while a turn streams. + + Covers the runtime ``AgentStatusHook`` (model-call and tool-call + boundaries), the ``agent_status`` SSE event the stream coordinator drains + from it, and the SPA's live status line + per-tool durations. **Default ON + with a kill switch** (house style, mirroring ``mid_turn_steering_enabled``): + unset or empty resolves to enabled; only the literal ``"false"`` + (case-insensitive) disables. + + While off the hook is still registered but every callback returns + immediately and the drain yields nothing, leaving the loading indicator on + its cycling phrases and tool rows with no duration — exactly the + pre-feature behaviour. + + Costs nothing against the model: the hook observes boundaries the event + loop already crosses and writes to an in-process list. Nothing it produces + reaches the prompt, so the cacheable prefix is untouched. + """ + return os.environ.get("AGENT_STATUS_ENABLED", "").strip().lower() != "false" + + +def tool_summaries_enabled() -> bool: + """Whether tool batches get a model-generated one-line summary. + + Covers the Nova Micro summarizer that runs as a side-channel task at each + tool boundary, the ``tool_group_summary`` SSE event, the ``TSUM#`` + persistence rows, and their replay on ``GET /messages``. **Default ON with + a kill switch** (house style): unset or empty resolves to enabled; only the + literal ``"false"`` (case-insensitive) disables. + + This flag gates the *model-generated* summary only. The SPA's deterministic + per-tool formatters are client-side, cost nothing, and keep working with + this off — turning the flag off degrades the rail from prose ("Found the + Syllabus Acknowledgment assignment in BIO 101") to the formatter line + ("Listed 4 assignments"), never to a bare tool name. + + Cost note (CLAUDE.md token-effectiveness tenet): the summarizer is a + **side-channel**, structured exactly like ``session_title`` — its own + Bedrock call on its own messages, concurrent with the agent stream. It + never appends to the agent's conversation, so it adds nothing to the + cacheable prefix and cannot cause a cache re-write. Its own spend is one + bounded Nova Micro call per tool batch (inputs and results are truncated + before they are sent), which is why it is affordable to leave on. + """ + return os.environ.get("TOOL_SUMMARIES_ENABLED", "").strip().lower() != "false" + + +def config_cache_enabled() -> bool: + """Whether tenant-global config catalogs are served from the in-process cache. + + Covers the model catalog, tool catalog, system-prompt list and connector + list — the lists every user's first load reads and that only an admin edit + changes. **Default ON with a kill switch** (house style, mirroring + ``scheduled_runs_enabled``): unset or empty resolves to enabled; only the + literal ``"false"`` (case-insensitive) disables. + + While off, every read goes straight to DynamoDB exactly as before — the + loaders are unchanged and still correct, they simply stop being memoized. + Turning this off costs latency and read units, never correctness, which is + what makes it a safe switch to flip if a stale catalog is ever suspected. + + Note the cache is per process (see ``apis.shared.caching.config_cache``), so + a write invalidates only the task that served it; other tasks catch up + within ``CONFIG_CACHE_TTL_SECONDS`` (default 60). + """ + return os.environ.get("CONFIG_CACHE_ENABLED", "").strip().lower() != "false" diff --git a/backend/src/apis/shared/images/__init__.py b/backend/src/apis/shared/images/__init__.py new file mode 100644 index 000000000..6639e2c6d --- /dev/null +++ b/backend/src/apis/shared/images/__init__.py @@ -0,0 +1,8 @@ +"""Shared image handling. + +Square-icon validation, normalization and object storage, used by anything that +lets an admin or author attach a small square image to a record (Agents, managed +models). Format sniffing, EXIF stripping and the size ladder live here exactly +once — a second copy would drift, and the one that drifted would be the one that +published somebody's GPS coordinates. +""" diff --git a/backend/src/apis/shared/images/icons.py b/backend/src/apis/shared/images/icons.py new file mode 100644 index 000000000..dbf33083e --- /dev/null +++ b/backend/src/apis/shared/images/icons.py @@ -0,0 +1,280 @@ +"""Square-icon validation, normalization and object storage. + +The owner-specific parts of an icon — what its S3 key looks like, what URL serves +it — belong to whatever owns the record (``apis.shared.assistants.icons`` for +Agents, ``apis.shared.models.model_icons`` for managed models). Everything below +is the part that must not be written twice. + +Validation +---------- +Everything a caller sends is untrusted, so the ``Content-Type`` header is ignored +and the format is sniffed from the bytes. Beyond the limits (PNG or JPEG, +≤ 400 KB, square), the image is **always re-encoded** even when it already +measures 512×512. That is not redundant work: re-encoding is what strips EXIF, +and someone uploading a phone photo as an icon would otherwise publish its GPS +coordinates to the whole institution. + +Storage +------- +:class:`IconStore` is the generic put/get/delete against one bucket. Keys are +**content-addressed** by the caller, which buys two things: re-uploading the same +image is a no-op rather than a new object, and the digest doubles as the cache +version — a serve route hands it out as the ETag and read shapes hang it off +``?v=`` so an icon can be cached ``immutable`` and still change the moment a new +one is uploaded. +""" + +from __future__ import annotations + +import hashlib +import io +import logging +import os +from typing import Optional, Tuple + +try: # boto3 is absent in some local-dev setups + import boto3 + from botocore.exceptions import ClientError +except ImportError: # pragma: no cover - exercised only without boto3 + boto3 = None + ClientError = Exception # type: ignore[assignment, misc] + +logger = logging.getLogger(__name__) + +# D5 limits (Agent Marketplace), reused verbatim for every other icon: the +# ceiling exists because the *record* has a 400 KB DynamoDB item limit, and the +# rendering sizes are the same tiles everywhere. +ICON_MAX_BYTES = 400 * 1024 +ICON_SIZE = 512 +# Below this, upscaling to 512 produces a soft tile that reads worse than the +# generated gradient it replaced — so we decline rather than accept a downgrade. +ICON_MIN_SOURCE = 256 +# A hand-cropped square is often off by a pixel; a 4:3 photo is not. +ICON_SQUARE_TOLERANCE_PX = 2 + +_FORMAT_EXT = {"PNG": "png", "JPEG": "jpg"} +_EXT_CONTENT_TYPE = {"png": "image/png", "jpg": "image/jpeg"} + +# AWS-managed (SSE-S3 / AES256) encryption, matching the bucket default. +_SSE_ALGORITHM = "AES256" + + +class IconError(ValueError): + """An icon the caller cannot store, with a message written for that caller. + + Every message names the limit *and* what was actually supplied, because + "invalid image" sends someone back to a file picker with nothing to change. + """ + + +class IconStoreError(RuntimeError): + """Storage is unavailable or the object could not be read/written.""" + + +# ── validation / normalization ─────────────────────────────────────────────── + + +def normalize_icon(content: bytes) -> Tuple[bytes, str, str]: + """Validate and normalize an uploaded icon. + + Returns ``(bytes, ext, content_type)`` for a 512×512, metadata-free PNG or + JPEG. Raises :class:`IconError` with a caller-facing message on anything it + declines. + """ + from PIL import Image, UnidentifiedImageError # lazy: keeps PIL off every importer + + if not content: + raise IconError("The uploaded file is empty.") + if len(content) > ICON_MAX_BYTES: + raise IconError( + f"Icons must be {ICON_MAX_BYTES // 1024} KB or smaller " + f"(this one is {len(content) // 1024} KB)." + ) + + try: + image = Image.open(io.BytesIO(content)) + source_format = image.format + image.load() + except (UnidentifiedImageError, OSError, ValueError) as e: + raise IconError("Icons must be a PNG or JPEG image.") from e + + if source_format not in _FORMAT_EXT: + raise IconError( + f"Icons must be a PNG or JPEG image (this one is {source_format or 'an unknown format'})." + ) + + width, height = image.size + if abs(width - height) > ICON_SQUARE_TOLERANCE_PX: + raise IconError(f"Icons must be square (this one is {width}×{height}).") + if min(width, height) < ICON_MIN_SOURCE: + raise IconError( + f"Icons must be at least {ICON_MIN_SOURCE}×{ICON_MIN_SOURCE} " + f"(this one is {width}×{height})." + ) + + ext = _FORMAT_EXT[source_format] + # Re-encoding always happens — see the module docstring on EXIF. LANCZOS + # because a 28px tile is an 18× downscale of the stored icon and cheaper + # filters alias badly. + image = image.convert("RGBA" if ext == "png" else "RGB") + if (width, height) != (ICON_SIZE, ICON_SIZE): + image = image.resize((ICON_SIZE, ICON_SIZE), Image.Resampling.LANCZOS) + + encoded = _encode_within_limit(image, ext) + if encoded is None: + raise IconError( + f"This icon could not be stored under {ICON_MAX_BYTES // 1024} KB. " + "Try a simpler image or fewer colors." + ) + data, ext = encoded + return data, ext, _EXT_CONTENT_TYPE[ext] + + +def _encode_within_limit(image, ext: str) -> Optional[Tuple[bytes, str]]: + """Encode at 512×512 under the size ceiling, degrading in defined steps. + + A downscale to 512 almost always lands well under 400 KB; this ladder exists + for the input that arrives *already* 512×512 and near the ceiling, where + re-encoding could push it over. Each rung is deliberate rather than a retry + loop: JPEG loses quality, an opaque PNG becomes a JPEG, and a transparent PNG + loses colors but keeps its alpha. + """ + from PIL import Image + + if ext == "jpg": + for quality in (92, 85, 78): + data = _save(image, "JPEG", quality=quality, optimize=True, progressive=True) + if len(data) <= ICON_MAX_BYTES: + return data, "jpg" + return None + + data = _save(image, "PNG", optimize=True) + if len(data) <= ICON_MAX_BYTES: + return data, "png" + + has_alpha = image.getchannel("A").getextrema()[0] < 255 + if not has_alpha: + for quality in (92, 85): + data = _save(image.convert("RGB"), "JPEG", quality=quality, optimize=True) + if len(data) <= ICON_MAX_BYTES: + return data, "jpg" + return None + + # FASTOCTREE, not the default MEDIANCUT: it is the only method Pillow will + # quantize an RGBA image with, and this rung exists precisely to keep the alpha. + quantized = image.quantize(colors=256, method=Image.Quantize.FASTOCTREE) + data = _save(quantized, "PNG", optimize=True) + return (data, "png") if len(data) <= ICON_MAX_BYTES else None + + +def _save(image, fmt: str, **kwargs) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format=fmt, **kwargs) + return buffer.getvalue() + + +def content_digest(content: bytes) -> str: + """The 16-hex-char content address used in the key and as the cache version.""" + return hashlib.sha256(content).hexdigest()[:16] + + +def key_version(icon_key: Optional[str]) -> Optional[str]: + """The cache version carried by a content-addressed key: its digest segment. + + Used as the ``?v=`` on an icon URL and as the ETag on a serve route, so a + stored icon can be cached ``immutable`` while a replacement busts it + immediately. + """ + if not icon_key: + return None + return icon_key.rsplit("/", 1)[-1].rsplit(".", 1)[0] + + +# ── S3 ─────────────────────────────────────────────────────────────────────── + + +class IconStore: + """Put / get / delete icon objects in one bucket. + + ``bucket_env`` is read lazily at construction; ``label`` only shapes log + lines and the "storage is not configured" message, so an operator reading a + log knows which feature went quiet. + """ + + def __init__( + self, + *, + bucket_env: str = "S3_ASSISTANTS_DOCUMENTS_BUCKET_NAME", + label: str = "icons", + bucket_name: Optional[str] = None, + s3_client: Optional[object] = None, + ) -> None: + self.bucket_env = bucket_env + self.label = label + self.bucket_name = bucket_name or os.environ.get(bucket_env) + # Lazily constructed so importing this module never needs AWS credentials; + # tests inject a client. + self._s3 = s3_client + + @property + def enabled(self) -> bool: + return bool(self.bucket_name) and boto3 is not None + + def _client(self): + if self._s3 is None: + if boto3 is None: # pragma: no cover - import-guarded above + raise IconStoreError("icon storage unavailable: boto3 is not installed") + self._s3 = boto3.client("s3") + return self._s3 + + def _require_enabled(self) -> None: + if not self.enabled: + raise IconStoreError( + f"{self.label}: icon storage is not configured ({self.bucket_env} is unset)" + ) + + def put_object(self, *, key: str, content: bytes, content_type: str) -> str: + """Store bytes at ``key`` and return it.""" + self._require_enabled() + try: + self._client().put_object( + Bucket=self.bucket_name, + Key=key, + Body=content, + ContentType=content_type, + ServerSideEncryption=_SSE_ALGORITHM, + # The object is immutable by construction (the key is its digest), + # so the cache directive belongs on the object as much as on the + # serve response. + CacheControl="public, max-age=31536000, immutable", + ) + except ClientError as e: # pragma: no cover - network/permission path + logger.error(f"{self.label}: put failed for key={key}: {e}") + raise IconStoreError(f"failed to store icon at key '{key}'") from e + + logger.info(f"🖼️ {self.label}: stored key={key} ({len(content)} bytes)") + return key + + def get(self, icon_key: str) -> Tuple[bytes, str]: + """Return ``(bytes, content_type)`` for a stored icon.""" + self._require_enabled() + try: + response = self._client().get_object(Bucket=self.bucket_name, Key=icon_key) + body = response["Body"].read() + return body, response.get("ContentType") or "application/octet-stream" + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in ("NoSuchKey", "404"): + raise IconStoreError(f"icon not found at key '{icon_key}'") from e + logger.error(f"{self.label}: get failed for key={icon_key}: {e}") + raise IconStoreError(f"failed to read icon at key '{icon_key}'") from e + + def delete(self, icon_key: str) -> None: + """Best-effort delete. Never raises: a replaced icon's old object going + missing is not a reason to fail the upload that replaced it.""" + if not self.enabled or not icon_key: + return + try: + self._client().delete_object(Bucket=self.bucket_name, Key=icon_key) + except ClientError as e: # pragma: no cover - network/permission path + logger.warning(f"{self.label}: delete failed for key={icon_key}: {e}") diff --git a/backend/src/apis/shared/kb_backend/byte_cap.py b/backend/src/apis/shared/kb_backend/byte_cap.py index dd5e532a4..bb261463a 100644 --- a/backend/src/apis/shared/kb_backend/byte_cap.py +++ b/backend/src/apis/shared/kb_backend/byte_cap.py @@ -127,6 +127,24 @@ def per_kb_ceiling() -> int: return _env_int("MANAGED_KB_PER_KB_CEILING_BYTES", DEFAULT_PER_KB_CEILING_BYTES) +def effective_cap(elevated: bool = False) -> int: + """The single binding cap for an owner's knowledge base this phase. + + ``App_KB_Id == assistant_id`` today, so one owner has exactly one managed + knowledge base and both limits — the per-owner allowance and the per-KB + ceiling — apply to the *same* accounting record. The binding limit is + therefore the smaller of the two. + + Returning ``min`` and reserving against it lets a single atomic + :func:`reserve` enforce both caps at once (Requirement 12.1). The obvious + alternative — reserve against the owner cap, then a second read-and-compare + against the ceiling — is not atomic: two concurrent uploads could each pass a + separate ceiling check and collectively breach it, which is the exact race a + cap exists to close. Folding both into one conditional write keeps 12.5. + """ + return min(per_owner_cap(elevated), per_kb_ceiling()) + + def _table(): import boto3 @@ -234,6 +252,45 @@ def release(assistant_id: str, app_kb_id: str, n_bytes: int) -> None: ) +def settle_once(assistant_id: str, document_id: str) -> bool: + """Claim the one-time right to settle a document's reservation. + + Returns ``True`` for exactly one caller per document and ``False`` for every + caller after it, by atomically stamping ``byteCapSettled`` on the ``DOC#`` row + under ``attribute_not_exists``. + + This is what makes commit/release idempotent. A reservation taken at request + time is settled — converted to stored bytes, or returned — on whichever + terminal path the document actually reaches: the ingestion consumer, a + client-reported upload failure, or the stale-document sweep. But those paths + are not mutually exclusive under concurrency, and the ingestion consumer in + particular is *redelivered* (its whole design turns on EventBridge's 2-retry + cap), so a document that reaches ``INDEXED`` is re-examined on every + redelivery. Without this guard a redelivery would commit the same bytes twice + — driving ``reservedBytes`` negative and inflating ``storedBytes`` — and two + racing failure paths would release the same reservation twice, over-crediting + the allowance and defeating the cap. Both break the invariant + ``totalBytes == storedBytes + reservedBytes`` that Property 5 rests on. + + Keyed on the ``DOC#`` row rather than a separate ledger so the claim shares the + document's own lifetime: delete the document and the marker goes with it. + """ + from botocore.exceptions import ClientError + + try: + _table().update_item( + Key={"PK": f"AST#{assistant_id}", "SK": f"DOC#{document_id}"}, + UpdateExpression="SET byteCapSettled = :true", + ConditionExpression="attribute_not_exists(byteCapSettled)", + ExpressionAttributeValues={":true": True}, + ) + return True + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") == "ConditionalCheckFailedException": + return False + raise + + def reserve_snapshot( assistant_id: str, app_kb_id: str, diff --git a/backend/src/apis/shared/kb_backend/provisioning.py b/backend/src/apis/shared/kb_backend/provisioning.py index 47280d698..82496c286 100644 --- a/backend/src/apis/shared/kb_backend/provisioning.py +++ b/backend/src/apis/shared/kb_backend/provisioning.py @@ -625,6 +625,47 @@ def _resource_name(app_kb_id: str, project_prefix: Optional[str] = None) -> str: return f"{tag_prefix(project_prefix)}-kb-{app_kb_id}" +def new_managed_kb_record( + app_kb_id: str, + owner_user_id: str, + *, + client_token: str, + visibility: str = "PRIVATE", +): + """The KB_Record a not-yet-built managed knowledge base starts life as. + + Factored out rather than inlined because born-managed + (``MANAGED_KB_NEW_DEFAULT``) writes this record from the API on first upload, + minutes before :func:`provision_managed_kb` ever runs. Two call sites building + the same record by hand is exactly the drift that produced the tag-contract + defect: they would agree on the day they were written and diverge on the day + one of them gained a field. ``parser_config`` in particular is not decoration — + a corpus indexed without image extraction is not comparable to one indexed + with it, and this record is where that fact is captured. + """ + from apis.shared.kb_backend import records as r + + return r.KbRecord( + app_kb_id=app_kb_id, + owner_user_id=owner_user_id, + visibility=visibility, + provisioning_state=r.PROVISIONING, + client_token=client_token, + # Not recorded: with no pin, Bedrock chooses the embedding model and + # nothing here would know which. A field naming Titan on a knowledge + # base Bedrock embedded with something else is worse than an absent + # one — nothing reads these for retrieval, so they can only mislead. + embedding_model_id=None, + embedding_dimensions=0, + image_extraction=True, + parser_config={ + "imageExtractionStatus": IMAGE_EXTRACTION_STATUS, + "connectorType": CONNECTOR_TYPE, + "embeddingDataType": EMBEDDING_DATA_TYPE, + }, + ) + + async def provision_managed_kb( assistant_id: str, app_kb_id: Optional[str] = None, @@ -697,23 +738,8 @@ async def provision_managed_kb( f"{existing.get('provisioningState')} record" ) else: - record = r.KbRecord( - app_kb_id=app_kb_id, - owner_user_id=owner_user_id, - provisioning_state=r.PROVISIONING, - client_token=kb_token, - # Not recorded: with no pin, Bedrock chooses the embedding model and - # nothing here would know which. A field naming Titan on a knowledge - # base Bedrock embedded with something else is worse than an absent - # one — nothing reads these for retrieval, so they can only mislead. - embedding_model_id=None, - embedding_dimensions=0, - image_extraction=True, - parser_config={ - "imageExtractionStatus": IMAGE_EXTRACTION_STATUS, - "connectorType": CONNECTOR_TYPE, - "embeddingDataType": EMBEDDING_DATA_TYPE, - }, + record = new_managed_kb_record( + app_kb_id, owner_user_id, client_token=kb_token ) try: # DDB before AWS. See the module docstring; this ordering is the diff --git a/backend/src/apis/shared/kb_backend/records.py b/backend/src/apis/shared/kb_backend/records.py index 5a5af07e2..ec5a4fed5 100644 --- a/backend/src/apis/shared/kb_backend/records.py +++ b/backend/src/apis/shared/kb_backend/records.py @@ -76,6 +76,18 @@ RETAIN = "retain" MIGRATION_FAILED = "failed" +#: Born-managed provisioning (``MANAGED_KB_NEW_DEFAULT``). Not part of the +#: shadow→verify→promote migration at all: there is no legacy corpus to carry +#: across and nothing to verify against, because the knowledge base is managed +#: from its first document. It borrows this column purely to reuse the sparse +#: work-key queue and the worker lease — the two things that make a minutes-long +#: provision survive a crash — rather than growing a second dispatcher. +#: +#: Deliberately NOT named ``provisioning``: that string is already the +#: ``provisioningState`` value, and two attributes carrying the same word with +#: different meanings is how the wrong one gets read. +BORN_MANAGED = "born_managed" + #: Reserved in the enum so a stored value round-trips, but never entered in this #: phase. Reclaiming legacy vectors is explicitly a follow-up spec; a worker that #: found itself here would delete data this phase has promised to retain. @@ -83,7 +95,12 @@ #: States that keep a record in the dispatcher's queue. Work keys are written on #: entering one of these. -WORK_ELIGIBLE_STATES = frozenset({SHADOW, VERIFY, PROMOTE}) +#: +#: ``BORN_MANAGED`` is here for the same reason the migration states are: it is +#: work the dispatcher must keep handing back until it reaches a terminal state. +#: Adding it here is what makes the dispatcher sweep it — ``_work_states`` derives +#: from this set rather than restating it. +WORK_ELIGIBLE_STATES = frozenset({BORN_MANAGED, SHADOW, VERIFY, PROMOTE}) #: States that take a record out of the queue for good. Work keys are removed on #: entering one of these. ``RETAIN`` is the terminal state this phase reaches; @@ -92,7 +109,7 @@ TERMINAL_STATES = frozenset({RETAIN, MIGRATION_FAILED}) ALL_MIGRATION_STATES = frozenset( - {SHADOW, VERIFY, PROMOTE, RETAIN, MIGRATION_FAILED, RECLAIM} + {BORN_MANAGED, SHADOW, VERIFY, PROMOTE, RETAIN, MIGRATION_FAILED, RECLAIM} ) @@ -426,6 +443,65 @@ def set_resource_policy_state( ) +def adopt_managed_engine( + assistant_id: str, + app_kb_id: str, + now_iso: str, +) -> None: + """Declare a brand-new knowledge base managed BEFORE it has been built. + + Born-managed's counterpart to :func:`promote_engine`, and deliberately not + the same function. ``promote_engine`` is guarded on ``migrationState = + promote`` and on the catch-up pass having converged, because it is a + **cutover**: a corpus already exists on legacy and must be proven carried + across before anything is switched. Here there is no corpus and nothing to + carry — the engine is declared first precisely so the first document is + picked up by the managed pipeline instead of the legacy one. + + That ordering is the whole point. The legacy ingestion handler skips a + document only when its record already resolves to ``managed`` + (``handler._resolve_engine``), so a knowledge base that became managed + *after* its first upload would have that document indexed on legacy, + answered from legacy, and then indexed a second time on managed. + + Two guards, both necessary: + + * ``attribute_not_exists(retrievalEngine)`` — never re-declare. A record that + is already managed (born that way, or promoted by a migration) must not have + its ``promotedAt``/``bornManagedAt`` rewritten by a retry, and a concurrent + second first-upload must lose rather than both "win". + * ``provisioningState = provisioning`` — only a record that is still being + built may be declared this way. Without it a torn-down (``deleting``) record + could be resurrected as managed with no knowledge base behind it. + + ``upgradeNoticeDismissedAt`` is stamped here on purpose. The upgrade card + reads a managed record as ``phase="succeeded"`` and offers the one-time "your + knowledge base was upgraded" notice; a knowledge base that was never on legacy + has nothing to be congratulated about, so the notice is retired before it can + ever be shown. + + Raises :class:`TransitionLost` when either guard fails, which every caller + treats as "somebody else got here first" rather than an error. + """ + _conditional( + _table().update_item, + Key={"PK": kb_pk(assistant_id), "SK": kb_sk(app_kb_id)}, + UpdateExpression=( + "SET retrievalEngine = :managed, bornManagedAt = :now, " + "upgradeNoticeDismissedAt = :now, updatedAt = :now" + ), + ConditionExpression=( + "attribute_not_exists(retrievalEngine) " + "AND provisioningState = :provisioning" + ), + ExpressionAttributeValues={ + ":managed": ENGINE_MANAGED, + ":provisioning": PROVISIONING, + ":now": now_iso, + }, + ) + + def promote_engine( assistant_id: str, app_kb_id: str, diff --git a/backend/src/apis/shared/middleware/proxied_redirect.py b/backend/src/apis/shared/middleware/proxied_redirect.py new file mode 100644 index 000000000..3d07c1bde --- /dev/null +++ b/backend/src/apis/shared/middleware/proxied_redirect.py @@ -0,0 +1,135 @@ +"""ProxiedRedirectMiddleware — keep app-generated redirects on the public URL. + +app-api never sees the URL the browser actually asked for. CloudFront's +`/api/*` behaviour strips the `/api` prefix, sends the request to the ALB +under the origin's own hostname (`ALL_VIEWER_EXCEPT_HOST_HEADER`), and the +ALB terminates TLS — so a request the user made as + + GET https://dev.boisestate.ai/api/agents/ + +arrives at Starlette as path `/agents/`, `Host: api.dev.boisestate.ai`, +scheme `http`. Starlette's `redirect_slashes` then answers it with an +*absolute* `Location` built from what it can see: + + Location: http://api.dev.boisestate.ai/agents + +which is wrong three times over. The browser refuses it outright as mixed +content ("was loaded over HTTPS, but requested an insecure resource"), so +the caller silently gets nothing; the internal ALB hostname leaks into a +page the user can read; and even if a client did follow it, the host is a +different origin, so the `__Host-` BFF session cookies would not be sent +and the redirected request would 401. + +This middleware rewrites exactly those self-referential redirects into a +root-relative `Location`, restoring the proxy's stripped prefix from +`X-Forwarded-Prefix` (stamped by the CloudFront path-strip Function): + + Location: /api/agents + +A root-relative `Location` inherits the browser's own scheme and host, so it +is correct under CloudFront, behind a bare ALB, and on localhost without the +app having to be told which one it is under. + +**Only self-referential redirects are touched.** A `Location` pointing at +another host is left exactly as-is — that is the BFF's OAuth flow bouncing +to Cognito/Entra and back to the SPA origin, and rewriting those would break +sign-in. + +Implemented as raw ASGI rather than `BaseHTTPMiddleware` on purpose: it needs +nothing but the response headers, and `BaseHTTPMiddleware` would wrap every +response body in an extra stream — including the SSE chat streams, which are +the one thing in this service least worth putting another layer around. +""" + +from __future__ import annotations + +import logging +from urllib.parse import urlsplit + +from starlette.datastructures import Headers, MutableHeaders +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +logger = logging.getLogger(__name__) + +#: Set by the CloudFront path-strip Function to the prefix it removed. +FORWARDED_PREFIX_HEADER = "x-forwarded-prefix" + + +def _sanitize_prefix(raw: str | None) -> str: + """Normalise `X-Forwarded-Prefix` to `''` or `/segment[/segment...]`. + + The header reaches us through CloudFront, which overwrites it on every + request to this origin, so a viewer cannot choose its value. It is still + normalised here rather than trusted: this middleware writes the result + into a `Location`, and a value like `//evil.example` would turn a + same-origin redirect into a scheme-relative jump off-site. + """ + if not raw: + return "" + prefix = raw.strip().rstrip("/") + if not prefix.startswith("/") or prefix.startswith("//"): + return "" + return prefix + + +def rewrite_location(location: str, request_headers: Headers) -> str | None: + """Return the rewritten `Location`, or `None` to leave it alone. + + `None` covers everything that is already correct or none of our business: + an already-relative `Location`, and any absolute one aimed at a different + host than the one this request arrived on. + """ + if not location: + return None + + parts = urlsplit(location) + if not parts.netloc: + # Already relative — the browser resolves it against the public URL. + return None + + request_host = request_headers.get("host", "") + if parts.netloc.lower() != request_host.lower(): + # Some other origin (Cognito, Entra, the SPA). Not ours to touch. + return None + + prefix = _sanitize_prefix(request_headers.get(FORWARDED_PREFIX_HEADER)) + rewritten = f"{prefix}{parts.path}" + if parts.query: + rewritten = f"{rewritten}?{parts.query}" + if parts.fragment: + rewritten = f"{rewritten}#{parts.fragment}" + + # A redirect to the host root with no prefix leaves nothing to anchor on. + if not rewritten.startswith("/"): + rewritten = f"/{rewritten}" + return rewritten + + +class ProxiedRedirectMiddleware: + """Rewrite self-referential absolute redirects to root-relative ones.""" + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + request_headers = Headers(scope=scope) + + async def send_wrapper(message: Message) -> None: + if message["type"] == "http.response.start": + status = message["status"] + if 300 <= status < 400: + headers = MutableHeaders(scope=message) + location = headers.get("location") + rewritten = rewrite_location(location or "", request_headers) + if rewritten is not None: + logger.debug( + "Rewrote proxied redirect %s -> %s", location, rewritten + ) + headers["location"] = rewritten + await send(message) + + await self.app(scope, receive, send_wrapper) diff --git a/backend/src/apis/shared/models/managed_models.py b/backend/src/apis/shared/models/managed_models.py index 7994c2e1d..5ed1df7cf 100644 --- a/backend/src/apis/shared/models/managed_models.py +++ b/backend/src/apis/shared/models/managed_models.py @@ -4,6 +4,7 @@ Requires DynamoDB storage via DYNAMODB_MANAGED_MODELS_TABLE_NAME. """ +import asyncio import logging import os import uuid @@ -14,6 +15,7 @@ import boto3 from botocore.exceptions import ClientError +from apis.shared.caching import config_cache from .models import ManagedModel, ManagedModelCreate, ManagedModelUpdate logger = logging.getLogger(__name__) @@ -224,7 +226,11 @@ async def create_managed_model(model_data: ManagedModelCreate) -> ManagedModel: managed_models_table = os.environ.get('DYNAMODB_MANAGED_MODELS_TABLE_NAME') if not managed_models_table: raise RuntimeError("DYNAMODB_MANAGED_MODELS_TABLE_NAME environment variable is required") - return await _create_managed_model_cloud(model_data, managed_models_table) + created = await _create_managed_model_cloud(model_data, managed_models_table) + # Invalidate here rather than in the admin route: every write to this table + # funnels through these three functions, so a future caller cannot forget. + config_cache.invalidate(config_cache.MANAGED_MODELS) + return created async def _create_managed_model_cloud(model_data: ManagedModelCreate, table_name: str) -> ManagedModel: @@ -275,6 +281,8 @@ async def _create_managed_model_cloud(model_data: ManagedModelCreate, table_name id=model_id, model_id=model_data.model_id, model_name=model_data.model_name, + short_description=model_data.short_description, + icon_slug=model_data.icon_slug, provider=model_data.provider, provider_name=model_data.provider_name, input_modalities=model_data.input_modalities, @@ -293,6 +301,7 @@ async def _create_managed_model_cloud(model_data: ManagedModelCreate, table_name knowledge_cutoff_date=model_data.knowledge_cutoff_date, supports_caching=_resolve_supports_caching(model_data.supports_caching, model_data.provider), is_default=model_data.is_default, + is_featured=model_data.is_featured, mantle_api_mode=_resolve_mantle_api_mode(model_data.mantle_api_mode, model_data.provider), mantle_region=_resolve_mantle_region(model_data.mantle_region, model_data.provider), supported_params=model_data.supported_params, @@ -320,6 +329,7 @@ async def _create_managed_model_cloud(model_data: ManagedModelCreate, table_name 'outputPricePerMillionTokens': model_data.output_price_per_million_tokens, 'supportsCaching': _resolve_supports_caching(model_data.supports_caching, model_data.provider), 'isDefault': model_data.is_default, + 'isFeatured': model_data.is_featured, 'createdAt': now.isoformat(), 'updatedAt': now.isoformat(), } @@ -333,6 +343,10 @@ async def _create_managed_model_cloud(model_data: ManagedModelCreate, table_name item['cacheReadPricePerMillionTokens'] = model_data.cache_read_price_per_million_tokens if model_data.knowledge_cutoff_date is not None: item['knowledgeCutoffDate'] = model_data.knowledge_cutoff_date + if model_data.short_description: + item['shortDescription'] = model_data.short_description + if model_data.icon_slug: + item['iconSlug'] = model_data.icon_slug resolved_api_mode = _resolve_mantle_api_mode(model_data.mantle_api_mode, model_data.provider) if resolved_api_mode is not None: item['apiMode'] = resolved_api_mode @@ -465,41 +479,56 @@ async def list_all_managed_models() -> List[ManagedModel]: return await _list_managed_models_cloud(managed_models_table) +def _scan_managed_model_items(table_name: str) -> List[dict]: + """Scan the raw MODEL# items. Blocking; call via ``asyncio.to_thread``.""" + table = dynamodb.Table(table_name) + + response = table.scan( + FilterExpression='begins_with(PK, :pk_prefix)', + ExpressionAttributeValues={ + ':pk_prefix': 'MODEL#' + } + ) + + items = response.get('Items', []) + + # Handle pagination + while 'LastEvaluatedKey' in response: + response = table.scan( + FilterExpression='begins_with(PK, :pk_prefix)', + ExpressionAttributeValues={ + ':pk_prefix': 'MODEL#' + }, + ExclusiveStartKey=response['LastEvaluatedKey'] + ) + items.extend(response.get('Items', [])) + + return items + + async def _list_managed_models_cloud(table_name: str) -> List[ManagedModel]: """ List all managed models from DynamoDB + The scan is cached per process (see ``apis.shared.caching.config_cache``); + parsing is not. Callers mutate the models they receive — the admin list + route hands them straight to ``hydrate_model_roles``, which writes + ``allowed_app_roles`` in place — so each caller must get objects it owns. + Args: table_name: DynamoDB table name Returns: List of ManagedModel objects """ - table = dynamodb.Table(table_name) models = [] try: - # Scan table for all models (PK starts with MODEL#) - response = table.scan( - FilterExpression='begins_with(PK, :pk_prefix)', - ExpressionAttributeValues={ - ':pk_prefix': 'MODEL#' - } + items = await config_cache.get_or_load( + config_cache.MANAGED_MODELS, + lambda: asyncio.to_thread(_scan_managed_model_items, table_name), ) - items = response.get('Items', []) - - # Handle pagination - while 'LastEvaluatedKey' in response: - response = table.scan( - FilterExpression='begins_with(PK, :pk_prefix)', - ExpressionAttributeValues={ - ':pk_prefix': 'MODEL#' - }, - ExclusiveStartKey=response['LastEvaluatedKey'] - ) - items.extend(response.get('Items', [])) - # Convert items to ManagedModel objects for item in items: try: @@ -557,7 +586,9 @@ async def update_managed_model(model_id: str, updates: ManagedModelUpdate) -> Op managed_models_table = os.environ.get('DYNAMODB_MANAGED_MODELS_TABLE_NAME') if not managed_models_table: raise RuntimeError("DYNAMODB_MANAGED_MODELS_TABLE_NAME environment variable is required") - return await _update_managed_model_cloud(model_id, updates, managed_models_table) + updated = await _update_managed_model_cloud(model_id, updates, managed_models_table) + config_cache.invalidate(config_cache.MANAGED_MODELS) + return updated async def _update_managed_model_cloud(model_id: str, updates: ManagedModelUpdate, table_name: str) -> Optional[ManagedModel]: @@ -626,9 +657,19 @@ async def _update_managed_model_cloud(model_id: str, updates: ManagedModelUpdate # Build update expression update_expression_parts = [] + remove_expression_parts = [] expression_attribute_names = {} expression_attribute_values = {} + # '' on iconSlug is the wire value for "clear it" — None can't be, because + # the model_dump above drops None fields, which is what makes a PATCH a + # PATCH. Removing the attribute rather than storing '' keeps the record + # shaped like one that never had an icon. + if update_data.get('iconSlug') == '': + update_data.pop('iconSlug') + remove_expression_parts.append('#iconSlug') + expression_attribute_names['#iconSlug'] = 'iconSlug' + # Add updatedAt timestamp update_data['updatedAt'] = datetime.now(timezone.utc).isoformat() @@ -650,6 +691,8 @@ async def _update_managed_model_cloud(model_id: str, updates: ManagedModelUpdate update_expression_parts.append('#GSI1PK = :GSI1PK') update_expression = "SET " + ", ".join(update_expression_parts) + if remove_expression_parts: + update_expression += " REMOVE " + ", ".join(remove_expression_parts) try: response = table.update_item( @@ -689,6 +732,53 @@ async def _update_managed_model_cloud(model_id: str, updates: ManagedModelUpdate raise +async def write_model_icon_key(model_id: str, icon_key: Optional[str]) -> None: + """Set or clear a model's uploaded-icon key, and nothing else. + + A dedicated writer rather than a field on ``ManagedModelUpdate`` because the + key is not admin-supplied data: it is produced by the upload path from the + bytes it just stored. Routing it through the general update model would make + it forgeable from the model form — an admin could point one model's record at + another's object, or at any key in the bucket. + + Invalidates the catalog cache, or the new icon would not appear for up to a + minute on the task that served the upload. + """ + table_name = os.environ.get('DYNAMODB_MANAGED_MODELS_TABLE_NAME') + if not table_name: + raise RuntimeError("DYNAMODB_MANAGED_MODELS_TABLE_NAME environment variable is required") + + table = dynamodb.Table(table_name) + key = {'PK': f'MODEL#{model_id}', 'SK': f'MODEL#{model_id}'} + now = datetime.now(timezone.utc).isoformat() + + try: + if icon_key: + table.update_item( + Key=key, + UpdateExpression='SET #iconKey = :iconKey, #updatedAt = :updatedAt', + ExpressionAttributeNames={'#iconKey': 'iconKey', '#updatedAt': 'updatedAt'}, + ExpressionAttributeValues={':iconKey': icon_key, ':updatedAt': now}, + ConditionExpression='attribute_exists(PK)', + ) + else: + table.update_item( + Key=key, + UpdateExpression='SET #updatedAt = :updatedAt REMOVE #iconKey', + ExpressionAttributeNames={'#iconKey': 'iconKey', '#updatedAt': 'updatedAt'}, + ExpressionAttributeValues={':updatedAt': now}, + ConditionExpression='attribute_exists(PK)', + ) + except ClientError as e: + if e.response['Error']['Code'] == 'ConditionalCheckFailedException': + raise ValueError(f"Model not found: {model_id}") from e + logger.error(f"Failed to write icon key for model {model_id}: {e}") + raise + + config_cache.invalidate(config_cache.MANAGED_MODELS) + logger.info(f"🖼️ model-icons: record {model_id} now points at {icon_key or '(none)'}") + + async def delete_managed_model(model_id: str) -> bool: """ Delete an managed model @@ -702,7 +792,9 @@ async def delete_managed_model(model_id: str) -> bool: managed_models_table = os.environ.get('DYNAMODB_MANAGED_MODELS_TABLE_NAME') if not managed_models_table: raise RuntimeError("DYNAMODB_MANAGED_MODELS_TABLE_NAME environment variable is required") - return await _delete_managed_model_cloud(model_id, managed_models_table) + deleted = await _delete_managed_model_cloud(model_id, managed_models_table) + config_cache.invalidate(config_cache.MANAGED_MODELS) + return deleted async def _delete_managed_model_cloud(model_id: str, table_name: str) -> bool: @@ -729,7 +821,16 @@ async def _delete_managed_model_cloud(model_id: str, table_name: str) -> bool: ) # Check if item was actually deleted - if response.get('Attributes'): + attributes = response.get('Attributes') + if attributes: + # The record is gone; its uploaded icon should go with it. Best-effort + # and after the fact — an orphaned object costs pennies, while failing + # the delete over one would leave the admin with a model they can't + # remove. A built-in iconSlug has no object to clean up. + icon_key = attributes.get('iconKey') + if icon_key: + from apis.shared.models.model_icons import get_model_icon_store + get_model_icon_store().delete(icon_key) logger.info(f"🗑️ Deleted managed model from DynamoDB: {model_id}") return True return False diff --git a/backend/src/apis/shared/models/model_icons.py b/backend/src/apis/shared/models/model_icons.py new file mode 100644 index 000000000..274450b23 --- /dev/null +++ b/backend/src/apis/shared/models/model_icons.py @@ -0,0 +1,142 @@ +"""Icons for managed models — the built-in logo slug and the uploaded override. + +A model in the chat picker gets a left-aligned avatar, and there are two ways an +admin supplies one: + +**A built-in logo slug** (``iconSlug``). The SPA already ships a light/dark SVG +pair per vendor under ``public/img/provider-logos/{slug}/``, which is what the +admin model catalog's quick-add cards render. Pointing at one of those is a +string on the record and costs no storage, no upload and no serve round trip — +and it stays a crisp, theme-correct vector, which a stored raster cannot be. +That is why this is not merely the fallback: for the vendors we ship, it is the +*better* answer. + +**An uploaded image** (``iconKey`` → ``iconUrl``). For everything else — a +fine-tuned in-house model, a vendor we ship no logo for — the bytes go to S3 and +the record carries only the content-addressed key, exactly like Agent icons +(:mod:`apis.shared.assistants.icons`). Validation and storage are the shared +machinery in :mod:`apis.shared.images.icons`. + +Precedence is upload → slug → (client-side) provider-name match → generic glyph. +An upload wins because it is the more specific, more deliberate act: an admin who +uploaded a file after choosing a slug meant the file. + +Storage +------- +Objects land in the assistants asset bucket +(``S3_ASSISTANTS_DOCUMENTS_BUCKET_NAME`` → ``{prefix}-rag-documents``), under a +prefix of their own: + + models/{model_id}/icons/{sha256[:16]}.{png|jpg} + +``model_id`` here is the record's UUID (``ManagedModel.id``), not the Bedrock +model identifier — the latter contains ``:`` and ``.`` and changes when an admin +re-points a record at a new model version, which would orphan the object. +""" + +from __future__ import annotations + +import logging +from typing import Optional + +from apis.shared.images.icons import ( # noqa: F401 - part of this module's surface + ICON_MAX_BYTES, + ICON_MIN_SOURCE, + ICON_SIZE, + IconError, + IconStore, + IconStoreError, + content_digest, + key_version, + normalize_icon, +) + +logger = logging.getLogger(__name__) + +#: Built-in vendor logos shipped with the SPA. Each slug must have a +#: ``frontend/ai.client/public/img/provider-logos/{slug}/{light,dark}.svg`` pair — +#: adding a vendor means dropping the pair in *and* listing it here and in the +#: SPA's ``BUILTIN_MODEL_ICONS``. Validated on write so a typo is rejected at the +#: admin form rather than rendering an invisible tile for every user. +BUILTIN_MODEL_ICONS: tuple[str, ...] = ("amazon", "anthropic", "meta", "openai") + + +def is_builtin_icon_slug(slug: Optional[str]) -> bool: + """Whether ``slug`` names a logo the SPA actually ships. Empty/None is valid + input meaning "no built-in icon" — see :func:`normalize_icon_slug`.""" + return slug in BUILTIN_MODEL_ICONS + + +def normalize_icon_slug(slug: Optional[str], *, keep_clear_sentinel: bool = False) -> Optional[str]: + """Validate a built-in slug, raising :class:`ValueError` for an unknown one. + + ``""`` is the wire value for *clear this*, and on a PATCH it must stay ``""`` + rather than collapse to ``None``: the update path drops ``None`` fields + (``exclude_none``), so an admin who set a slug could otherwise never remove + it. Hence ``keep_clear_sentinel`` — on for the update model, off everywhere + else, where an absent slug and a cleared one mean the same thing. + """ + if slug is None: + return None + if slug.strip() == "": + return "" if keep_clear_sentinel else None + slug = slug.strip().lower() + if not is_builtin_icon_slug(slug): + raise ValueError( + f"Unknown icon '{slug}'. Built-in icons are: {', '.join(BUILTIN_MODEL_ICONS)}." + ) + return slug + + +# ── keys and URLs ──────────────────────────────────────────────────────────── + + +def build_model_icon_key(model_id: str, digest: str, ext: str) -> str: + """``models/{model_id}/icons/{digest}.{ext}`` — the content-addressed key.""" + return f"models/{model_id}/icons/{digest}.{ext}" + + +def model_icon_version(icon_key: Optional[str]) -> Optional[str]: + """The key's digest segment, served as the ETag and as ``iconUrl``'s ``?v=``.""" + return key_version(icon_key) + + +def model_icon_url(model_id: str, icon_key: Optional[str]) -> Optional[str]: + """The read-shape ``iconUrl`` for a stored key, or ``None`` when unset. + + A relative app-api path (the SPA prefixes ``config.appApiUrl()``), pointing at + the *user-facing* serve route rather than an admin one: every signed-in user + renders these in the model picker, and only admins can reach ``/admin/*``. + The ``?v=`` digest is what lets the response be cached ``immutable`` while a + replacement takes effect immediately. + """ + version = model_icon_version(icon_key) + if not version: + return None + return f"/models/{model_id}/icon?v={version}" + + +# ── S3 ─────────────────────────────────────────────────────────────────────── + + +class ModelIconStore(IconStore): + """Put / get / delete managed-model icons in the assistants asset bucket.""" + + def __init__(self, bucket_name: Optional[str] = None, s3_client: Optional[object] = None) -> None: + super().__init__(label="model-icons", bucket_name=bucket_name, s3_client=s3_client) + + def put(self, *, model_id: str, content: bytes, ext: str, content_type: str) -> str: + """Store normalized bytes and return the content-addressed key.""" + key = build_model_icon_key(model_id, content_digest(content), ext) + return self.put_object(key=key, content=content, content_type=content_type) + + +_store: Optional[ModelIconStore] = None + + +def get_model_icon_store() -> ModelIconStore: + """Process-wide store, bound on first use (the env is set by the time routes run).""" + global _store + if _store is None: + _store = ModelIconStore() + return _store diff --git a/backend/src/apis/shared/models/models.py b/backend/src/apis/shared/models/models.py index 6b861efa5..efc7d8968 100644 --- a/backend/src/apis/shared/models/models.py +++ b/backend/src/apis/shared/models/models.py @@ -4,10 +4,12 @@ app API and inference API deployments. """ -from pydantic import BaseModel, Field, ConfigDict, model_validator +from pydantic import BaseModel, Field, ConfigDict, computed_field, field_validator, model_validator from typing import Any, Dict, List, Optional from datetime import datetime +from apis.shared.models.model_icons import model_icon_url, normalize_icon_slug + class ModelParamSpec(BaseModel): """Capability + bounds for a single inference parameter on a model. @@ -151,6 +153,29 @@ class ManagedModelCreate(BaseModel): model_id: str = Field(..., alias="modelId", min_length=1) model_name: str = Field(..., alias="modelName", min_length=1) + short_description: Optional[str] = Field( + None, + alias="shortDescription", + max_length=80, + description="One-line reason a user would pick this model, shown under its name " + "in the chat model picker. Keep it short — the picker truncates.", + ) + icon_slug: Optional[str] = Field( + None, + alias="iconSlug", + description="Built-in vendor logo to show beside this model in the chat " + "picker (e.g. 'anthropic'). A crisp, theme-aware SVG the SPA " + "already ships — prefer it over an upload when we have one. " + "Send '' to clear it; an uploaded icon takes precedence.", + ) + + @field_validator("icon_slug") + @classmethod + def _validate_icon_slug(cls, value: Optional[str]) -> Optional[str]: + # Normalizes case/whitespace and rejects a slug we ship no asset for, + # which would otherwise render an invisible tile for every user. + return normalize_icon_slug(value) + provider: str = Field(..., min_length=1) provider_name: str = Field(..., alias="providerName", min_length=1) input_modalities: List[str] = Field(..., alias="inputModalities", min_length=1) @@ -201,6 +226,14 @@ class ManagedModelCreate(BaseModel): alias="isDefault", description="Whether this is the default model for new sessions. Only one model can be default." ) + is_featured: bool = Field( + True, + alias="isFeatured", + description="Whether the model appears at the top level of the chat model " + "picker. False collapses it into the picker's 'More models' " + "submenu. Defaults to True so an uncurated catalog keeps showing " + "every model where it always has." + ) mantle_api_mode: Optional[str] = Field( None, alias="apiMode", @@ -247,6 +280,29 @@ class ManagedModelUpdate(BaseModel): model_id: Optional[str] = Field(None, alias="modelId", min_length=1) model_name: Optional[str] = Field(None, alias="modelName") + short_description: Optional[str] = Field( + None, + alias="shortDescription", + max_length=80, + description="One-line reason a user would pick this model, shown under its name " + "in the chat model picker. Keep it short — the picker truncates.", + ) + icon_slug: Optional[str] = Field( + None, + alias="iconSlug", + description="Built-in vendor logo to show beside this model in the chat " + "picker (e.g. 'anthropic'). A crisp, theme-aware SVG the SPA " + "already ships — prefer it over an upload when we have one. " + "Send '' to clear it; an uploaded icon takes precedence.", + ) + + @field_validator("icon_slug") + @classmethod + def _validate_icon_slug(cls, value: Optional[str]) -> Optional[str]: + # Same validation as create, except '' survives as '': on a PATCH it is + # the only way to say "remove the slug", since None means "don't touch". + return normalize_icon_slug(value, keep_clear_sentinel=True) + provider: Optional[str] = None provider_name: Optional[str] = Field(None, alias="providerName") input_modalities: Optional[List[str]] = Field(None, alias="inputModalities") @@ -293,6 +349,14 @@ class ManagedModelUpdate(BaseModel): alias="isDefault", description="Whether this is the default model for new sessions." ) + is_featured: Optional[bool] = Field( + None, + alias="isFeatured", + description="Whether the model appears at the top level of the chat model " + "picker. False collapses it into the picker's 'More models' " + "submenu. Defaults to True so an uncurated catalog keeps showing " + "every model where it always has." + ) mantle_api_mode: Optional[str] = Field( None, alias="apiMode", @@ -332,6 +396,32 @@ class ManagedModel(BaseModel): id: str model_id: str = Field(..., alias="modelId") model_name: str = Field(..., alias="modelName") + short_description: Optional[str] = Field( + None, + alias="shortDescription", + # Deliberately NOT length-capped here, unlike the create/update models. + # This is the READ model: a stored value longer than the write-path cap + # (hand-edited record, or a future cap that shrinks) would fail + # validation and take the whole /models listing down with it. Bound the + # input, be permissive about what is already persisted; the picker + # truncates visually anyway. + description="One-line reason a user would pick this model, shown under its name " + "in the chat model picker.", + ) + icon_slug: Optional[str] = Field( + None, + alias="iconSlug", + description="Built-in vendor logo slug (e.g. 'anthropic'). The SPA resolves " + "it to its shipped light/dark SVG pair. Superseded by iconUrl " + "when an icon has been uploaded.", + ) + icon_key: Optional[str] = Field( + None, + alias="iconKey", + description="S3 object key for an uploaded icon. Internal — clients read " + "iconUrl, which is derived from this.", + ) + provider: str provider_name: str = Field(..., alias="providerName") input_modalities: List[str] = Field(..., alias="inputModalities") @@ -385,6 +475,14 @@ class ManagedModel(BaseModel): alias="isDefault", description="Whether this is the default model for new sessions. Only one model can be default." ) + is_featured: bool = Field( + True, + alias="isFeatured", + description="Whether the model appears at the top level of the chat model " + "picker. False collapses it into the picker's 'More models' " + "submenu. Defaults to True so an uncurated catalog keeps showing " + "every model where it always has." + ) mantle_api_mode: Optional[str] = Field( None, alias="apiMode", @@ -411,6 +509,17 @@ class ManagedModel(BaseModel): alias="supportedParams", description="Per-model inference parameter capabilities." ) + @computed_field(alias="iconUrl", return_type=Optional[str]) # type: ignore[prop-decorator] + @property + def icon_url(self) -> Optional[str]: + """Path that serves the uploaded icon, or ``None`` when there isn't one. + + Derived rather than stored so the ``?v=`` cache-buster can never disagree + with the key it is meant to describe. Clients choose: ``iconUrl`` first, + then ``iconSlug``, then their own provider-name fallback. + """ + return model_icon_url(self.id, self.icon_key) + created_at: datetime = Field(..., alias="createdAt") updated_at: datetime = Field(..., alias="updatedAt") diff --git a/backend/src/apis/shared/oauth/provider_repository.py b/backend/src/apis/shared/oauth/provider_repository.py index 79df9d868..75ee53c11 100644 --- a/backend/src/apis/shared/oauth/provider_repository.py +++ b/backend/src/apis/shared/oauth/provider_repository.py @@ -5,6 +5,7 @@ registered directly with AgentCore Identity by the admin route. """ +import asyncio import logging import os from typing import List, Optional @@ -12,6 +13,7 @@ import boto3 from botocore.exceptions import ClientError +from apis.shared.caching import config_cache from .models import OAuthProvider, OAuthProviderUpdate from apis.shared.timestamps import utc_now_iso @@ -67,34 +69,19 @@ async def list_providers(self, enabled_only: bool = False) -> List[OAuthProvider return [] try: + # Two entries, because the branches return different rows. Both are + # dropped on any write — see `invalidate_oauth_providers`. Parsing + # stays per-call so each caller owns its OAuthProvider objects. if enabled_only: - response = self._table.query( - IndexName="EnabledProvidersIndex", - KeyConditionExpression="GSI1PK = :pk", - ExpressionAttributeValues={":pk": "ENABLED#true"}, + items = await config_cache.get_or_load( + config_cache.OAUTH_PROVIDERS_ENABLED, + lambda: asyncio.to_thread(self._query_enabled_provider_items), ) - items = response.get("Items", []) - while "LastEvaluatedKey" in response: - response = self._table.query( - IndexName="EnabledProvidersIndex", - KeyConditionExpression="GSI1PK = :pk", - ExpressionAttributeValues={":pk": "ENABLED#true"}, - ExclusiveStartKey=response["LastEvaluatedKey"], - ) - items.extend(response.get("Items", [])) else: - response = self._table.scan( - FilterExpression="SK = :sk", - ExpressionAttributeValues={":sk": "CONFIG"}, + items = await config_cache.get_or_load( + config_cache.OAUTH_PROVIDERS_ALL, + lambda: asyncio.to_thread(self._scan_provider_items), ) - items = response.get("Items", []) - while "LastEvaluatedKey" in response: - response = self._table.scan( - FilterExpression="SK = :sk", - ExpressionAttributeValues={":sk": "CONFIG"}, - ExclusiveStartKey=response["LastEvaluatedKey"], - ) - items.extend(response.get("Items", [])) providers = [OAuthProvider.from_dynamo_item(item) for item in items] providers.sort(key=lambda p: p.display_name.lower()) @@ -103,6 +90,40 @@ async def list_providers(self, enabled_only: bool = False) -> List[OAuthProvider logger.error("Error listing providers: %s", e) raise + def _query_enabled_provider_items(self) -> List[dict]: + """Query raw enabled-provider items. Blocking; call via ``to_thread``.""" + response = self._table.query( + IndexName="EnabledProvidersIndex", + KeyConditionExpression="GSI1PK = :pk", + ExpressionAttributeValues={":pk": "ENABLED#true"}, + ) + items = response.get("Items", []) + while "LastEvaluatedKey" in response: + response = self._table.query( + IndexName="EnabledProvidersIndex", + KeyConditionExpression="GSI1PK = :pk", + ExpressionAttributeValues={":pk": "ENABLED#true"}, + ExclusiveStartKey=response["LastEvaluatedKey"], + ) + items.extend(response.get("Items", [])) + return items + + def _scan_provider_items(self) -> List[dict]: + """Scan all raw provider items. Blocking; call via ``to_thread``.""" + response = self._table.scan( + FilterExpression="SK = :sk", + ExpressionAttributeValues={":sk": "CONFIG"}, + ) + items = response.get("Items", []) + while "LastEvaluatedKey" in response: + response = self._table.scan( + FilterExpression="SK = :sk", + ExpressionAttributeValues={":sk": "CONFIG"}, + ExclusiveStartKey=response["LastEvaluatedKey"], + ) + items.extend(response.get("Items", [])) + return items + # ------------------------------------------------------------------ writes async def put_provider(self, provider: OAuthProvider) -> OAuthProvider: """Upsert a fully-formed provider record. @@ -114,6 +135,7 @@ async def put_provider(self, provider: OAuthProvider) -> OAuthProvider: raise RuntimeError("OAuth provider repository is not enabled") self._table.put_item(Item=provider.to_dynamo_item()) + config_cache.invalidate_oauth_providers() logger.info("Upserted OAuth provider: %s", provider.provider_id) return provider @@ -165,6 +187,7 @@ async def apply_metadata_update( existing.updated_at = utc_now_iso() self._table.put_item(Item=existing.to_dynamo_item()) + config_cache.invalidate_oauth_providers() logger.info("Updated OAuth provider metadata: %s", provider_id) return existing @@ -180,6 +203,7 @@ async def delete_provider(self, provider_id: str) -> bool: self._table.delete_item( Key={"PK": f"PROVIDER#{provider_id}", "SK": "CONFIG"} ) + config_cache.invalidate_oauth_providers() logger.info("Deleted OAuth provider: %s", provider_id) return True except ClientError as e: diff --git a/backend/src/apis/shared/rbac/service.py b/backend/src/apis/shared/rbac/service.py index e2b42b2f2..7fa8a353d 100644 --- a/backend/src/apis/shared/rbac/service.py +++ b/backend/src/apis/shared/rbac/service.py @@ -262,11 +262,17 @@ async def _tool_grant_set(self, user: User) -> Set[str]: return granted | set(await get_public_tool_ids()) async def can_access_tool(self, user: User, tool_id: str) -> bool: - """Check if user can access a specific tool. + """Check if user can access a specific tool, bare or scoped. - Exact-match on the id by design: callers pass a bare catalog id - (an Agent's ``binding.ref``, validated against the author's palette - at design time), never a scoped ``base::tool`` id. + ``tool_id`` may be a bare catalog id or a scoped ``base::tool`` id + referencing one tool within an MCP server (an Agent's ``binding.ref`` + carries either). A scoped id is accessible when its **base server id** + is granted: scoping narrows a grant, it never widens one, so a role + that grants the whole server necessarily admits any subset of it. + + This is the same predicate ``filter_requested_tools`` applies on the + ``enabled_tools`` axis — keep the two in agreement, or a subset the + picker admits will be denied on the bindings axis (and vice versa). """ allowed = await self._tool_grant_set(user) @@ -274,7 +280,7 @@ async def can_access_tool(self, user: User, tool_id: str) -> bool: if "*" in allowed: return True - return tool_id in allowed + return tool_id in allowed or base_tool_id(tool_id) in allowed async def can_access_model(self, user: User, model_id: str) -> bool: """Check if user can access a specific model.""" diff --git a/backend/src/apis/shared/sessions/messages.py b/backend/src/apis/shared/sessions/messages.py index ced26b578..13a2d785c 100644 --- a/backend/src/apis/shared/sessions/messages.py +++ b/backend/src/apis/shared/sessions/messages.py @@ -436,14 +436,33 @@ async def fetch_ui_resources(): user_id=user_id, ) - # Run fetches in parallel - messages_raw, metadata_index, pending_interrupts, ui_resource_rows = ( - await asyncio.gather( - fetch_messages(), - fetch_metadata(), - fetch_pending_interrupts(), - fetch_ui_resources(), + async def fetch_tool_summaries(): + """Fetch persisted model-generated tool-batch summaries. + + Sync DynamoDB query off the session GSI; runs in a thread pool. + Empty when the table is absent (dev) or nothing was summarized. + """ + from apis.shared.tool_summaries import get_tool_summary_store + + return await asyncio.to_thread( + get_tool_summary_store().list_for_session, + session_id=session_id, + user_id=user_id, ) + + # Run fetches in parallel + ( + messages_raw, + metadata_index, + pending_interrupts, + ui_resource_rows, + tool_summary_rows, + ) = await asyncio.gather( + fetch_messages(), + fetch_metadata(), + fetch_pending_interrupts(), + fetch_ui_resources(), + fetch_tool_summaries(), ) messages_raw = list(messages_raw or []) @@ -519,11 +538,30 @@ async def fetch_ui_resources(): } ) + # Tool-batch summaries: the `tool_group_summary` SSE is emitted once + # mid-turn and never re-streams, so without this replay a reloaded + # conversation silently downgrades from the model's prose line to the + # SPA's deterministic formatter. First page only — the SPA keys them by + # toolUseId and holds them all regardless of which page renders the + # correlated tool_use block (same reasoning as uiResources above). + tool_summaries: List[Dict[str, Any]] = [] + if not next_token: + tool_summaries = [ + { + "batchId": row.get("batchId", ""), + "toolUseIds": row.get("toolUseIds") or [], + "summary": row.get("summary", ""), + } + for row in tool_summary_rows + if row.get("summary") + ] + return MessagesListResponse( messages=message_responses, next_token=next_page_token, pending_interrupts=pending_interrupts, ui_resources=ui_resources, + tool_summaries=tool_summaries, ) except Exception as e: diff --git a/backend/src/apis/shared/sessions/models.py b/backend/src/apis/shared/sessions/models.py index 47235c332..40c11d4d0 100644 --- a/backend/src/apis/shared/sessions/models.py +++ b/backend/src/apis/shared/sessions/models.py @@ -687,3 +687,8 @@ class MessagesListResponse(BaseModel): alias="uiResources", description="Persisted MCP App UI resources (SEP-1865) for this session, each shaped like the inline `ui_resource` SSE event ({type, toolUseId, resourceUri, html, mimeType, csp, permissions, sandboxOrigin}). Replayed on load to re-seed McpAppStateService and re-instantiate the mcp-app-frame iframe. Returned only on the first page.", ) + tool_summaries: List[Dict[str, Any]] = Field( + default_factory=list, + alias="toolSummaries", + description="Persisted model-generated tool-batch summaries for this session, each shaped like the live `tool_group_summary` SSE event ({batchId, toolUseIds, summary}). Replayed on load so a reloaded conversation keeps the prose line the user saw live instead of downgrading to the client-side deterministic formatter. Returned only on the first page.", + ) diff --git a/backend/src/apis/shared/system_prompts/repository.py b/backend/src/apis/shared/system_prompts/repository.py index ec6ca4996..3dc169379 100644 --- a/backend/src/apis/shared/system_prompts/repository.py +++ b/backend/src/apis/shared/system_prompts/repository.py @@ -1,5 +1,6 @@ """DynamoDB repository for admin-managed system prompts.""" +import asyncio import logging import os import uuid @@ -8,6 +9,7 @@ import boto3 from botocore.exceptions import ClientError +from apis.shared.caching import config_cache from .models import SystemPrompt, SystemPromptCreate, SystemPromptUpdate from apis.shared.timestamps import utc_now_iso @@ -53,18 +55,12 @@ async def list_prompts(self, enabled_only: bool = False) -> List[SystemPrompt]: return [] try: - response = self._table.scan( - FilterExpression="SK = :sk", - ExpressionAttributeValues={":sk": "METADATA"}, + # Cached per process; parsing stays per-call so each caller owns + # the SystemPrompt objects it gets back. + items = await config_cache.get_or_load( + config_cache.SYSTEM_PROMPTS, + lambda: asyncio.to_thread(self._scan_prompt_items), ) - items = response.get("Items", []) - while "LastEvaluatedKey" in response: - response = self._table.scan( - FilterExpression="SK = :sk", - ExpressionAttributeValues={":sk": "METADATA"}, - ExclusiveStartKey=response["LastEvaluatedKey"], - ) - items.extend(response.get("Items", [])) except ClientError: logger.error("Error listing system prompts", exc_info=True) raise @@ -75,6 +71,22 @@ async def list_prompts(self, enabled_only: bool = False) -> List[SystemPrompt]: prompts.sort(key=lambda p: p.name.lower()) return prompts + def _scan_prompt_items(self) -> List[dict]: + """Scan the raw PROMPT# items. Blocking; call via ``asyncio.to_thread``.""" + response = self._table.scan( + FilterExpression="SK = :sk", + ExpressionAttributeValues={":sk": "METADATA"}, + ) + items = response.get("Items", []) + while "LastEvaluatedKey" in response: + response = self._table.scan( + FilterExpression="SK = :sk", + ExpressionAttributeValues={":sk": "METADATA"}, + ExclusiveStartKey=response["LastEvaluatedKey"], + ) + items.extend(response.get("Items", [])) + return items + async def get_prompt(self, prompt_id: str) -> Optional[SystemPrompt]: """Return a single prompt by ID, or None if not found.""" if not self._enabled: @@ -119,6 +131,7 @@ async def create_prompt( logger.error("Error creating system prompt", exc_info=True) raise + config_cache.invalidate(config_cache.SYSTEM_PROMPTS) logger.info(f"Created system prompt: {prompt.prompt_id} name={prompt.name!r}") return prompt @@ -156,6 +169,7 @@ async def update_prompt( logger.error("Error updating system prompt", exc_info=True) raise + config_cache.invalidate(config_cache.SYSTEM_PROMPTS) logger.info(f"Updated system prompt: {prompt_id}") return existing @@ -173,6 +187,7 @@ async def delete_prompt(self, prompt_id: str) -> bool: except ClientError: logger.error("Error deleting system prompt", exc_info=True) raise + config_cache.invalidate(config_cache.SYSTEM_PROMPTS) logger.info(f"Deleted system prompt: {prompt_id}") return True diff --git a/backend/src/apis/shared/tool_summaries/__init__.py b/backend/src/apis/shared/tool_summaries/__init__.py new file mode 100644 index 000000000..7bf778e20 --- /dev/null +++ b/backend/src/apis/shared/tool_summaries/__init__.py @@ -0,0 +1,19 @@ +"""Model-generated one-line summaries for finished tool batches. + +The write side runs from the agents stream coordinator (where the batch is +born); the read side runs on the app-api messages endpoint. Both reach this +shared package and neither imports the other — import-boundary safe +(``tests/architecture/test_import_boundaries.py``). +""" + +from apis.shared.tool_summaries.store import ( + ToolSummaryStore, + get_tool_summary_store, +) +from apis.shared.tool_summaries.summarizer import summarize_tool_batch + +__all__ = [ + "ToolSummaryStore", + "get_tool_summary_store", + "summarize_tool_batch", +] diff --git a/backend/src/apis/shared/tool_summaries/store.py b/backend/src/apis/shared/tool_summaries/store.py new file mode 100644 index 000000000..96f96cfc7 --- /dev/null +++ b/backend/src/apis/shared/tool_summaries/store.py @@ -0,0 +1,203 @@ +"""Reload persistence for model-generated tool-batch summaries. + +The `tool_group_summary` SSE event is emitted once, mid-turn, as the +summarizer task finishes. It never re-streams, so on a page reload the SPA +would fall back to its deterministic client-side formatter and the prose line +the user saw ("Found the Syllabus Acknowledgment assignment in BIO 101") would +silently downgrade to "Listed 4 assignments". This store closes that gap the +same way `mcp_apps/ui_resource_store.py` does for App frames: a small +per-session side-channel record the messages endpoint replays on load. + +WHY NOT ON THE MESSAGE ITSELF +----------------------------- +A summary describes a `toolUse` block, so the obvious home is a field on that +block. It is the wrong home, twice over: + +1. The message content blocks ARE the Bedrock Converse payload. A non-standard + key on a `toolUse` block is at best ignored and at worst rejected. +2. Far more expensive: conversation history is the cacheable prefix. Writing a + summary into it would re-write the prefix on the next turn at the + cache-write premium (CLAUDE.md prompt-cache contract) — paying model rates, + every turn, for a display string. A side-channel row costs one small + DynamoDB write and nothing at inference time. + +STORAGE +------- +Reuses the existing `sessions-metadata` table — its `SessionLookupIndex` GSI +(`GSI_PK=SESSION#`, Projection ALL) and the app-api task role's Query grant +already exist, so this needs **zero new infra**. New `TSUM#` SK prefix +alongside `C#` (cost), `META`, `APPCARD#` and `UIRES#`: + + PK: USER# + SK: TSUM# (last-write-wins per batch) + GSI_PK: SESSION# (SessionLookupIndex) + GSI_SK: TSUM# + +`batch_id` is the first `toolUseId` of the batch, so a re-run of the same +invocation overwrites its own row rather than accumulating. The row also +carries every `toolUseId` in the batch, which is what the SPA keys on: it +groups the rail by tool-use id, not by batch, and needs to find the summary +from any call in the group. + +Rows are tiny (a sentence plus a handful of ids), so unlike the UI-resource +store there is no compression and no size gate — only a defensive length clamp +on the summary text itself. +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +try: # boto3 is absent in some local-dev setups + import boto3 + from boto3.dynamodb.conditions import Key + from botocore.exceptions import ClientError +except ImportError: # pragma: no cover - exercised only without boto3 + boto3 = None + Key = None # type: ignore[assignment] + ClientError = Exception # type: ignore[assignment, misc] + +logger = logging.getLogger(__name__) + +# Summaries expire with the conversation; 90d matches the sibling stores. +_SUMMARY_TTL_DAYS = 90 +# Defensive clamp. The summarizer already caps output tokens; this guards +# against a model that ignores the instruction, so one bad generation cannot +# bloat the row. +_MAX_SUMMARY_CHARS = 240 +# A batch cannot realistically fan out past this; the cap keeps the id list +# from turning a tiny row into a large one. +_MAX_TOOL_USE_IDS = 24 +_KEY_ATTRS = ("PK", "SK", "GSI_PK", "GSI_SK", "ttl") + + +class ToolSummaryStore: + """Per-session store of model-generated tool-batch summaries.""" + + def __init__(self) -> None: + self._table = None + if boto3 is None: + return + table_name = os.environ.get("DYNAMODB_SESSIONS_METADATA_TABLE_NAME") + if not table_name: + return + try: + self._table = boto3.resource("dynamodb").Table(table_name) + except Exception: # noqa: BLE001 - dev without AWS creds + logger.warning( + "tool-summary store: DynamoDB unavailable; persistence " + "disabled (summaries will be live-only).", + exc_info=True, + ) + self._table = None + + @property + def enabled(self) -> bool: + return self._table is not None + + def store( + self, + *, + user_id: str, + session_id: str, + batch_id: str, + tool_use_ids: List[str], + summary: str, + ) -> None: + """Persist one batch summary. Best-effort, never raises. + + A failed write costs only reload survival — the user already saw the + summary live, and the SPA's deterministic formatter covers the reload. + That is a good enough fallback that it is never worth failing a turn. + """ + if self._table is None or not summary or not batch_id: + return + + created_at = datetime.now(timezone.utc).isoformat() + ttl = int( + (datetime.now(timezone.utc) + timedelta(days=_SUMMARY_TTL_DAYS)).timestamp() + ) + item = { + "PK": f"USER#{user_id}", + "SK": f"TSUM#{batch_id}", + "GSI_PK": f"SESSION#{session_id}", + "GSI_SK": f"TSUM#{created_at}", + "userId": user_id, + "sessionId": session_id, + "batchId": batch_id, + "toolUseIds": list(tool_use_ids)[:_MAX_TOOL_USE_IDS], + "summary": summary[:_MAX_SUMMARY_CHARS], + "createdAt": created_at, + "ttl": ttl, + } + try: + self._table.put_item(Item=item) + except Exception: # noqa: BLE001 - persistence is best-effort + logger.warning( + "tool-summary store: failed to persist summary " + "(session=%s, batch=%s)", + session_id, + batch_id, + exc_info=True, + ) + + def list_for_session( + self, *, session_id: str, user_id: str + ) -> List[Dict[str, Any]]: + """Return this user's tool-batch summaries for a session. + + Queried off the session GSI then re-filtered by `userId`, so a guessed + session id cannot surface another user's summaries (mirrors the + UI-resource store's ownership re-check). Oldest-first for stable order. + """ + if self._table is None: + return [] + try: + items: List[Dict[str, Any]] = [] + kwargs: Dict[str, Any] = { + "IndexName": "SessionLookupIndex", + "KeyConditionExpression": Key("GSI_PK").eq(f"SESSION#{session_id}") + & Key("GSI_SK").begins_with("TSUM#"), + "ScanIndexForward": True, + } + while True: + resp = self._table.query(**kwargs) + items.extend(resp.get("Items", [])) + lek = resp.get("LastEvaluatedKey") + if not lek: + break + kwargs["ExclusiveStartKey"] = lek + except ClientError: + logger.warning( + "tool-summary store: query failed (session=%s)", + session_id, + exc_info=True, + ) + return [] + + summaries: List[Dict[str, Any]] = [] + for item in items: + if item.get("userId") != user_id: + continue # ownership re-check (guessed session id) + summaries.append( + { + "batchId": str(item.get("batchId", "")), + "toolUseIds": [str(t) for t in (item.get("toolUseIds") or [])], + "summary": str(item.get("summary", "")), + } + ) + return summaries + + +_store: Optional[ToolSummaryStore] = None + + +def get_tool_summary_store() -> ToolSummaryStore: + """Get or create the process-global tool-summary store.""" + global _store + if _store is None: + _store = ToolSummaryStore() + return _store diff --git a/backend/src/apis/shared/tool_summaries/summarizer.py b/backend/src/apis/shared/tool_summaries/summarizer.py new file mode 100644 index 000000000..bfa6a0bc6 --- /dev/null +++ b/backend/src/apis/shared/tool_summaries/summarizer.py @@ -0,0 +1,198 @@ +"""Turn a finished batch of tool calls into one line a person would say. + +The SPA can already describe a tool call deterministically ("Listed 4 +assignments") from the input and result it holds. What it cannot do is read +the *content* of those results and say which thing was found — "Found the +Syllabus Acknowledgment assignment in BIO 101 (25 pts, due Aug 29)". That +needs a model, and this is the cheapest honest way to get one. + +SIDE-CHANNEL, NOT AGENT CONTEXT +------------------------------- +Structured exactly like `generate_conversation_title`: its own Bedrock call, +on its own messages, run as an asyncio task concurrent with the agent stream. +It never touches `agent.messages`, never appends to the conversation, and +therefore adds **nothing** to the cacheable prefix — the CLAUDE.md +prompt-cache contract is untouched and no turn pays a cache re-write for a +display string. + +Its own spend is bounded by construction: Nova Micro, a hard cap on the number +of calls described, and per-call input/result truncation applied twice (once at +capture in `AgentStatusHook`, once here). A wide batch of large results costs +roughly the same as a narrow one. + +FAILURE IS A DOWNGRADE, NOT AN ERROR +------------------------------------ +Every failure path returns `None`. The SPA then keeps the deterministic line +it has been showing since the tools started — the user sees a slightly less +specific sentence and nothing else changes. That is why this is safe to leave +on by default, and why nothing here is allowed to raise into the stream. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + +# Nova Micro: the cheapest Bedrock model that reliably follows a one-line +# instruction. Same model the conversation-title side-channel uses. +_MODEL_ID = "us.amazon.nova-micro-v1:0" +# The summary is one short sentence; anything longer is the model ignoring the +# brief, and the store clamps it again on write. Headroom matters more than +# tightness here: a generation that hits this ceiling is DISCARDED (see +# `stopReason` below), so a cap set too low silently costs summaries rather +# than shortening them. +_MAX_OUTPUT_TOKENS = 100 +# Second-stage truncation, applied to what the hook already captured. Keeps a +# wide batch's prompt flat regardless of how chatty the tools were. +_MAX_INPUT_CHARS = 300 +_MAX_RESULT_CHARS = 700 +_MAX_CALLS = 8 + +_SUMMARY_SYSTEM_PROMPT = """You describe what an AI assistant just did, in one short past-tense line. + +You are given the tool calls the assistant made and what they returned. Write a single line a non-technical person would understand, naming the specific things that were found. + +Guidelines: +- One line, maximum 90 characters. No trailing period. +- Past tense, starting with a verb: "Found", "Listed", "Searched", "Created", "Updated", "Read". +- Name the specific result: the course, the file, the assignment, the count. That specificity is the entire point. +- Never mention tool names, parameters, IDs, JSON, or "the API". +- Never mention that you are an AI or that tools were used. +- If a call failed, say so plainly: "Couldn't reach Canvas". + +Examples: +Calls: list_courses -> 3 courses (BIO 101, HIST 210, CS 121) +Output: Found 3 active courses + +Calls: list_assignments -> 12 assignments; get_assignment_details -> "Syllabus Acknowledgment", 25 pts, due Aug 29 +Output: Found the Syllabus Acknowledgment assignment in BIO 101 + +Calls: search_knowledge_base -> 4 passages about parking permits +Output: Searched the knowledge base for parking permit rules + +Calls: create_rubric -> error: 422 rubric association required +Output: Couldn't create the rubric — Canvas rejected it""" + + +def _truncate(text: str, limit: int) -> str: + if not text: + return "" + if len(text) <= limit: + return text + return text[:limit] + "..." + + +def _build_prompt(calls: List[Dict[str, Any]]) -> str: + """Render the batch as the compact "Calls: ..." block the prompt models.""" + lines: List[str] = [] + for call in calls[:_MAX_CALLS]: + name = str(call.get("toolName") or "tool") + tool_input = _truncate(str(call.get("input") or ""), _MAX_INPUT_CHARS) + result = _truncate(str(call.get("result") or ""), _MAX_RESULT_CHARS) + if not call.get("ok", True): + lines.append(f"{name}({tool_input}) -> FAILED: {result}") + else: + lines.append(f"{name}({tool_input}) -> {result}") + overflow = len(calls) - _MAX_CALLS + if overflow > 0: + lines.append(f"...and {overflow} more call(s)") + return "Calls:\n" + "\n".join(lines) + + +def _unwrap_quotes(summary: str) -> str: + """Remove quotes that wrap the WHOLE line, and only those. + + ``str.strip('"')`` was the original implementation and it was wrong in a + way that showed up in production: it strips from both ends independently, + so a summary that legitimately ends in a quoted name lost its closing + quote. Observed live on dev — the model produced + + Found the course "Faculty Demo: Intro to MCP" + + and what persisted was + + Found the course "Faculty Demo: Intro to MCP + + which then rendered in the rail with a dangling quote, reading as a + truncation bug. Quoting the specific thing that was found is exactly what + makes these summaries useful, so those quotes have to survive. + """ + for quote in ('"', "'"): + if len(summary) >= 2 and summary.startswith(quote) and summary.endswith(quote): + return summary[1:-1].strip() + return summary + + +def _clean(text: str) -> str: + """Strip the wrappers small models like to add around a one-liner.""" + summary = (text or "").strip() + # Models sometimes answer with "Output: ..." because the prompt shows it. + for prefix in ("Output:", "Summary:", "Line:"): + if summary.lower().startswith(prefix.lower()): + summary = summary[len(prefix) :].strip() + summary = _unwrap_quotes(summary) + # One line only — a model that explains itself gets its first sentence used. + summary = summary.splitlines()[0].strip() if summary else "" + return summary.rstrip(".").strip() + + +async def summarize_tool_batch(calls: List[Dict[str, Any]]) -> Optional[str]: + """Summarize one finished tool batch, or return ``None``. + + ``calls`` are the bounded records captured by ``AgentStatusHook`` — + ``{toolUseId, toolName, input, result, ok, durationMs}``. + + Returns ``None`` on an empty batch, a disabled flag, a missing Bedrock + client, a model error, or an empty generation. Every one of those means + "keep showing the deterministic line", which is why they share a return + value instead of raising. + """ + from apis.shared.feature_flags import tool_summaries_enabled + + if not calls or not tool_summaries_enabled(): + return None + + try: + import boto3 + except ImportError: # pragma: no cover - dev without boto3 + return None + + try: + region = os.environ.get("AWS_REGION", "us-west-2") + client = boto3.client("bedrock-runtime", region_name=region) + + # boto3's converse() is synchronous — awaited inline it would block + # the event loop for the whole Nova round trip, stalling the agent + # stream this task runs concurrently with. + response = await asyncio.to_thread( + client.converse, + modelId=_MODEL_ID, + messages=[{"role": "user", "content": [{"text": _build_prompt(calls)}]}], + system=[{"text": _SUMMARY_SYSTEM_PROMPT}], + inferenceConfig={ + "temperature": 0.2, + "maxTokens": _MAX_OUTPUT_TOKENS, + "topP": 0.9, + }, + ) + # A generation cut off at the token ceiling is a fragment, not a + # summary, and cleaning cannot rescue one — the missing half is the + # specific thing the line was naming. Drop it and let the + # deterministic formatter speak. (Defensive: the dangling quotes seen + # on dev turned out to be `_unwrap_quotes`, not truncation, but + # nothing guarded this boundary and a fragment must never persist.) + if response.get("stopReason") == "max_tokens": + logger.debug("Tool-batch summary hit the token ceiling; discarding") + return None + + summary = _clean(response["output"]["message"]["content"][0]["text"]) + if not summary: + return None + return summary + except Exception: # noqa: BLE001 - a summary is never worth an error + logger.debug("Tool-batch summary generation skipped", exc_info=True) + return None diff --git a/backend/src/apis/shared/tools/freshness.py b/backend/src/apis/shared/tools/freshness.py index 139076529..3deeeaf60 100644 --- a/backend/src/apis/shared/tools/freshness.py +++ b/backend/src/apis/shared/tools/freshness.py @@ -37,6 +37,7 @@ import time from typing import Dict, FrozenSet, List, Optional, Tuple from apis.shared.timestamps import to_iso +from apis.shared.tools.scoped_ids import base_tool_ids logger = logging.getLogger(__name__) @@ -99,11 +100,19 @@ async def get_freshness_hash(tool_ids: List[str]) -> str: Changes when any of the given tools' config is edited. Empty list returns the empty string so callers can short-circuit. + + Scoped ids (`base::tool`, selecting one tool of an MCP server) are + collapsed to their base first: freshness is a property of the catalog + record, and the catalog only holds the base. Hashing a scoped id + verbatim looks it up, misses, and pins that entry to a constant + `none` — so an admin edit to a server would never evict an agent that + had bound a subset of it. Collapsing also de-duplicates, so an agent + binding seven tools of one server costs one catalog read, not seven. """ if not tool_ids: return "" - sorted_ids = sorted(tool_ids) + sorted_ids = sorted(base_tool_ids(tool_ids)) values = await asyncio.gather( *(get_tool_updated_at(tid) for tid in sorted_ids) ) diff --git a/backend/src/apis/shared/tools/models.py b/backend/src/apis/shared/tools/models.py index daccc400c..682c001d2 100644 --- a/backend/src/apis/shared/tools/models.py +++ b/backend/src/apis/shared/tools/models.py @@ -7,11 +7,20 @@ from datetime import datetime, timezone from enum import Enum -from typing import Any, Dict, List, Literal, Optional, Set +from typing import Any, Dict, List, Literal, Optional, Set, Union from pydantic import BaseModel, Field, field_validator, model_validator from apis.shared.timestamps import from_iso, to_iso +# EntityTypeIndex (GSI5) partition value for tool-catalog rows. +# +# The index is generic — "list every item of type X" on a table that mixes +# tools, skills, roles, role grants and one preferences row per user. Only the +# tool partition is written and read today; a second entity type can be added +# without another GSI, which matters because DynamoDB allows only one GSI +# creation per UpdateTable. +ENTITY_TYPE_TOOL = "ENTITY#TOOL" + class ToolCategory(str, Enum): """Categories for organizing tools in the UI.""" @@ -743,6 +752,13 @@ def to_dynamo_item(self) -> dict: "SK": "METADATA", "GSI1PK": f"CATEGORY#{self.category}", "GSI1SK": f"TOOL#{self.tool_id}", + # EntityTypeIndex — lets "list every tool" be a Query on one + # partition instead of a Scan of a table shared with roles, skills + # and a preferences row per user. Sparse: a row without these two + # attributes simply is not in the index, which is why existing rows + # need backfill_tool_catalog_index.py. + "GSI5PK": ENTITY_TYPE_TOOL, + "GSI5SK": f"TOOL#{self.tool_id}", "toolId": self.tool_id, "displayName": self.display_name, "description": self.description, @@ -1469,3 +1485,240 @@ class GatewayTargetStatusResponse(BaseModel): model_config = {"populate_by_name": True} + + +# ============================================================================= +# MCP capability snapshot +# ============================================================================= +# +# An MCP server exposes three listings: tools, prompts and resources. The stack +# has only ever called ``tools/list``, so prompts and resources were invisible +# to every surface in the product. +# +# The snapshot is stored beside the catalog row (``PK=TOOL#, SK=CAPABILITIES``) +# rather than inside it, on purpose. The catalog row is read on the agent build +# path; prompts and resources are of no use to the agent today, and folding a few +# KB of prompt text into an item read on every turn would be a latency and cost +# regression for a feature the agent does not consume. + + +# A single stored snapshot is bounded well under the 400KB DynamoDB item limit. +# Servers are free to expose hundreds of resources, and a description can be a +# whole docstring, so both the per-entry text and the entry counts are capped. +MAX_CAPABILITY_ENTRIES = 200 +MAX_CAPABILITY_TEXT = 500 +# Guards against a server that paginates forever. +MAX_CAPABILITY_PAGES = 20 + + +def _clip(value: Optional[str]) -> Optional[str]: + """Bound a single description/title so one verbose entry can't blow the item.""" + if value is None: + return None + text = value.strip() + if len(text) <= MAX_CAPABILITY_TEXT: + return text + return text[: MAX_CAPABILITY_TEXT - 1] + "…" + + +class MCPPromptArgument(BaseModel): + """One argument an MCP prompt accepts (``PromptArgument``). + + Capture originally flattened this to the name alone, which is enough to + *describe* a prompt and not enough to *fill one in*: a form needs + ``required`` to validate and ``description`` for the field's hint. + + ``required`` defaults to False because that is what the MCP type says — + ``required`` is ``bool | None`` and absent means not required. A snapshot + taken before this model existed stored bare strings; those rehydrate here + with the same default, so an old snapshot under-constrains a form rather + than blocking the user on a field we never actually learned about. + """ + + name: str + description: Optional[str] = None + required: bool = False + + def to_dict(self) -> dict: + return { + "name": self.name, + "description": self.description, + "required": self.required, + } + + @classmethod + def from_dict(cls, data: Union[str, dict]) -> "MCPPromptArgument": + # Pre-widening snapshots stored the name as a bare string. + if isinstance(data, str): + return cls(name=data) + return cls( + name=data.get("name", ""), + description=data.get("description"), + required=bool(data.get("required", False)), + ) + + +class MCPPromptEntry(BaseModel): + """A prompt template exposed by an MCP server (``prompts/list``).""" + + name: str + title: Optional[str] = None + description: Optional[str] = None + arguments: List[MCPPromptArgument] = Field( + default_factory=list, + description="Arguments the prompt accepts, in the order the server listed them.", + ) + + def to_dict(self) -> dict: + return { + "name": self.name, + "title": self.title, + "description": self.description, + "arguments": [a.to_dict() for a in self.arguments], + } + + @classmethod + def from_dict(cls, data: dict) -> "MCPPromptEntry": + return cls( + name=data.get("name", ""), + title=data.get("title"), + description=data.get("description"), + arguments=[ + MCPPromptArgument.from_dict(a) for a in (data.get("arguments") or []) + ], + ) + + +class MCPResourceEntry(BaseModel): + """A resource exposed by an MCP server (``resources/list``). + + ``uri_template`` is set for entries that came from + ``resources/templates/list`` — those are patterns such as + ``canvas://courses/{course_id}/syllabus`` rather than concrete URIs, and a + caller has to fill the placeholders before reading one. + """ + + uri: str + name: Optional[str] = None + description: Optional[str] = None + mime_type: Optional[str] = Field(None, alias="mimeType") + uri_template: bool = Field(default=False, alias="uriTemplate") + + model_config = {"populate_by_name": True} + + def to_dict(self) -> dict: + return { + "uri": self.uri, + "name": self.name, + "description": self.description, + "mimeType": self.mime_type, + "uriTemplate": self.uri_template, + } + + @classmethod + def from_dict(cls, data: dict) -> "MCPResourceEntry": + return cls( + uri=data.get("uri", ""), + name=data.get("name"), + description=data.get("description"), + mime_type=data.get("mimeType"), + uri_template=bool(data.get("uriTemplate", False)), + ) + + +class ToolCapabilitySnapshot(BaseModel): + """What one MCP server told us it offers, and when it said so. + + Persisted so a detail view never has to open a live MCP session to render. + A 31-card catalogue opening one session per server would be unusable, and + OAuth-gated servers cannot be reached at all without a consent token. + + ``supports_prompts`` / ``supports_resources`` record whether the server + answered the listing at all. A server that does not implement prompts + returns a JSON-RPC "method not found", which is a different fact from a + server that implements prompts and has none — and the UI should say + different things about each. + """ + + tool_id: str = Field(..., alias="toolId") + prompts: List[MCPPromptEntry] = Field(default_factory=list) + resources: List[MCPResourceEntry] = Field(default_factory=list) + supports_prompts: bool = Field(default=False, alias="supportsPrompts") + supports_resources: bool = Field(default=False, alias="supportsResources") + discovered_at: Optional[str] = Field(None, alias="discoveredAt") + discovered_by: Optional[str] = Field(None, alias="discoveredBy") + #: Set when the last attempt failed, so the UI can distinguish "this server + #: offers nothing" from "we could not ask". + error: Optional[str] = None + #: True when a listing was cut short by the entry cap above. + truncated: bool = Field(default=False) + + model_config = {"populate_by_name": True} + + def to_dynamo_item(self) -> dict: + return { + "PK": f"TOOL#{self.tool_id}", + "SK": "CAPABILITIES", + "toolId": self.tool_id, + "prompts": [p.to_dict() for p in self.prompts], + "resources": [r.to_dict() for r in self.resources], + "supportsPrompts": self.supports_prompts, + "supportsResources": self.supports_resources, + "discoveredAt": self.discovered_at, + "discoveredBy": self.discovered_by, + "error": self.error, + "truncated": self.truncated, + } + + @classmethod + def from_dynamo_item(cls, item: dict) -> "ToolCapabilitySnapshot": + return cls( + tool_id=item.get("toolId", ""), + prompts=[MCPPromptEntry.from_dict(p) for p in item.get("prompts") or []], + resources=[ + MCPResourceEntry.from_dict(r) for r in item.get("resources") or [] + ], + supports_prompts=bool(item.get("supportsPrompts", False)), + supports_resources=bool(item.get("supportsResources", False)), + discovered_at=item.get("discoveredAt"), + discovered_by=item.get("discoveredBy"), + error=item.get("error"), + truncated=bool(item.get("truncated", False)), + ) + + +# ============================================================================= +# Resolved prompt (prompts/get) +# ============================================================================= +# +# Unlike the capability snapshot, a resolved prompt is never persisted. It is +# composed from arguments the user just typed, it can be large, and it is of no +# use to anyone but the person who asked for it — storing it would be a cost +# with no reader. + +#: A resolved prompt is shown to a person, so it is bounded by what a person +#: will actually read rather than by the DynamoDB item limit. +MAX_RESOLVED_PROMPT_CHARS = 20000 +MAX_RESOLVED_PROMPT_MESSAGES = 20 + + +class ResolvedPromptMessage(BaseModel): + """One message a server composed for a prompt. + + ``kind`` is the MCP content type. Anything other than ``text`` carries no + body — see ``_message_text`` — and the UI says so rather than rendering an + empty message. + """ + + role: str + kind: str = "text" + text: str = "" + + +class ResolvedPrompt(BaseModel): + """The result of ``prompts/get`` for one prompt.""" + + description: Optional[str] = None + messages: List[ResolvedPromptMessage] = Field(default_factory=list) + #: True when the message list or its text was cut short by the caps above. + truncated: bool = Field(default=False) diff --git a/backend/src/apis/shared/tools/repository.py b/backend/src/apis/shared/tools/repository.py index 5c48ff55c..0213ee525 100644 --- a/backend/src/apis/shared/tools/repository.py +++ b/backend/src/apis/shared/tools/repository.py @@ -5,6 +5,7 @@ Uses the same table as AppRoles with different PK patterns. """ +import asyncio import os import logging from datetime import datetime, timezone @@ -13,10 +14,22 @@ import boto3 from botocore.exceptions import ClientError -from .models import ToolDefinition, UserToolPreference, ToolStatus +from apis.shared.caching import config_cache +from apis.shared.dynamo_errors import is_missing_index_error +from .models import ( + ENTITY_TYPE_TOOL, + ToolCapabilitySnapshot, + ToolDefinition, + UserToolPreference, + ToolStatus, +) logger = logging.getLogger(__name__) +# The GSI that makes "list every tool" a Query instead of a Scan of a table +# shared with roles, skills and a preferences row per user. +ENTITY_TYPE_INDEX = "EntityTypeIndex" + class ToolCatalogRepository: """ @@ -64,6 +77,53 @@ async def get_tool(self, tool_id: str) -> Optional[ToolDefinition]: logger.error(f"Error getting tool {tool_id}: {e}") raise + # ========================================================================= + # MCP capability snapshots (PK=TOOL#{id}, SK=CAPABILITIES) + # ========================================================================= + + async def get_capabilities( + self, tool_id: str + ) -> Optional["ToolCapabilitySnapshot"]: + """The stored prompts/resources snapshot for a tool, or None if never discovered.""" + try: + response = self._table.get_item( + Key={"PK": f"TOOL#{tool_id}", "SK": "CAPABILITIES"} + ) + item = response.get("Item") + if not item: + return None + return ToolCapabilitySnapshot.from_dynamo_item(item) + except ClientError as e: + logger.error(f"Error getting capabilities for {tool_id}: {e}") + raise + + async def put_capabilities( + self, snapshot: "ToolCapabilitySnapshot" + ) -> "ToolCapabilitySnapshot": + """Write a capability snapshot, replacing any previous one. + + Replace rather than merge: the snapshot is a point-in-time answer from + the server, and merging would keep prompts the server has since removed. + """ + try: + self._table.put_item(Item=snapshot.to_dynamo_item()) + return snapshot + except ClientError as e: + logger.error( + f"Error writing capabilities for {snapshot.tool_id}: {e}" + ) + raise + + async def delete_capabilities(self, tool_id: str) -> None: + """Drop a tool's snapshot — used when the tool itself is deleted.""" + try: + self._table.delete_item( + Key={"PK": f"TOOL#{tool_id}", "SK": "CAPABILITIES"} + ) + except ClientError as e: + logger.error(f"Error deleting capabilities for {tool_id}: {e}") + raise + async def list_tools( self, status: Optional[str] = None, category: Optional[str] = None ) -> List[ToolDefinition]: @@ -79,42 +139,17 @@ async def list_tools( """ try: if category: - # Use GSI1 for category queries - response = self._table.query( - IndexName="JwtRoleMappingIndex", - KeyConditionExpression="GSI1PK = :pk", - ExpressionAttributeValues={":pk": f"CATEGORY#{category}"}, - ) - items = response.get("Items", []) - - # Handle pagination - while "LastEvaluatedKey" in response: - response = self._table.query( - IndexName="JwtRoleMappingIndex", - KeyConditionExpression="GSI1PK = :pk", - ExpressionAttributeValues={":pk": f"CATEGORY#{category}"}, - ExclusiveStartKey=response["LastEvaluatedKey"], - ) - items.extend(response.get("Items", [])) + items = await asyncio.to_thread(self._query_tool_items_by_category, category) else: - # Scan for all tools - filter_expr = "begins_with(PK, :pk_prefix) AND SK = :sk" - expr_values = {":pk_prefix": "TOOL#", ":sk": "METADATA"} - - response = self._table.scan( - FilterExpression=filter_expr, - ExpressionAttributeValues=expr_values, + # The whole-catalog read is the one on the SPA's first-load path + # (GET /tools/), so it is cached per process; parsing below is + # not, because callers mutate what they get back — see + # `list_tools_with_roles`, which writes `allowed_app_roles` onto + # each ToolDefinition in place. + items = await config_cache.get_or_load( + config_cache.TOOL_CATALOG, + self._load_all_tool_items, ) - items = response.get("Items", []) - - # Handle pagination - while "LastEvaluatedKey" in response: - response = self._table.scan( - FilterExpression=filter_expr, - ExpressionAttributeValues=expr_values, - ExclusiveStartKey=response["LastEvaluatedKey"], - ) - items.extend(response.get("Items", [])) tools = [ToolDefinition.from_dynamo_item(item) for item in items] @@ -131,6 +166,131 @@ async def list_tools( logger.error(f"Error listing tools: {e}") raise + async def _load_all_tool_items(self) -> List[dict]: + """Every tool row, by Query on EntityTypeIndex, with Scan as the safety net. + + The Query is the point of the index: this table is shared with roles, + skills, role grants and one tool-preferences row PER USER, so the Scan + reads the whole table to return the tool rows and its cost grows with + enrollment rather than with the catalog. Measured on dev, 95 items read + to return 24. + + Two ways the index can fail to answer, and neither may take the catalog + down with it — an empty tool list is not a degraded experience here, it + is a broken one: + + **The index is not there.** `platform.yml` and `backend.yml` are ordered + by nothing, a GSI is still CREATING after CloudFormation reports success, + and a rolled-back stack ships its images anyway. Unlike the surfaces + `dynamo_errors` was written for, we have a *correct* answer available, so + we fall back to it rather than degrading to empty. + + **The index is there but unpopulated.** The keys are sparse, so a catalog + whose backfill has not run indexes nothing and the Query succeeds with + zero rows — no error to catch. A zero result is therefore treated as + suspect and re-read via Scan: if the table really is empty (a fresh + install before seeding) both agree and it costs one extra read per cache + fill; if it is not, we serve the truth and say loudly why. + + A partial backfill is NOT covered — detecting it would mean scanning + every time, which is the cost being removed. That is what the release + gate and the backfill's own `skipped=0 failed=0` report are for. + """ + try: + items = await asyncio.to_thread(self._query_tool_items) + except ClientError as exc: + if not is_missing_index_error(exc): + raise + logger.warning( + "⚠️ DynamoDB index '%s' does not exist — falling back to Scan for " + "the tool catalog. Expected transiently while the GSI is CREATING " + "or a deploy is incomplete; if it persists, every catalog read is " + "paying a full table scan.", + ENTITY_TYPE_INDEX, + ) + return await asyncio.to_thread(self._scan_tool_items) + + if items: + return items + + # Zero rows from a sparse index is indistinguishable from "no tools", so + # confirm against the base table before believing it. + scanned = await asyncio.to_thread(self._scan_tool_items) + if scanned: + logger.error( + "⚠️ DynamoDB index '%s' returned 0 tools but the table holds %d — " + "the %s backfill has not been run in this environment. Serving the " + "Scan result so the catalog is correct; run the backfill.", + ENTITY_TYPE_INDEX, + len(scanned), + "backfill_tool_catalog_index.py", + ) + return scanned + + def _query_tool_items(self) -> List[dict]: + """Query the raw tool rows off EntityTypeIndex. Blocking; via ``to_thread``.""" + kwargs = { + "IndexName": ENTITY_TYPE_INDEX, + "KeyConditionExpression": "GSI5PK = :pk", + "ExpressionAttributeValues": {":pk": ENTITY_TYPE_TOOL}, + } + response = self._table.query(**kwargs) + items = response.get("Items", []) + + while "LastEvaluatedKey" in response: + response = self._table.query( + **kwargs, ExclusiveStartKey=response["LastEvaluatedKey"] + ) + items.extend(response.get("Items", [])) + + return items + + def _scan_tool_items(self) -> List[dict]: + """Scan the raw TOOL#/METADATA items. Blocking; call via ``to_thread``.""" + filter_expr = "begins_with(PK, :pk_prefix) AND SK = :sk" + expr_values = {":pk_prefix": "TOOL#", ":sk": "METADATA"} + + response = self._table.scan( + FilterExpression=filter_expr, + ExpressionAttributeValues=expr_values, + ) + items = response.get("Items", []) + + while "LastEvaluatedKey" in response: + response = self._table.scan( + FilterExpression=filter_expr, + ExpressionAttributeValues=expr_values, + ExclusiveStartKey=response["LastEvaluatedKey"], + ) + items.extend(response.get("Items", [])) + + return items + + def _query_tool_items_by_category(self, category: str) -> List[dict]: + """Query raw items for one category. Blocking; call via ``to_thread``. + + Not cached: it is a bounded GSI query off the first-load path, and + caching per category would multiply the invalidation surface for no + measured gain. + """ + response = self._table.query( + IndexName="JwtRoleMappingIndex", + KeyConditionExpression="GSI1PK = :pk", + ExpressionAttributeValues={":pk": f"CATEGORY#{category}"}, + ) + items = response.get("Items", []) + + while "LastEvaluatedKey" in response: + response = self._table.query( + IndexName="JwtRoleMappingIndex", + KeyConditionExpression="GSI1PK = :pk", + ExpressionAttributeValues={":pk": f"CATEGORY#{category}"}, + ExclusiveStartKey=response["LastEvaluatedKey"], + ) + items.extend(response.get("Items", [])) + + return items + async def create_tool(self, tool: ToolDefinition) -> ToolDefinition: """ Create a new tool catalog entry. @@ -162,6 +322,9 @@ async def create_tool(self, tool: ToolDefinition) -> ToolDefinition: ConditionExpression="attribute_not_exists(PK)", ) + # Invalidate in the repository, not the admin route: every write + # to the catalog lands here, so a new caller cannot forget to. + config_cache.invalidate(config_cache.TOOL_CATALOG) logger.info(f"Created tool: {tool.tool_id}") return tool @@ -204,6 +367,7 @@ async def update_tool( item = existing.to_dynamo_item() self._table.put_item(Item=item) + config_cache.invalidate(config_cache.TOOL_CATALOG) logger.info(f"Updated tool: {tool_id}") return existing @@ -230,6 +394,7 @@ async def delete_tool(self, tool_id: str) -> bool: Key={"PK": f"TOOL#{tool_id}", "SK": "METADATA"} ) + config_cache.invalidate(config_cache.TOOL_CATALOG) logger.info(f"Deleted tool: {tool_id}") return True @@ -444,6 +609,7 @@ async def batch_create_tools( item = tool.to_dynamo_item() batch.put_item(Item=item) + config_cache.invalidate(config_cache.TOOL_CATALOG) logger.info(f"Batch created {len(tools)} tools") return tools diff --git a/backend/tests/agents/main_agent/session/test_agent_status_hook.py b/backend/tests/agents/main_agent/session/test_agent_status_hook.py new file mode 100644 index 000000000..56894d9bc --- /dev/null +++ b/backend/tests/agents/main_agent/session/test_agent_status_hook.py @@ -0,0 +1,296 @@ +"""Tests for AgentStatusHook — live narration of a streaming turn. + +The properties under test, in order of how expensive they are to get wrong: + +1. **Per-turn reset.** The interrupt path (OAuth consent, tool approval) + unwinds the event loop without draining. Without the ``BeforeInvocationEvent`` + reset, the next turn's first drain replays stale transitions and the SPA + narrates tools that are not running. +2. **Drain-once semantics.** A transition handed to the stream coordinator must + not be handed over again, or the status line stutters backwards. +3. **Duration normalization.** Strands has expressed ``duration`` as both a + float of seconds and a ``timedelta``; a wrong number on a tool row is worse + than no number, so anything unrecognized must degrade to ``None``. +4. **Batch capture is bounded** and carries what the summarizer needs. +5. Fail-soft: flag off narrates nothing, and a malformed event never raises. +""" + +from datetime import timedelta +from unittest.mock import MagicMock + +import pytest + +from agents.main_agent.session.hooks.agent_status import ( + _MAX_CALLS_PER_BATCH, + _MAX_INPUT_CHARS, + _MAX_RESULT_CHARS, + AgentStatusHook, + _duration_ms, +) + + +@pytest.fixture(autouse=True) +def status_enabled(monkeypatch): + monkeypatch.setenv("AGENT_STATUS_ENABLED", "true") + + +@pytest.fixture +def hook(): + return AgentStatusHook() + + +def _tool_event(name="list_courses", tool_use_id="t1", tool_input=None, **kwargs): + event = MagicMock() + event.tool_use = { + "toolUseId": tool_use_id, + "name": name, + "input": tool_input if tool_input is not None else {"term": "fall"}, + } + for key, value in kwargs.items(): + setattr(event, key, value) + return event + + +def _after_tool_event(name="list_courses", tool_use_id="t1", **kwargs): + defaults = { + "duration": 0.25, + "exception": None, + "result": {"status": "success", "content": [{"text": "3 courses"}]}, + } + defaults.update(kwargs) + return _tool_event(name=name, tool_use_id=tool_use_id, **defaults) + + +# -- 1. per-turn reset ---------------------------------------------------- + + +def test_turn_start_drops_state_left_by_an_interrupted_turn(hook): + hook._on_before_model_call(MagicMock()) + hook._on_before_tool_call(_tool_event()) + hook._on_after_tool_call(_after_tool_event()) + assert hook._statuses, "precondition: the aborted turn recorded transitions" + + hook._on_turn_start(MagicMock()) + + assert hook.drain_statuses() == [] + assert hook.drain_batches() == [] + + +def test_turn_start_resets_the_cycle_counter(hook): + hook._on_before_model_call(MagicMock()) + hook._on_before_model_call(MagicMock()) + hook._on_turn_start(MagicMock()) + hook._on_before_model_call(MagicMock()) + + assert hook.drain_statuses() == [{"phase": "thinking", "cycle": 1}] + + +# -- 2. drain-once -------------------------------------------------------- + + +def test_statuses_drain_once(hook): + hook._on_before_model_call(MagicMock()) + first = hook.drain_statuses() + + assert first == [{"phase": "thinking", "cycle": 1}] + assert hook.drain_statuses() == [] + + +def test_cycle_increments_across_tool_round_trips(hook): + hook._on_before_model_call(MagicMock()) # cycle 1: decide to call a tool + hook._on_before_tool_call(_tool_event()) + hook._on_after_tool_call(_after_tool_event()) + hook._on_before_model_call(MagicMock()) # cycle 2: read results, answer + + phases = [(s["phase"], s["cycle"]) for s in hook.drain_statuses()] + assert phases == [ + ("thinking", 1), + ("tool_start", 1), + ("tool_end", 1), + ("thinking", 2), + ] + + +def test_tool_start_carries_the_name_the_ui_renders(hook): + hook._on_before_tool_call(_tool_event(name="list_assignments", tool_use_id="tu-9")) + + (status,) = hook.drain_statuses() + assert status["phase"] == "tool_start" + assert status["toolName"] == "list_assignments" + assert status["toolUseId"] == "tu-9" + + +# -- 3. duration normalization ------------------------------------------- + + +@pytest.mark.parametrize( + "raw,expected", + [ + (0.25, 250), + (2, 2000), + (timedelta(milliseconds=1500), 1500), + (None, None), + ("quarter of a second", None), + (object(), None), + ], +) +def test_duration_normalizes_or_degrades_to_none(raw, expected): + assert _duration_ms(raw) == expected + + +def test_negative_duration_clamps_to_zero(): + assert _duration_ms(-1.0) == 0 + + +def test_tool_end_reports_the_measured_duration(hook): + hook._on_after_tool_call(_after_tool_event(duration=timedelta(seconds=1.2))) + + (status,) = hook.drain_statuses() + assert status["durationMs"] == 1200 + assert status["ok"] is True + + +def test_tool_failure_inside_a_successful_invocation_is_not_ok(hook): + """Strands reports a tool-level failure as a result status, not a raise. + + The rail's red dot depends on catching both, so a result marked "error" + must flip `ok` even though no exception reached the hook. + """ + hook._on_after_tool_call( + _after_tool_event(result={"status": "error", "content": [{"text": "422"}]}) + ) + + (status,) = hook.drain_statuses() + assert status["ok"] is False + + +def test_raised_exception_is_not_ok(hook): + hook._on_after_tool_call(_after_tool_event(exception=RuntimeError("boom"))) + + (status,) = hook.drain_statuses() + assert status["ok"] is False + + +# -- 4. batch capture ----------------------------------------------------- + + +def test_batch_closes_with_what_the_summarizer_needs(hook): + hook._on_before_tool_call(_tool_event(name="list_courses", tool_use_id="t1")) + hook._on_after_tool_call(_after_tool_event(name="list_courses", tool_use_id="t1")) + hook._on_after_tools(MagicMock()) + + (batch,) = hook.drain_batches() + assert batch["batchId"] == "t1" + assert batch["toolUseIds"] == ["t1"] + (call,) = batch["calls"] + assert call["toolName"] == "list_courses" + assert call["ok"] is True + assert "3 courses" in call["result"] + assert "fall" in call["input"] + + +def test_batches_drain_once(hook): + hook._on_after_tool_call(_after_tool_event()) + hook._on_after_tools(MagicMock()) + + assert len(hook.drain_batches()) == 1 + assert hook.drain_batches() == [] + + +def test_empty_batch_is_not_parked(hook): + """A batch cancelled before any tool ran has nothing to summarize.""" + hook._on_after_tools(MagicMock()) + + assert hook.drain_batches() == [] + + +def test_consecutive_batches_are_separate(hook): + hook._on_after_tool_call(_after_tool_event(tool_use_id="t1")) + hook._on_after_tools(MagicMock()) + hook._on_after_tool_call(_after_tool_event(tool_use_id="t2")) + hook._on_after_tools(MagicMock()) + + batches = hook.drain_batches() + assert [b["batchId"] for b in batches] == ["t1", "t2"] + + +def test_batch_capture_is_width_bounded(hook): + for i in range(_MAX_CALLS_PER_BATCH + 10): + hook._on_after_tool_call(_after_tool_event(tool_use_id=f"t{i}")) + hook._on_after_tools(MagicMock()) + + (batch,) = hook.drain_batches() + assert len(batch["calls"]) == _MAX_CALLS_PER_BATCH + + +def test_oversized_payloads_are_truncated_at_capture(hook): + """An 8MB tool result must not sit in memory for the life of the turn.""" + hook._on_after_tool_call( + _after_tool_event( + tool_input={"q": "x" * 50_000}, + result={"status": "success", "content": [{"text": "y" * 500_000}]}, + ) + ) + hook._on_after_tools(MagicMock()) + + (batch,) = hook.drain_batches() + (call,) = batch["calls"] + assert len(call["input"]) <= _MAX_INPUT_CHARS + 3 + assert len(call["result"]) <= _MAX_RESULT_CHARS + 3 + + +def test_batches_are_captured_even_with_narration_off(hook, monkeypatch): + """Summaries and the status line are gated separately. + + A deployment can want summaries without the live line; the batch record is + what the summarizer consumes, so it must not ride the narration flag. + """ + monkeypatch.setenv("AGENT_STATUS_ENABLED", "false") + hook._on_after_tool_call(_after_tool_event()) + hook._on_after_tools(MagicMock()) + + assert hook.drain_statuses() == [], "the live line is off" + (batch,) = hook.drain_batches() + assert batch["calls"], "but the batch is still there to summarize" + + +# -- 5. fail-soft --------------------------------------------------------- + + +def test_flag_off_narrates_nothing(hook, monkeypatch): + monkeypatch.setenv("AGENT_STATUS_ENABLED", "false") + + hook._on_before_model_call(MagicMock()) + hook._on_before_tool_call(_tool_event()) + hook._on_after_tool_call(_after_tool_event()) + + assert hook.drain_statuses() == [] + + +def test_malformed_tool_event_never_raises(hook): + broken = MagicMock() + broken.tool_use = "not-a-dict" + + hook._on_before_tool_call(broken) + hook._on_after_tool_call(broken) + + assert hook.drain_statuses() == [] + + +def test_tool_event_without_a_name_is_skipped(hook): + """A nameless tool has nothing the UI could say about it.""" + event = MagicMock() + event.tool_use = {"toolUseId": "t1"} + + hook._on_before_tool_call(event) + + assert hook.drain_statuses() == [] + + +def test_status_queue_is_capped(hook): + from agents.main_agent.session.hooks.agent_status import _MAX_QUEUED_STATUSES + + for _ in range(_MAX_QUEUED_STATUSES + 50): + hook._on_before_model_call(MagicMock()) + + assert len(hook.drain_statuses()) == _MAX_QUEUED_STATUSES diff --git a/backend/tests/agents/main_agent/session/test_compaction_stability.py b/backend/tests/agents/main_agent/session/test_compaction_stability.py index bc69080bd..bef500736 100644 --- a/backend/tests/agents/main_agent/session/test_compaction_stability.py +++ b/backend/tests/agents/main_agent/session/test_compaction_stability.py @@ -5,8 +5,10 @@ a pure function of (stored messages, persisted compaction state). The old design truncated tool contents behind a sliding protected-turns window, which re-mutated the turn that just aged past the window on every restore — -breaking the cached prefix and forcing a full prefix re-write (~$2.5/MTok on -a 35k–150k prefix) nearly every turn (observed in prod session aecd387d: +breaking the cached prefix and forcing a full re-write of a 35k–150k prefix +nearly every turn, at the cache-write premium (1.25x the model's own base +input rate, not a flat per-MTok figure — see CLAUDE.md's prompt-cache +contract), observed in prod session aecd387d: -382/-1035/-1513 inter-turn prefix-token shrinkages with cacheRead=0 well inside the cache TTL). diff --git a/backend/tests/apis/app_api/admin/oauth/test_update_provider_discovery_guard.py b/backend/tests/apis/app_api/admin/oauth/test_update_provider_discovery_guard.py new file mode 100644 index 000000000..d3171b355 --- /dev/null +++ b/backend/tests/apis/app_api/admin/oauth/test_update_provider_discovery_guard.py @@ -0,0 +1,244 @@ +"""Route tests for the discovery-change guard on `PATCH /admin/oauth-providers/{id}`. + +A discovery-config change requires a credential rotation, because AgentCore's +update API demands the full config and never echoes the stored client secret +back. The guard must fire on a *real* change only: clients that round-trip the +whole record resend the unchanged discovery URL on every save, and rejecting +those makes metadata-only edits (scopes, display name, icon, enabled) +impossible for any provider that has one — the admin has no way to satisfy the +rotation requirement. See `apis/app_api/admin/oauth/routes.py`. +""" + +from __future__ import annotations + +import pytest +from fastapi import HTTPException + +from apis.app_api.admin.oauth import routes +from apis.shared.auth.models import User +from apis.shared.oauth.agentcore_registrar import CredentialProviderInfo +from apis.shared.oauth.models import ( + OAuthProvider, + OAuthProviderType, + OAuthProviderUpdate, +) + +_DISCOVERY_URL = "https://boisestate.instructure.com/.well-known/openid-configuration" + + +def _admin() -> User: + return User( + email="admin@example.edu", + user_id="u1", + name="Admin", + roles=["system_admin"], + raw_token="admin-tok", + ) + + +def _provider(**overrides) -> OAuthProvider: + defaults = dict( + provider_id="canvas-faculty", + display_name="Canvas (Faculty)", + provider_type=OAuthProviderType.CANVAS, + scopes=["url:GET|/api/v1/courses"], + allowed_roles=["faculty"], + oauth_discovery_url=_DISCOVERY_URL, + credential_provider_arn="arn:aws:bedrock-agentcore:us-west-2:1:provider/canvas", + ) + defaults.update(overrides) + return OAuthProvider(**defaults) + + +class _FakeProviderRepo: + """Minimal stand-in for `OAuthProviderRepository`. + + `apply_metadata_update` mirrors the real repository closely enough for + the guard's purposes: it copies the populated fields onto the record and + stamps `updated_at`. + """ + + def __init__(self, provider: OAuthProvider | None): + self._provider = provider + self.metadata_updates: list[OAuthProviderUpdate] = [] + + async def get_provider(self, provider_id: str) -> OAuthProvider | None: + return self._provider + + async def apply_metadata_update( + self, provider_id: str, updates: OAuthProviderUpdate + ) -> OAuthProvider | None: + self.metadata_updates.append(updates) + if self._provider is None: + return None + if updates.scopes is not None: + self._provider.scopes = updates.scopes + if updates.display_name is not None: + self._provider.display_name = updates.display_name + if updates.oauth_discovery_url is not None: + self._provider.oauth_discovery_url = updates.oauth_discovery_url + self._provider.updated_at = "2026-09-10T00:00:00Z" + return self._provider + + async def put_provider(self, provider: OAuthProvider) -> None: + self._provider = provider + + +class _FakeRegistrar: + def __init__(self): + self.update_calls: list[dict] = [] + + def update_credential_provider(self, **kwargs) -> CredentialProviderInfo: + self.update_calls.append(kwargs) + return CredentialProviderInfo( + provider_id=kwargs["provider_id"], + vendor="CustomOauth2", + credential_provider_arn="arn:aws:bedrock-agentcore:us-west-2:1:provider/new", + client_secret_arn="arn:aws:secretsmanager:us-west-2:1:secret/new", + callback_url="https://example.edu/auth/callback", + ) + + +async def _update(updates: OAuthProviderUpdate, repo, registrar): + return await routes.update_provider( + provider_id="canvas-faculty", + updates=updates, + admin=_admin(), + provider_repo=repo, + registrar=registrar, + ) + + +class TestDiscoveryGuard: + @pytest.mark.asyncio + async def test_unchanged_discovery_url_allows_metadata_only_edit(self): + """The SPA resends the discovery URL verbatim; that must not 400.""" + repo = _FakeProviderRepo(_provider()) + registrar = _FakeRegistrar() + + response = await _update( + OAuthProviderUpdate( + display_name="Canvas (Faculty)", + scopes=["url:GET|/api/v1/courses", "url:GET|/api/v1/users/:id"], + oauth_discovery_url=_DISCOVERY_URL, + ), + repo, + registrar, + ) + + assert response.scopes == [ + "url:GET|/api/v1/courses", + "url:GET|/api/v1/users/:id", + ] + # No AgentCore call — nothing about the credential config changed. + assert registrar.update_calls == [] + + @pytest.mark.asyncio + async def test_scopes_only_edit_succeeds(self): + """The documented workaround payload — no discovery field at all.""" + repo = _FakeProviderRepo(_provider()) + registrar = _FakeRegistrar() + + response = await _update( + OAuthProviderUpdate(scopes=["url:GET|/api/v1/courses"]), repo, registrar + ) + + assert response.scopes == ["url:GET|/api/v1/courses"] + assert registrar.update_calls == [] + + @pytest.mark.asyncio + async def test_changed_discovery_url_without_credentials_still_400s(self): + repo = _FakeProviderRepo(_provider()) + registrar = _FakeRegistrar() + + with pytest.raises(HTTPException) as exc: + await _update( + OAuthProviderUpdate( + oauth_discovery_url="https://other.example.edu/.well-known/openid-configuration" + ), + repo, + registrar, + ) + + assert exc.value.status_code == 400 + assert "credential rotation" in exc.value.detail + assert registrar.update_calls == [] + + @pytest.mark.asyncio + async def test_unchanged_metadata_dict_allows_metadata_only_edit(self): + """Same rule for the explicit-metadata flavor of the discovery config.""" + metadata = {"issuer": "https://idp.example.edu", "authorization_endpoint": "/a"} + repo = _FakeProviderRepo( + _provider(oauth_discovery_url=None, authorization_server_metadata=metadata) + ) + registrar = _FakeRegistrar() + + response = await _update( + OAuthProviderUpdate( + scopes=["openid"], + authorization_server_metadata=dict(metadata), + ), + repo, + registrar, + ) + + assert response.scopes == ["openid"] + assert registrar.update_calls == [] + + @pytest.mark.asyncio + async def test_changed_metadata_dict_without_credentials_still_400s(self): + repo = _FakeProviderRepo( + _provider( + oauth_discovery_url=None, + authorization_server_metadata={"issuer": "https://idp.example.edu"}, + ) + ) + registrar = _FakeRegistrar() + + with pytest.raises(HTTPException) as exc: + await _update( + OAuthProviderUpdate( + authorization_server_metadata={"issuer": "https://new.example.edu"} + ), + repo, + registrar, + ) + + assert exc.value.status_code == 400 + assert registrar.update_calls == [] + + @pytest.mark.asyncio + async def test_changed_discovery_url_with_rotation_reaches_agentcore(self): + repo = _FakeProviderRepo(_provider()) + registrar = _FakeRegistrar() + new_url = "https://other.example.edu/.well-known/openid-configuration" + + response = await _update( + OAuthProviderUpdate( + client_id="new-client", + client_secret="new-secret", + oauth_discovery_url=new_url, + ), + repo, + registrar, + ) + + assert len(registrar.update_calls) == 1 + assert registrar.update_calls[0]["discovery_url"] == new_url + assert registrar.update_calls[0]["client_id"] == "new-client" + assert response.callback_url == "https://example.edu/auth/callback" + + @pytest.mark.asyncio + async def test_rotation_alone_carries_the_existing_discovery_url_forward(self): + """AgentCore needs the full config, so the stored URL is resent.""" + repo = _FakeProviderRepo(_provider()) + registrar = _FakeRegistrar() + + await _update( + OAuthProviderUpdate(client_id="new-client", client_secret="new-secret"), + repo, + registrar, + ) + + assert len(registrar.update_calls) == 1 + assert registrar.update_calls[0]["discovery_url"] == _DISCOVERY_URL diff --git a/backend/tests/apis/app_api/agent_designer/test_binding_validation.py b/backend/tests/apis/app_api/agent_designer/test_binding_validation.py index 14fb19878..aa9c3389f 100644 --- a/backend/tests/apis/app_api/agent_designer/test_binding_validation.py +++ b/backend/tests/apis/app_api/agent_designer/test_binding_validation.py @@ -38,12 +38,25 @@ def _mem_svc(space, role) -> MagicMock: return svc -def _tool_svc(*accessible_ids: str) -> MagicMock: - # Mirror the palette: get_user_accessible_tools returns objects carrying .tool_id. +def _tool_svc(*accessible: object) -> MagicMock: + """Mirror the palette: ``get_user_accessible_tools`` returns ``UserToolAccess``. + + Each entry is a bare id (a tool with no discovered per-tool list, the shape of a + local tool or an MCP server that has never been discovered) or a + ``(id, [tool names])`` pair carrying that server's ``server_tools`` — the field + ``_validate_tool`` checks a scoped ref's tool name against. + """ svc = MagicMock() - svc.get_user_accessible_tools = AsyncMock( - return_value=[SimpleNamespace(tool_id=t) for t in accessible_ids] - ) + items = [] + for entry in accessible: + tool_id, names = entry if isinstance(entry, tuple) else (entry, ()) + items.append( + SimpleNamespace( + tool_id=tool_id, + server_tools=[SimpleNamespace(name=n) for n in names], + ) + ) + svc.get_user_accessible_tools = AsyncMock(return_value=items) return svc @@ -334,6 +347,79 @@ async def test_no_tool_binding_skips_tool_fetch(self, monkeypatch): ) svc.get_user_accessible_tools.assert_not_awaited() + # -- scoped refs (``toolId::mcpToolName``) ----------------------------------- + @pytest.mark.asyncio + async def test_scoped_ref_passes_when_base_accessible_and_name_exposed(self): + # The whole point: bind 2 of a 3-tool server. The base carries the grant, the + # discovered serverTools list carries the name. + svc = _tool_svc(("canvas_faculty", ["list_courses", "list_rubrics", "grade_submission"])) + await validate_agent_write( + _user(), + bindings=[ + AgentBinding(kind="tool", ref="canvas_faculty::list_courses"), + AgentBinding(kind="tool", ref="canvas_faculty::list_rubrics"), + ], + tool_service=svc, + ) + svc.get_user_accessible_tools.assert_awaited_once() + + @pytest.mark.asyncio + async def test_scoped_ref_403_when_base_inaccessible(self): + # Scoping narrows a grant; it can never conjure one. + with pytest.raises(BindingValidationError) as ei: + await validate_agent_write( + _user(), + bindings=[AgentBinding(kind="tool", ref="secret_server::peek")], + tool_service=_tool_svc(("canvas_faculty", ["list_courses"])), + ) + assert ei.value.status_code == 403 + # The message names the base — that is what an admin would grant. + assert "secret_server" in ei.value.message + assert "::" not in ei.value.message + + @pytest.mark.asyncio + async def test_scoped_ref_400_when_name_not_exposed(self): + with pytest.raises(BindingValidationError) as ei: + await validate_agent_write( + _user(), + bindings=[AgentBinding(kind="tool", ref="canvas_faculty::no_such_tool")], + tool_service=_tool_svc(("canvas_faculty", ["list_courses"])), + ) + assert ei.value.status_code == 400 + assert "no_such_tool" in ei.value.message + + @pytest.mark.asyncio + async def test_scoped_ref_allowed_when_server_never_discovered(self): + # An empty serverTools list means "never discovered", not "exposes nothing" — + # mirrors ToolCatalogService.save_user_preferences, which skips the name check + # in exactly this case rather than rejecting every scoped ref. + await validate_agent_write( + _user(), + bindings=[AgentBinding(kind="tool", ref="canvas_faculty::list_courses")], + tool_service=_tool_svc("canvas_faculty"), + ) + + @pytest.mark.asyncio + async def test_ref_with_empty_tool_name_400(self): + # "base::" parses to a bare ref, so accepting it would store a whole-server + # binding under a ref the author wrote to narrow one. + with pytest.raises(BindingValidationError) as ei: + await validate_agent_write( + _user(), + bindings=[AgentBinding(kind="tool", ref="canvas_faculty::")], + tool_service=_tool_svc(("canvas_faculty", ["list_courses"])), + ) + assert ei.value.status_code == 400 + + @pytest.mark.asyncio + async def test_bare_ref_still_binds_whole_server(self): + # Additive: a bare ref is unaffected by the discovered list. + await validate_agent_write( + _user(), + bindings=[AgentBinding(kind="tool", ref="canvas_faculty")], + tool_service=_tool_svc(("canvas_faculty", ["list_courses", "grade_submission"])), + ) + # --------------------------------------------------------------------------- KB class TestKnowledgeBase: diff --git a/backend/tests/apis/app_api/test_user_skills_routes.py b/backend/tests/apis/app_api/test_user_skills_routes.py index a1cf23cf8..8c6f2dd88 100644 --- a/backend/tests/apis/app_api/test_user_skills_routes.py +++ b/backend/tests/apis/app_api/test_user_skills_routes.py @@ -1,4 +1,9 @@ -"""Route tests for the user-facing skills API (GET /skills/, PUT /skills/preferences).""" +"""Route tests for the user-facing skills API. + +Covers the picker (GET /skills/, PUT /skills/preferences) and the +read-only detail surface behind Customize → Skills → one skill +(GET /skills/{id}, GET /skills/{id}/resources/{filename}). +""" from __future__ import annotations @@ -10,7 +15,12 @@ from apis.shared.auth import get_current_user_from_session from apis.shared.auth.models import User -from apis.shared.skills.models import SkillDefinition, SkillStatus, UserSkillPreference +from apis.shared.skills.models import ( + SkillDefinition, + SkillResourceRef, + SkillStatus, + UserSkillPreference, +) from apis.app_api.skills import routes as skills_routes @@ -104,6 +114,22 @@ def test_lists_active_accessible_skills_with_prefs_merged(self, monkeypatch): "web_research", ] + def test_serves_the_runtime_activation_slug(self, monkeypatch): + """The `/` command menu writes this slug into the message verbatim. + + It has to be the same string the ``AgentSkills`` plugin injects as + ``Skill.name``, so it is derived here from the one shared slug rule + rather than re-implemented client-side. + """ + repo = _FakeRepo( + skills=[_skill("pdf_workflows_v2", "PDF Workflows")], + prefs={}, + ) + client = _make_client(monkeypatch, ["pdf_workflows_v2"], repo) + + body = client.get("/skills/").json() + assert body["skills"][0]["slug"] == "pdf-workflows-v2" + def test_non_active_skills_are_hidden(self, monkeypatch): repo = _FakeRepo( skills=[ @@ -151,3 +177,194 @@ def test_rejects_inaccessible_skill_ids(self, monkeypatch): assert resp.status_code == 400 assert "forbidden_skill" in resp.json()["detail"] assert repo.saved is None + + +class _FakeCatalogService: + """Duck-typed stand-in for SkillCatalogService (detail + resource read).""" + + def __init__(self, skills: List[SkillDefinition], blobs: Dict[str, bytes] = None): + self._skills = {s.skill_id: s for s in skills} + self._blobs = dict(blobs or {}) + + async def get_skill(self, skill_id: str): + return self._skills.get(skill_id) + + async def read_resource(self, skill_id: str, filename: str): + skill = self._skills.get(skill_id) + if skill is None: + raise ValueError(f"Skill '{skill_id}' not found") + ref = next((r for r in skill.resources if r.filename == filename), None) + if ref is None: + raise ValueError(f"Reference file '{filename}' not found") + return ref, self._blobs.get(filename, b"") + + +def _make_detail_client( + monkeypatch: pytest.MonkeyPatch, + accessible: List[str], + repo: _FakeRepo, + catalog: _FakeCatalogService, +) -> TestClient: + client = _make_client(monkeypatch, accessible, repo) + monkeypatch.setattr(skills_routes, "get_skill_catalog_service", lambda: catalog) + return client + + +class TestGetAccessibleSkill: + def test_returns_the_instructions_body_and_merged_preference(self, monkeypatch): + skill = _skill("web_research", "Web Research") + skill.instructions = "# Web Research\n\nSearch broadly, cite everything." + skill.category = "research" + skill.allowed_tools = ["web_search"] + repo = _FakeRepo(skills=[skill], prefs={"web_research": True}) + client = _make_detail_client( + monkeypatch, ["web_research"], repo, _FakeCatalogService([skill]) + ) + + body = client.get("/skills/web_research").json() + assert body["skillId"] == "web_research" + # The point of the endpoint: GET /skills/ carries none of this. + assert body["instructions"] == "# Web Research\n\nSearch broadly, cite everything." + assert body["allowedTools"] == ["web_search"] + assert body["category"] == "research" + assert body["userEnabled"] is True + assert body["isEnabled"] is True + + def test_untouched_skill_reads_off_like_the_picker(self, monkeypatch): + """D6 opt-in. The detail page must not contradict the list it came from.""" + skill = _skill("pdf_workflows", "PDF Workflows") + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, ["pdf_workflows"], repo, _FakeCatalogService([skill]) + ) + + body = client.get("/skills/pdf_workflows").json() + assert body["userEnabled"] is None + assert body["isEnabled"] is False + + def test_inaccessible_skill_is_404_not_403(self, monkeypatch): + """A skill you weren't granted must be indistinguishable from one that + does not exist — a 403 would confirm it is out there.""" + skill = _skill("someone_elses", "Someone Else's") + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, ["web_research"], repo, _FakeCatalogService([skill]) + ) + + assert client.get("/skills/someone_elses").status_code == 404 + + def test_non_active_catalog_skill_is_404(self, monkeypatch): + """GET /skills/ filters to ACTIVE, so a drilled-in DRAFT must not be + reachable by id from a surface that never listed it.""" + skill = _skill("draft_one", "Draft One", status=SkillStatus.DRAFT) + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, ["draft_one"], repo, _FakeCatalogService([skill]) + ) + + assert client.get("/skills/draft_one").status_code == 404 + + def test_owner_still_reads_their_own_non_active_skill(self, monkeypatch): + """Ownership is its own grant: a user's own draft is theirs to open.""" + skill = _skill("my_draft", "My Draft", status=SkillStatus.DRAFT) + skill.owner_id = "user-1" + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, ["my_draft"], repo, _FakeCatalogService([skill]) + ) + + body = client.get("/skills/my_draft").json() + assert body["isOwned"] is True + assert body["status"] == "draft" + + def test_catalog_skill_is_not_owned(self, monkeypatch): + skill = _skill("web_research", "Web Research") # owner_id defaults to "system" + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, ["web_research"], repo, _FakeCatalogService([skill]) + ) + + assert client.get("/skills/web_research").json()["isOwned"] is False + + def test_does_not_leak_owner_id_or_role_topology(self, monkeypatch): + """`ownerId` would name another user; `allowedAppRoles` is an + admin-display projection of RBAC (CLAUDE.md) and has no business on a + surface any granted user can open.""" + skill = _skill("web_research", "Web Research") + skill.owner_id = "someone-else" + skill.allowed_app_roles = ["faculty", "staff"] + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, ["web_research"], repo, _FakeCatalogService([skill]) + ) + + body = client.get("/skills/web_research").json() + assert "ownerId" not in body + assert "allowedAppRoles" not in body + + def test_mine_still_routes_to_the_list_not_a_skill_called_mine(self, monkeypatch): + """⚠️ Registration-order guard. SKILL_ID_PATTERN matches the literal + 'mine', so declaring /{skill_id} above the /mine routes would turn + GET /skills/mine into a lookup for a skill named "mine".""" + repo = _FakeRepo(skills=[], prefs={}) + client = _make_detail_client(monkeypatch, [], repo, _FakeCatalogService([])) + + async def fake_list(user): + return [] + + monkeypatch.setattr( + skills_routes.get_user_skill_service(), "list_my_skills", fake_list + ) + resp = client.get("/skills/mine") + assert resp.status_code == 200 + assert resp.json() == {"skills": [], "totalCount": 0} + + +class TestReadAccessibleSkillResource: + def _skill_with_file(self) -> SkillDefinition: + skill = _skill("web_research", "Web Research") + skill.resources = [ + SkillResourceRef( + filename="forms.md", + content_hash="abc123", + size=11, + content_type="text/markdown", + s3_key="skills/web_research/references/forms.md", + ) + ] + return skill + + def test_serves_a_granted_skills_file_the_user_does_not_own(self, monkeypatch): + skill = self._skill_with_file() + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, + ["web_research"], + repo, + _FakeCatalogService([skill], {"forms.md": b"hello world"}), + ) + + resp = client.get("/skills/web_research/resources/forms.md") + assert resp.status_code == 200 + assert resp.content == b"hello world" + # Served inert: never a script-bearing document on the SPA's origin. + assert resp.headers["x-content-type-options"] == "nosniff" + assert "attachment" in resp.headers["content-disposition"] + + def test_inaccessible_skills_file_is_404(self, monkeypatch): + skill = self._skill_with_file() + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, [], repo, _FakeCatalogService([skill], {"forms.md": b"x"}) + ) + + assert client.get("/skills/web_research/resources/forms.md").status_code == 404 + + def test_unknown_filename_is_404(self, monkeypatch): + skill = self._skill_with_file() + repo = _FakeRepo(skills=[skill], prefs={}) + client = _make_detail_client( + monkeypatch, ["web_research"], repo, _FakeCatalogService([skill]) + ) + + assert client.get("/skills/web_research/resources/nope.md").status_code == 404 diff --git a/backend/tests/apis/app_api/tools/test_capability_discovery.py b/backend/tests/apis/app_api/tools/test_capability_discovery.py new file mode 100644 index 000000000..9281e3a53 --- /dev/null +++ b/backend/tests/apis/app_api/tools/test_capability_discovery.py @@ -0,0 +1,275 @@ +"""Capability discovery: prompts + resources from a saved MCP server. + +The stack only ever called ``tools/list``. These cover the part that is easy to +get wrong — a server that implements *some* of the three listings. Answering +"method not found" to ``prompts/list`` is normal and must not cost us the +resources listing, or the snapshot. + +The MCP client is stubbed; the logic under test is the guarding, pagination and +capping around it. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from apis.app_api.tools.discovery import discover_capabilities_for_saved_tool +from apis.shared.tools.models import ( + MAX_CAPABILITY_ENTRIES, + MCPServerConfig, + ToolDefinition, + ToolProtocol, + ToolStatus, +) + + +class _StubClient: + """Stands in for strands' MCPClient as a context manager.""" + + def __init__(self, prompts=None, resources=None, templates=None): + self._prompts = prompts + self._resources = resources + self._templates = templates + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def _answer(self, payload, kind): + if payload is None: + # What a server that doesn't implement the method actually does. + raise RuntimeError(f"Method not found: {kind}") + return payload + + def list_prompts_sync(self, pagination_token=None): + return self._answer(self._prompts, "prompts/list") + + def list_resources_sync(self, pagination_token=None): + return self._answer(self._resources, "resources/list") + + def list_resource_templates_sync(self, pagination_token=None): + return self._answer(self._templates, "resources/templates/list") + + +def _tool(tool_id="canvas_faculty", protocol=ToolProtocol.MCP_EXTERNAL): + return ToolDefinition( + tool_id=tool_id, + display_name=tool_id, + description="x", + protocol=protocol, + status=ToolStatus.ACTIVE, + mcp_config=MCPServerConfig(server_url="https://example.com/mcp", tools=[]), + ) + + +def _prompts(*names): + return SimpleNamespace( + prompts=[ + SimpleNamespace( + name=n, + title=None, + description=f"desc for {n}", + arguments=[ + SimpleNamespace( + name="course_id", description="Which course", required=True + ) + ], + ) + for n in names + ], + nextCursor=None, + ) + + +def _resources(*uris): + return SimpleNamespace( + resources=[ + SimpleNamespace(uri=u, name=None, description=None, mimeType="text/plain") + for u in uris + ], + nextCursor=None, + ) + + +def _templates(*uris): + return SimpleNamespace( + resourceTemplates=[ + SimpleNamespace( + uriTemplate=u, name=None, description=None, mimeType=None + ) + for u in uris + ], + nextCursor=None, + ) + + +async def _run(tool, client): + """Patch at the source module: discovery imports it inside the function.""" + with patch( + "agents.main_agent.integrations.external_mcp_client.create_external_mcp_client", + return_value=client, + ): + return await discover_capabilities_for_saved_tool(tool) + + +@pytest.mark.asyncio +async def test_collects_prompts_and_resources(): + snapshot = await _run( + _tool(), + _StubClient( + prompts=_prompts("grade_summary"), + resources=_resources("canvas://a"), + templates=_templates("canvas://courses/{id}/syllabus"), + ), + ) + assert snapshot.supports_prompts is True + assert snapshot.supports_resources is True + assert [p.name for p in snapshot.prompts] == ["grade_summary"] + # Arguments carry `required` and `description`, not just a name: a form + # cannot be built from names alone. + argument = snapshot.prompts[0].arguments[0] + assert (argument.name, argument.required, argument.description) == ( + "course_id", + True, + "Which course", + ) + assert {r.uri for r in snapshot.resources} == { + "canvas://a", + "canvas://courses/{id}/syllabus", + } + # A template is not a readable URI — the UI has to say so. + assert [r.uri_template for r in snapshot.resources if "{id}" in r.uri] == [True] + assert snapshot.error is None + + +@pytest.mark.asyncio +async def test_a_server_without_prompts_still_yields_resources(): + # The regression this guards: one unsupported listing taking the whole + # snapshot down with it. + snapshot = await _run( + _tool(), + _StubClient(prompts=None, resources=_resources("x://1"), templates=None), + ) + assert snapshot.supports_prompts is False + assert snapshot.prompts == [] + assert snapshot.supports_resources is True + assert len(snapshot.resources) == 1 + assert snapshot.error is None + + +@pytest.mark.asyncio +async def test_a_server_with_neither_is_not_an_error(): + snapshot = await _run(_tool(), _StubClient()) + assert snapshot.supports_prompts is False + assert snapshot.supports_resources is False + # "Offers nothing" and "we couldn't ask" are different facts. + assert snapshot.error is None + + +@pytest.mark.asyncio +async def test_unreachable_server_records_an_error(): + class _Boom(_StubClient): + def __enter__(self): + raise RuntimeError("connection refused") + + snapshot = await _run(_tool(), _Boom()) + assert snapshot.error is not None + assert "connection refused" in snapshot.error + + +@pytest.mark.asyncio +async def test_gateway_tools_are_not_probed(): + # A Gateway target exposes no prompt or resource surface to ask. + snapshot = await _run(_tool(protocol=ToolProtocol.MCP_GATEWAY), _StubClient(prompts=_prompts("p"))) + assert snapshot.prompts == [] + assert snapshot.error is not None + + +@pytest.mark.asyncio +async def test_entries_are_capped_so_one_server_cannot_blow_the_item(): + many = _resources(*[f"x://{i}" for i in range(MAX_CAPABILITY_ENTRIES + 50)]) + snapshot = await _run(_tool(), _StubClient(resources=many)) + assert len(snapshot.resources) == MAX_CAPABILITY_ENTRIES + assert snapshot.truncated is True + + +@pytest.mark.asyncio +async def test_pagination_follows_the_cursor(): + pages = [ + SimpleNamespace( + resources=[ + SimpleNamespace( + uri="x://1", name=None, description=None, mimeType=None + ) + ], + nextCursor="c1", + ), + SimpleNamespace( + resources=[ + SimpleNamespace( + uri="x://2", name=None, description=None, mimeType=None + ) + ], + nextCursor=None, + ), + ] + + class _Paged(_StubClient): + def __init__(self): + super().__init__() + self.calls = 0 + + def list_resources_sync(self, pagination_token=None): + page = pages[self.calls] + self.calls += 1 + return page + + def list_prompts_sync(self, pagination_token=None): + raise RuntimeError("no prompts") + + def list_resource_templates_sync(self, pagination_token=None): + raise RuntimeError("no templates") + + snapshot = await _run(_tool(), _Paged()) + assert {r.uri for r in snapshot.resources} == {"x://1", "x://2"} + + +@pytest.mark.asyncio +async def test_a_broken_cursor_cannot_loop_forever(): + class _Endless(_StubClient): + def list_resources_sync(self, pagination_token=None): + return SimpleNamespace( + resources=[ + SimpleNamespace( + uri="x://same", name=None, description=None, mimeType=None + ) + ], + nextCursor="always", + ) + + def list_prompts_sync(self, pagination_token=None): + raise RuntimeError("no prompts") + + def list_resource_templates_sync(self, pagination_token=None): + raise RuntimeError("no templates") + + snapshot = await _run(_tool(), _Endless()) + assert snapshot.truncated is True + + +@pytest.mark.asyncio +async def test_long_descriptions_are_clipped(): + long_desc = "x" * 5000 + payload = SimpleNamespace( + prompts=[ + SimpleNamespace( + name="p", title=None, description=long_desc, arguments=[] + ) + ], + nextCursor=None, + ) + snapshot = await _run(_tool(), _StubClient(prompts=payload)) + assert len(snapshot.prompts[0].description) < 600 diff --git a/backend/tests/apis/app_api/tools/test_oauth_provider_surfaced.py b/backend/tests/apis/app_api/tools/test_oauth_provider_surfaced.py new file mode 100644 index 000000000..afde4aae4 --- /dev/null +++ b/backend/tests/apis/app_api/tools/test_oauth_provider_surfaced.py @@ -0,0 +1,84 @@ +"""`requiresOauthProvider` must reach the user-facing /tools payload. + +`UserToolAccess` has always declared the field, but the service never passed it, +so every tool reported ``requiresOauthProvider: null``. The SPA therefore had no +way to tell that 13 of the 31 tools in prod need an OAuth connection before they +will run — the user only found out when a turn failed mid-answer. + +Dependencies are mocked; the logic under test is pure. +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from apis.app_api.tools.service import ToolCatalogService +from apis.shared.auth.models import User +from apis.shared.rbac.models import UserEffectivePermissions +from apis.shared.tools.models import ( + MCPServerConfig, + ToolDefinition, + ToolProtocol, + ToolStatus, + UserToolPreference, +) + + +def _user(): + return User(user_id="u1", email="u@x.com", name="U", roles=["User"], raw_token="t") + + +def _service(tools): + repo = MagicMock() + repo.list_tools = AsyncMock(return_value=tools) + repo.get_user_preferences = AsyncMock( + return_value=UserToolPreference(user_id="u1", tool_preferences={}) + ) + role_service = MagicMock() + role_service.resolve_user_permissions = AsyncMock( + return_value=UserEffectivePermissions( + user_id="u1", + app_roles=["User"], + tools=["*"], + models=["*"], + quota_tier=None, + resolved_at="2024-01-01T00:00:00Z", + ) + ) + return ToolCatalogService(repository=repo, app_role_service=role_service) + + +def _tool(tool_id: str, provider: str | None): + return ToolDefinition( + tool_id=tool_id, + display_name=tool_id, + description="x", + protocol=ToolProtocol.MCP_EXTERNAL, + status=ToolStatus.ACTIVE, + requires_oauth_provider=provider, + mcp_config=MCPServerConfig(server_url="https://example.com/mcp", tools=[]), + ) + + +@pytest.mark.asyncio +async def test_oauth_provider_reaches_the_user_payload(): + service = _service([_tool("canvas_faculty", "canvas-faculty")]) + tools = await service.get_user_accessible_tools(_user()) + assert tools[0].requires_oauth_provider == "canvas-faculty" + + +@pytest.mark.asyncio +async def test_tool_without_a_provider_reports_none(): + service = _service([_tool("calculator", None)]) + tools = await service.get_user_accessible_tools(_user()) + assert tools[0].requires_oauth_provider is None + + +@pytest.mark.asyncio +async def test_provider_survives_serialization_under_its_alias(): + # The SPA reads `requiresOauthProvider`; a field that only exists under its + # snake_case name would be just as invisible as not being set at all. + service = _service([_tool("gmail_employee", "gmail-employee")]) + tools = await service.get_user_accessible_tools(_user()) + dumped = tools[0].model_dump(by_alias=True) + assert dumped["requiresOauthProvider"] == "gmail-employee" diff --git a/backend/tests/apis/app_api/tools/test_prompt_resolution.py b/backend/tests/apis/app_api/tools/test_prompt_resolution.py new file mode 100644 index 000000000..e6ae38751 --- /dev/null +++ b/backend/tests/apis/app_api/tools/test_prompt_resolution.py @@ -0,0 +1,181 @@ +"""Prompt resolution: composing one of a server's prompts (``prompts/get``). + +The listings are a stored snapshot; this is a live call whose result depends on +arguments the user just typed. What is easy to get wrong is the flattening — an +MCP prompt message can carry an image, a resource link or an embedded blob, and +none of those survive into text a person can read. Dropping them silently would +be worse than saying so. + +The MCP client is stubbed; the logic under test is the capping and flattening +around it. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from apis.app_api.tools.discovery import resolve_prompt_for_saved_tool +from apis.shared.tools.models import ( + MAX_RESOLVED_PROMPT_CHARS, + MAX_RESOLVED_PROMPT_MESSAGES, + MCPServerConfig, + ToolDefinition, + ToolProtocol, + ToolStatus, +) + + +class _StubClient: + """Stands in for strands' MCPClient as a context manager.""" + + def __init__(self, result=None, error=None): + self._result = result + self._error = error + self.called_with = None + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def get_prompt_sync(self, prompt_id, args): + self.called_with = (prompt_id, args) + if self._error: + raise RuntimeError(self._error) + return self._result + + +def _tool(protocol=ToolProtocol.MCP_EXTERNAL): + return ToolDefinition( + tool_id="canvas_faculty", + display_name="Canvas Faculty", + description="x", + protocol=protocol, + status=ToolStatus.ACTIVE, + mcp_config=MCPServerConfig(server_url="https://example.com/mcp", tools=[]), + ) + + +def _text(text): + return SimpleNamespace(type="text", text=text) + + +def _result(*messages, description=None): + return SimpleNamespace(description=description, messages=list(messages)) + + +def _message(content, role="user"): + return SimpleNamespace(role=role, content=content) + + +async def _resolve(client, arguments=None, tool=None): + with patch( + "agents.main_agent.integrations.external_mcp_client.create_external_mcp_client", + return_value=client, + ): + return await resolve_prompt_for_saved_tool( + tool or _tool(), "grade_submission", arguments or {} + ) + + +@pytest.mark.asyncio +async def test_composes_text_messages_and_passes_arguments(): + client = _StubClient( + _result( + _message(_text("Grade BIO 101.")), + _message(_text("Sure."), role="assistant"), + description="Rubric-guided feedback", + ) + ) + + resolved = await _resolve(client, {"course": "BIO 101"}) + + assert client.called_with == ("grade_submission", {"course": "BIO 101"}) + assert resolved.description == "Rubric-guided feedback" + assert [(m.role, m.text) for m in resolved.messages] == [ + ("user", "Grade BIO 101."), + ("assistant", "Sure."), + ] + assert resolved.truncated is False + + +@pytest.mark.asyncio +async def test_non_text_content_is_reported_by_kind_not_dropped_silently(): + """An image has no readable body, but the message must still be visible.""" + client = _StubClient( + _result( + _message(SimpleNamespace(type="image", data="…", mimeType="image/png")), + ) + ) + + resolved = await _resolve(client) + + assert len(resolved.messages) == 1 + assert resolved.messages[0].kind == "image" + assert resolved.messages[0].text == "" + + +@pytest.mark.asyncio +async def test_embedded_text_resource_is_readable_but_a_blob_is_not(): + client = _StubClient( + _result( + _message( + SimpleNamespace( + type="resource", + resource=SimpleNamespace(uri="file://a.md", text="# Rubric"), + ) + ), + _message( + SimpleNamespace( + type="resource", + resource=SimpleNamespace(uri="file://b.pdf", text=None), + ) + ), + ) + ) + + resolved = await _resolve(client) + + # A text resource flattens to readable text... + assert (resolved.messages[0].kind, resolved.messages[0].text) == ("text", "# Rubric") + # ...a binary one names itself and carries no payload. + assert resolved.messages[1].kind == "resource" + assert resolved.messages[1].text == "file://b.pdf" + + +@pytest.mark.asyncio +async def test_oversized_text_is_capped_and_flagged(): + client = _StubClient(_result(_message(_text("x" * (MAX_RESOLVED_PROMPT_CHARS + 500))))) + + resolved = await _resolve(client) + + assert len(resolved.messages[0].text) == MAX_RESOLVED_PROMPT_CHARS + assert resolved.truncated is True + + +@pytest.mark.asyncio +async def test_too_many_messages_are_capped_and_flagged(): + client = _StubClient( + _result(*[_message(_text("hi")) for _ in range(MAX_RESOLVED_PROMPT_MESSAGES + 5)]) + ) + + resolved = await _resolve(client) + + assert len(resolved.messages) == MAX_RESOLVED_PROMPT_MESSAGES + assert resolved.truncated is True + + +@pytest.mark.asyncio +async def test_server_failure_raises_for_the_route_to_turn_into_a_502(): + client = _StubClient(error="Unknown prompt: grade_submission") + + with pytest.raises(RuntimeError, match="could not compose"): + await _resolve(client) + + +@pytest.mark.asyncio +async def test_a_gateway_tool_has_no_prompt_surface_to_ask(): + with pytest.raises(RuntimeError, match="not an external MCP server"): + await resolve_prompt_for_saved_tool(_tool(ToolProtocol.MCP_GATEWAY), "p", {}) diff --git a/backend/tests/apis/inference_api/test_agent_binding_resolver.py b/backend/tests/apis/inference_api/test_agent_binding_resolver.py index 046edc4fc..9bf57a40d 100644 --- a/backend/tests/apis/inference_api/test_agent_binding_resolver.py +++ b/backend/tests/apis/inference_api/test_agent_binding_resolver.py @@ -258,6 +258,77 @@ async def test_tool_access_checked_against_invoker(self, monkeypatch): args = svc.can_access_tool.await_args.args assert args[0].user_id == "u-bob" and args[1] == "web_search" + # -- scoped refs (``toolId::mcpToolName``) ----------------------------------- + @pytest.mark.asyncio + async def test_scoped_refs_survive_resolution_verbatim(self, monkeypatch): + # The scoped id IS the enforcement: it must reach enabled_tools intact for + # collect_tool_name_filters to narrow the server. Collapsing it to the base + # here would silently restore all of the server's tools. + _patch_tool_access(monkeypatch, True) + plan = await resolve_agent_invocation( + _assistant( + bindings=[ + _tool_binding("canvas_faculty::list_courses"), + _tool_binding("canvas_faculty::list_rubrics"), + ] + ), + _user(), + ) + assert plan.tools.tool_ids == [ + "canvas_faculty::list_courses", + "canvas_faculty::list_rubrics", + ] + + @pytest.mark.asyncio + async def test_bare_ref_still_resolves_to_whole_server(self, monkeypatch): + # Additive: an existing whole-server binding is untouched. + _patch_tool_access(monkeypatch, True) + plan = await resolve_agent_invocation( + _assistant(bindings=[_tool_binding("canvas_faculty")]), _user() + ) + assert plan.tools.tool_ids == ["canvas_faculty"] + + @pytest.mark.asyncio + async def test_scoped_ref_blocks_when_base_inaccessible(self, monkeypatch): + # Block-with-message, not a silent drop (D5). The gate answers on the base, so + # this asserts the real shape: the invoker is granted neither, and the message + # names the server an administrator would grant. + _patch_tool_access(monkeypatch, {"calculator"}) + with pytest.raises(AgentBindingBlockedError) as ei: + await resolve_agent_invocation( + _assistant(bindings=[_tool_binding("canvas_faculty::list_courses")]), + _user(), + ) + assert "canvas_faculty" in ei.value.message + assert "::" not in ei.value.message + + @pytest.mark.asyncio + async def test_scoped_ref_allowed_when_base_server_granted(self, monkeypatch): + # A grant on the server admits any subset of it — the gate is handed the scoped + # id and AppRoleService.can_access_tool base-collapses it (tested against real + # role records in tests/shared/test_scoped_tool_grants.py). + _patch_tool_access(monkeypatch, {"canvas_faculty::list_courses"}) + plan = await resolve_agent_invocation( + _assistant(bindings=[_tool_binding("canvas_faculty::list_courses")]), _user() + ) + assert plan.tools.tool_ids == ["canvas_faculty::list_courses"] + + @pytest.mark.asyncio + async def test_access_checked_once_per_server(self, monkeypatch): + # Seven tools of one server is the normal shape; it should cost one gate call. + svc = _patch_tool_access(monkeypatch, True) + await resolve_agent_invocation( + _assistant( + bindings=[ + _tool_binding("canvas_faculty::list_courses"), + _tool_binding("canvas_faculty::list_rubrics"), + _tool_binding("web_search"), + ] + ), + _user(), + ) + assert svc.can_access_tool.await_count == 2 + class TestSkillResolution: @pytest.mark.asyncio diff --git a/backend/tests/apis/inference_api/test_agent_binding_scoped_tools_e2e.py b/backend/tests/apis/inference_api/test_agent_binding_scoped_tools_e2e.py new file mode 100644 index 000000000..0b49ce04e --- /dev/null +++ b/backend/tests/apis/inference_api/test_agent_binding_scoped_tools_e2e.py @@ -0,0 +1,161 @@ +"""An Agent's scoped tool bindings must reach the MCP client as a name filter. + +The layers either side of this seam are each covered on their own — the resolver +in ``test_agent_binding_resolver.py``, the client filter in +``tests/agents/main_agent/integrations/test_external_mcp_client.py``. What neither +proves is that they are *connected*: a resolver that collapsed a scoped ref to its +base would pass its own tests and quietly hand the turn all 44 of a server's tools, +which is invisible from outside because the agent still works. It just stops being +fenced, and the tool definitions it does not need stay in the cacheable prefix on +every turn for the life of the session. + +So this drives the real chain — ``Assistant.bindings`` → ``resolve_agent_invocation`` +→ ``enabled_tools`` → ``ToolFilter`` classification → ``load_external_tools`` — and +asserts the filter that actually reaches ``create_external_mcp_client``. Only the +catalog and the client constructor are stubbed. +""" + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from agents.main_agent.integrations.external_mcp_client import ExternalMCPIntegration +from agents.main_agent.tools.tool_filter import ToolFilter +from apis.inference_api.chat.agent_binding_resolver import resolve_agent_invocation +from apis.shared.assistants.models import AgentBinding, Assistant +from apis.shared.auth.models import User + +RESOLVER = "apis.inference_api.chat.agent_binding_resolver" +SERVER = "canvas_faculty" + +# The seven the Rubric Builder agent actually needs, of the forty-four the server has. +BOUND = [ + "list_courses", + "list_assignments", + "get_assignment_details", + "list_rubrics", + "get_rubric", + "create_rubric", + "associate_rubric", +] +# The ones its system prompt currently asks it not to call. +UNBOUND = ["grade_submission", "bulk_grade_submissions", "create_assignment", "delete_rubric"] + + +def _user() -> User: + return User(email="prof@x.edu", user_id="u-prof", name="Prof", roles=[]) + + +def _agent(refs) -> Assistant: + return Assistant( + assistantId="ast-rubric", + ownerId="u-alice", + ownerName="Alice", + name="Rubric Builder", + description="d", + instructions="i", + vectorIndexId="idx", + visibility="SHARED", + createdAt="t", + updatedAt="t", + status="COMPLETE", + bindings=[AgentBinding(kind="tool", ref=r) for r in refs], + ) + + +def _grant_whole_server(monkeypatch): + """The invoker's role grants the server; the binding decides the subset.""" + svc = MagicMock() + svc.can_access_tool = AsyncMock(return_value=True) + monkeypatch.setattr(f"{RESOLVER}.get_app_role_service", lambda: svc) + + +def _catalog_tool(): + return SimpleNamespace( + tool_id=SERVER, + protocol="mcp_external", + mcp_config=SimpleNamespace( + server_url="https://example.com/mcp", + approval_required_names=lambda: set(), + ), + forward_auth_token=False, + requires_oauth_provider=None, + updated_at=datetime(2025, 1, 1, tzinfo=timezone.utc), + ) + + +async def _allowed_names_for(tool_ids): + """Run ``tool_ids`` through the runtime and return the client's name filter.""" + integration = ExternalMCPIntegration() + repo = SimpleNamespace(get_tool=AsyncMock(return_value=_catalog_tool())) + client = SimpleNamespace(load_tools=AsyncMock(return_value=[])) + with patch( + "apis.shared.tools.repository.get_tool_catalog_repository", return_value=repo + ), patch( + "agents.main_agent.integrations.external_mcp_client.create_external_mcp_client", + return_value=client, + ) as create_mock: + await integration.load_external_tools(tool_ids) + assert create_mock.call_count == 1, "one server ⇒ one client" + return create_mock.call_args.kwargs["allowed_tool_names"] + + +class TestScopedBindingsNarrowTheTurn: + @pytest.mark.asyncio + async def test_scoped_bindings_yield_a_filtered_client(self, monkeypatch): + _grant_whole_server(monkeypatch) + plan = await resolve_agent_invocation( + _agent([f"{SERVER}::{name}" for name in BOUND]), _user() + ) + + allowed = await _allowed_names_for(plan.tools.tool_ids) + + assert allowed == set(BOUND) + # The point of the exercise: the destructive tools are absent from the turn, + # not merely unused by it. + for name in UNBOUND: + assert name not in allowed + + @pytest.mark.asyncio + async def test_bare_binding_still_loads_the_whole_server(self, monkeypatch): + """Additive — every Agent binding a server today keeps all of its tools.""" + _grant_whole_server(monkeypatch) + plan = await resolve_agent_invocation(_agent([SERVER]), _user()) + + assert await _allowed_names_for(plan.tools.tool_ids) is None + + @pytest.mark.asyncio + async def test_a_bare_binding_alongside_scoped_ones_wins(self, monkeypatch): + """Documented ``collect_tool_name_filters`` precedence, asserted end to end. + + An author who binds the server *and* two of its tools has asked for the + server; the filter must not silently narrow to the two. + """ + _grant_whole_server(monkeypatch) + plan = await resolve_agent_invocation( + _agent([SERVER, f"{SERVER}::list_courses", f"{SERVER}::get_rubric"]), _user() + ) + + assert await _allowed_names_for(plan.tools.tool_ids) is None + + @pytest.mark.asyncio + async def test_scoped_ids_classify_as_external_mcp_tools(self, monkeypatch): + """The step between: ``ToolFilter`` must route a scoped id to the MCP loader. + + It classifies on the *base* id, so a scoped ref that failed to match would + fall through to the "not a known tool id" warning and be dropped silently — + the agent would run with no Canvas tools at all. + """ + _grant_whole_server(monkeypatch) + plan = await resolve_agent_invocation( + _agent([f"{SERVER}::{name}" for name in BOUND]), _user() + ) + + tool_filter = ToolFilter(registry=SimpleNamespace(has_tool=lambda _: False)) + tool_filter.set_external_mcp_tools([SERVER]) + result = tool_filter.filter_tools_extended(plan.tools.tool_ids) + + assert result.external_mcp_tool_ids == plan.tools.tool_ids + assert result.local_tools == [] and result.gateway_tool_ids == [] diff --git a/backend/tests/apis/inference_api/test_skill_slash_commands.py b/backend/tests/apis/inference_api/test_skill_slash_commands.py new file mode 100644 index 000000000..f214fdf3b --- /dev/null +++ b/backend/tests/apis/inference_api/test_skill_slash_commands.py @@ -0,0 +1,72 @@ +"""Slash-command skill invocation (`/skill-name` in the composer). + +Two halves, both in inference-api chat routes: + +* ``_resolve_invoked_skill_slugs`` — narrow-never-grant, the same rule + ``_apply_enabled_skills_filter`` applies to ``enabled_skills``, re-run + against the turn's FINAL effective set because an Agent's skill bindings can + replace that set after the first filter runs. +* ``_build_skill_invocation_note`` — the directive appended to the user + message. A slash command has to be expressed as an instruction because the + only activation path is the plugin's own ``skills`` tool, which the model + calls. +""" + +from apis.inference_api.chat.routes import ( + _build_skill_invocation_note, + _resolve_invoked_skill_slugs, +) + + +class TestResolveInvokedSkillSlugs: + def test_returns_activation_slugs_not_catalog_ids(self): + # The id never appears in anything the model can see; the slug is what + # the `skills` tool takes and what `` lists. + assert _resolve_invoked_skill_slugs(["pdf_workflows"], ["pdf_workflows"]) == [ + "pdf-workflows" + ] + + def test_drops_a_skill_the_turn_does_not_disclose(self): + # Narrow, never grant. A directive naming a skill absent from + # would cost the model a tool call to discover. + assert _resolve_invoked_skill_slugs(["web_research"], ["pdf_workflows"]) == [] + + def test_orders_by_the_effective_set_not_the_request(self): + # The directive is persisted in the message, so two turns naming the + # same skills must produce byte-identical text regardless of the order + # the client happened to send them in. + effective = ["alpha", "beta", "gamma"] + assert _resolve_invoked_skill_slugs(effective, ["gamma", "alpha"]) == [ + "alpha", + "gamma", + ] + + def test_no_skills_on_the_turn_means_no_invocation(self): + # An Agent binding that replaced the skill set with nothing, or a turn + # that never asked for skills at all. + assert _resolve_invoked_skill_slugs(None, ["web_research"]) == [] + assert _resolve_invoked_skill_slugs([], ["web_research"]) == [] + + def test_absent_selection_is_inert(self): + assert _resolve_invoked_skill_slugs(["web_research"], None) == [] + assert _resolve_invoked_skill_slugs(["web_research"], []) == [] + + +class TestBuildSkillInvocationNote: + def test_names_the_slug_and_the_activation_tool(self): + note = _build_skill_invocation_note(["pdf-workflows"]) + assert "`pdf-workflows`" in note + assert "`skills`" in note + assert "skill with a slash command" in note + + def test_pluralizes_for_more_than_one_skill(self): + note = _build_skill_invocation_note(["pdf-workflows", "web-research"]) + assert "skills with a slash command" in note + assert "Activate each" in note + + def test_is_one_bounded_line(self): + # It rides the user message: paid as input this turn and as cached + # history on every later turn of the session. Keep it small. + note = _build_skill_invocation_note(["pdf-workflows"]) + assert "\n" not in note + assert len(note) < 250 diff --git a/backend/tests/apis/shared/middleware/test_proxied_redirect_middleware.py b/backend/tests/apis/shared/middleware/test_proxied_redirect_middleware.py new file mode 100644 index 000000000..67bbdb7b4 --- /dev/null +++ b/backend/tests/apis/shared/middleware/test_proxied_redirect_middleware.py @@ -0,0 +1,149 @@ +"""Tests for ProxiedRedirectMiddleware. + +The regression these lock down: loading a conversation page on dev logged + + Mixed Content: ... requested an insecure resource + 'http://api.dev.boisestate.ai/agents'. This request has been blocked. + +That URL is not built anywhere in the SPA — it is Starlette's own +`redirect_slashes` answer to `GET /api/agents/`, rendered from what app-api +can see behind CloudFront: the ALB's hostname, plain HTTP, and a path with +the `/api` prefix already stripped off. +""" + +from __future__ import annotations + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from starlette.responses import RedirectResponse + +from apis.shared.middleware.proxied_redirect import ( + FORWARDED_PREFIX_HEADER, + ProxiedRedirectMiddleware, +) + +PROXY_HOST = "api.dev.boisestate.ai" + + +@pytest.fixture +def app() -> FastAPI: + app = FastAPI() + app.add_middleware(ProxiedRedirectMiddleware) + + # Declared without a trailing slash, exactly like the real `/agents` + # router — so `GET /agents/` is answered by `redirect_slashes`. + @app.get("/agents") + def list_agents() -> dict: + return {"agents": []} + + @app.get("/agents/pins") + def list_pins() -> dict: + return {"pins": []} + + @app.get("/auth/login") + def login() -> RedirectResponse: + return RedirectResponse( + url="https://login.microsoftonline.com/oauth2/authorize?client_id=x", + status_code=302, + ) + + return app + + +@pytest.fixture +def client(app: FastAPI) -> TestClient: + return TestClient(app, follow_redirects=False) + + +def _proxied(path: str) -> dict: + """Headers as CloudFront's `/api/*` behaviour delivers them to the ALB.""" + return {"host": PROXY_HOST, FORWARDED_PREFIX_HEADER: "/api"} + + +class TestTrailingSlashRedirects: + def test_agents_redirect_is_relative_and_keeps_the_api_prefix( + self, client: TestClient + ) -> None: + response = client.get("/agents/", headers=_proxied("/agents/")) + + assert response.status_code == 307 + assert response.headers["location"] == "/api/agents" + + def test_location_never_names_the_origin_host_or_plain_http( + self, client: TestClient + ) -> None: + response = client.get("/agents/", headers=_proxied("/agents/")) + + location = response.headers["location"] + assert "http://" not in location + assert PROXY_HOST not in location + + def test_nested_path_keeps_its_full_path(self, client: TestClient) -> None: + response = client.get("/agents/pins/", headers=_proxied("/agents/pins/")) + + assert response.headers["location"] == "/api/agents/pins" + + def test_query_string_survives_the_rewrite(self, client: TestClient) -> None: + response = client.get( + "/agents/?include_drafts=false", headers=_proxied("/agents/") + ) + + assert response.headers["location"] == "/api/agents?include_drafts=false" + + +class TestWithoutTheProxy: + """Local dev and direct-to-ALB calls: no prefix was stripped, so none is added.""" + + def test_redirect_is_root_relative_when_no_prefix_header_is_present( + self, client: TestClient + ) -> None: + response = client.get("/agents/", headers={"host": "localhost:8000"}) + + assert response.headers["location"] == "/agents" + + +class TestRedirectsWeMustNotTouch: + def test_cross_host_redirect_is_left_alone(self, client: TestClient) -> None: + """The BFF's OAuth bounce to Entra/Cognito must stay absolute.""" + response = client.get("/auth/login", headers=_proxied("/auth/login")) + + assert response.status_code == 302 + assert response.headers["location"] == ( + "https://login.microsoftonline.com/oauth2/authorize?client_id=x" + ) + + def test_non_redirect_responses_are_untouched(self, client: TestClient) -> None: + response = client.get("/agents", headers=_proxied("/agents")) + + assert response.status_code == 200 + assert "location" not in response.headers + + +class TestPrefixSanitization: + """The header is overwritten by CloudFront, but a `Location` is worth guarding.""" + + @pytest.mark.parametrize( + "spoofed", + ["//evil.example", "evil.example", "https://evil.example", ""], + ) + def test_a_prefix_that_could_escape_the_origin_is_dropped( + self, client: TestClient, spoofed: str + ) -> None: + response = client.get( + "/agents/", + headers={"host": PROXY_HOST, FORWARDED_PREFIX_HEADER: spoofed}, + ) + + location = response.headers["location"] + assert location == "/agents" + assert not location.startswith("//") + + def test_trailing_slash_on_the_prefix_does_not_double_up( + self, client: TestClient + ) -> None: + response = client.get( + "/agents/", headers={"host": PROXY_HOST, FORWARDED_PREFIX_HEADER: "/api/"} + ) + + assert response.headers["location"] == "/api/agents" diff --git a/backend/tests/apis/shared/tool_summaries/test_summarizer.py b/backend/tests/apis/shared/tool_summaries/test_summarizer.py new file mode 100644 index 000000000..4d116975b --- /dev/null +++ b/backend/tests/apis/shared/tool_summaries/test_summarizer.py @@ -0,0 +1,173 @@ +"""Tests for the tool-batch summarizer side-channel. + +The property that matters most here is that a BAD summary is worse than no +summary. The SPA always has a deterministic line to fall back on, so every +failure mode must return ``None`` rather than surface a fragment — a half +sentence in the rail reads as a bug in the product, where the fallback reads +as normal. + +The truncation case is not hypothetical: observed live on dev 2026-09-11, +Nova began ``Found the course "Faculty Demo: Intro to MCP"`` and was hard-cut +at the token ceiling, persisting ``Found the course "Faculty Demo: Intro to +MCP`` with a dangling quote. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from apis.shared.tool_summaries.summarizer import ( + _build_prompt, + _clean, + _MAX_CALLS, + summarize_tool_batch, +) + + +def _response(text: str, stop_reason: str = "end_turn") -> dict: + return { + "output": {"message": {"content": [{"text": text}]}}, + "stopReason": stop_reason, + } + + +def _calls(n: int = 1): + return [ + { + "toolUseId": f"t{i}", + "toolName": "list_courses", + "input": '{"term": "fall"}', + "result": '{"courses": ["BIO 101"]}', + "ok": True, + "durationMs": 120, + } + for i in range(n) + ] + + +@pytest.fixture(autouse=True) +def summaries_enabled(monkeypatch): + monkeypatch.setenv("TOOL_SUMMARIES_ENABLED", "true") + + +@pytest.fixture +def bedrock(monkeypatch): + """Patch boto3.client so no test ever reaches Bedrock.""" + client = MagicMock() + module = MagicMock() + module.client.return_value = client + monkeypatch.setitem(__import__("sys").modules, "boto3", module) + return client + + +# -- the truncation regression ------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_generation_cut_at_the_token_ceiling_is_discarded(bedrock): + bedrock.converse.return_value = _response( + 'Found the course "Faculty Demo: Intro to MCP', stop_reason="max_tokens" + ) + + assert await summarize_tool_batch(_calls()) is None + + +@pytest.mark.asyncio +async def test_a_complete_generation_is_kept(bedrock): + bedrock.converse.return_value = _response("Found 3 active courses") + + assert await summarize_tool_batch(_calls()) == "Found 3 active courses" + + +@pytest.mark.asyncio +async def test_truncation_is_judged_by_stop_reason_not_by_length(bedrock): + """A short line is not evidence of completeness, and vice versa. + + `stopReason` is the only signal that distinguishes "the model finished" + from "we cut it off", so a long-but-finished summary must survive. + """ + long_but_finished = "Found the Syllabus Acknowledgment and Homework 1 assignments" + bedrock.converse.return_value = _response(long_but_finished) + + assert await summarize_tool_batch(_calls()) == long_but_finished + + +# -- every other failure is also a None ----------------------------------- + + +@pytest.mark.asyncio +async def test_empty_batch_returns_none(bedrock): + assert await summarize_tool_batch([]) is None + bedrock.converse.assert_not_called() + + +@pytest.mark.asyncio +async def test_flag_off_returns_none_without_calling_bedrock(bedrock, monkeypatch): + monkeypatch.setenv("TOOL_SUMMARIES_ENABLED", "false") + + assert await summarize_tool_batch(_calls()) is None + bedrock.converse.assert_not_called() + + +@pytest.mark.asyncio +async def test_a_model_error_returns_none(bedrock): + bedrock.converse.side_effect = RuntimeError("throttled") + + assert await summarize_tool_batch(_calls()) is None + + +@pytest.mark.asyncio +async def test_an_empty_generation_returns_none(bedrock): + bedrock.converse.return_value = _response(" ") + + assert await summarize_tool_batch(_calls()) is None + + +@pytest.mark.asyncio +async def test_a_malformed_response_returns_none(bedrock): + bedrock.converse.return_value = {"output": {}} + + assert await summarize_tool_batch(_calls()) is None + + +# -- cleaning -------------------------------------------------------------- + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("Output: Found 3 courses", "Found 3 courses"), + ("Summary: Found 3 courses", "Found 3 courses"), + ('"Found 3 courses"', "Found 3 courses"), + ("Found 3 courses.", "Found 3 courses"), + ("Found 3 courses\nAnd some commentary", "Found 3 courses"), + (" Found 3 courses ", "Found 3 courses"), + ], +) +def test_clean_strips_the_wrappers_small_models_add(raw, expected): + assert _clean(raw) == expected + + +def test_clean_keeps_an_interior_quote(): + # Stripping only the OUTER quotes matters: the specific thing being named + # is often quoted, and eating those quotes loses the specificity that is + # the whole point of the summary. + assert _clean('Found the course "Intro to MCP"') == 'Found the course "Intro to MCP"' + + +# -- prompt bounds --------------------------------------------------------- + + +def test_prompt_is_bounded_for_a_wide_batch(): + prompt = _build_prompt(_calls(_MAX_CALLS + 5)) + + assert prompt.count("list_courses(") == _MAX_CALLS + assert "and 5 more call(s)" in prompt + + +def test_prompt_marks_a_failed_call_as_failed(): + calls = _calls() + calls[0]["ok"] = False + calls[0]["result"] = "422 rubric association required" + + assert "FAILED" in _build_prompt(calls) diff --git a/backend/tests/auth/test_dependencies.py b/backend/tests/auth/test_dependencies.py index 5e4fcf890..4e2a62a95 100644 --- a/backend/tests/auth/test_dependencies.py +++ b/backend/tests/auth/test_dependencies.py @@ -7,11 +7,13 @@ Requirements: 10.5, 10.6 """ +import asyncio from unittest.mock import MagicMock, patch import pytest from fastapi import HTTPException +from apis.shared.auth import dependencies as deps from apis.shared.auth.dependencies import ( get_current_user_id, get_current_user_trusted, @@ -163,3 +165,131 @@ async def test_returns_string(self, make_user): assert user_id == "uid-42" assert isinstance(user_id, str) + + +# --------------------------------------------------------------------------- +# Background user-sync throttle tests +# --------------------------------------------------------------------------- + + +class TestUserSyncThrottle: + """The per-request profile upsert must collapse to one write per window. + + Regression cover for the first-load fan-out: 12 concurrent SPA calls each + fired `sync_user_from_jwt`, and that upsert is a GetItem + PutItem, so a + single page load issued 24 DynamoDB operations against one item. + """ + + def setup_method(self): + deps.reset_user_sync_throttle() + + def teardown_method(self): + deps.reset_user_sync_throttle() + + def test_first_claim_wins_rest_are_throttled(self): + """Only the first of a burst claims the sync.""" + claims = [deps._claim_user_sync("uid-1") for _ in range(12)] + + assert claims[0] is True + assert not any(claims[1:]) + + def test_throttle_is_per_user(self): + """One user's claim must not suppress another's — a classroom of + students signing in together each need their own row created.""" + assert deps._claim_user_sync("uid-a") is True + assert deps._claim_user_sync("uid-b") is True + + def test_claim_allowed_again_after_window(self, monkeypatch): + """Past the window the sync resumes, so the refresh still happens.""" + monkeypatch.setattr(deps, "_USER_SYNC_THROTTLE_SECONDS", 300) + + clock = {"now": 1_000.0} + monkeypatch.setattr(deps.time, "monotonic", lambda: clock["now"]) + + assert deps._claim_user_sync("uid-1") is True + clock["now"] += 299 + assert deps._claim_user_sync("uid-1") is False + clock["now"] += 2 # now 301s past the first claim + assert deps._claim_user_sync("uid-1") is True + + def test_reset_forces_immediate_resync(self): + """An explicit reset lets the next request through without waiting.""" + assert deps._claim_user_sync("uid-1") is True + assert deps._claim_user_sync("uid-1") is False + + deps.reset_user_sync_throttle("uid-1") + + assert deps._claim_user_sync("uid-1") is True + + def test_tracker_is_pruned_when_it_grows(self, monkeypatch): + """A long-lived container that has served many users stays bounded.""" + monkeypatch.setattr(deps, "_USER_SYNC_THROTTLE_SECONDS", 300) + monkeypatch.setattr(deps, "_USER_SYNC_TRACKER_MAX", 10) + + clock = {"now": 1_000.0} + monkeypatch.setattr(deps.time, "monotonic", lambda: clock["now"]) + + for i in range(10): + deps._claim_user_sync(f"old-{i}") + + # Move past the window so every recorded entry is now stale, then add + # one more to trip the prune. + clock["now"] += 301 + deps._claim_user_sync("fresh") + + assert list(deps._user_sync_last_run) == ["fresh"] + + def test_schedule_is_noop_when_sync_disabled(self, make_user, monkeypatch): + """No sync service configured means no task and no claim consumed.""" + monkeypatch.setattr(deps, "_get_user_sync_service", lambda: None) + + deps._schedule_user_sync(make_user(user_id="uid-1")) + + assert deps._user_sync_last_run == {} + assert deps._user_sync_tasks == set() + + @pytest.mark.asyncio + async def test_schedule_dispatches_once_per_window(self, make_user): + """End to end: a 12-call page load produces exactly one sync.""" + service = MagicMock() + service.enabled = True + calls = [] + + async def _record(user): + calls.append(user.user_id) + + service.sync_user_from_jwt = _record + user = make_user(user_id="uid-1") + + with patch.object(deps, "_get_user_sync_service", return_value=service): + for _ in range(12): + deps._schedule_user_sync(user) + # Let the dispatched task run to completion. + await asyncio.gather(*list(deps._user_sync_tasks)) + + assert calls == ["uid-1"] + + @pytest.mark.asyncio + async def test_task_reference_is_held_then_released(self, make_user): + """The task is strongly referenced while in flight (so the GC cannot + collect it mid-await) and discarded once it finishes.""" + service = MagicMock() + service.enabled = True + started = asyncio.Event() + release = asyncio.Event() + + async def _block(user): + started.set() + await release.wait() + + service.sync_user_from_jwt = _block + + with patch.object(deps, "_get_user_sync_service", return_value=service): + deps._schedule_user_sync(make_user(user_id="uid-1")) + await started.wait() + assert len(deps._user_sync_tasks) == 1 + + release.set() + await asyncio.gather(*list(deps._user_sync_tasks)) + + assert deps._user_sync_tasks == set() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index f84420de6..8d9038c95 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -105,6 +105,30 @@ def _clear_aws_profile_env(): ) +@pytest.fixture(autouse=True) +def _clear_config_cache(): + """Drop the process-wide config-catalog cache between tests. + + ``apis.shared.caching.config_cache`` memoizes the model / tool / + system-prompt / provider catalogs for the life of the process. In + production that is invalidated by the write paths themselves, but tests + swap the whole table out underneath it — fixtures already reset the + module-level repo and service singletons for the same reason, and this is + the same class of state. Without it, a test that seeds a catalog leaves the + next test reading the previous one's rows. + + A backstop, not the primary contract: production correctness comes from the + invalidation in the repositories, not from here. + """ + from apis.shared.caching import config_cache + + config_cache.get_config_cache().clear() + try: + yield + finally: + config_cache.get_config_cache().clear() + + @pytest.fixture(autouse=True) def _clear_env_config_bleed(): saved = { diff --git a/backend/tests/fine_tuning/test_checkpointing.py b/backend/tests/fine_tuning/test_checkpointing.py new file mode 100644 index 000000000..4535289ae --- /dev/null +++ b/backend/tests/fine_tuning/test_checkpointing.py @@ -0,0 +1,171 @@ +"""Checkpointing: what makes a restarted training job resume rather than restart. + +Two things restart a job: a spot interruption, and SageMaker killing it at +MaxRuntimeInSeconds — which the dollar-quota clamp makes routine for long runs. +Before this existed, `save_strategy="no"` meant the adapter was only written +after `trainer.train()` returned, so either restart produced *nothing* for the +money already spent. +""" + +import os + +import pytest +from unittest.mock import MagicMock + +from apis.app_api.fine_tuning import task_types +from apis.app_api.fine_tuning.sagemaker_scripts import task_common +from apis.app_api.fine_tuning.sagemaker_scripts import train as train_script +from apis.app_api.fine_tuning import sagemaker_service as sm_service + + +class TestResolveSaveSteps: + """A fixed save_steps cannot serve both ends of this catalog.""" + + def test_vlm_shaped_run_still_checkpoints(self): + """The trap this function exists for. + + A 34B VLM trains at batch 1 with 16-step accumulation, so a + 90-sample epoch is ~6 optimizer steps. Against a hardcoded + save_steps=50 the longest, most interruption-exposed job in the + catalog would never checkpoint at all. + """ + steps = task_common.resolve_save_steps( + num_examples=90, batch_size=1, gradient_accumulation_steps=16, epochs=1 + ) + assert steps >= 1 + assert steps < 50 + + def test_scales_with_a_long_run(self): + """A text classifier with thousands of steps must not checkpoint constantly.""" + steps = task_common.resolve_save_steps( + num_examples=50_000, batch_size=16, gradient_accumulation_steps=1, epochs=3 + ) + # ~9375 optimizer steps / 10 targets + assert steps > 100 + + def test_targets_roughly_ten_checkpoints(self): + total_steps = 1000 + steps = task_common.resolve_save_steps( + num_examples=total_steps, batch_size=1, gradient_accumulation_steps=1, epochs=1 + ) + assert 1 <= total_steps // steps <= 20 + + def test_never_returns_zero(self): + """save_steps=0 is rejected by TrainingArguments.""" + assert task_common.resolve_save_steps(1, 64, 64, 1) >= 1 + + def test_tolerates_degenerate_inputs(self): + """Guards against a division-by-zero on an unset hyperparameter.""" + assert task_common.resolve_save_steps(10, 0, 0, 0) >= 1 + + def test_accumulation_reduces_the_step_count(self): + """Accumulation means fewer optimizer steps for the same data.""" + without = task_common.resolve_save_steps(1000, 1, 1, 1) + with_accum = task_common.resolve_save_steps(1000, 1, 16, 1) + assert with_accum < without + + +class TestCheckpointArguments: + + def test_enabled_produces_step_strategy(self): + args = task_common.checkpoint_arguments(25) + assert args["save_strategy"] == "steps" + assert args["save_steps"] == 25 + + def test_keeps_only_the_newest(self): + """Resume needs the latest checkpoint; older ones only grow the mirror.""" + assert task_common.checkpoint_arguments(25)["save_total_limit"] == 1 + + def test_kill_switch_restores_the_old_behaviour(self): + args = task_common.checkpoint_arguments(25, enabled=False) + assert args == {"save_strategy": "no"} + + def test_zero_steps_disables(self): + assert task_common.checkpoint_arguments(0)["save_strategy"] == "no" + + +class TestLatestCheckpoint: + + def test_missing_directory_is_a_fresh_start(self, tmp_path): + assert task_common.latest_checkpoint(str(tmp_path / "nope")) is None + + def test_empty_directory_is_a_fresh_start(self, tmp_path): + assert task_common.latest_checkpoint(str(tmp_path)) is None + + def test_an_unreadable_checkpoint_does_not_fail_the_job(self, tmp_path): + """Losing progress is survivable; failing the whole run is not. + + In the backend venv transformers is absent by design, so this + exercises the real import-failure path: a directory that *looks* like + it holds a checkpoint must still yield a quiet fresh start rather than + an ImportError escaping into the trainer. + """ + (tmp_path / "checkpoint-10").mkdir() + assert task_common.latest_checkpoint(str(tmp_path)) is None + + +class TestCheckpointConfigWiring: + """The container writing checkpoints is useless if S3 never mirrors them.""" + + def _service(self): + return sm_service.SageMakerService( + sagemaker_client=MagicMock(), logs_client=MagicMock() + ) + + def _create(self, **overrides): + service = self._service() + kwargs = dict( + job_name="j", hyperparameters={"a": "b"}, + input_s3_uri="s3://b/in", output_s3_uri="s3://b/out", + instance_type="ml.g6e.xlarge", max_runtime=3600, + source_dir_s3_uri="s3://b/src.tar.gz", + task_type=task_types.IMAGE_TEXT_TO_TEXT, + ) + kwargs.update(overrides) + service.create_training_job(**kwargs) + return service._sagemaker.create_training_job.call_args[1] + + def test_checkpoint_config_is_sent(self): + params = self._create(checkpoint_s3_uri="s3://bucket/checkpoints/u/j") + assert params["CheckpointConfig"]["S3Uri"] == "s3://bucket/checkpoints/u/j" + + def test_local_path_matches_what_the_trainer_writes(self): + """SageMaker only mirrors the directory it is told about.""" + params = self._create(checkpoint_s3_uri="s3://bucket/checkpoints/u/j") + assert params["CheckpointConfig"]["LocalPath"] == task_common.CHECKPOINT_DIR + + def test_omitted_when_no_uri_is_given(self): + """Absent config must not become an empty one — SageMaker rejects that.""" + assert "CheckpointConfig" not in self._create() + + +class TestCheckpointS3Uri: + + def _s3(self): + from apis.app_api.fine_tuning.s3_service import FineTuningS3Service + + return FineTuningS3Service(s3_client=MagicMock(), bucket_name="ft-bucket") + + def test_is_scoped_per_user_and_job(self): + uri = self._s3().get_checkpoint_s3_uri("user-1", "job-9") + assert uri == "s3://ft-bucket/checkpoints/user-1/job-9" + + def test_does_not_collide_with_the_output_prefix(self): + """model.tar.gz lands under output/; a live mirror there is ambiguous.""" + s3 = self._s3() + assert not s3.get_checkpoint_s3_uri("u", "j").startswith( + s3.get_output_s3_uri("u", "j") + ) + + +class TestCheckpointingHyperparameter: + + def test_defaults_on(self): + args = train_script.parse_args(["--model_name_or_path", "x"]) + assert args.checkpointing is True + + def test_can_be_disabled(self): + args = train_script.parse_args( + ["--model_name_or_path", "x", "--checkpointing", "false"] + ) + assert args.checkpointing is False diff --git a/backend/tests/fine_tuning/test_job_routes.py b/backend/tests/fine_tuning/test_job_routes.py index 2672a29ca..dfe25ddac 100644 --- a/backend/tests/fine_tuning/test_job_routes.py +++ b/backend/tests/fine_tuning/test_job_routes.py @@ -178,6 +178,84 @@ def test_returns_201_on_success(self, make_user): body = resp.json() assert body["model_id"] == "distilgpt2" + def _spot_app(self, make_user): + app = _create_app() + user = make_user(email="user@example.com") + + mock_s3 = MagicMock() + mock_s3.check_object_exists.return_value = True + mock_s3.get_output_s3_prefix.return_value = "output/user-001/job-abc" + mock_s3.get_output_s3_uri.return_value = "s3://bucket/output/user-001/job-abc" + mock_s3.get_checkpoint_s3_uri.return_value = "s3://bucket/checkpoints/user-001/job-abc" + mock_s3.bucket_name = "test-bucket" + + mock_jobs = MagicMock() + mock_jobs.create_job.return_value = SAMPLE_JOB + mock_jobs.update_job_status.return_value = {**SAMPLE_JOB, "status": "TRAINING"} + + mock_sm = MagicMock() + mock_sm.create_training_job.return_value = {} + + mock_script = MagicMock() + mock_script.ensure_scripts_uploaded.return_value = "s3://test-bucket/scripts/sourcedir-text.tar.gz" + + _setup_deps(app, user, SAMPLE_GRANT, mock_jobs, mock_s3, mock_sm, + MagicMock(), mock_script) + return TestClient(app), mock_sm + + def test_spot_is_off_unless_asked_for(self, make_user): + client, mock_sm = self._spot_app(make_user) + resp = client.post("/fine-tuning/jobs", json={ + "model_id": "distilgpt2", + "dataset_s3_key": "datasets/user-001/abc/train.jsonl", + }) + assert resp.status_code == 201 + assert mock_sm.create_training_job.call_args[1]["use_spot"] is False + + def test_spot_is_forwarded_to_sagemaker(self, make_user): + client, mock_sm = self._spot_app(make_user) + resp = client.post("/fine-tuning/jobs", json={ + "model_id": "distilgpt2", + "dataset_s3_key": "datasets/user-001/abc/train.jsonl", + "use_spot": True, + }) + assert resp.status_code == 201 + assert mock_sm.create_training_job.call_args[1]["use_spot"] is True + + def test_spot_with_checkpointing_disabled_is_refused(self, make_user): + """Spot restarts from the last checkpoint. With none, a long run + restarts from zero and may never finish while billing every attempt — + refuse the combination rather than sell it.""" + client, mock_sm = self._spot_app(make_user) + resp = client.post("/fine-tuning/jobs", json={ + "model_id": "distilgpt2", + "dataset_s3_key": "datasets/user-001/abc/train.jsonl", + "use_spot": True, + "hyperparameters": {"checkpointing": "false"}, + }) + assert resp.status_code == 400 + assert "checkpointing" in resp.json()["detail"].lower() + mock_sm.create_training_job.assert_not_called() + + def test_checkpointing_off_without_spot_is_allowed(self, make_user): + """The kill switch stays usable on on-demand.""" + client, mock_sm = self._spot_app(make_user) + resp = client.post("/fine-tuning/jobs", json={ + "model_id": "distilgpt2", + "dataset_s3_key": "datasets/user-001/abc/train.jsonl", + "hyperparameters": {"checkpointing": "false"}, + }) + assert resp.status_code == 201 + + def test_checkpoint_uri_reaches_sagemaker(self, make_user): + client, mock_sm = self._spot_app(make_user) + client.post("/fine-tuning/jobs", json={ + "model_id": "distilgpt2", + "dataset_s3_key": "datasets/user-001/abc/train.jsonl", + }) + uri = mock_sm.create_training_job.call_args[1]["checkpoint_s3_uri"] + assert uri == "s3://bucket/checkpoints/user-001/job-abc" + def test_rejects_dataset_the_trainer_cannot_read(self, make_user): """Last gate before SageMaker: no GPU is provisioned for a doomed job.""" app = _create_app() diff --git a/backend/tests/fine_tuning/test_managed_spot.py b/backend/tests/fine_tuning/test_managed_spot.py new file mode 100644 index 000000000..91dc26bc4 --- /dev/null +++ b/backend/tests/fine_tuning/test_managed_spot.py @@ -0,0 +1,122 @@ +"""Managed spot training. + +Spot trades a large discount for a longer queue and the risk of being +interrupted. It is only safe here because checkpointing landed first: without +it an interrupted job restarts from zero, and a run longer than the mean time +between interruptions almost never completes while being billed for every +attempt. +""" + +import pytest +from unittest.mock import MagicMock + +from apis.app_api.fine_tuning import pricing, task_types +from apis.app_api.fine_tuning import sagemaker_service as sm_service +from apis.app_api.fine_tuning.sagemaker_scripts import task_common + + +def _params(**overrides): + service = sm_service.SageMakerService( + sagemaker_client=MagicMock(), logs_client=MagicMock() + ) + kwargs = dict( + job_name="j", hyperparameters={"a": "b"}, + input_s3_uri="s3://b/in", output_s3_uri="s3://b/out", + instance_type="ml.g6e.xlarge", max_runtime=3600, + source_dir_s3_uri="s3://b/src.tar.gz", + task_type=task_types.IMAGE_TEXT_TO_TEXT, + checkpoint_s3_uri="s3://b/checkpoints/u/j", + ) + kwargs.update(overrides) + service.create_training_job(**kwargs) + return service._sagemaker.create_training_job.call_args[1] + + +class TestSpotStoppingCondition: + + def test_off_by_default(self): + """Opt-in: a researcher who needs a result today pays for certainty.""" + params = _params() + assert "EnableManagedSpotTraining" not in params + assert "MaxWaitTimeInSeconds" not in params["StoppingCondition"] + + def test_enables_the_flag(self): + assert _params(use_spot=True)["EnableManagedSpotTraining"] is True + + def test_max_wait_strictly_exceeds_max_runtime(self): + """The API rejects MaxWaitTime <= MaxRuntime.""" + params = _params(use_spot=True, max_runtime=3600) + assert params["StoppingCondition"]["MaxWaitTimeInSeconds"] > 3600 + + def test_max_wait_leaves_room_for_the_capacity_queue(self): + """MaxWaitTime covers waiting AND training. + + Measured on-demand waits for these GPU families ran 28-58 minutes; + spot draws from the surplus of the same constrained pools, so a wait + allowance shorter than that fails the job with MaxWaitTimeExceeded + after it has already queued. + """ + params = _params(use_spot=True, max_runtime=3600) + slack = params["StoppingCondition"]["MaxWaitTimeInSeconds"] - 3600 + assert slack >= 3600 + + def test_max_runtime_is_unchanged_by_spot(self): + """Spot must not quietly extend the budget-clamped stopping condition.""" + params = _params(use_spot=True, max_runtime=1234) + assert params["StoppingCondition"]["MaxRuntimeInSeconds"] == 1234 + + def test_spot_still_carries_checkpoint_config(self): + """The pairing that makes spot survivable.""" + params = _params(use_spot=True) + assert params["CheckpointConfig"]["LocalPath"] == task_common.CHECKPOINT_DIR + + +class TestSpotCostAccounting: + """AWS shrinks BillableTimeInSeconds rather than discounting the rate.""" + + def test_on_demand_rate_is_correct_for_spot(self): + """The documented savings formula is + (1 - BillableTimeInSeconds / TrainingTimeInSeconds) * 100 — i.e. the + discount is already inside billable_seconds, so multiplying it by the + on-demand rate is right and needs no spot branch.""" + full = pricing.calculate_cost("ml.g6e.xlarge", 3600) + discounted = pricing.calculate_cost("ml.g6e.xlarge", 1200) + assert discounted == pytest.approx(full / 3) + + def test_multiplies_by_instance_count(self): + """AWS documents BillableTimeInSeconds as per-instance.""" + one = pricing.calculate_cost("ml.g6e.xlarge", 3600, instance_count=1) + four = pricing.calculate_cost("ml.g6e.xlarge", 3600, instance_count=4) + assert four == pytest.approx(one * 4) + + def test_default_instance_count_is_one(self): + assert pricing.calculate_cost("ml.g6e.xlarge", 3600) == pytest.approx( + pricing.calculate_cost("ml.g6e.xlarge", 3600, instance_count=1) + ) + + def test_a_zero_count_does_not_zero_the_bill(self): + """Defensive: a bad count must not silently make GPU time free.""" + assert pricing.calculate_cost("ml.g6e.xlarge", 3600, instance_count=0) > 0 + + +class TestSharedBooleanParsing: + """app-api and the container must agree, or app-api admits a job the + trainer then runs with different settings.""" + + @pytest.mark.parametrize("value", ["true", "True", "1", "yes", "on", True]) + def test_truthy(self, value): + assert task_types.str2bool(value) is True + + @pytest.mark.parametrize("value", ["false", "False", "0", "no", "", False]) + def test_falsy(self, value): + assert task_types.str2bool(value) is False + + def test_the_string_False_is_not_truthy(self): + """SageMaker JSON-parses "false" and passes back Python-style "False".""" + assert bool("False") is True + assert task_types.str2bool("False") is False + + def test_train_script_reexports_the_same_function(self): + from apis.app_api.fine_tuning.sagemaker_scripts import train + + assert train.str2bool is task_types.str2bool diff --git a/backend/tests/fine_tuning/test_vlm_task.py b/backend/tests/fine_tuning/test_vlm_task.py index a386f94c5..185190731 100644 --- a/backend/tests/fine_tuning/test_vlm_task.py +++ b/backend/tests/fine_tuning/test_vlm_task.py @@ -283,3 +283,92 @@ def test_shorter_dataset_than_the_sample_size(self): def test_empty_dataset_is_a_no_op(self): vlm.check_collation(lambda batch: 1 / 0, []) + + +class TestResolveEffectiveContext: + """A fixed context length cannot be right for every model. + + How much of the sequence an image consumes depends on the checkpoint's + tiling AND on the resolution of the images the user uploaded, so the + trainer measures a sample and raises the budget to fit. + """ + + def test_keeps_the_request_when_it_already_fits(self): + effective, raised = vlm.resolve_effective_context(2048, 900, 8192) + assert (effective, raised) == (2048, False) + + def test_raises_to_fit_a_longer_record(self): + effective, raised = vlm.resolve_effective_context(1024, 1500, 8192) + assert raised + assert effective == 1500 + vlm.TEXT_TOKEN_HEADROOM + + def test_smolvlm_1377_token_image_regression(self): + """The real failure this exists to prevent. + + SmolVLM-Instruct spends 1377 tokens on one image. Against the old + 1024 default, truncation cut the image placeholder run to 891 and the + job died on a billed GPU with a processor-level token-count mismatch. + """ + effective, raised = vlm.resolve_effective_context(1024, 1377, 8192) + assert raised + assert effective > 1377 + + def test_clamps_to_the_model_maximum(self): + """Never ask for more context than the checkpoint supports.""" + effective, _ = vlm.resolve_effective_context(4096, 4000, 2048) + assert effective == 2048 + + def test_clamp_wins_over_the_raise(self): + effective, raised = vlm.resolve_effective_context(512, 9000, 4096) + assert raised + assert effective == 4096 + + def test_unknown_model_max_is_not_a_clamp(self): + """resolve_max_context_length returns None when nothing is readable.""" + effective, _ = vlm.resolve_effective_context(2048, 3000, None) + assert effective == 3000 + vlm.TEXT_TOKEN_HEADROOM + + def test_unmeasurable_sample_keeps_the_request(self): + """Measurement is best-effort; check_collation is the backstop.""" + effective, raised = vlm.resolve_effective_context(2048, None, 8192) + assert (effective, raised) == (2048, False) + + +class TestMeasureRequiredContext: + + def test_returns_none_when_a_record_cannot_be_processed(self, monkeypatch): + """A measurement failure must not abort the job on its own.""" + processor = MagicMock() + processor.chat_template = "t" + processor.side_effect = RuntimeError("boom") + dataset = [{"image": "/nope.png", "prompt": "p", "response": "r"}] + assert vlm.measure_required_context(processor, SPEC, dataset) is None + + def test_samples_no_more_than_the_dataset_holds(self, monkeypatch): + """A 1-record dataset must not index past the end.""" + processor = MagicMock() + processor.chat_template = None + assert vlm.measure_required_context(processor, SPEC, []) is None + + +class TestCatalogContextDefaults: + """Every catalog default must clear its model's own image-token cost.""" + + def test_smolvlm_default_fits_its_own_image(self): + from apis.app_api.fine_tuning.job_models import MODEL_CATALOG + + default = int( + MODEL_CATALOG["smolvlm-instruct"].default_hyperparameters["context_length"] + ) + # Measured on dev: 1377 tokens for one image, plus prompt and response. + assert default > 1377 + + def test_anyres_models_get_a_larger_budget(self): + """LLaVA-NeXT tiling and Qwen dynamic resolution both exceed 2048.""" + from apis.app_api.fine_tuning.job_models import MODEL_CATALOG + + for model_id in ("llava-1.6-mistral-7b", "qwen25-vl-7b-instruct", "llava-1.6-34b"): + default = int( + MODEL_CATALOG[model_id].default_hyperparameters["context_length"] + ) + assert default >= 4096, model_id diff --git a/backend/tests/lambdas/test_kb_born_managed.py b/backend/tests/lambdas/test_kb_born_managed.py new file mode 100644 index 000000000..cd696892b --- /dev/null +++ b/backend/tests/lambdas/test_kb_born_managed.py @@ -0,0 +1,615 @@ +"""Born-managed: provisioning stacked onto the first document upload. + +Feature: managed-kb-migration, ``MANAGED_KB_NEW_DEFAULT`` (rollout ladder step 2). +Spec: ``.kiro/specs/managed-kb-migration/born-managed-provision-then-ingest.md``. + +Four failure modes are what this file is actually for. None of them raises an +error on its own, which is why each needs a test that fails when the guard is +removed: + +**1. Double indexing.** The legacy pipeline skips a document only when the record +already resolves to ``managed``. Declare the engine after the first upload instead +of before it and that document is indexed on legacy *and* on managed, with two +writers racing over one ``status`` field. + +**2. A dead-lettered first document.** The S3 event fires in seconds; +``CreateKnowledgeBase`` takes minutes. Lambda's async retry is capped at 2 +attempts, so a consumer that raises "not provisioned" burns the whole window and +dead-letters — the §5.37 shape, where the bytes are fine and the document is +invisible forever. + +**3. A stranded agent.** Managed intent with no knowledge base is a dead end in +both directions: legacy skips it because the record says managed, managed cannot +serve because there is nothing to serve from. Every provisioning failure must +REMOVE ``retrievalEngine``, not merely stop. + +**4. A rollout ladder that skipped a rung.** Born-managed is served by the +migration dispatcher, so gating that dispatcher on ``MANAGED_KB_MIGRATION_ENABLED`` +would make step 2 either useless alone or a back door that migrates the existing +fleet. Each work state is gated on its own flag, and both halves are asserted. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import boto3 +import pytest +from moto import mock_aws + +from apis.app_api.kb_migration import ingestion_consumer as ic +from apis.app_api.kb_migration import provisioner as pv +from apis.app_api.kb_upgrade import born_managed as bm +from apis.shared.kb_backend import records as r + +REGION = "us-east-1" +TABLE = "test-born-managed" +ASSISTANT_ID = "ast-born01" +OWNER = "user-born01" +DOCUMENT_ID = "DOC-born01" +BUCKET = "docs-bucket" +KEY = f"assistants/{ASSISTANT_ID}/documents/{DOCUMENT_ID}/first.pdf" +OBJECT_BYTES = 4096 + + +@pytest.fixture() +def table(monkeypatch): + monkeypatch.setenv("AWS_DEFAULT_REGION", REGION) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_SESSION_TOKEN", "testing") + monkeypatch.setenv("DYNAMODB_ASSISTANTS_TABLE_NAME", TABLE) + monkeypatch.setenv("S3_ASSISTANTS_DOCUMENTS_BUCKET_NAME", BUCKET) + + with mock_aws(): + ddb = boto3.client("dynamodb", region_name=REGION) + ddb.create_table( + TableName=TABLE, + KeySchema=[ + {"AttributeName": "PK", "KeyType": "HASH"}, + {"AttributeName": "SK", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "PK", "AttributeType": "S"}, + {"AttributeName": "SK", "AttributeType": "S"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket(Bucket=BUCKET) + s3.put_object(Bucket=BUCKET, Key=KEY, Body=b"x" * OBJECT_BYTES) + yield boto3.resource("dynamodb", region_name=REGION).Table(TABLE) + + +@pytest.fixture() +def flag_on(monkeypatch): + monkeypatch.setenv("MANAGED_KB_NEW_DEFAULT", "true") + + +def _kb(table): + got = table.get_item(Key={"PK": r.kb_pk(ASSISTANT_ID), "SK": r.kb_sk(ASSISTANT_ID)}) + return got.get("Item") + + +def _seed_doc(table, status=bm.STATUS_PROVISIONING, document_id=DOCUMENT_ID, **extra): + item = { + "PK": f"AST#{ASSISTANT_ID}", + "SK": f"DOC#{document_id}", + "documentId": document_id, + "status": status, + "s3Key": KEY, + "createdAt": "2026-09-10T00:00:00Z", + } + item.update(extra) + table.put_item(Item=item) + + +def _doc(table, document_id=DOCUMENT_ID): + return table.get_item( + Key={"PK": f"AST#{ASSISTANT_ID}", "SK": f"DOC#{document_id}"} + ).get("Item") + + +# ── 1. The first-upload trigger ────────────────────────────────────────────── +class TestFirstUploadTrigger: + @pytest.mark.asyncio + async def test_flag_off_writes_nothing_at_all(self, table, monkeypatch): + """MUTATION GUARD: drop the new_default_enabled() gate and this fails — + every deployment would start creating Bedrock knowledge bases against a + ~10,000-per-account quota without anybody turning anything on.""" + monkeypatch.delenv("MANAGED_KB_NEW_DEFAULT", raising=False) + + assert await bm.begin_born_managed(ASSISTANT_ID, owner_user_id=OWNER) is False + assert _kb(table) is None + + @pytest.mark.asyncio + async def test_first_upload_declares_managed_and_queues_the_job(self, table, flag_on): + assert await bm.begin_born_managed(ASSISTANT_ID, owner_user_id=OWNER) is True + + record = _kb(table) + # Declared managed BEFORE the object lands — this is what makes the legacy + # pipeline stand down for the very first document. + assert record["retrievalEngine"] == r.ENGINE_MANAGED + assert record["provisioningState"] == r.PROVISIONING + assert record["ownerUserId"] == OWNER + # Queued: the sparse work keys ARE the queue, so their absence would mean a + # managed-intent record nothing ever provisions. + assert record["migrationState"] == r.BORN_MANAGED + assert record["GSI7_PK"] == r.work_pk(r.BORN_MANAGED) + assert record["GSI7_SK"] + # Not yet built, so no identifiers to ingest against. + assert "awsKbId" not in record + # The parser configuration is captured at creation, from the same factory + # provision_managed_kb uses — a corpus indexed without image extraction is + # not comparable to one indexed with it. + assert record["imageExtraction"] is True + assert record["parserConfig"]["imageExtractionStatus"] + # A knowledge base that was never on legacy has nothing to be congratulated + # about, so the one-time upgrade notice is retired before it can be shown. + assert record["upgradeNoticeDismissedAt"] + + @pytest.mark.asyncio + async def test_a_second_upload_during_provisioning_also_waits(self, table, flag_on): + await bm.begin_born_managed(ASSISTANT_ID, owner_user_id=OWNER) + before = _kb(table)["GSI7_SK"] + + # Same answer, and it must not re-declare the engine or bump anything. + assert await bm.begin_born_managed(ASSISTANT_ID, owner_user_id=OWNER) is True + after = _kb(table) + assert after["GSI7_SK"] == before + assert after["migrationState"] == r.BORN_MANAGED + + @pytest.mark.asyncio + async def test_upload_to_a_built_knowledge_base_takes_the_ordinary_path( + self, table, flag_on + ): + """Once the identifiers are attached this is a plain managed upload: the S3 + event and the ingestion consumer own it, the byte cap applies, and the row + starts at `uploading` like every other.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.ACTIVE, + "awsKbId": "KB123", + "awsDataSourceId": "DS456", + } + ) + assert await bm.begin_born_managed(ASSISTANT_ID, owner_user_id=OWNER) is False + + @pytest.mark.asyncio + async def test_legacy_record_is_left_alone(self, table, flag_on): + """An existing legacy knowledge base is never converted by this path. + Upgrading an existing corpus is the Upgrade flow's job, and it verifies.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "provisioningState": r.ACTIVE, + } + ) + assert await bm.begin_born_managed(ASSISTANT_ID, owner_user_id=OWNER) is False + assert "retrievalEngine" not in _kb(table) + + @pytest.mark.asyncio + async def test_managed_intent_with_no_job_is_requeued(self, table, flag_on): + """The crash-recovery case: _start died between declaring the engine and + writing the work keys. Without the re-queue the agent could never ingest + anything again — legacy skips every upload, and no knowledge base exists.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.PROVISIONING, + "migrationGeneration": 0, + } + ) + assert await bm.begin_born_managed(ASSISTANT_ID, owner_user_id=OWNER) is True + + record = _kb(table) + assert record["migrationState"] == r.BORN_MANAGED + assert record["GSI7_PK"] == r.work_pk(r.BORN_MANAGED) + + @pytest.mark.asyncio + async def test_a_failure_never_fails_the_upload(self, table, flag_on): + """Provisioning decides WHICH engine serves a knowledge base; it is never a + precondition for storing a document. So the trigger swallows everything and + the upload proceeds on legacy.""" + with patch.object( + r, "create_provisioning", side_effect=RuntimeError("dynamo is having a day") + ): + assert await bm.begin_born_managed(ASSISTANT_ID, owner_user_id=OWNER) is False + + +# ── 2. The legacy pipeline stands down ─────────────────────────────────────── +class TestLegacyPipelineSkips: + def test_legacy_handler_skips_a_document_bound_for_a_provisioning_kb(self, table): + """The reason the engine is declared BEFORE the object lands. `_resolve_engine` + reads the same records.resolve_engine the consumer does, so a managed-intent + record makes this pipeline stand down even though the knowledge base does not + exist yet — which is exactly when the provisioner wants it to.""" + from apis.app_api.documents.ingestion import handler as legacy + + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.PROVISIONING, + } + ) + assert legacy._resolve_engine(ASSISTANT_ID) == r.ENGINE_MANAGED + + +# ── 3. The consumer defers instead of dead-lettering ───────────────────────── +class TestConsumerDefers: + def test_defers_while_the_knowledge_base_is_being_provisioned(self, table): + """MUTATION GUARD: remove the born-managed branch from handle_object and + this raises IngestionRoutingError — which, capped at 2 Lambda retries, + dead-letters the first document of every born-managed agent and leaves it + invisible forever (§5.37).""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.PROVISIONING, + "migrationState": r.BORN_MANAGED, + } + ) + _seed_doc(table) + + result = ic.handle_object(BUCKET, KEY) + + assert result == { + "routed": "managed", + "ingested": False, + "document_id": DOCUMENT_ID, + "note": "deferred-provisioning", + } + # Untouched: the provisioning job owns this row, and a status written here + # would either lie to the user or race the job. + assert _doc(table)["status"] == bm.STATUS_PROVISIONING + + def test_still_raises_for_managed_intent_with_no_provisioner_behind_it(self, table): + """The defer is scoped to born-managed on purpose. A record that is managed + with no identifiers and NO provisioning job is a genuine fault: falling + silently back to legacy there would create the dual-index the consumer + exists to prevent, so it must still fail loudly.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.PROVISIONING, + } + ) + _seed_doc(table, status="uploading") + + with pytest.raises(ic.IngestionRoutingError): + ic.handle_object(BUCKET, KEY) + + +# ── 4. The engine-declaration write ────────────────────────────────────────── +class TestAdoptManagedEngine: + def test_declares_managed_on_a_provisioning_record(self, table): + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "provisioningState": r.PROVISIONING, + } + ) + r.adopt_managed_engine(ASSISTANT_ID, ASSISTANT_ID, "2026-09-10T12:00:00Z") + + record = _kb(table) + assert record["retrievalEngine"] == r.ENGINE_MANAGED + assert record["bornManagedAt"] == "2026-09-10T12:00:00Z" + + def test_refuses_to_redeclare_an_already_managed_record(self, table): + """Guards the promotion timestamps of a knowledge base that got to managed + the other way — and makes a concurrent second first-upload lose cleanly.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "provisioningState": r.PROVISIONING, + "retrievalEngine": r.ENGINE_MANAGED, + "promotedAt": "2026-01-01T00:00:00Z", + } + ) + with pytest.raises(r.TransitionLost): + r.adopt_managed_engine(ASSISTANT_ID, ASSISTANT_ID, "2026-09-10T12:00:00Z") + assert _kb(table)["promotedAt"] == "2026-01-01T00:00:00Z" + + def test_refuses_a_record_that_is_not_being_provisioned(self, table): + """A torn-down record must not be resurrected as managed with no knowledge + base behind it.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "provisioningState": r.DELETING, + } + ) + with pytest.raises(r.TransitionLost): + r.adopt_managed_engine(ASSISTANT_ID, ASSISTANT_ID, "2026-09-10T12:00:00Z") + + def test_born_managed_is_work_eligible_and_not_terminal(self): + """The state has to be in both sets correctly or the queue misbehaves in one + of two silent ways: not work-eligible and the job is never dispatched; + terminal and its work keys are stripped the moment it is written.""" + assert r.BORN_MANAGED in r.WORK_ELIGIBLE_STATES + assert r.BORN_MANAGED in r.ALL_MIGRATION_STATES + assert r.BORN_MANAGED not in r.TERMINAL_STATES + + +# ── 5. The provisioning job ────────────────────────────────────────────────── +class TestProvisioningJob: + @pytest.mark.asyncio + async def test_provisions_then_ingests_the_waiting_document(self, table): + """The whole point of the design: the JOB triggers ingestion, so correctness + never depends on the two-try S3 redelivery window.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "ownerUserId": OWNER, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.PROVISIONING, + "migrationState": r.BORN_MANAGED, + "migrationGeneration": 0, + "GSI7_PK": r.work_pk(r.BORN_MANAGED), + "GSI7_SK": "2026-09-10T00:00:00Z", + } + ) + _seed_doc(table) + + record = _kb(table) + with patch.object(pv, "_provision", new=AsyncMock(return_value=True)), patch.object( + ic, "handle_object", return_value={"routed": "managed"} + ) as handled: + result = await pv.run_born_managed(ASSISTANT_ID, ASSISTANT_ID, record) + + # Reuses the consumer outright rather than reimplementing ingest → + # wait-indexed → wait-retrievable → terminal, which is also what carries the + # authoritative S3-HEAD byte-cap reconcile (Requirement 12.3). + handled.assert_called_once_with(BUCKET, KEY) + assert result.documents_migrated == 1 + assert result.to_state == r.RETAIN + + # Terminal, so the work keys are gone: the record has left the queue by + # physics rather than by filter. + done = _kb(table) + assert done["migrationState"] == r.RETAIN + assert "GSI7_PK" not in done + # The row left "Provisioning knowledge base…" before the indexing wait, so + # the author sees an ordinary upload rather than a stuck one. + assert _doc(table)["status"] == "uploading" + + @pytest.mark.asyncio + async def test_more_documents_than_one_invocation_can_finish_are_requeued(self, table): + """One document per invocation: the worker's timeout is 15 minutes and one + document's indexing budget is already 10.5, so a second could not finish and + being killed mid-wait costs a whole dispatcher interval.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.PROVISIONING, + "migrationState": r.BORN_MANAGED, + "migrationGeneration": 0, + } + ) + _seed_doc(table, document_id="DOC-a", createdAt="2026-09-10T00:00:00Z") + _seed_doc(table, document_id="DOC-b", createdAt="2026-09-10T00:00:01Z") + + record = _kb(table) + with patch.object(pv, "_provision", new=AsyncMock(return_value=True)), patch.object( + ic, "handle_object", return_value={} + ) as handled: + result = await pv.run_born_managed(ASSISTANT_ID, ASSISTANT_ID, record) + + assert handled.call_count == 1 + assert result.to_state == r.BORN_MANAGED + # Still queued, so the dispatcher brings it back for the second document. + assert _kb(table)["GSI7_PK"] == r.work_pk(r.BORN_MANAGED) + + @pytest.mark.asyncio + async def test_provisioning_failure_falls_back_to_legacy(self, table): + """MUTATION GUARD: drop the rollback and this fails with retrievalEngine + still set — managed intent, no knowledge base, legacy skipping every upload + and nothing left in the queue. The agent could never ingest again.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.PROVISIONING, + "migrationState": r.BORN_MANAGED, + "migrationGeneration": 0, + "GSI7_PK": r.work_pk(r.BORN_MANAGED), + "GSI7_SK": "2026-09-10T00:00:00Z", + } + ) + _seed_doc(table, sizeBytes=OBJECT_BYTES) + + record = _kb(table) + with patch( + "apis.shared.kb_backend.provisioning.provision_managed_kb", + new=AsyncMock(side_effect=RuntimeError("CreateKnowledgeBase refused")), + ): + result = await pv.run_born_managed(ASSISTANT_ID, ASSISTANT_ID, record) + + assert result.to_state == r.MIGRATION_FAILED + + after = _kb(table) + # Legacy by ABSENCE again, byte-identical to a record that never tried — so + # the next upload takes the legacy pipeline and simply works. + assert "retrievalEngine" not in after + assert after["rolledBackAt"] + assert after["migrationState"] == r.MIGRATION_FAILED + assert "GSI7_PK" not in after + + # The waiting document is failed with copy the author can act on, rather + # than left spinning on a knowledge base that will never exist. + document = _doc(table) + assert document["status"] == "failed" + assert "upload it again" in document["ingestionError"] + + @pytest.mark.asyncio + async def test_a_record_already_rolled_back_is_closed_out_not_reprovisioned( + self, table + ): + """A previous attempt fell back to legacy but was killed before clearing the + work keys. Provisioning a knowledge base for a record that has stopped + pointing at one would be a paid resource nothing uses.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "provisioningState": r.PROVISIONING, + "migrationState": r.BORN_MANAGED, + "migrationGeneration": 0, + "GSI7_PK": r.work_pk(r.BORN_MANAGED), + "GSI7_SK": "2026-09-10T00:00:00Z", + } + ) + record = _kb(table) + with patch.object(pv, "_provision", new=AsyncMock()) as provision: + result = await pv.run_born_managed(ASSISTANT_ID, ASSISTANT_ID, record) + + provision.assert_not_awaited() + assert result.to_state == r.MIGRATION_FAILED + assert "GSI7_PK" not in _kb(table) + + @pytest.mark.asyncio + async def test_a_concurrent_provisioner_makes_this_one_step_aside(self, table): + """Losing the provisioning race must NOT be read as a failure: rolling back + to legacy while another worker is successfully building the knowledge base + would undo good work.""" + table.put_item( + Item={ + "PK": r.kb_pk(ASSISTANT_ID), + "SK": r.kb_sk(ASSISTANT_ID), + "appKbId": ASSISTANT_ID, + "retrievalEngine": r.ENGINE_MANAGED, + "provisioningState": r.PROVISIONING, + "migrationState": r.BORN_MANAGED, + "migrationGeneration": 0, + } + ) + from apis.shared.kb_backend.provisioning import ProvisioningInProgress + + record = _kb(table) + with patch( + "apis.shared.kb_backend.provisioning.provision_managed_kb", + new=AsyncMock(side_effect=ProvisioningInProgress("someone else has it")), + ): + result = await pv.run_born_managed(ASSISTANT_ID, ASSISTANT_ID, record) + + assert result.to_state == r.BORN_MANAGED + after = _kb(table) + assert after["retrievalEngine"] == r.ENGINE_MANAGED + assert after["GSI7_PK"] == r.work_pk(r.BORN_MANAGED) + + def test_pending_documents_only_sees_the_waiting_ones(self, table): + _seed_doc(table, document_id="DOC-waiting") + _seed_doc(table, document_id="DOC-done", status="complete") + _seed_doc(table, document_id="DOC-gone", status="deleting") + + found = [d["documentId"] for d in pv.pending_documents(ASSISTANT_ID)] + assert found == ["DOC-waiting"] + + +# ── 6. The rollout ladder's rungs stay independent ─────────────────────────── +class TestDispatcherFlagGating: + def test_born_managed_is_swept_under_new_default_alone(self, monkeypatch): + from apis.app_api.kb_migration import dispatcher as d + + monkeypatch.setenv("MANAGED_KB_NEW_DEFAULT", "true") + monkeypatch.delenv("MANAGED_KB_MIGRATION_ENABLED", raising=False) + + assert d.dispatcher_enabled() is True + assert d._enabled_work_states() == [r.BORN_MANAGED] + + def test_migration_states_are_not_swept_under_new_default_alone(self, monkeypatch): + """MUTATION GUARD: gate the states on ``migration_enabled or + new_default_enabled`` instead of each on its own flag, and this fails — + turning on step 2 of the ladder would silently start migrating the entire + existing fleet, which is step 3 and a different blast radius.""" + from apis.app_api.kb_migration import dispatcher as d + + monkeypatch.setenv("MANAGED_KB_NEW_DEFAULT", "true") + monkeypatch.delenv("MANAGED_KB_MIGRATION_ENABLED", raising=False) + + swept = d._enabled_work_states() + assert r.SHADOW not in swept + assert r.VERIFY not in swept + assert r.PROMOTE not in swept + + def test_born_managed_is_not_swept_under_migration_alone(self, monkeypatch): + from apis.app_api.kb_migration import dispatcher as d + + monkeypatch.setenv("MANAGED_KB_MIGRATION_ENABLED", "true") + monkeypatch.delenv("MANAGED_KB_NEW_DEFAULT", raising=False) + + swept = d._enabled_work_states() + assert r.BORN_MANAGED not in swept + assert swept == [r.PROMOTE, r.VERIFY, r.SHADOW] + + def test_born_managed_is_served_first(self, monkeypatch): + """Somebody is watching an upload spinner for it; the migration states are + background work.""" + from apis.app_api.kb_migration import dispatcher as d + + monkeypatch.setenv("MANAGED_KB_NEW_DEFAULT", "true") + monkeypatch.setenv("MANAGED_KB_MIGRATION_ENABLED", "true") + + assert d._enabled_work_states()[0] == r.BORN_MANAGED + + @pytest.mark.asyncio + async def test_tick_is_a_no_op_with_both_flags_off(self, monkeypatch): + from apis.app_api.kb_migration import dispatcher as d + + monkeypatch.delenv("MANAGED_KB_NEW_DEFAULT", raising=False) + monkeypatch.delenv("MANAGED_KB_MIGRATION_ENABLED", raising=False) + + with patch.object(d, "_invoke_worker") as invoke: + counts = await d.dispatch_once() + + invoke.assert_not_called() + assert counts == {"Due": 0, "Dispatched": 0, "Failed": 0} + + +# ── 7. The document reconciler backstop ────────────────────────────────────── +def test_provisioning_is_a_reconciler_candidate(): + """The long-horizon backstop for a first document whose provisioning job was + killed between building the knowledge base and handing the document over. Safe + to include because the sweep already skips any record with no awsKbId, so a + document is never probed against a knowledge base that does not exist.""" + from apis.app_api.kb_migration.document_reconciler import NON_TERMINAL_STATUSES + + assert bm.STATUS_PROVISIONING in NON_TERMINAL_STATUSES + assert "complete" not in NON_TERMINAL_STATUSES + assert "deleting" not in NON_TERMINAL_STATUSES + + +def test_the_two_halves_agree_on_the_status_string(): + """The trigger writes it, the consumer defers on it, the provisioner clears it. + They live in different Lambda images, so the constant is defined once and + re-exported rather than spelled twice.""" + assert bm.STATUS_PROVISIONING == ic.STATUS_PROVISIONING == "provisioning" diff --git a/backend/tests/lambdas/test_kb_ingestion_consumer.py b/backend/tests/lambdas/test_kb_ingestion_consumer.py index 909b04527..0ca52b812 100644 --- a/backend/tests/lambdas/test_kb_ingestion_consumer.py +++ b/backend/tests/lambdas/test_kb_ingestion_consumer.py @@ -30,6 +30,13 @@ BUCKET = "docs-bucket" KEY = f"assistants/{ASSISTANT_ID}/documents/{DOCUMENT_ID}/report.pdf" +#: Size of the object the fixture stages in S3. The reconcile HEADs the real +#: object, so a managed document that reaches `complete` must have real bytes to +#: measure. The DOC# rows are seeded WITHOUT a declared sizeBytes (declared == 0), +#: so the common managed path exercises the "client under-reported" reconcile +#: branch: reserve the real size, then commit it — net-zero on reservedBytes. +OBJECT_BYTES = 2048 + @pytest.fixture() def table(monkeypatch): @@ -53,6 +60,13 @@ def table(monkeypatch): ], BillingMode="PAY_PER_REQUEST", ) + # The RAG documents bucket, with the uploaded object in place. The reconcile + # takes the authoritative size from an S3 HEAD, so the object must exist for + # a managed document to complete. + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket(Bucket=BUCKET) + s3.put_object(Bucket=BUCKET, Key=KEY, Body=b"x" * OBJECT_BYTES) + t = boto3.resource("dynamodb", region_name=REGION).Table(TABLE) t.put_item( Item={ @@ -779,3 +793,202 @@ def test_text_indexed_is_a_classified_status_not_an_unknown_one(self, table, cap "TEXT_INDEXED was handled by the unknown-status fallback; it is a state " "we have observed in production and it should be classified explicitly" ) + + +# --------------------------------------------------------------------------- +# The byte cap is settled against the AUTHORITATIVE S3 size (Req 12.3/12.4/12.6) +# --------------------------------------------------------------------------- +class TestByteCapReconcileAtIngestion: + """Enforcement of Requirement 12.11 on the ingestion side. + + The request-time reservation used the client-declared size, which is not + trustworthy. When a managed document reaches ``complete`` the reservation is + reconciled against the real S3 size; on every terminal FAILURE the reservation + is returned so a failed upload never permanently shrinks the owner's allowance. + """ + + def _seed_managed(self, table): + _seed_kb(table, retrievalEngine="managed", awsKbId="KB123", awsDataSourceId="DS456") + + def _reserve(self, table, declared): + """Model the request-time state: DOC# declares ``declared`` and the KB + record already holds that many reserved bytes.""" + table.update_item( + Key={"PK": f"AST#{ASSISTANT_ID}", "SK": f"DOC#{DOCUMENT_ID}"}, + UpdateExpression="SET sizeBytes = :d", + ExpressionAttributeValues={":d": declared}, + ) + table.update_item( + Key={"PK": f"AST#{ASSISTANT_ID}", "SK": f"KB#{ASSISTANT_ID}"}, + UpdateExpression="SET reservedBytes = :d, storedBytes = :z, totalBytes = :d", + ExpressionAttributeValues={":d": declared, ":z": 0}, + ) + + def _put_object(self, size): + boto3.client("s3", region_name=REGION).put_object( + Bucket=BUCKET, Key=KEY, Body=b"x" * size + ) + + def _kb(self, table): + return table.get_item( + Key={"PK": f"AST#{ASSISTANT_ID}", "SK": f"KB#{ASSISTANT_ID}"} + ).get("Item") or {} + + def _run(self, table, statuses=("NOT_FOUND", "INDEXED"), backend_cls=_FakeBackend): + fake = backend_cls(statuses=list(statuses)) + with patch( + "apis.shared.kb_backend.managed_backend.ManagedKbBackend", return_value=fake + ): + return fake, ic.handle_object(BUCKET, KEY) + + def test_real_equals_declared_commits_the_reservation(self, table): + self._seed_managed(table) + self._reserve(table, 2048) + self._put_object(2048) + + self._run(table) + + kb = self._kb(table) + assert int(kb["storedBytes"]) == 2048 + assert int(kb["reservedBytes"]) == 0 + assert int(kb["totalBytes"]) == 2048 + assert _doc(table)["status"] == "complete" + + def test_real_smaller_than_declared_commits_real_and_releases_the_difference(self, table): + """Client over-reported: charge only what was stored, return the rest.""" + self._seed_managed(table) + self._reserve(table, 4096) + self._put_object(2048) + + self._run(table) + + kb = self._kb(table) + assert int(kb["storedBytes"]) == 2048 + assert int(kb["reservedBytes"]) == 0 + assert int(kb["totalBytes"]) == 2048 # 4096 reserved, 2048 released + + def test_real_larger_than_declared_reserves_the_shortfall_then_commits(self, table): + """Client under-reported but still under the cap: reserve the extra, commit.""" + self._seed_managed(table) + self._reserve(table, 1024) + self._put_object(2048) + + self._run(table) + + kb = self._kb(table) + assert int(kb["storedBytes"]) == 2048 + assert int(kb["reservedBytes"]) == 0 + assert int(kb["totalBytes"]) == 2048 + assert _doc(table)["status"] == "complete" + + def test_real_over_cap_fails_the_document_deletes_object_and_releases(self, table, monkeypatch): + """Client under-reported AND the true size overshoots the cap. + + The document must be failed, the orphaned S3 object deleted, and the + original reservation returned — no bytes charged, no stranded source. + """ + monkeypatch.setenv("MANAGED_KB_PER_OWNER_DEFAULT_BYTES", "2048") + monkeypatch.setenv("MANAGED_KB_PER_KB_CEILING_BYTES", "2048") + self._seed_managed(table) + self._reserve(table, 1024) + self._put_object(4096) # real; 1024 declared -> shortfall 3072 over the 2048 cap + + _, result = self._run(table) + + assert result["status"] == "failed" + assert result.get("note") == "byte-cap-exceeded" + assert _doc(table)["status"] == "failed" + # The reservation is returned in full. + kb = self._kb(table) + assert int(kb["reservedBytes"]) == 0 + assert int(kb["totalBytes"]) == 0 + assert int(kb.get("storedBytes") or 0) == 0 + # The orphaned object is gone. + with pytest.raises(Exception): + boto3.client("s3", region_name=REGION).head_object(Bucket=BUCKET, Key=KEY) + + def test_a_failed_ingestion_releases_the_reservation(self, table): + """NAMED mutation guard: dropping the release-on-failure in the Bedrock- + FAILED path must fail here. A failed upload may not permanently shrink the + owner's allowance (Requirement 12.6).""" + self._seed_managed(table) + self._reserve(table, 2048) + + self._run(table, statuses=["FAILED"]) + + kb = self._kb(table) + assert int(kb["reservedBytes"]) == 0, ( + "a Bedrock-FAILED document did not release its reservation; a failed " + "upload has permanently shrunk the owner's cap" + ) + assert int(kb["totalBytes"]) == 0 + assert _doc(table)["status"] == "failed" + + def test_an_ingest_exception_releases_the_reservation(self, table): + """The submit itself raising is also terminal and must release.""" + self._seed_managed(table) + self._reserve(table, 2048) + + class _Failing(_FakeBackend): + async def ingest(self, kb_ref, source): + raise RuntimeError("bedrock unavailable") + + with patch( + "apis.shared.kb_backend.managed_backend.ManagedKbBackend", + return_value=_Failing(statuses=["NOT_FOUND"]), + ): + with pytest.raises(RuntimeError): + ic.handle_object(BUCKET, KEY) + + kb = self._kb(table) + assert int(kb["reservedBytes"]) == 0 + assert int(kb["totalBytes"]) == 0 + + def test_a_still_indexing_document_does_NOT_release(self, table): + """The non-terminal 'leave for redelivery' path must keep the bytes + reserved — the delivery that completes the document will settle them.""" + self._seed_managed(table) + self._reserve(table, 2048) + + with patch( + "apis.shared.kb_backend.managed_backend.ManagedKbBackend", + return_value=_FakeBackend(statuses=["IN_PROGRESS"]), + ): + with pytest.raises(ic.IngestionRoutingError): + ic.handle_object(BUCKET, KEY) + + kb = self._kb(table) + assert int(kb["reservedBytes"]) == 2048, ( + "a document still indexing released its reservation; the completing " + "delivery would then find nothing reserved" + ) + + def test_a_redelivery_does_not_double_commit(self, table): + """settle_once + the terminal early-exit make the byte accounting + idempotent: a second delivery of an already-complete document must not + drive reservedBytes negative.""" + self._seed_managed(table) + self._reserve(table, 2048) + self._put_object(2048) + + # Delivery 1 completes and commits. + self._run(table, statuses=["NOT_FOUND", "INDEXED"]) + kb1 = self._kb(table) + assert int(kb1["storedBytes"]) == 2048 + assert int(kb1["reservedBytes"]) == 0 + + # Delivery 2: Bedrock still reports INDEXED. Must be a no-op for accounting. + _, result = self._run(table, statuses=["INDEXED"]) + assert result.get("note") == "already-settled" + kb2 = self._kb(table) + assert int(kb2["storedBytes"]) == 2048 + assert int(kb2["reservedBytes"]) == 0, ( + "a redelivery committed a second time and drove reservedBytes negative" + ) + + def test_a_legacy_document_is_never_byte_accounted(self, table): + """Legacy KBs stay uncapped: no reservation is created or touched.""" + _seed_kb(table) # no retrievalEngine -> legacy + ic.handle_object(BUCKET, KEY) + kb = self._kb(table) + assert "reservedBytes" not in kb and "storedBytes" not in kb diff --git a/backend/tests/lambdas/test_kb_migration_worker.py b/backend/tests/lambdas/test_kb_migration_worker.py index aa3553169..a1343783a 100644 --- a/backend/tests/lambdas/test_kb_migration_worker.py +++ b/backend/tests/lambdas/test_kb_migration_worker.py @@ -219,10 +219,19 @@ def test_a_state_added_to_the_records_module_is_still_swept(self): # Appended, not promoted ahead of the known order. assert states[-1] == "reindex" - def test_promote_is_swept_first(self): + def test_promote_is_swept_before_the_other_migration_states(self): """A record in ``promote`` is one conditional write from finished, so - draining beats starting new shadow work.""" - assert dispatcher._work_states()[0] == r.PROMOTE + draining beats starting new shadow work. + + ``born_managed`` now leads the whole list — a first upload has somebody + watching a spinner for it, whereas every migration state is background work + — so this asserts the ordering among the MIGRATION states, which is what the + drain-first argument was ever about. + """ + states = dispatcher._work_states() + migration_states = [s for s in states if s != r.BORN_MANAGED] + assert migration_states[0] == r.PROMOTE + assert states[0] == r.BORN_MANAGED def test_no_terminal_state_is_swept(self): assert not set(dispatcher._work_states()) & set(r.TERMINAL_STATES) diff --git a/backend/tests/routes/test_document_chunk_inspector.py b/backend/tests/routes/test_document_chunk_inspector.py new file mode 100644 index 000000000..0603b20d0 --- /dev/null +++ b/backend/tests/routes/test_document_chunk_inspector.py @@ -0,0 +1,375 @@ +"""The chunk inspector: show an owner what the knowledge base actually extracted. + +Feature: `kb-chunk-inspector`. Tooling half of the task-16.2 decision on +`managed-kb-migration` §5.41. + +The failure this endpoint exists to make visible is not a crash — it is a *confident +wrong answer*. The managed backend flattens a column-structured diagram at ingestion, +so "how many credits in semester 4" comes back plausible and wrong with no trace. 16.2 +decided against fixing the parser and in favour of user guidance; guidance the user +cannot verify is not guidance, so this endpoint is what lets them look. + +Two things here are security properties rather than features, and both are mutation- +guarded below: + +**The document filter is the isolation boundary.** It must be `equals` on +`document_id`. A prefix or substring operator matching `DOC-1` also admits `DOC-10`, +which means one owner's inspector renders another document's content. That is a leak, +not a display bug — hence `ISOLATION_SAFE_FILTER_OPERATORS`. + +**A classic knowledge base must refuse rather than approximate.** The legacy adapter +accepts no filter and ignores `top_k`; it always returns five results from the *whole* +knowledge base. Running it anyway would show the owner other documents' chunks under +the heading of theirs. +""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from apis.app_api.documents.routes import router +from apis.app_api.documents.services import chunk_inspector as ci +from apis.shared.auth import get_current_user_from_session +from apis.shared.auth.models import User +from apis.shared.kb_backend.managed_backend import ISOLATION_SAFE_FILTER_OPERATORS +from apis.shared.kb_backend.protocol import Chunk + +ROUTES_MODULE = "apis.app_api.documents.routes" +INSPECTOR_MODULE = "apis.app_api.documents.services.chunk_inspector" +RESOLVER_MODULE = "apis.shared.kb_backend.resolver" +ASSISTANT_ID = "ast-insp01" +DOCUMENT_ID = "DOC-insp01" +OTHER_DOCUMENT_ID = "DOC-insp01-other" +USER_ID = "user-insp" +FILENAME = "4-yr-flowchart-v2026.pdf" + + +@pytest.fixture() +def app(): + _app = FastAPI() + _app.include_router(router) + _app.dependency_overrides[get_current_user_from_session] = lambda: User( + user_id=USER_ID, email=f"{USER_ID}@example.com", name="Test User", roles=["User"] + ) + return _app + + +def _owner(): + return (SimpleNamespace(owner_id=USER_ID), "owner") + + +def _viewer(): + return (SimpleNamespace(owner_id="somebody-else"), "viewer") + + +def _document(status="complete", filename=FILENAME): + return SimpleNamespace( + document_id=DOCUMENT_ID, filename=filename, status=status, s3_key="k" + ) + + +def _chunk(text, document_id=DOCUMENT_ID, relevance=0.5, metadata=None): + return Chunk( + text=text, + relevance=relevance, + document_id=document_id, + metadata=metadata or {}, + key=f"{document_id}#0", + ) + + +class _FakeBackend: + """Records the retrieval filter it was handed and returns canned chunks. + + Modelling the filter is the whole point: the isolation guarantee lives in the + argument, not in the response, so a fake that ignored it could not tell a scoped + query from an unscoped one. + + ``honour_filter=False`` models a backend that accepts the filter and does not + apply it — a service-side regression, or an adapter that drops the argument. That + is not hypothetical enough to skip: it is the failure the post-filter exists for, + and it is silent. + """ + + def __init__(self, chunks, honour_filter=True): + self._chunks = chunks + self._honour_filter = honour_filter + self.calls = [] + + async def search(self, kb_ref, query, top_k=5, retrieval_filter=None): + self.calls.append( + {"kb_ref": kb_ref, "query": query, "top_k": top_k, "filter": retrieval_filter} + ) + if not retrieval_filter or not self._honour_filter: + # Model the real world: unfiltered, the index returns everything it has, + # including other documents. A fake that returned only the "right" chunks + # would hide exactly the bug being guarded against. + return self._chunks + wanted = retrieval_filter.get("equals", {}).get("value") + return [c for c in self._chunks if c.document_id == wanted] + + +def _get(app): + return TestClient(app).get( + f"/assistants/{ASSISTANT_ID}/documents/{DOCUMENT_ID}/chunks" + ) + + +def _inspect(app, *, document, engine="managed", backend=None, record=None): + """Drive the endpoint with the resolver and document lookup stubbed.""" + return ( + patch( + f"{ROUTES_MODULE}.resolve_assistant_permission", + new_callable=AsyncMock, + return_value=_owner(), + ), + patch( + f"{ROUTES_MODULE}.get_document_service", + new_callable=AsyncMock, + return_value=document, + ), + # Patched on the resolver module, not on the inspector: the inspector imports + # it function-locally (stdlib-only module scope is the house rule for anything + # the size-constrained Lambda images touch), so there is no module attribute + # to patch on this side. + patch(f"{RESOLVER_MODULE}.load_record", return_value=record or {}), + patch(f"{RESOLVER_MODULE}.resolve_engine_for", return_value=engine), + patch(f"{RESOLVER_MODULE}.resolve_backend", return_value=backend), + ) + + +class TestTheHappyPath: + def test_an_owner_sees_the_full_extracted_text(self, app): + backend = _FakeBackend( + [ + _chunk("Semester 1 | ENGL 101 | 3 credits", metadata={"page": "1"}), + _chunk("x" * 900), # longer than the 500-char citation cap + ] + ) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + resp = _get(app) + + assert resp.status_code == 200 + body = resp.json() + assert body["available"] is True + assert body["engine"] == "managed" + assert body["returned"] == 2 + assert body["capReached"] is False + # NOT truncated to 500 — the whole reason the citation trace cannot serve this. + assert len(body["chunks"][1]["text"]) == 900 + assert body["chunks"][0]["page"] == 1 + + def test_the_filter_sent_to_the_backend_scopes_to_one_document(self, app): + """MUTATION GUARD: drop `retrieval_filter=document_filter(...)` from + `inspect_document_chunks` and this fails. + + This is the guard for the filter itself, asserted on the ARGUMENT rather than + on the response, and that distinction was earned the hard way. The obvious + guard — seed a second document's chunks and assert they do not appear — passes + with the filter removed, because the post-filter in `inspect_document_chunks` + catches them on the way out. A test that green-lights the mutated code is not + a guard, however sensible it reads. See + `test_a_backend_that_ignores_the_filter_still_cannot_leak` for the other layer. + """ + backend = _FakeBackend([_chunk("only mine")]) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + _get(app) + + call = backend.calls[0] + assert call["query"] == FILENAME + assert call["top_k"] == ci.INSPECT_TOP_K + assert call["filter"] == { + "equals": {"key": "document_id", "value": DOCUMENT_ID} + } + + def test_repeated_passages_are_collapsed(self, app): + """`Retrieve` is query-ranked, not a cursor, so it can return a passage twice. + Showing a mangled table three times would read as three bad ingestions.""" + backend = _FakeBackend([_chunk("same text"), _chunk("same text"), _chunk("other")]) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + body = _get(app).json() + + assert [c["text"] for c in body["chunks"]] == ["same text", "other"] + + def test_cap_reached_is_reported_when_the_ceiling_is_hit(self, app): + """Honesty, not paging (Req 3). Bedrock has no chunk-enumeration API, so the + UI must say 'up to N' rather than 'this document has N chunks'.""" + backend = _FakeBackend([_chunk(f"chunk {i}") for i in range(ci.INSPECT_TOP_K)]) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + body = _get(app).json() + + assert body["capReached"] is True + assert body["returned"] == ci.INSPECT_TOP_K + + def test_a_page_number_is_never_invented(self, app): + """A fabricated page would be indistinguishable from a real one and would make + an unordered set look authoritatively ordered.""" + backend = _FakeBackend([_chunk("no page metadata", metadata={"junk": "x"})]) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + body = _get(app).json() + + assert body["chunks"][0]["page"] is None + + +class TestIsolation: + def test_a_backend_that_ignores_the_filter_still_cannot_leak(self, app): + """MUTATION GUARD for the SECOND layer: delete the `scoped = [...]` post-filter + in `inspect_document_chunks` and this fails. + + The fake here deliberately ignores the filter it is handed, modelling a backend + that stops honouring it — a service-side regression, or a future adapter that + accepts the argument and drops it. Neither would raise. The response would just + quietly start carrying another document's content under this document's + filename, and nothing else in the system would notice. + + Belt and braces is the right call here precisely because the failure is silent + and the blast radius is one owner reading another's document. + """ + backend = _FakeBackend( + [ + _chunk("mine", document_id=DOCUMENT_ID), + _chunk("SOMEBODY ELSE'S CONTENT", document_id=OTHER_DOCUMENT_ID), + ], + honour_filter=False, + ) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + body = _get(app).json() + + texts = [c["text"] for c in body["chunks"]] + assert texts == ["mine"] + assert "SOMEBODY ELSE'S CONTENT" not in texts + + def test_only_the_requested_document_reaches_the_owner(self, app): + """The observable guarantee, through whichever layer delivers it. Both the + filter and the post-filter would have to fail for this to break.""" + backend = _FakeBackend( + [ + _chunk("mine", document_id=DOCUMENT_ID), + _chunk("SOMEBODY ELSE'S CONTENT", document_id=OTHER_DOCUMENT_ID), + ] + ) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + body = _get(app).json() + + assert [c["text"] for c in body["chunks"]] == ["mine"] + + def test_the_filter_operator_is_isolation_safe(self): + """The operator choice IS the isolation boundary. A prefix match for `DOC-1` + admits `DOC-10`, so this asserts the operator rather than trusting it.""" + built = ci.document_filter(DOCUMENT_ID) + assert list(built) == ["equals"] + assert set(built) <= ISOLATION_SAFE_FILTER_OPERATORS + assert built["equals"] == {"key": "document_id", "value": DOCUMENT_ID} + + def test_a_prefix_style_document_id_cannot_over_match(self, app): + """`DOC-insp01` must not admit `DOC-insp01-other`, which is exactly what a + substring operator would do. Driven through a filter-honouring fake so this + exercises the real `equals` semantics rather than the post-filter.""" + backend = _FakeBackend( + [ + _chunk("mine", document_id=DOCUMENT_ID), + _chunk("longer id, must not match", document_id=OTHER_DOCUMENT_ID), + ] + ) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + body = _get(app).json() + + assert [c["text"] for c in body["chunks"]] == ["mine"] + + def test_a_viewer_is_refused(self, app): + with patch( + f"{ROUTES_MODULE}.resolve_assistant_permission", + new_callable=AsyncMock, + return_value=_viewer(), + ): + resp = _get(app) + assert resp.status_code == 403 + + +class TestWhatCannotBeInspected: + def test_a_classic_knowledge_base_refuses_rather_than_approximating(self, app): + """MUTATION GUARD: let the legacy engine through to `search` and this fails. + The legacy adapter takes no filter and ignores top_k — it returns five results + from the WHOLE knowledge base — so 'approximating' means rendering other + documents' content under this document's name.""" + backend = _FakeBackend([_chunk("whole-KB result", document_id=OTHER_DOCUMENT_ID)]) + p, d, lr, re_, rb = _inspect( + app, document=_document(), engine="s3vectors", backend=backend + ) + with p, d, lr, re_, rb: + resp = _get(app) + + assert resp.status_code == 200 # an answer, not an error + body = resp.json() + assert body["available"] is False + assert body["chunks"] == [] + assert "classic" in body["reason"].lower() + # The decisive assertion: the backend was never asked. + assert backend.calls == [] + + @pytest.mark.parametrize( + "status", ["uploading", "chunking", "embedding", "provisioning"] + ) + def test_a_document_still_processing_is_409_not_404(self, app, status): + """It exists; it simply has no content yet. A 404 would tell the owner their + file is missing, which is both wrong and alarming.""" + p, d, lr, re_, rb = _inspect(app, document=_document(status=status)) + with p, d, lr, re_, rb: + resp = _get(app) + + assert resp.status_code == 409 + + def test_provisioning_says_the_knowledge_base_is_being_created(self, app): + """Born-managed's leading status. 'Still processing' would understate it — the + wait is on the knowledge base itself, not on this file.""" + p, d, lr, re_, rb = _inspect(app, document=_document(status="provisioning")) + with p, d, lr, re_, rb: + detail = _get(app).json()["detail"] + + assert "knowledge base" in detail.lower() + assert "being created" in detail.lower() + + def test_a_failed_document_explains_itself(self, app): + p, d, lr, re_, rb = _inspect(app, document=_document(status="failed")) + with p, d, lr, re_, rb: + resp = _get(app) + + assert resp.status_code == 409 + assert "could not be processed" in resp.json()["detail"].lower() + + def test_a_missing_document_is_404(self, app): + p, d, lr, re_, rb = _inspect(app, document=None) + with p, d, lr, re_, rb: + resp = _get(app) + assert resp.status_code == 404 + + def test_a_soft_deleted_document_is_404_not_its_content(self, app): + """`deleting` means removal is under way on purpose. Rendering its content is + resurrecting it in the one place the user was told it is gone.""" + p, d, lr, re_, rb = _inspect(app, document=_document(status="deleting")) + with p, d, lr, re_, rb: + resp = _get(app) + assert resp.status_code == 404 + + +class TestBoundedCost: + def test_exactly_one_retrieve_per_request(self, app): + """Req 6. `Retrieve` was measured at 662–695 ms p50 and offers no cursor, so a + second call would cost that again for an overlapping arbitrary subset.""" + backend = _FakeBackend([_chunk("a"), _chunk("b")]) + p, d, lr, re_, rb = _inspect(app, document=_document(), backend=backend) + with p, d, lr, re_, rb: + _get(app) + + assert len(backend.calls) == 1 diff --git a/backend/tests/routes/test_document_upload_byte_cap.py b/backend/tests/routes/test_document_upload_byte_cap.py new file mode 100644 index 000000000..1201e1cd4 --- /dev/null +++ b/backend/tests/routes/test_document_upload_byte_cap.py @@ -0,0 +1,219 @@ +"""Request-time byte-cap enforcement on the document upload-URL endpoint. + +Feature: managed-kb-migration — enforce the managed-KB Byte_Cap at document +upload time (completes Requirement 12.11; the interactive upload path was the one +byte-adding path left uncapped). + +The endpoint reserves the client-declared size against the binding cap BEFORE it +creates the DOC# row or issues a presigned URL, so an over-cap upload is refused +with a 413 carrying the numbers (Req 12.12) rather than after the bytes are +staged. The reservation is provisional — the authoritative gate is the S3-HEAD +reconcile at ingestion (Req 12.3) — but it gives fast, friendly feedback and is +released if a later step of the same request fails (Req 12.6). Legacy KBs are +never checked. +""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import boto3 +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from moto import mock_aws + +from apis.app_api.documents.routes import router +from apis.shared.auth import get_current_user_from_session +from apis.shared.auth.models import User + +ROUTES_MODULE = "apis.app_api.documents.routes" +REGION = "us-east-1" +TABLE = "test-upload-byte-cap" +ASSISTANT_ID = "ast-cap01" +USER_ID = "user-001" + + +@pytest.fixture() +def table(monkeypatch): + monkeypatch.setenv("AWS_DEFAULT_REGION", REGION) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_SESSION_TOKEN", "testing") + monkeypatch.setenv("DYNAMODB_ASSISTANTS_TABLE_NAME", TABLE) + + with mock_aws(): + ddb = boto3.client("dynamodb", region_name=REGION) + ddb.create_table( + TableName=TABLE, + KeySchema=[ + {"AttributeName": "PK", "KeyType": "HASH"}, + {"AttributeName": "SK", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "PK", "AttributeType": "S"}, + {"AttributeName": "SK", "AttributeType": "S"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + yield boto3.resource("dynamodb", region_name=REGION).Table(TABLE) + + +@pytest.fixture() +def app(): + _app = FastAPI() + _app.include_router(router) + _app.dependency_overrides[get_current_user_from_session] = lambda: User( + user_id=USER_ID, email=f"{USER_ID}@example.com", name="Test User", roles=["User"] + ) + return _app + + +def _seed_managed_kb(table, **overrides): + item = { + "PK": f"AST#{ASSISTANT_ID}", + "SK": f"KB#{ASSISTANT_ID}", + "appKbId": ASSISTANT_ID, + "ownerUserId": USER_ID, + "retrievalEngine": "managed", + "storedBytes": 0, + "reservedBytes": 0, + "totalBytes": 0, + } + item.update(overrides) + table.put_item(Item=item) + + +def _kb(table): + return table.get_item( + Key={"PK": f"AST#{ASSISTANT_ID}", "SK": f"KB#{ASSISTANT_ID}"} + ).get("Item") + + +def _post(app, size_bytes): + return TestClient(app).post( + f"/assistants/{ASSISTANT_ID}/documents/upload-url", + json={"filename": "report.pdf", "contentType": "application/pdf", "sizeBytes": size_bytes}, + ) + + +def _owner(): + return (SimpleNamespace(owner_id=USER_ID), "owner") + + +class TestManagedUploadIsCapped: + def test_an_over_cap_upload_is_rejected_413_with_the_numbers(self, table, app, monkeypatch): + """NAMED mutation guard: dropping the request-time reserve makes this pass + as a 200. The cap is 5000 bytes and the file is 10000.""" + monkeypatch.setenv("MANAGED_KB_PER_OWNER_DEFAULT_BYTES", "5000") + monkeypatch.setenv("MANAGED_KB_PER_KB_CEILING_BYTES", "5000") + _seed_managed_kb(table) + + with patch( + f"{ROUTES_MODULE}.resolve_assistant_permission", + new_callable=AsyncMock, + return_value=_owner(), + ), patch( + f"{ROUTES_MODULE}.create_document", new_callable=AsyncMock + ) as create, patch( + f"{ROUTES_MODULE}.generate_upload_url", new_callable=AsyncMock + ) as gen: + resp = _post(app, 10000) + + assert resp.status_code == 413 + detail = resp.json()["detail"] + assert "10000" in detail and "5000" in detail # requested and cap (Req 12.12) + # The row was never created and no URL was issued. + create.assert_not_awaited() + gen.assert_not_awaited() + # No bytes were reserved (the reserve raised before mutating on a fits check; + # n > cap short-circuits without a write). + assert int(_kb(table)["reservedBytes"]) == 0 + + def test_an_under_cap_upload_succeeds_and_reserves_the_declared_size(self, table, app): + _seed_managed_kb(table) + + with patch( + f"{ROUTES_MODULE}.resolve_assistant_permission", + new_callable=AsyncMock, + return_value=_owner(), + ), patch( + f"{ROUTES_MODULE}.create_document", new_callable=AsyncMock + ), patch( + f"{ROUTES_MODULE}.generate_upload_url", + new_callable=AsyncMock, + return_value=("https://signed.example/put", None), + ): + resp = _post(app, 2048) + + assert resp.status_code == 200 + body = resp.json() + assert body["uploadUrl"] == "https://signed.example/put" + assert body["documentId"] + assert int(_kb(table)["reservedBytes"]) == 2048 # provisional reservation + + def test_the_elevated_tier_is_read_from_the_record(self, table, app, monkeypatch): + """A 10000-byte file that fails the 5000 default cap succeeds when the + owner carries elevatedByteCap=true and the elevated cap is 100000.""" + monkeypatch.setenv("MANAGED_KB_PER_OWNER_DEFAULT_BYTES", "5000") + monkeypatch.setenv("MANAGED_KB_PER_OWNER_ELEVATED_BYTES", "100000") + _seed_managed_kb(table, elevatedByteCap=True) + + with patch( + f"{ROUTES_MODULE}.resolve_assistant_permission", + new_callable=AsyncMock, + return_value=_owner(), + ), patch( + f"{ROUTES_MODULE}.create_document", new_callable=AsyncMock + ), patch( + f"{ROUTES_MODULE}.generate_upload_url", + new_callable=AsyncMock, + return_value=("https://signed.example/put", None), + ): + resp = _post(app, 10000) + + assert resp.status_code == 200 + assert int(_kb(table)["reservedBytes"]) == 10000 + + def test_a_later_step_failing_releases_the_reservation(self, table, app): + """Req 12.6: if URL generation fails after the reserve, the bytes are + returned rather than leaked.""" + _seed_managed_kb(table) + + with patch( + f"{ROUTES_MODULE}.resolve_assistant_permission", + new_callable=AsyncMock, + return_value=_owner(), + ), patch( + f"{ROUTES_MODULE}.create_document", new_callable=AsyncMock + ), patch( + f"{ROUTES_MODULE}.generate_upload_url", + new_callable=AsyncMock, + side_effect=RuntimeError("s3 signer down"), + ): + resp = _post(app, 2048) + + assert resp.status_code == 500 + assert int(_kb(table)["reservedBytes"]) == 0, "reservation leaked on a failed request" + + +class TestLegacyUploadIsNotCapped: + def test_a_legacy_kb_is_never_reserved_against(self, table, app): + """No KB_Record -> legacy -> uncapped. The upload succeeds and nothing is + written to a KB accounting record.""" + # deliberately no _seed_managed_kb + + with patch( + f"{ROUTES_MODULE}.resolve_assistant_permission", + new_callable=AsyncMock, + return_value=_owner(), + ), patch( + f"{ROUTES_MODULE}.create_document", new_callable=AsyncMock + ), patch( + f"{ROUTES_MODULE}.generate_upload_url", + new_callable=AsyncMock, + return_value=("https://signed.example/put", None), + ): + resp = _post(app, 10_000_000_000) # 10 GB, would fail any managed cap + + assert resp.status_code == 200 + assert _kb(table) is None, "a legacy upload created a byte-cap record" diff --git a/backend/tests/routes/test_kb_upgrade.py b/backend/tests/routes/test_kb_upgrade.py index 2df285fcb..1fb1e7e4b 100644 --- a/backend/tests/routes/test_kb_upgrade.py +++ b/backend/tests/routes/test_kb_upgrade.py @@ -30,7 +30,7 @@ from decimal import Decimal from typing import Any, Dict, List, Optional -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, patch, AsyncMock import pytest from fastapi import FastAPI @@ -731,3 +731,27 @@ def test_stranded_document_serialises_camel_case(self): retryable=True, ).model_dump(by_alias=True) assert payload["documentId"] == "d1" + + +# ── Born-managed (MANAGED_KB_NEW_DEFAULT) ──────────────────────────────────── +class TestNewDefaultFlag: + @pytest.mark.parametrize( + "raw", ["", " ", "false", "no", "off", "0", "disabled", None] + ) + def test_absent_empty_or_negative_reads_as_off(self, monkeypatch, raw): + if raw is None: + monkeypatch.delenv(s.FLAG_NEW_DEFAULT, raising=False) + else: + monkeypatch.setenv(s.FLAG_NEW_DEFAULT, raw) + assert s.new_default_enabled() is False + + @pytest.mark.parametrize("raw", ["1", "true", "yes", "on", "enabled", "TRUE"]) + def test_affirmative_spellings_read_as_on(self, monkeypatch, raw): + monkeypatch.setenv(s.FLAG_NEW_DEFAULT, raw) + assert s.new_default_enabled() is True + + def test_the_flag_is_read_at_call_time(self, monkeypatch): + monkeypatch.setenv(s.FLAG_NEW_DEFAULT, "true") + assert s.new_default_enabled() is True + monkeypatch.setenv(s.FLAG_NEW_DEFAULT, "false") + assert s.new_default_enabled() is False diff --git a/backend/tests/shared/test_agent_icons.py b/backend/tests/shared/test_agent_icons.py index 7833cbfe4..942c27fda 100644 --- a/backend/tests/shared/test_agent_icons.py +++ b/backend/tests/shared/test_agent_icons.py @@ -144,7 +144,9 @@ def test_the_encode_ladder_degrades_an_opaque_png_to_jpeg(monkeypatch): input — a 512² of pure noise — is ~770 KB and never gets past the upload gate. The ladder is still what runs; only the number it is measured against moves. """ - import apis.shared.assistants.icons as icons_module + # The ladder lives in the shared image module now; the agent module re-exports + # the constant but the ceiling that matters is the one the ladder itself reads. + import apis.shared.images.icons as icons_module monkeypatch.setattr(icons_module, "ICON_MAX_BYTES", 300_000) noise = Image.open(io.BytesIO(_noise_png())).convert("RGBA") @@ -157,7 +159,9 @@ def test_the_encode_ladder_degrades_an_opaque_png_to_jpeg(monkeypatch): def test_the_encode_ladder_keeps_alpha_by_quantizing(monkeypatch): """A transparent PNG cannot become a JPEG without losing its alpha, so it loses colors instead.""" - import apis.shared.assistants.icons as icons_module + # The ladder lives in the shared image module now; the agent module re-exports + # the constant but the ceiling that matters is the one the ladder itself reads. + import apis.shared.images.icons as icons_module monkeypatch.setattr(icons_module, "ICON_MAX_BYTES", 300_000) noise = Image.open(io.BytesIO(_noise_png())).convert("RGBA") diff --git a/backend/tests/shared/test_config_cache.py b/backend/tests/shared/test_config_cache.py new file mode 100644 index 000000000..b563a2850 --- /dev/null +++ b/backend/tests/shared/test_config_cache.py @@ -0,0 +1,240 @@ +"""Tests for the tenant-global config catalog cache. + +The behaviours worth pinning here are the ones that would silently regress: +single-flight under a burst (the case the cache exists for), invalidation on +write, and the raw-items contract that keeps a mutating caller from poisoning +what every other user sees. +""" + +import asyncio + +import pytest + +from apis.shared.caching import config_cache +from apis.shared.caching.config_cache import ConfigListCache + + +@pytest.fixture(autouse=True) +def _clean_cache(monkeypatch): + """Every test starts on an empty, enabled, default-TTL cache.""" + monkeypatch.delenv("CONFIG_CACHE_ENABLED", raising=False) + monkeypatch.delenv("CONFIG_CACHE_TTL_SECONDS", raising=False) + config_cache.get_config_cache().clear() + yield + config_cache.get_config_cache().clear() + + +def _counting_loader(items, calls): + async def loader(): + calls.append(1) + return list(items) + return loader + + +class TestCaching: + @pytest.mark.asyncio + async def test_second_read_does_not_hit_the_loader(self): + cache = ConfigListCache() + calls = [] + loader = _counting_loader([{"id": "a"}], calls) + + first = await cache.get_or_load("k", loader) + second = await cache.get_or_load("k", loader) + + assert len(calls) == 1 + assert first == second == [{"id": "a"}] + + @pytest.mark.asyncio + async def test_keys_are_independent(self): + cache = ConfigListCache() + calls_a, calls_b = [], [] + + await cache.get_or_load("a", _counting_loader([{"id": "a"}], calls_a)) + await cache.get_or_load("b", _counting_loader([{"id": "b"}], calls_b)) + + assert len(calls_a) == 1 + assert len(calls_b) == 1 + + @pytest.mark.asyncio + async def test_expired_entry_reloads(self, monkeypatch): + monkeypatch.setenv("CONFIG_CACHE_TTL_SECONDS", "0") + cache = ConfigListCache() + calls = [] + loader = _counting_loader([{"id": "a"}], calls) + + await cache.get_or_load("k", loader) + await cache.get_or_load("k", loader) + + # TTL 0 means every entry is already expired on read. + assert len(calls) == 2 + + @pytest.mark.asyncio + async def test_malformed_ttl_falls_back_to_default(self, monkeypatch): + """A bad env var should degrade to sane caching, not break every read.""" + monkeypatch.setenv("CONFIG_CACHE_TTL_SECONDS", "not-a-number") + cache = ConfigListCache() + calls = [] + loader = _counting_loader([{"id": "a"}], calls) + + await cache.get_or_load("k", loader) + await cache.get_or_load("k", loader) + + assert len(calls) == 1 + + +class TestSingleFlight: + @pytest.mark.asyncio + async def test_concurrent_miss_loads_once(self): + """The stampede case: a cold cache and a classroom arriving together. + + Without single flight this is one scan per request, and the pileup + lands at exactly the moment the burst does. + """ + cache = ConfigListCache() + calls = [] + started = asyncio.Event() + release = asyncio.Event() + + async def slow_loader(): + calls.append(1) + started.set() + await release.wait() + return [{"id": "a"}] + + waiters = [asyncio.create_task(cache.get_or_load("k", slow_loader)) for _ in range(50)] + await started.wait() + release.set() + results = await asyncio.gather(*waiters) + + assert len(calls) == 1 + assert all(r == [{"id": "a"}] for r in results) + + @pytest.mark.asyncio + async def test_loader_failure_is_not_cached(self): + """A failed load must not strand later callers on a cached error.""" + cache = ConfigListCache() + calls = [] + + async def failing_loader(): + calls.append(1) + raise RuntimeError("dynamo down") + + with pytest.raises(RuntimeError): + await cache.get_or_load("k", failing_loader) + + # Next caller retries rather than inheriting the failure. + items = await cache.get_or_load("k", _counting_loader([{"id": "a"}], calls)) + assert items == [{"id": "a"}] + assert len(calls) == 2 + + @pytest.mark.asyncio + async def test_failure_propagates_to_every_concurrent_waiter(self): + cache = ConfigListCache() + release = asyncio.Event() + + async def failing_loader(): + await release.wait() + raise RuntimeError("dynamo down") + + waiters = [ + asyncio.create_task(cache.get_or_load("k", failing_loader)) + for _ in range(5) + ] + await asyncio.sleep(0) + release.set() + results = await asyncio.gather(*waiters, return_exceptions=True) + + # The leader raises; the queued waiters re-check, miss, and retry the + # loader themselves — every one of them ends in an error rather than a + # silently empty catalog, which is the property that matters. + assert all(isinstance(r, RuntimeError) for r in results) + + +class TestInvalidation: + @pytest.mark.asyncio + async def test_invalidate_forces_reload(self): + cache = ConfigListCache() + calls = [] + loader = _counting_loader([{"id": "a"}], calls) + + await cache.get_or_load("k", loader) + cache.invalidate("k") + await cache.get_or_load("k", loader) + + assert len(calls) == 2 + + @pytest.mark.asyncio + async def test_invalidate_oauth_drops_both_entries(self): + """`enabled` is part of the GSI key, so a write moves providers between + the enabled-only and full result sets — both must be dropped.""" + calls_enabled, calls_all = [], [] + await config_cache.get_or_load( + config_cache.OAUTH_PROVIDERS_ENABLED, + _counting_loader([{"id": "p"}], calls_enabled), + ) + await config_cache.get_or_load( + config_cache.OAUTH_PROVIDERS_ALL, + _counting_loader([{"id": "p"}], calls_all), + ) + + config_cache.invalidate_oauth_providers() + + await config_cache.get_or_load( + config_cache.OAUTH_PROVIDERS_ENABLED, + _counting_loader([{"id": "p"}], calls_enabled), + ) + await config_cache.get_or_load( + config_cache.OAUTH_PROVIDERS_ALL, + _counting_loader([{"id": "p"}], calls_all), + ) + + assert len(calls_enabled) == 2 + assert len(calls_all) == 2 + + +class TestIsolationContract: + @pytest.mark.asyncio + async def test_caller_cannot_mutate_the_cached_list(self): + """A caller that appends or sorts its result must not corrupt the entry. + + This is the guard that lets us cache at all: `hydrate_model_roles` and + `list_tools_with_roles` mutate what they are handed. + """ + cache = ConfigListCache() + calls = [] + loader = _counting_loader([{"id": "a"}], calls) + + first = await cache.get_or_load("k", loader) + first.append({"id": "injected"}) + first.clear() + + second = await cache.get_or_load("k", loader) + assert second == [{"id": "a"}] + assert len(calls) == 1 + + +class TestKillSwitch: + @pytest.mark.asyncio + async def test_disabled_bypasses_the_cache_entirely(self, monkeypatch): + monkeypatch.setenv("CONFIG_CACHE_ENABLED", "false") + cache = ConfigListCache() + calls = [] + loader = _counting_loader([{"id": "a"}], calls) + + await cache.get_or_load("k", loader) + await cache.get_or_load("k", loader) + + assert len(calls) == 2 + + @pytest.mark.asyncio + async def test_empty_value_is_still_enabled(self, monkeypatch): + """House style: unset or empty resolves to ON; only literal 'false' disables.""" + monkeypatch.setenv("CONFIG_CACHE_ENABLED", "") + cache = ConfigListCache() + calls = [] + loader = _counting_loader([{"id": "a"}], calls) + + await cache.get_or_load("k", loader) + await cache.get_or_load("k", loader) + + assert len(calls) == 1 diff --git a/backend/tests/shared/test_model_icons.py b/backend/tests/shared/test_model_icons.py new file mode 100644 index 000000000..616682466 --- /dev/null +++ b/backend/tests/shared/test_model_icons.py @@ -0,0 +1,395 @@ +"""Managed-model icons — the built-in slug, the uploaded override, and precedence. + +The interesting assertions are not the happy path: + +* **A cleared slug has to survive ``exclude_none``.** ``ManagedModelUpdate`` drops + ``None`` fields, which is what makes a PATCH a PATCH — so ``None`` cannot also + mean "remove it". ``''`` is that signal, and the update path has to turn it into + a DynamoDB REMOVE rather than storing an empty string. +* **The bytes never touch the record.** ``iconKey`` is a key; the object lives in + S3. Same 400 KB DynamoDB item-limit lesson the Agent icons learned. +* **An unknown slug is refused at the write.** A slug we ship no asset for renders + an invisible tile for every user, and nothing downstream would report it. +""" + +import io + +import boto3 +import pytest +from moto import mock_aws +from PIL import Image + +from apis.app_api.admin.services.model_icons import ( + ModelIconError, + read_model_icon, + remove_model_icon, + upload_model_icon, +) +from apis.shared.models.managed_models import ( + create_managed_model, + get_managed_model, + update_managed_model, +) +from apis.shared.models.model_icons import ( + BUILTIN_MODEL_ICONS, + ModelIconStore, + build_model_icon_key, + content_digest, + model_icon_url, + model_icon_version, + normalize_icon_slug, +) +from apis.shared.models.models import ManagedModelCreate, ManagedModelUpdate + +REGION = "us-west-2" +TABLE = "test-managed-models" +BUCKET = "test-rag-documents" + + +def _png(size=(512, 512), color=(30, 90, 200, 255)) -> bytes: + buffer = io.BytesIO() + Image.new("RGBA", size, color).save(buffer, format="PNG") + return buffer.getvalue() + + +@pytest.fixture +def aws(monkeypatch): + monkeypatch.setenv("AWS_DEFAULT_REGION", REGION) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("DYNAMODB_MANAGED_MODELS_TABLE_NAME", TABLE) + monkeypatch.setenv("S3_ASSISTANTS_DOCUMENTS_BUCKET_NAME", BUCKET) + + with mock_aws(): + ddb = boto3.resource("dynamodb", region_name=REGION) + ddb.create_table( + TableName=TABLE, + KeySchema=[ + {"AttributeName": "PK", "KeyType": "HASH"}, + {"AttributeName": "SK", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "PK", "AttributeType": "S"}, + {"AttributeName": "SK", "AttributeType": "S"}, + {"AttributeName": "GSI1PK", "AttributeType": "S"}, + {"AttributeName": "GSI1SK", "AttributeType": "S"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "ModelIdIndex", + "KeySchema": [ + {"AttributeName": "GSI1PK", "KeyType": "HASH"}, + {"AttributeName": "GSI1SK", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + }, + ], + BillingMode="PAY_PER_REQUEST", + ) + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket( + Bucket=BUCKET, CreateBucketConfiguration={"LocationConstraint": REGION} + ) + + # The managed-models repository binds its DynamoDB resource at import time, + # and the icon store is a module-level singleton bound on first use. Rebind + # both so they see this moto session rather than a previous test's. + import apis.shared.models.managed_models as repo + import apis.shared.models.model_icons as icons_module + + monkeypatch.setattr(repo, "dynamodb", ddb) + monkeypatch.setattr( + icons_module, "_store", ModelIconStore(bucket_name=BUCKET, s3_client=s3) + ) + # The catalog scan is cached per process; a test writing two models in a row + # would otherwise read the first one's snapshot. + from apis.shared.caching import config_cache + + config_cache.invalidate(config_cache.MANAGED_MODELS) + yield {"table": ddb.Table(TABLE), "s3": s3} + + +def _create(**extra): + payload = { + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "modelName": "Claude Haiku 4.5", + "provider": "bedrock", + "providerName": "Anthropic", + "inputModalities": ["TEXT"], + "outputModalities": ["TEXT"], + "maxInputTokens": 200000, + "inputPricePerMillionTokens": 1.0, + "outputPricePerMillionTokens": 5.0, + } + payload.update(extra) + return create_managed_model(ManagedModelCreate.model_validate(payload)) + + +# ── the built-in slug ──────────────────────────────────────────────────────── + + +def test_a_known_slug_is_normalized_and_persisted(): + assert normalize_icon_slug(" Anthropic ") == "anthropic" + + +def test_an_unknown_slug_is_refused_with_the_list_of_real_ones(): + with pytest.raises(ValueError) as excinfo: + normalize_icon_slug("acme-labs") + + message = str(excinfo.value) + assert "acme-labs" in message + for slug in BUILTIN_MODEL_ICONS: + assert slug in message + + +def test_clearing_a_slug_survives_the_patch_models_exclude_none(): + """``None`` means "don't touch" on a PATCH, so ``''`` has to carry "remove it" + all the way through ``model_dump(exclude_none=True)``.""" + cleared = ManagedModelUpdate.model_validate({"iconSlug": ""}) + + assert cleared.model_dump(exclude_none=True, by_alias=True) == {"iconSlug": ""} + + +def test_an_absent_slug_stays_absent_from_the_patch(): + assert ManagedModelUpdate.model_validate({}).model_dump(exclude_none=True, by_alias=True) == {} + + +@pytest.mark.asyncio +async def test_a_cleared_slug_removes_the_attribute_rather_than_storing_empty(aws): + model = await _create(iconSlug="anthropic") + assert model.icon_slug == "anthropic" + + await update_managed_model(model.id, ManagedModelUpdate.model_validate({"iconSlug": ""})) + + item = aws["table"].get_item( + Key={"PK": f"MODEL#{model.id}", "SK": f"MODEL#{model.id}"} + )["Item"] + assert "iconSlug" not in item + + +# ── keys and URLs ──────────────────────────────────────────────────────────── + + +def test_the_key_is_content_addressed_under_its_own_models_prefix(): + digest = content_digest(_png()) + key = build_model_icon_key("m-1", digest, "png") + + assert key == f"models/m-1/icons/{digest}.png" + + +def test_the_url_carries_the_digest_so_a_replacement_busts_an_immutable_cache(): + key = build_model_icon_key("m-1", "0123456789abcdef", "png") + + assert model_icon_url("m-1", key) == "/models/m-1/icon?v=0123456789abcdef" + assert model_icon_version(key) == "0123456789abcdef" + + +def test_no_key_means_no_url(): + assert model_icon_url("m-1", None) is None + + +# ── upload / read / remove ─────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_upload_stores_bytes_in_s3_and_only_a_key_on_the_record(aws): + model = await _create() + + icon_key, icon_url = await upload_model_icon(model.id, _png()) + + assert icon_key.startswith(f"models/{model.id}/icons/") + assert icon_url == f"/models/{model.id}/icon?v={model_icon_version(icon_key)}" + + item = aws["table"].get_item( + Key={"PK": f"MODEL#{model.id}", "SK": f"MODEL#{model.id}"} + )["Item"] + # A short key, not anything resembling image data — the 400 KB DynamoDB item + # limit is the whole reason the bytes live in S3. + assert item["iconKey"] == icon_key + assert len(item["iconKey"]) < 100 + + stored = aws["s3"].get_object(Bucket=BUCKET, Key=icon_key)["Body"].read() + assert Image.open(io.BytesIO(stored)).size == (512, 512) + + +@pytest.mark.asyncio +async def test_the_read_model_derives_icon_url_from_the_stored_key(aws): + model = await _create() + icon_key, _ = await upload_model_icon(model.id, _png()) + + reread = await get_managed_model(model.id) + + assert reread.icon_key == icon_key + assert reread.model_dump(by_alias=True)["iconUrl"] == model_icon_url(model.id, icon_key) + + +@pytest.mark.asyncio +async def test_reuploading_the_same_image_is_idempotent(aws): + model = await _create() + first, _ = await upload_model_icon(model.id, _png()) + + second, _ = await upload_model_icon(model.id, _png()) + + assert first == second + # And the object is still there: the "delete the previous one" step must not + # fire when the content-addressed key is unchanged. + aws["s3"].get_object(Bucket=BUCKET, Key=second) + + +@pytest.mark.asyncio +async def test_replacing_an_icon_deletes_the_object_it_replaced(aws): + model = await _create() + first, _ = await upload_model_icon(model.id, _png(color=(200, 30, 30, 255))) + + second, _ = await upload_model_icon(model.id, _png(color=(30, 200, 90, 255))) + + assert first != second + with pytest.raises(aws["s3"].exceptions.NoSuchKey): + aws["s3"].get_object(Bucket=BUCKET, Key=first) + + +@pytest.mark.asyncio +async def test_removing_an_icon_clears_the_key_and_leaves_the_slug_alone(aws): + model = await _create(iconSlug="anthropic") + await upload_model_icon(model.id, _png()) + + icon_key, icon_url = await remove_model_icon(model.id) + + assert icon_key is None and icon_url is None + reread = await get_managed_model(model.id) + assert reread.icon_key is None + # The fallback the removal returns the model to — clearing an upload is not + # the same act as clearing the built-in logo. + assert reread.icon_slug == "anthropic" + + +@pytest.mark.asyncio +async def test_reading_an_icon_returns_the_bytes_and_its_cache_version(aws): + model = await _create() + icon_key, _ = await upload_model_icon(model.id, _png()) + + data, content_type, version = await read_model_icon(model.id) + + assert content_type == "image/png" + assert version == model_icon_version(icon_key) + assert Image.open(io.BytesIO(data)).size == (512, 512) + + +@pytest.mark.asyncio +async def test_reading_a_model_with_no_icon_is_a_404_not_a_500(aws): + """So the SPA's error path falls through to the slug rather than + rendering a broken tile.""" + model = await _create() + + with pytest.raises(ModelIconError) as excinfo: + await read_model_icon(model.id) + + assert excinfo.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_a_key_that_outlived_its_object_is_also_a_404(aws): + model = await _create() + icon_key, _ = await upload_model_icon(model.id, _png()) + aws["s3"].delete_object(Bucket=BUCKET, Key=icon_key) + + with pytest.raises(ModelIconError) as excinfo: + await read_model_icon(model.id) + + assert excinfo.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_uploading_to_a_missing_model_is_a_404(aws): + with pytest.raises(ModelIconError) as excinfo: + await upload_model_icon("no-such-model", _png()) + + assert excinfo.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_a_rejected_image_names_the_limit_it_broke(aws): + """An upload gate's failure mode is not "it let something through", it is + "it said no and the admin cannot tell why".""" + model = await _create() + + with pytest.raises(ModelIconError) as excinfo: + await upload_model_icon(model.id, _png(size=(512, 256))) + + assert excinfo.value.status_code == 400 + assert "square" in excinfo.value.message and "512×256" in excinfo.value.message + + +# ── the serve route's cache directives ─────────────────────────────────────── +# +# Driven through the real router, not a re-implementation of its branch: a test +# that mirrors the logic keeps passing when the route changes, which is exactly +# when it needs to fail. + + +def _icon_app(monkeypatch, *, version: str = "abc123"): + """The user-facing models router with auth stubbed and one icon on disk.""" + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from apis.app_api.models import routes + from apis.shared.auth.models import User + + app = FastAPI() + app.include_router(routes.router) + app.dependency_overrides[routes.get_current_user_from_session] = lambda: User( + user_id="u-1", email="reader@example.edu", name="Reader", roles=["User"] + ) + + async def _fake_read(model_id: str): + return b"\x89PNG-bytes", "image/png", version + + monkeypatch.setattr(routes, "read_model_icon", _fake_read) + return TestClient(app) + + +def test_the_versioned_url_is_cached_immutably(monkeypatch): + # ?v= names one specific object and can never mean anything else. + client = _icon_app(monkeypatch) + + response = client.get("/models/m-1/icon", params={"v": "abc123"}) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=31536000, immutable" + assert response.headers["etag"] == '"abc123"' + + +def test_the_bare_url_revalidates_instead_of_pinning_a_year(monkeypatch): + """The bare path tracks whatever the record points at now. + + Serving it ``immutable`` keeps a removed or replaced icon alive in every + cache that saw it — the removal simply never becomes visible. Caught in the + browser: a year-long response for the un-versioned path kept serving an icon + that had already been deleted. + """ + client = _icon_app(monkeypatch) + + response = client.get("/models/m-1/icon") + + assert response.status_code == 200 + assert response.headers["cache-control"] == "no-cache" + + +def test_a_stale_version_also_revalidates(monkeypatch): + # An old ?v= off a cached page must not be answered as if it were current. + client = _icon_app(monkeypatch) + + response = client.get("/models/m-1/icon", params={"v": "outdated"}) + + assert response.headers["cache-control"] == "no-cache" + + +def test_a_matching_etag_is_answered_304_without_the_bytes(monkeypatch): + client = _icon_app(monkeypatch) + + response = client.get( + "/models/m-1/icon", params={"v": "abc123"}, headers={"If-None-Match": '"abc123"'} + ) + + assert response.status_code == 304 + assert response.content == b"" diff --git a/backend/tests/shared/test_prompt_cache_observability.py b/backend/tests/shared/test_prompt_cache_observability.py index 8c2c25611..efda7e7af 100644 --- a/backend/tests/shared/test_prompt_cache_observability.py +++ b/backend/tests/shared/test_prompt_cache_observability.py @@ -464,7 +464,13 @@ class TestIncidentReplay: """ # Pricing implied by the incident's own numbers: 10.95M write tokens billed - # at $27.39 → $2.50/MTok write, against a $0.20/MTok read. + # at $27.39 → $2.50/MTok write, against a $0.20/MTok read. These are a + # HISTORICAL record of one August 2026 session and are load-bearing for this + # replay — do not "correct" them to current rates or the acceptance band + # stops meaning anything. They are not the cost model: the live rule is that + # cache write is 1.25x the model's own base input rate and cache read ~0.1x + # of it, with no flat per-MTok figure (see CLAUDE.md's prompt-cache + # contract). INCIDENT_PRICING = { "cacheWritePricePerMtok": 2.50, "cacheReadPricePerMtok": 0.20, diff --git a/backend/tests/shared/test_scoped_tool_grants.py b/backend/tests/shared/test_scoped_tool_grants.py new file mode 100644 index 000000000..f9e6bd32d --- /dev/null +++ b/backend/tests/shared/test_scoped_tool_grants.py @@ -0,0 +1,128 @@ +"""A scoped tool id (``base::tool``) is granted by a grant on its **base** server. + +Scoping narrows a grant, it never widens one: an Agent binding +``canvas_faculty::list_courses`` asks for strictly less than one binding +``canvas_faculty``, so the same role grant must admit it. Before this, the +binding gate ``AppRoleService.can_access_tool`` exact-matched the id, so a +scoped binding was denied for every user — including one holding the whole +server — while the sibling ``filter_requested_tools`` on the ``enabled_tools`` +axis already base-collapsed correctly. The two must stay in agreement, or the +chat picker and the Agent Designer disagree about the same subset. + +Real ``AppRole`` records throughout (no predicate stubs), so these fail if the +grant semantics move. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from apis.shared.tools import freshness + +GRANTED_SERVER = "canvas_faculty" +PUBLIC_SERVER = "fetch_url_content" +PRIVATE_SERVER = "gmail_employee" + + +def _catalog_tool(tool_id: str, is_public: bool): + repo_tool = MagicMock() + repo_tool.tool_id = tool_id + repo_tool.is_public = is_public + return repo_tool + + +@pytest.fixture(autouse=True) +def _catalog(): + freshness._reset_for_tests() + repo = MagicMock() + repo.list_tools = AsyncMock( + return_value=[ + _catalog_tool(GRANTED_SERVER, is_public=False), + _catalog_tool(PUBLIC_SERVER, is_public=True), + _catalog_tool(PRIVATE_SERVER, is_public=False), + ] + ) + with patch( + "apis.shared.tools.repository.get_tool_catalog_repository", + return_value=repo, + ): + yield + freshness._reset_for_tests() + + +def _service(granted_tools): + from apis.shared.rbac.cache import AppRoleCache + from apis.shared.rbac.models import AppRole, EffectivePermissions + from apis.shared.rbac.service import AppRoleService + + repo = AsyncMock() + repo.get_roles_for_jwt_role.return_value = ["faculty"] + repo.get_role.return_value = AppRole( + role_id="faculty", + display_name="faculty", + description="test", + jwt_role_mappings=["faculty"], + priority=10, + enabled=True, + effective_permissions=EffectivePermissions(tools=granted_tools, models=["*"]), + ) + return AppRoleService(repository=repo, cache=AppRoleCache()) + + +@pytest.fixture +def svc(): + """A role granting one MCP server and nothing else.""" + return _service([GRANTED_SERVER]) + + +def _user(): + user = MagicMock() + user.user_id = "u1" + user.email = "prof@example.edu" + user.roles = ["faculty"] + return user + + +class TestCanAccessTool: + """The gate an Agent's tool binding is re-resolved through (D5).""" + + @pytest.mark.asyncio + async def test_scoped_id_admitted_by_a_grant_on_its_base_server(self, svc): + assert await svc.can_access_tool(_user(), f"{GRANTED_SERVER}::list_courses") is True + + @pytest.mark.asyncio + async def test_bare_id_still_admitted(self, svc): + assert await svc.can_access_tool(_user(), GRANTED_SERVER) is True + + @pytest.mark.asyncio + async def test_scoped_id_denied_when_its_base_is_not_granted(self, svc): + """Scoping narrows a grant; it cannot manufacture one.""" + assert await svc.can_access_tool(_user(), f"{PRIVATE_SERVER}::send_mail") is False + + @pytest.mark.asyncio + async def test_scoped_id_admitted_when_its_base_is_public(self, svc): + """A public tool is a grant, so it admits subsets like any other.""" + assert await svc.can_access_tool(_user(), f"{PUBLIC_SERVER}::fetch") is True + + @pytest.mark.asyncio + async def test_wildcard_admits_a_scoped_id(self): + assert await _service(["*"]).can_access_tool(_user(), f"{PRIVATE_SERVER}::send_mail") is True + + @pytest.mark.asyncio + async def test_agrees_with_filter_requested_tools(self, svc): + """The bindings axis and the enabled_tools axis must answer alike. + + They diverged before this change — the reason a subset the chat picker + happily sent was rejected as an Agent binding. + """ + candidates = [ + GRANTED_SERVER, + f"{GRANTED_SERVER}::list_courses", + PUBLIC_SERVER, + f"{PUBLIC_SERVER}::fetch", + PRIVATE_SERVER, + f"{PRIVATE_SERVER}::send_mail", + ] + by_filter = set(await svc.filter_requested_tools(_user(), candidates)) + for tool_id in candidates: + assert await svc.can_access_tool(_user(), tool_id) is (tool_id in by_filter), tool_id diff --git a/backend/tests/test_backfill_tool_catalog_index.py b/backend/tests/test_backfill_tool_catalog_index.py new file mode 100644 index 000000000..825a32ee6 --- /dev/null +++ b/backend/tests/test_backfill_tool_catalog_index.py @@ -0,0 +1,202 @@ +"""Tests for the tool-catalog EntityTypeIndex backfill. + +The stakes are asymmetric: a row this script misses is not stale in the index, +it is absent from a sparse index forever — and once `list_tools` queries that +index, an absent row is a tool that silently vanished from every user's +catalog. So the tests care most about *which rows get stamped* and *which are +left alone*, not just the happy path. +""" + +import os +import sys + +import boto3 +import pytest +from moto import mock_aws + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) + +from backfill_tool_catalog_index import ( # noqa: E402 + ENTITY_TYPE_TOOL, + backfill, + plan_row, +) + +REGION = "us-east-1" +TABLE = "test-app-roles" + + +@pytest.fixture() +def aws(monkeypatch): + monkeypatch.setenv("AWS_DEFAULT_REGION", REGION) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_SESSION_TOKEN", "testing") + with mock_aws(): + yield + + +@pytest.fixture() +def table(aws): + ddb = boto3.client("dynamodb", region_name=REGION) + ddb.create_table( + TableName=TABLE, + KeySchema=[ + {"AttributeName": "PK", "KeyType": "HASH"}, + {"AttributeName": "SK", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "PK", "AttributeType": "S"}, + {"AttributeName": "SK", "AttributeType": "S"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + return boto3.resource("dynamodb", region_name=REGION).Table(TABLE) + + +def _seed_shared_table(table): + """The app-roles table as it really is — tools mixed with everything else.""" + rows = [ + {"PK": "TOOL#calculator", "SK": "METADATA", "toolId": "calculator"}, + {"PK": "TOOL#web_search", "SK": "METADATA", "toolId": "web_search"}, + # Not tools, and must not be stamped: + {"PK": "TOOL#calculator", "SK": "CAPABILITIES", "prompts": []}, + {"PK": "SKILL#research", "SK": "METADATA", "skillId": "research"}, + {"PK": "ROLE#student", "SK": "DEFINITION", "roleId": "student"}, + {"PK": "ROLE#student", "SK": "TOOL_GRANT#calculator"}, + {"PK": "USER#u1", "SK": "TOOL_PREFERENCES", "userId": "u1"}, + ] + for r in rows: + table.put_item(Item=r) + + +class TestPlanRow: + def test_derives_both_keys_from_the_pk(self): + plan = plan_row({"PK": "TOOL#calculator", "SK": "METADATA"}) + assert plan == {"gsi5pk": ENTITY_TYPE_TOOL, "gsi5sk": "TOOL#calculator"} + + def test_already_stamped_row_needs_nothing(self): + assert plan_row( + {"PK": "TOOL#calculator", "SK": "METADATA", "GSI5PK": ENTITY_TYPE_TOOL} + ) is None + + def test_unexpected_pk_is_skipped_not_guessed(self): + assert "skip" in plan_row({"PK": "SKILL#research", "SK": "METADATA"}) + + def test_empty_tool_id_is_skipped(self): + assert "skip" in plan_row({"PK": "TOOL#", "SK": "METADATA"}) + + def test_key_ignores_a_disagreeing_toolid_attribute(self): + """Identity comes from the PK, so the index cannot diverge from the + base table even if `toolId` is wrong or missing.""" + plan = plan_row({"PK": "TOOL#real", "SK": "METADATA", "toolId": "WRONG"}) + assert plan["gsi5sk"] == "TOOL#real" + + +class TestBackfill: + def test_dry_run_writes_nothing(self, table): + _seed_shared_table(table) + + stats = backfill(table, apply=False) + + assert stats["stamped"] == 2 + for tool_id in ("calculator", "web_search"): + item = table.get_item( + Key={"PK": f"TOOL#{tool_id}", "SK": "METADATA"} + )["Item"] + assert "GSI5PK" not in item + + def test_stamps_only_tool_metadata_rows(self, table): + _seed_shared_table(table) + + stats = backfill(table, apply=True) + + assert stats["tool_rows"] == 2 + assert stats["stamped"] == 2 + assert stats["skipped"] == 0 + assert stats["failed"] == 0 + + for tool_id in ("calculator", "web_search"): + item = table.get_item( + Key={"PK": f"TOOL#{tool_id}", "SK": "METADATA"} + )["Item"] + assert item["GSI5PK"] == ENTITY_TYPE_TOOL + assert item["GSI5SK"] == f"TOOL#{tool_id}" + + # Everything that is not a tool stays untouched — the tool partition of + # the index must contain exactly the tool catalog. + for key in ( + {"PK": "TOOL#calculator", "SK": "CAPABILITIES"}, + {"PK": "SKILL#research", "SK": "METADATA"}, + {"PK": "ROLE#student", "SK": "DEFINITION"}, + {"PK": "ROLE#student", "SK": "TOOL_GRANT#calculator"}, + {"PK": "USER#u1", "SK": "TOOL_PREFERENCES"}, + ): + assert "GSI5PK" not in table.get_item(Key=key)["Item"] + + def test_is_idempotent(self, table): + _seed_shared_table(table) + backfill(table, apply=True) + + second = backfill(table, apply=True) + + assert second["stamped"] == 0 + assert second["already"] == 2 + assert second["failed"] == 0 + + def test_leaves_a_row_the_writer_already_stamped(self, table): + _seed_shared_table(table) + table.put_item( + Item={ + "PK": "TOOL#fresh", + "SK": "METADATA", + "toolId": "fresh", + "GSI5PK": ENTITY_TYPE_TOOL, + "GSI5SK": "TOOL#fresh", + } + ) + + stats = backfill(table, apply=True) + + assert stats["already"] == 1 + assert stats["stamped"] == 2 + + def test_paginates_past_one_scan_page(self, table): + """A short table would hide a missing ExclusiveStartKey, and the rows + beyond page one would be the ones silently dropped from the index.""" + for i in range(120): + table.put_item( + Item={"PK": f"TOOL#t{i:03d}", "SK": "METADATA", "toolId": f"t{i:03d}"} + ) + + stats = backfill(table, apply=True) + + assert stats["tool_rows"] == 120 + assert stats["stamped"] == 120 + + +class TestContractWithTheWriter: + def test_partition_value_matches_the_model(self): + """The script hardcodes the constant so it can run standalone; this is + what stops the two drifting.""" + from apis.shared.tools.models import ENTITY_TYPE_TOOL as MODEL_CONSTANT + + assert ENTITY_TYPE_TOOL == MODEL_CONSTANT + + def test_backfilled_keys_match_what_the_writer_would_write(self): + """A stamped row must be byte-identical to a freshly written one, or + the index would sort or resolve differently for old vs new tools.""" + from apis.shared.tools.models import ToolDefinition + + written = ToolDefinition( + tool_id="calculator", + display_name="Calculator", + description="d", + category="utility", + protocol="local", + ).to_dynamo_item() + + planned = plan_row({"PK": "TOOL#calculator", "SK": "METADATA"}) + + assert planned["gsi5pk"] == written["GSI5PK"] + assert planned["gsi5sk"] == written["GSI5SK"] diff --git a/backend/tests/test_seed_tool_index_keys.py b/backend/tests/test_seed_tool_index_keys.py new file mode 100644 index 000000000..a54ead737 --- /dev/null +++ b/backend/tests/test_seed_tool_index_keys.py @@ -0,0 +1,70 @@ +"""The bootstrap seeder's tool rows must carry the same index keys the model writes. + +`seed_bootstrap_data.py` hand-builds its tool item instead of going through +`ToolDefinition.to_dynamo_item`, so every index key exists in TWO places and a +new one can be added to the model while the seeder is forgotten. + +That omission is invisible in the worst way. A freshly bootstrapped deployment — +a fork, a new environment, a rebuilt dev — would seed tools with no index keys, +and once the catalog read moves to `EntityTypeIndex` it would list ZERO tools. No +error: a sparse index answers "nothing matched", not "something is wrong". And a +backfill would not be the fix, because nothing about that install is legacy. + +This test is the thing that fails instead. +""" + +import os +import re +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) + +from apis.shared.tools.models import ENTITY_TYPE_TOOL, ToolDefinition # noqa: E402 + +SEED_SCRIPT = os.path.join( + os.path.dirname(__file__), "..", "scripts", "seed_bootstrap_data.py" +) + + +def _model_index_keys() -> set[str]: + """Every GSI key attribute `to_dynamo_item` writes on a tool row.""" + item = ToolDefinition( + tool_id="probe", + display_name="Probe", + description="d", + category="utility", + protocol="local", + ).to_dynamo_item() + return {k for k in item if re.fullmatch(r"GSI\d+(PK|SK)", k)} + + +def _seeded_tool_block() -> str: + """The seeder's hand-built tool item literal.""" + source = open(SEED_SCRIPT, encoding="utf-8").read() + start = source.index('"GSI1PK": f"CATEGORY#') + return source[start : start + 1200] + + +class TestSeederMirrorsTheModel: + def test_seeder_writes_every_index_key_the_model_does(self): + block = _seeded_tool_block() + missing = sorted(k for k in _model_index_keys() if f'"{k}"' not in block) + + assert not missing, ( + f"seed_bootstrap_data.py does not write {missing} on its tool rows. " + "It hand-builds the item rather than calling " + "ToolDefinition.to_dynamo_item, so a new index key must be added in " + "both places — otherwise a freshly bootstrapped deployment is absent " + "from that index, silently." + ) + + def test_seeder_uses_the_shared_entity_type_constant(self): + """A literal that drifts from ENTITY_TYPE_TOOL puts seeded tools in a + partition the reader never queries — the same silent-empty failure.""" + assert f'"{ENTITY_TYPE_TOOL}"' in _seeded_tool_block() + + def test_model_is_the_source_of_truth_for_this_test(self): + """Guards the test itself: if the probe stops producing index keys the + assertions above would pass vacuously.""" + keys = _model_index_keys() + assert {"GSI1PK", "GSI1SK", "GSI5PK", "GSI5SK"} <= keys diff --git a/backend/tests/test_tool_catalog_index_read.py b/backend/tests/test_tool_catalog_index_read.py new file mode 100644 index 000000000..a88bda570 --- /dev/null +++ b/backend/tests/test_tool_catalog_index_read.py @@ -0,0 +1,240 @@ +"""`list_tools` reads EntityTypeIndex, and must never answer "no tools" wrongly. + +The Query is the optimization; the fallbacks are the reason it is safe to ship. +An empty tool catalog is not a degraded experience — every user loses every +tool — and both ways this index can fail to answer produce exactly that unless +something catches them: + +* the index is absent (deploy race), which raises; and +* the index is present but unpopulated (backfill not run), which does NOT raise — + a sparse index simply matches nothing. + +These tests exist for the fallbacks more than for the happy path. +""" + +import boto3 +import pytest +from botocore.exceptions import ClientError +from moto import mock_aws + +from apis.shared.caching import config_cache +from apis.shared.tools.models import ENTITY_TYPE_TOOL, ToolDefinition +from apis.shared.tools.repository import ENTITY_TYPE_INDEX, ToolCatalogRepository + +REGION = "us-east-1" +TABLE = "test-app-roles-index-read" + +KEY_SCHEMA = [ + {"AttributeName": "PK", "KeyType": "HASH"}, + {"AttributeName": "SK", "KeyType": "RANGE"}, +] +ATTRS = [ + {"AttributeName": "PK", "AttributeType": "S"}, + {"AttributeName": "SK", "AttributeType": "S"}, + {"AttributeName": "GSI5PK", "AttributeType": "S"}, + {"AttributeName": "GSI5SK", "AttributeType": "S"}, + {"AttributeName": "GSI1PK", "AttributeType": "S"}, + {"AttributeName": "GSI1SK", "AttributeType": "S"}, +] + +# The category path queries this one; it exists on the real table and is +# deliberately untouched by this change. +CATEGORY_INDEX = { + "IndexName": "JwtRoleMappingIndex", + "KeySchema": [ + {"AttributeName": "GSI1PK", "KeyType": "HASH"}, + {"AttributeName": "GSI1SK", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, +} +INDEX = { + "IndexName": ENTITY_TYPE_INDEX, + "KeySchema": [ + {"AttributeName": "GSI5PK", "KeyType": "HASH"}, + {"AttributeName": "GSI5SK", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, +} + + +@pytest.fixture() +def aws(monkeypatch): + monkeypatch.setenv("AWS_DEFAULT_REGION", REGION) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_SESSION_TOKEN", "testing") + with mock_aws(): + yield + + +def _make_table(with_index: bool): + # Without EntityTypeIndex the table still has the category index, matching + # the real pre-deploy shape rather than a table with no indexes at all. + attrs = ATTRS if with_index else [a for a in ATTRS if not a["AttributeName"].startswith("GSI5")] + kwargs = { + "TableName": TABLE, + "KeySchema": KEY_SCHEMA, + "AttributeDefinitions": attrs, + "BillingMode": "PAY_PER_REQUEST", + "GlobalSecondaryIndexes": [CATEGORY_INDEX] + ([INDEX] if with_index else []), + } + boto3.client("dynamodb", region_name=REGION).create_table(**kwargs) + return boto3.resource("dynamodb", region_name=REGION).Table(TABLE) + + +def _tool(tool_id: str, stamped: bool = True) -> dict: + item = ToolDefinition( + tool_id=tool_id, + display_name=tool_id.title(), + description="d", + category="utility", + protocol="local", + ).to_dynamo_item() + if not stamped: + # A row written before the keys existed — the pre-backfill state. + item.pop("GSI5PK", None) + item.pop("GSI5SK", None) + return item + + +def _seed_noise(table): + """The other tenants of this shared table. None may reach the catalog.""" + table.put_item(Item={"PK": "SKILL#research", "SK": "METADATA"}) + table.put_item(Item={"PK": "ROLE#student", "SK": "DEFINITION"}) + table.put_item(Item={"PK": "USER#u1", "SK": "TOOL_PREFERENCES"}) + table.put_item(Item={"PK": "TOOL#calculator", "SK": "CAPABILITIES"}) + + +def _repo(monkeypatch): + monkeypatch.setenv("DYNAMODB_APP_ROLES_TABLE_NAME", TABLE) + config_cache.get_config_cache().clear() + return ToolCatalogRepository(table_name=TABLE) + + +class TestQueryPath: + @pytest.mark.asyncio + async def test_reads_the_catalog_off_the_index(self, aws, monkeypatch): + table = _make_table(with_index=True) + _seed_noise(table) + for t in ("calculator", "web_search"): + table.put_item(Item=_tool(t)) + + tools = await _repo(monkeypatch).list_tools() + + assert sorted(t.tool_id for t in tools) == ["calculator", "web_search"] + + @pytest.mark.asyncio + async def test_index_excludes_the_shared_table_noise(self, aws, monkeypatch): + """The whole point: skills, roles and per-user rows never come back.""" + table = _make_table(with_index=True) + _seed_noise(table) + table.put_item(Item=_tool("calculator")) + + tools = await _repo(monkeypatch).list_tools() + + assert [t.tool_id for t in tools] == ["calculator"] + + @pytest.mark.asyncio + async def test_a_genuinely_empty_catalog_is_empty(self, aws, monkeypatch): + table = _make_table(with_index=True) + _seed_noise(table) + + assert await _repo(monkeypatch).list_tools() == [] + + +class TestMissingIndexFallback: + @pytest.mark.asyncio + async def test_serves_the_catalog_when_the_index_does_not_exist( + self, aws, monkeypatch + ): + """The deploy race: backend ships before platform. Must NOT go empty — + we have a correct answer available, so serve it.""" + table = _make_table(with_index=False) + _seed_noise(table) + for t in ("calculator", "web_search"): + table.put_item(Item=_tool(t)) + + tools = await _repo(monkeypatch).list_tools() + + assert sorted(t.tool_id for t in tools) == ["calculator", "web_search"] + + @pytest.mark.asyncio + async def test_a_real_query_error_still_propagates(self, aws, monkeypatch): + """Only 'index missing' is absorbed. Throttling must not read as + 'there are no tools' — that is a lie about the data.""" + _make_table(with_index=True) + repo = _repo(monkeypatch) + + def boom(**_kwargs): + raise ClientError( + {"Error": {"Code": "ProvisionedThroughputExceededException", + "Message": "slow down"}}, + "Query", + ) + + monkeypatch.setattr(repo._table, "query", boom) + + with pytest.raises(ClientError): + await repo.list_tools() + + +class TestUnbackfilledIndexFallback: + @pytest.mark.asyncio + async def test_serves_the_catalog_when_the_index_is_unpopulated( + self, aws, monkeypatch + ): + """The silent case: index exists, backfill never ran, Query succeeds with + zero rows and raises nothing. Without the guard every user loses every + tool and no error is logged anywhere.""" + table = _make_table(with_index=True) + _seed_noise(table) + for t in ("calculator", "web_search"): + table.put_item(Item=_tool(t, stamped=False)) + + tools = await _repo(monkeypatch).list_tools() + + assert sorted(t.tool_id for t in tools) == ["calculator", "web_search"] + + @pytest.mark.asyncio + async def test_says_loudly_that_the_backfill_has_not_run( + self, aws, monkeypatch, caplog + ): + table = _make_table(with_index=True) + table.put_item(Item=_tool("calculator", stamped=False)) + + with caplog.at_level("ERROR"): + await _repo(monkeypatch).list_tools() + + assert "backfill_tool_catalog_index.py" in caplog.text + + @pytest.mark.asyncio + async def test_empty_table_logs_nothing_alarming(self, aws, monkeypatch, caplog): + """A fresh install before seeding is not a broken backfill.""" + _make_table(with_index=True) + + with caplog.at_level("ERROR"): + assert await _repo(monkeypatch).list_tools() == [] + + assert "backfill" not in caplog.text + + +class TestCategoryPathUnchanged: + @pytest.mark.asyncio + async def test_category_filter_still_uses_its_own_index(self, aws, monkeypatch): + """The category read is a separate bounded GSI query and is deliberately + untouched — it must not start returning the whole catalog.""" + table = _make_table(with_index=True) + table.put_item(Item=_tool("calculator")) + + repo = _repo(monkeypatch) + called = {"n": 0} + original = repo._query_tool_items_by_category + + def spy(category): + called["n"] += 1 + return original(category) + + monkeypatch.setattr(repo, "_query_tool_items_by_category", spy) + await repo.list_tools(category="utility") + + assert called["n"] == 1 diff --git a/backend/uv.lock b/backend/uv.lock index a0afe6749..679f9dcee 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.15'", @@ -10,9 +10,12 @@ resolution-markers = [ "python_full_version < '3.11'", ] +[manifest] +constraints = [{ name = "mcp", specifier = "<2" }] + [[package]] name = "agentcore-stack" -version = "1.20.0" +version = "1.21.0" source = { editable = "." } dependencies = [ { name = "aiofiles" }, diff --git a/docs/kaizen/research/2026-09-11.md b/docs/kaizen/research/2026-09-11.md new file mode 100644 index 000000000..d2d936d36 --- /dev/null +++ b/docs/kaizen/research/2026-09-11.md @@ -0,0 +1,363 @@ +# Kaizen Research — Friday, September 11, 2026 + +> Scan window: **September 4 – September 11, 2026 (7 days)**. +> Web budget: **15 / 50** used (well under target — seven of eleven subagents ran entirely through the authenticated `gh` CLI and `curl`, which cost nothing against the web budget). + +## TL;DR + +**The nightly build has been red for seven consecutive nights and nobody noticed, because the thing that broke can only break at night.** `Nightly Build & Test` was green through 2026-09-04 and has failed every run since ([33957119927](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/33957119927) → [34582692457](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/34582692457)). Two independent root causes, neither a flake. The nightly coverage job runs `ng test --no-watch --coverage`; PR CI runs `ng test --watch=false`. Under `--coverage` the six filesystem-reading branding guard/golden specs that landed with PR #933 resolve their paths **two directory levels shallower** — `rebranding-guide.doc-presence.spec.ts` reads `frontend/ai.client/README.md` (`# AiClient / This project was genera…`) instead of `src/branding/README.md` (`# Rebranding Guide`), and `prestart-generator-parity.spec.ts` fails at *collection* on a missing `../../scripts/branding`. **PR CI structurally cannot catch this**, because it never passes `--coverage`. Separately, `nightly-develop-PlatformStack` is wedged in `DELETE_FAILED` and every nightly deploy now dies on `is in DELETE_FAILED state and can not be updated`. + +**Upstream finally paid out on the standing prompt-caching watch, and the payout is already inside our pin.** Strands `CacheConfig` at 1.55.0 carries `system_prompt_ttl` (default `True`) and `tools_ttl` alongside `strategy`/`ttl`/`cache_key`, `strands/models/_openai_cache.py` exists, and issue [#4168](https://github.com/strands-agents/harness-sdk/issues/4168) — the `AccessDeniedException` our `bedrock_cache_points_supported()` gate was defending against — **closed 2026-09-04**, fixed in the 1.55.x line. The upstream docstring says a hand-placed system cache point is *"honored rather than doubled"*, so the migration off our hand-placed `SystemContentBlock` cachePoint can be attempted without a double-write risk. + +**And the provenance question that has been marked ⚠️ in the queue for a week is now settled, against us.** A full diff of the republished `AmazonBedrock` offer file (`20260804165549` → `20260911124408`, 10,998 → 11,657 SKUs) reproduces last week's result exactly: the entire file contains **10 Claude SKUs, none newer than Claude 3**. `CLAUDE.md`'s assertion that our rates "come from the AWS Price List API, not the pricing page" is **false for every Claude model we bill against**. The authoritative route is the per-model AWS model card; the Price List API is authoritative for xAI, Google and AgentCore. + +Recommended **#1** is the internal one: **fix the nightly, both halves.** A gate that has been red for a week is not a gate. + +## External Scan + +### What's moving this week + +The week's shape is *a frontier tier arriving with a pricing trap attached, while the protocol layer holds still and the SDK layer quietly hands us subtractions.* + +The headline is **OpenAI GPT-6 Astra, GA on Bedrock 2026-09-08** — 1,050,000-token context, 128,000 max output, CRIS-only. But it ships with a **two-tier price** that is the single most consequential external fact in the scan: Short Context (≤272K) is $11.00/$13.75/$1.10/$55.00 per MTok (in / cache-write / cache-read / out) on Geo CRIS; Long Context (1.05M) is **$22.00/$27.50/$2.20/$82.50** — exactly 2× on input. The reference repo registered it on 2026-09-09 as `maxInputTokens: 1000000`. We already set a *deliberate* `maxInputTokens: 272000` pricing cap on GPT-5.6 for precisely this reason, so copying their catalog row would opt the whole fleet into the expensive tier. The model card does not say whether the tier is selected by declared window or by actual token count — which is the one thing you must know before enabling it. + +The protocol layer, by contrast, did nothing. No new MCP spec revision (still 2026-07-28), no movement on SEP-2567 / SEP-2549 / SEP-2243 — all three landed in July and their tracking issues closed 2026-08-23. FastMCP shipped one patch (4.0.3). The MCP Apps SDK went to **v2.0.0**, but the release notes are explicit that the **wire protocol is unchanged**, with a bidirectional interop test against 1.7.5 — so our `ui_resource` host path needs nothing. Buried in it, though, is a genuine subtraction: `schema.json` now documents `io.modelcontextprotocol/serverInfo` in result `_meta`, which is a standards-blessed route to the `serverName`/`icon` we currently steal via a monkeypatch on a Strands internal. + +The SDK layer is where the quiet value is. Strands 1.55.1 is a safe patch with two fixes on our surfaces; upstream `main` is building a first-party context-manager/offloading stack (stash, offloading strategies, session integration) that is a future collision with our `ContextOffloader`; and `cache_key` on `main` has widened to auto-derive `strands-` when a session manager is present, which will start transmitting on the next minor. + +One reported-as-urgent item **did not survive verification**, and that is worth as much as the positives — see AgentCore SDK #659 below. + +### Notable items by source + +#### AWS Bedrock / AgentCore + +- **AgentCore Memory: direct long-term ingestion (`IngestData`)** — submits content straight into long-term memory extraction without first writing it to short-term/event storage — https://aws.amazon.com/about-aws/whats-new/2026/09/agentcore-memory-direct-ingest — *relevance*: `TurnBasedSessionManager`, the compaction path, session restore — *unlocks*: a route to seed durable memory (compaction summaries, Memory Spaces markdown) **without** round-tripping through the conversation event stream, which is today the only way a fact reaches long-term memory. Directly relevant to the W5 memory-cost track, where the self-managed strategy rate is $0.25/1,000 records vs $0.75 built-in. +- **Managed Knowledge Base: document ACL debugging APIs** — `CheckIngestedDocumentAcl` / `GetIngestedDocumentAcl` inspect why a given document is or is not visible to a principal — https://aws.amazon.com/about-aws/whats-new/2026/09/amazon-bedrock-knowledge-base-debugging-document-access-control/ — *relevance*: the Classic→Managed KB migration and RBAC — *unlocks*: per-document access introspection during cutover, instead of inferring filtering behaviour from retrieval results. Lands next to the dead-letter reconciler shipped in 1.20.0. +- **Managed Knowledge Base: Confluence Data Center connector** — native crawl of self-hosted Confluence spaces (previously Cloud-only) — https://aws.amazon.com/about-aws/whats-new/2026/09/amazon-bedrock-managed-knowledge-base-confluence-data-center-native-data-source-connector/ — *relevance*: Managed KB / `rag-ingestion` — *unlocks*: on-prem Confluence as a first-class source with no custom ingestion path. +- **⚠️ Flagged, not confirmed in-window: Consent Portal for AgentCore Identity.** The devguide release-notes page lists it under September 2026 but dates entries only to the month, and its What's New post is dated **2026-09-01** — outside the window — https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/release-notes.html. Worth a first-class look next cycle given our hand-rolled `OAuthConsentHook` and the pre-flight `oauth_required` flow. Same page also lists *Evaluations: TypeScript agent framework support*. + +#### Strands Agents + +- **`strands-agents` 1.55.1 (2026-09-09) — patch-only, no breaking changes vs our 1.55.0 pin** — https://github.com/strands-agents/harness-sdk/releases/tag/python%2Fv1.55.1 — *relevance*: two of five fixes land on us — `fix(session): filter malformed immutable snapshot IDs (#4199)` on `take_snapshot`/`load_snapshot`, and `fix(context): context manager parity (#4228)`. Also `#4154`: orphaned `toolResults` are now removed at **any** index, not just the first — diff that against our `_repair_tool_pairing`. +- **Provider-agnostic `CacheConfig` is real and already shipped in 1.55.0** — fields at 1.55.0/1.55.1 are `strategy: Literal["auto","anthropic"]`, `ttl`, `system_prompt_ttl: bool | str = True`, `cache_key: str | None`, `tools_ttl: bool | str | None`; `strands/models/_openai_cache.py` exists — https://github.com/strands-agents/harness-sdk/blob/python/v1.55.1/strands-py/src/strands/models/model.py — *relevance*: `core/model_config.py` `CacheConfig(strategy="auto")` and the hand-placed system `cachePoint` at `core/agent_factory.py` — *unlocks*: `system_prompt_ttl` / `tools_ttl` are candidates to **replace** the hand-placed point. The docstring states a hand-placed system cache point is *"honored rather than doubled"*, which de-risks the migration. +- **Issue #4168 (Bedrock `cachePoint` `AccessDeniedException`) CLOSED 2026-09-04**, fixed by `fix: gate the deprecated cache_tools point on the resolved cache strategy (#4186)`, in the 1.55.x line — https://github.com/strands-agents/harness-sdk/issues/4168 — *relevance*: this is the exact failure our `bedrock_cache_points_supported()` gate exists to prevent, and last week's queue entry called it "load-bearing, not redundant." At our `develop` pin it is fixed upstream. The gate may now be redundant — **verify before deleting**, don't infer it. +- **⏳ PR #4193 (`cache_write_tokens` → `cacheWriteInputTokens`, ours) still OPEN** — opened 2026-09-05, `REVIEW_REQUIRED`, zero comments, unreleased — https://github.com/strands-agents/harness-sdk/pull/4193 — *relevance*: half of `usage_normalization.py` still cannot retire. +- **⏳ Issue #3546 (disjoint `Usage` convention) OPEN, touched 2026-09-07** — activity in-window, no resolution — https://github.com/strands-agents/harness-sdk/issues/3546 — *relevance*: the other half of `usage_normalization.py`. +- **On `main` (unreleased): `cache_key` widens to `str | Literal[False] | None` and auto-derives `strands-` when the agent has a session manager** (`feat: automatically use openAI prompt-cache keys from session id (#4083)`) — https://github.com/strands-agents/harness-sdk/blob/main/strands-py/src/strands/models/model.py — *relevance*: our `TurnBasedSessionManager` subclasses `AgentCoreMemorySessionManager`, so the next minor will start transmitting a session-derived cache key by default. Bedrock/Anthropic ignores it; note it before adopting any OpenAI-family provider. This is `build_prompt_cache_key()`, upstream. +- **Upstream is building a first-party context-manager / offloading stack (post-1.55.1, unreleased)** — `feat(context-py): add session manager integration (#4254)`, `port stash integration (#4187)`, `port offloading strategies (#4146)`, `add session support for L1 (#4118)`, `export context manager types as experimental (#4231)` — https://github.com/strands-agents/harness-sdk/commits/main — *relevance*: our `ContextOffloader` and `TurnBasedSessionManager`. Read it before extending ours. ⚠️ Note the standing decisions-log entry: a bare "adopt the built-in, delete ours" proposal is **out of scope** and must not be re-surfaced without a migration design covering tool-content truncation, LTM summary retrieval, DynamoDB checkpoint persistence, and the `compaction` SSE-once invariant. +- **MCP client moved under us** — `fix(mcp-py): added httpx2 adapter (#4183)`, **`feat(mcp-py): relax mcp version to <2.2 (#4151)`**, `feat(mcp/py): support SEP-2663 tasks (#4125)`, `feat(mcp): prefix_with_server_name + continue_on_error defaults on load_servers (#4177)` — same release URL — *relevance*: the relaxed `mcp` floor means a transitive resolve could pull **mcp 2.x** under us while `uv.lock` currently resolves 1.28.1. Our `ClientSession` monkeypatch must be re-verified against both major lines. + +#### Reference repo (aws-samples/sample-strands-agent-with-agentcore) + +Quiet week: 8 commits on `main`, 6 Dependabot, 2 substantive. + +- **PR #290 — `fix: allow duplicate document uploads in a conversation`** — adds `ensure_unique_document_names(messages, current_message)`, walking all of `agent.messages` plus the new turn and renaming case-folded `document.name` collisions to `report-2`, `report-3`… Reason: Bedrock Converse **400s the whole request** when two `document` blocks share a name — https://github.com/aws-samples/sample-strands-agent-with-agentcore/pull/290 — *applicability*: **the bug applies to us; their fix would be a cache regression if copied verbatim.** `DocumentHandler.create_content_block` (`backend/src/agents/main_agent/multimodal/document_handler.py:83`) sets `"name": sanitized_name` with no cross-turn dedup, and `apis/shared/sessions/messages.py:88` defaults to the literal `"document"` on restore — so two restored attachments with missing names collide. Their implementation **mutates prior messages in place**, rewriting the cacheable prefix whenever a new collision appears mid-conversation — exactly the `historyHash` drift our prompt-cache contract forbids. Port the *detection*; disambiguate **only the incoming turn's** blocks against a seen-set derived from history, and never rename a historical block. +- **PR #291 — `feat: add GPT-6 Astra model`** — catalog-only: `us.openai.gpt-6-astra`, `transport: bedrock_responses`, **`maxInputTokens: 1000000`**, `rejectsTemperature: true`, catalog `revision` bumped to `2026-09-09.1` — https://github.com/aws-samples/sample-strands-agent-with-agentcore/pull/291 — *applicability*: see Idea #3 — their `maxInputTokens` is the trap. Their `catalogRevision` stamped onto each delegation record (`record["modelSelection"]["catalogRevision"]`) is a pattern we lack and would complement `GET /admin/costs/sessions/{id}/calls` — it makes "which catalog generation priced this call?" answerable after the fact. +- **They pin `strands-agents[a2a,otel]==1.54.0` and `bedrock-agentcore==1.22.0`** — one minor behind our `develop`; no 1.54→1.55 commit in the window. +- **They do NOT hand-place a system cachePoint.** `prompt_builder.py` mentions `cachePoint` only as an optional TypedDict key; `build_text_system_prompt()` returns two plain text blocks and relies on Strands' default. ⚠️ Two consequences: their green build is **not** evidence for our migration (they have nothing to collide), and they put a **daily-rotating date string in the system prompt's own blocks**, invalidating the system prefix at the Pacific-midnight boundary for every live session — a worked example of the exact waste our `systemPromptHash` exists to detect. + +#### MCP ecosystem + +- **No new spec revision or changelog entry in the window** — current rev stays **2026-07-28**. Only two PRs merged into `modelcontextprotocol/modelcontextprotocol`: a dev-dep bump (#3344) and a working-group proposal (#3282) — https://github.com/modelcontextprotocol/modelcontextprotocol/pull/3282. Blog's newest posts are 2026-08-22 and 2026-07-28 — https://blog.modelcontextprotocol.io. +- **No movement on SEP-2567 / SEP-2549 / SEP-2243.** All three landed in the 2026-07-28 revision; tracking issues #2883, #2882, #2875 all closed 2026-08-23. Only residual open item is SEP-2549 #2939 ("is `cacheScope` required or does it default to `public`"), untouched since 2026-06-30 — https://github.com/modelcontextprotocol/modelcontextprotocol/issues/2939. +- **`modelcontextprotocol/ext-apps` v2.0.0 (2026-09-08) — SDK break, protocol unchanged** — migrates to MCP TypeScript SDK 2.0 split packages (`@modelcontextprotocol/client@^2` + `/server@^2`, zod ^4.2, Node 20+), with a bidirectional interop test against published 1.7.5 — https://github.com/modelcontextprotocol/ext-apps/releases/tag/v2.0.0 — *relevance*: our `ui_resource` / `ui_tool_input_partial` host path and server-side `resources/read` inlining need **no change**; only a TS App author upgrading its own SDK is affected. +- **`schema.json` now documents `io.modelcontextprotocol/serverInfo` in result `_meta`** (plus `structuredContent` as any JSON value, loose `toolInfo.tool.outputSchema`) — same URL — *relevance*: the `ClientSession` monkeypatch at `integrations/mcp_apps.py:23–32` that captures `serverInfo` from `initialize` for the App-frame header — *unlocks*: a standards-blessed `_meta` route to `serverName`/`icon` that could retire the monkeypatch **without** waiting on the `server/discover` migration. Verify the servers we actually call emit it before betting on it. +- **⚠️ Host-side error-code deltas in the same release** — a handler-thrown `-32002` now reaches the View as `-32602`; invalid params on `ui/*` methods move `-32603` → `-32602`; the `MCP error N:` message prefix is **gone** — same URL — *relevance*: the SPA's App-bridge error handling and the app-tool-error envelope shipped in #1009/#1013. Worth a grep for any error-string matching, given how much of that chain was built on message text. +- **New MCP Apps host adopter: Alpic Playground** (#729) — https://github.com/modelcontextprotocol/ext-apps/pull/729 — ecosystem signal only. + +#### FastMCP + +- **Repo moved to `PrefectHQ/fastmcp`** (`jlowin/fastmcp` still redirects) — https://github.com/PrefectHQ/fastmcp/releases — *implications*: pin/provenance checks and any hardcoded repo URL in our tooling, including this skill's own source list. +- **v4.0.3 "Once Is Enough" (2026-09-05) — the only release in the window, patch-level, no breaking changes** — https://github.com/PrefectHQ/fastmcp/releases/tag/v4.0.3 — *implications*: the notable fix is **tools returning unconstrained sequences no longer send images twice (#4999)** — a free per-turn token reduction on any of our MCP tools returning `list`/`Sequence` without a constrained type, straight at the "bounded, never unbounded pass-through" tenet. Also: mixed-era multi-server clients skip needless startup retries (#4971), task timing fields serialize strictly (#5003). Latest PyPI `fastmcp` = **4.0.3**, uploaded 2026-09-05. +- **No breaking changes since 4.0.0** — 4.0.1/4.0.2/4.0.3 are all patches. The breaking set remains 4.0.0's: server-initiated sampling and roots removed, `ctx.elicit()` old-protocol-only, FastMCP 3 deprecated APIs gone, MCP model fields snake_case, background tasks moved to `fastmcp-tasks`, spec-correct error codes, `Client("server.py")` bare-string deprecated for removal in 5. +- **4.0.0 (2026-08-31, just pre-window) remains the release that matters for us** — it implements `server/discover` on **both** sides (server: `on_discover`, `DiscoverRequest`/`DiscoverResult` in `fastmcp/server/low_level.py`; client: an `"auto"` probe that validates `resultType`/`ttlMs`/`cacheScope` and synthesizes a minimal result for servers without one), plus `UserSession`/`SessionId` (#4604), SEP-2549 cache hints and SEP-2243 routing headers — https://github.com/PrefectHQ/fastmcp/releases/tag/v4.0.0 — *implications*: this is the concrete basis for the queued MCP-Apps `server/discover` migration. It also means server-advertised `ttlMs`/`cacheScope` on list results is a *principled* fix for tool-listing token cost rather than a hand-rolled cache. + +#### Agentic UI/UX patterns + +- **Cursor "Projects" — a coordinator agent that writes no code, plus persistent project context** — https://www.cursor.com/blog/projects — *what it is*: a chat-facing coordinator directing subagents, backed by a Project-scoped file set synced across cloud/local that agents write learnings into; it also subscribes to external events (Slack, PR activity, schedules) and acts unprompted. — *fit for our stack*: pattern-only (Angular equivalent: a coordinator thread whose child runs render as collapsible sub-cards inside this week's one-card-per-run accordion; `signal()` per child run, `computed()` roll-up status). — *where it'd land*: `agent_status` + `tool_group_summary` are already the right carriers; agent marketplace for the delegate identities; Memory Spaces as the persistent per-project knowledge file. +- **Claude Code `/fork` sessions get their own identity in the resume list** — https://github.com/anthropics/claude-code/blob/main/CHANGELOG.md (2.1.268: "Fixed `/resume` listing a `/fork` background session under its parent's name instead of its own `⑂` fork name") — *what it is*: forked conversations as first-class, separately-named entries — the minimum viable UX for branching. — *fit for our stack*: pattern-only (Angular equivalent: `parentSessionId` + branch label on the session metadata row; sidebar renders forks as indented siblings with a `⑂` glyph). — *where it'd land*: the gap we have flagged as **missing entirely** (regenerate / edit-and-resend / branching). Needs **no new SSE event** — session-metadata + sidebar work only, which makes it much cheaper than the queued Strands-snapshots framing implies. +- **Claude Code: approval prompt leads with the ask's question; artifacts get per-page tab icons** — same URL (2.1.268) — *fit for our stack*: direct port, both are template/copy changes — *where it'd land*: the tool-approval interrupt dialog, and the artifact library page / docked panel header. +- **assistant-ui — nothing in window.** Latest releases (`@assistant-ui/react-streamdown@0.3.13` and siblings) are all 2026-09-03, one day before the window opens. +- **Not scanned: NN/g AI topic page** — HTTP 404 on https://www.nngroup.com/topic/artificial-intelligence/, not retried per budget. **anthropic.com/news** carried nothing UI/artifact/design-related in-window (the only in-window post, 2026-09-10, is threat intelligence). + +#### Frontier model announcements + +- **OpenAI GPT-6 Astra GA on Bedrock (2026-09-08)** — context 1,050,000, max output 128,000, knowledge cutoff 2026-04-30, model id `openai.gpt-6-astra`; on `bedrock-runtime` it is **CRIS-only** (`us.` Geo / `global.`), in-Region not supported; `bedrock-mantle` endpoint is **us-west-2 only**, which matches our region — https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, https://aws.amazon.com/about-aws/whats-new/2026/09/openai-gpt-6-astra-on-amazon-bedrock/ — *relevance*: `curated-models.ts`, model-access RBAC, the prompt-cache cost model — *unlocks*: a genuine 1M-token frontier tier alongside Fable 5.1. ⚠️ The reference repo's `rejectsTemperature: true` is **not stated on the model card** — unverified. +- **⚠️ Astra is priced in TWO context tiers, and the ref repo's `maxInputTokens: 1000000` puts every call in the expensive one** — Geo CRIS Short Context (272K): **$11.00** in / $13.75 cache-write / $1.10 cache-read / $55.00 out per MTok. Geo CRIS Long Context (1.05M): **$22.00 / $27.50 / $2.20 / $82.50** — exactly 2× on input. Global CRIS is $10.00/$12.50/$1.00/$50.00 and $20.00/$25.00/$2.00/$75.00, i.e. a flat 10% under Geo, consistent with our existing Claude pattern — [model card, Pricing](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html) — *relevance*: the 272K/1.05M split mirrors the deliberate `maxInputTokens: 272000` cap we already set on GPT-5.6. **The card does not say whether the tier is chosen by declared window or by actual token count** — that is the gating question. +- **Astra's cache ratios are conventional — 0.1× read, 1.25× write — and the write SKU is labelled a 30-minute TTL** ($1.10 ÷ $11.00 = 0.1×; $13.75 ÷ $11.00 = 1.25×; "Input — 30m cache write", not Anthropic's 5m/1h). Caching is **Responses-API-only**, implicit + explicit, `bedrock-mantle` feature set only — same URL — *relevance*: our existing "Mantle caches only over Responses, not Converse" constraint carries over unchanged. Fable 5.1 remains the 0.025× ratio outlier. +- **Astra harness gaps: no structured output, no CountTokens, tool use splits by endpoint, and output tokens burn quota at 10×** — on `bedrock-runtime`: server-side tool use **not** supported, structured outputs **not** supported, count tokens **not** supported, Guardrails and application inference profiles Converse-only. On `bedrock-mantle`: server-side tool calling supported, application inference profiles **not** supported. 1 output token = 10 TPM — same URL — *relevance*: no CountTokens extends the known gap from our context-attribution work (CountTokens already rejects `us.*` ids). The 10× output quota multiplier matters a lot given 1.20.0's finding that the campus ceiling is the TPM quota, not compute. +- **Anthropic: nothing model-facing in the window.** The only in-window post is "Detecting and countering misuse of AI: September 2026" (2026-09-10) — https://www.anthropic.com/threat-intelligence-report-september-2026. The Fable 5.1 / Mythos 5.1 announcement is dated 2026-09-01, outside the window and already in our catalog. **No Anthropic pricing or prompt-caching API change this week.** + +#### Agent harness patterns + +- **Claude Code 2.1.268 — three compaction-determinism fixes in one release** — "`/compact` and auto-compact mangling text that contained `$` sequences"; "**resuming a conversation that ended with `/compact`: its restored-file notes now load in the same order on every resume**"; "SDK prompt suggestions, side questions and `/rename` sending the conversation from before a compaction" — https://github.com/anthropics/claude-code/blob/main/CHANGELOG.md — *relevance*: `TurnBasedSessionManager` compaction and the persisted byte-stable truncation anchor. The middle fix is **our exact failure class** — non-deterministic ordering of restored context across resumes is a silent prefix re-write. The third is the "auxiliary caller reads pre-compaction state" bug, and **we have auxiliary callers on the same session**: title generation (Nova Micro, concurrent asyncio task) and the second cached `Agent` built by an `@`-mention. — *unlocks*: a targeted audit of every reader of restored history that is not the main turn loop. +- **Claude Code 2.1.268 — "Fixed prompt caching and extended thinking breaking mid-session for SDK sessions using `excludeDynamicSections`: the first message is no longer re-rendered each request"** — same URL — *relevance*: the prompt-cache fingerprint hooks and derived `cacheStatus`. Independent confirmation that the dominant real-world cache-buster is a **re-rendered first message**, not tool or system drift — which is precisely what `partial_miss` was built to catch. — *unlocks*: argues for a `historyHash`-only regression alarm (a first-message re-render shows as history drift with tools+system flat) as the cheapest possible detector. +- **Claude Code 2.1.268 — auto-mode denials now name the blocking rule and steer the model** — "the message Claude receives now names the rule that blocked the action and asks Claude to try a safer method and finish unrelated work before stopping to ask you" — same URL — *relevance*: permission/approval UX on the tool-approval interrupt path. The pattern: a denial is a **steer**, not a dead end — the refusal payload carries the rule identity plus an instruction to exhaust unrelated work first. — *unlocks*: fewer interrupt round-trips per turn, and each resume is a fresh model call against the cached prefix, so this is a latency *and* cost item, not only UX. +- **pydantic-ai v2.40–v2.42 — approval validation, per-subagent model fallback, and a parallel tool-pairing fix** — https://github.com/pydantic/pydantic-ai/releases — *relevance*: three of our surfaces at once. (1) "Reject invalid `DeferredToolResults.approvals` values" ([#8081](https://github.com/pydantic/pydantic-ai/pull/8081)) — same shape as our `interruptId` resume guard. (2) `fallback_model` → `fallback_subagent_model` ([#8077](https://github.com/pydantic/pydantic-ai/pull/8077)) — cheap-vs-capable routing scoped **per built-in subagent** rather than per top-level agent. (3) "Fix `defer_loading` reveal synthesis splitting a parallel batch's `tool_result` from its `tool_use`" ([#7879](https://github.com/pydantic/pydantic-ai/pull/7879)) — the same hazard class our `_repair_tool_pairing` exists for, arriving via tool *deferred loading* rather than interrupts. — *unlocks*: if we ever add lazy/deferred tool reveal for MCP token bloat, #7879 is the pre-written bug report. Also `prices.update_in_background()` ([#4841](https://github.com/pydantic/pydantic-ai/pull/4841)) — a background price-table refresh, pointed given our hand-maintained `curated-models.ts`. +- **langchain 1.4.0 — first-class `langchain.mcp` namespace + `MCPAdapter`; middleware trace inputs omitted** — https://github.com/langchain-ai/langchain/releases/tag/langchain%3D%3D1.4.0 (#39939, #40098, #38355) — *relevance*: ToolRegistry multi-protocol adapters; "include model destination in agent tool routing" maps to our per-model tool filtering at `_build_filtered_tools`. Cross-check that "route tools by which model is actually in play" is becoming a standard harness concern, not a local optimization. +- **anthropic.com/engineering — nothing in window** (nearest posts are September 2025). + +#### opencode (anomalyco/opencode) + +Three releases in the window (v1.18.28, v1.18.29, v1.18.30), all maintenance/provider plumbing. **Nothing in the window touches the tooling lens** — no changes to tool definitions, permission/gating UX, the built-in tool set, or sub-agent delegation. + +- **[v1.18.30, 2026-09-09] Per-model system prompt ("Astra") for GPT-6 models** — a distinct system prompt selected by model family rather than one shared harness prompt — https://github.com/anomalyco/opencode/releases/tag/v1.18.30 — *lens*: context engineering — *relevance*: our agent prompt assembly. A model-keyed system prompt is a prompt-cache prefix decision for us: fine only if deterministic per model, since the prefix is what gets cached. +- **[v1.18.30] Reasoning-effort variants exposed as selectable catalog entries** — effort tier picked at model-selection time rather than as a runtime parameter — same URL — *lens*: cost-effectiveness — *relevance*: model selection in `inference_api` and `curated-models.ts`. The same "effort tier as a catalog row" shape we would need to expose cheap-vs-capable tiers of one model; note our #915 guard already trips on empty `supportedParams`. +- **[v1.18.30] Bedrock model IDs preserved, including ARN-based IDs** — a resolution bug where Bedrock identifiers were being normalized and no longer resolved — same URL — *lens*: cost-effectiveness — *relevance*: our Bedrock model-id handling (`us.*` CRIS vs `global.*`, ARN-shaped ids) — the same class of bug that bit us on CountTokens rejecting `us.*` ids. +- **[v1.18.28, 2026-09-04] Session ID propagated as a provider interaction header** — the harness stamps its session id onto upstream provider requests so a multi-turn session is traceable at the provider — https://github.com/anomalyco/opencode/releases/tag/v1.18.28 — *lens*: cost-effectiveness (observability) — *relevance*: our per-session cost attribution (`C#` rows, `GET /admin/costs/sessions/{id}/calls`); a reminder that runtime session-id pinning was what made our agent-cache hits measurable at all. Converges with the Strands `main` change auto-deriving `cache_key` from session id. + +#### Pricing / quota + +- **xAI Grok 4.6 gained its first us-west-2 published rates — and it breaks both halves of our prompt-cache contract** — the Bedrock offer went 10,998 → 11,657 SKUs, 659 added, **18 of them `USW2-xai.grok-4.6-mantle-*` where there were previously zero Grok SKUs in Oregon**. Standard: **$2.20/MTok in, $6.60 out, $0.55 cache-read** in-Region and Geo CRIS; **$2.00 / $6.00 / $0.50** on Global CRIS (the familiar ~10% Regional premium). Priority = 1.75×, Flex = 0.5×. **Two independent sources agree to the cent** (Price List API + model card) — https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html — *relevance*: `curated-models.ts` can now carry real Grok 4.6 rates, but **there is no cache-write SKU at all**, the card lists only *Implicit* prompt caching (no explicit `cachePoint`), and **cache-read is 0.25× input, not 0.1×**. Our contract assumes explicit cache points at 1.25× write / 0.1× read; Grok has nothing to place and reads at 2.5× the assumed ratio. Treat Grok as a non-cached model in cost projections until implicit caching is measured against `cacheStatus`. Also: no in-Region option in us-west-2 on `bedrock-runtime` (only `us.*`/`global.*`), and it needs `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{acct}:project/default`. +- **Zero price change for any model we actually run in us-west-2** — all **48** price changes in the republish are `UGW1-*` = **AWS GovCloud (US-West)** only, every one of them `openai.gpt-5.6-terra`/`-luna` (Luna dropped hard: `UGW1-openai.gpt-5.6-luna-mantle-input-tokens-standard` $1.32 → **$0.264**/MTok; Terra a uniform −20%). Filtering the changed set to non-GovCloud returns an **empty list**. `google.gemma-4-31b` holds at 8 us-west-2 SKUs, unchanged. — *relevance*: no action, but AWS is repricing the GPT-5.6 family *somewhere*, so re-check Oregon next week. +- **✅ SETTLED: `CLAUDE.md`'s Price List API provenance claim is false for Claude.** The entire `AmazonBedrock` offer file contains **10 Claude SKUs, none newer than Claude 3** (`USW2-Claude3Haiku-input-tokens`, `USW2-Claude2.1-…`); Haiku 4.5, Sonnet 4.6, Fable 5.1, GPT-5.4 and Nova Micro have **no us-west-2 SKUs in the offer at all**. This **reproduces last week's result** and resolves the ⚠️ on the queued [2026-09-04] #1 entry. Reproducible commands are in the Sources appendix. — *relevance*: the authoritative route for Claude/Nova/GPT rates is the **per-model AWS model card**; the Price List API is authoritative for xAI, Google and AgentCore. `CLAUDE.md` needs a one-line correction. +- **AgentCore pricing: no change whatsoever** — `AmazonBedrockAgentCore` `20260901164424` → `20260911124408` is a pure metadata republish: 6,481 → 6,481 SKUs, 0 added, 0 removed, 0 price changes, and the `products` dicts compare **byte-equal**. Runtime consumption, the 889 `Runtime:Instance-based::Management-Hours` SKUs from 2026-09-01, Memory, Gateway, Browser, Code Interpreter and Identity are all untouched. — *relevance*: the instance-based SKUs are stable enough to model against, so the W5 runtime-memory track can be costed on these numbers without waiting for another republish. + +#### Community + GitHub issues + +- **⚠️→✅ VERIFIED NOT APPLICABLE: AgentCore SDK [#659](https://github.com/aws/bedrock-agentcore-sdk-python/issues/659)** — "streaming responses silently turn unserialisable events into JSON strings of their Python repr." `BedrockAgentCoreApp._safe_serialize_to_json_string` routes through `convert_complex_objects`, which does not handle `bytes`, so an event carrying binary falls through to `json.dumps(str(obj))` and the SSE line arrives as a JSON *string* holding a Python repr. Named triggers are `reasoningContent.redactedContent` (which GPT-5.6 Luna emits after tool calls) and image/document `source.bytes`. Reported against a stack profile nearly identical to ours. **It does not affect us, verified three ways:** (1) `grep -rn 'BedrockAgentCoreApp' backend/` returns **zero hits** — we run our own FastAPI app in the Runtime container and hand-roll SSE via `StreamingResponse` + `json.dumps`; (2) `stream_processor.py:117–121` already base64-encodes `bytes` explicitly ("ensures binary content like PNG images can be safely JSON serialized"); (3) `_create_event()` at `stream_processor.py:157` runs `_serialize_object(data)` on **every** emitted event, and `redactedContent` is routed through `_create_event` at line 910. — *relevance*: recorded so it is not re-proposed. The subagent that surfaced it ranked it "highest of the week for us"; it is not a finding, and the negative result is the finding. +- **AgentCore SDK [#661](https://github.com/aws/bedrock-agentcore-sdk-python/issues/661): port `AgentCoreMemoryStore` (Strands `MemoryStore` interface) from the TypeScript SDK to Python** — filed 2026-09-11 (today). The TS SDK (v0.4.1+) ships an `AgentCoreMemoryStore` so AgentCore Memory plugs into `MemoryManager` with recall, **automatic prompt injection**, and server-side extraction; Python has only `AgentCoreMemorySessionManager` — *state*: open — *relevance*: our `TurnBasedSessionManager` subclasses the only Python integration that exists. Worth watching for whether AWS starts treating the session-manager path as legacy. ⚠️ A `MemoryStore` that does **automatic prompt injection** is a prompt-cache-prefix concern for us: non-deterministic injected recall rewrites the cacheable prefix. +- **Starter-toolkit [#498](https://github.com/aws/bedrock-agentcore-starter-toolkit/issues/498): no API to list or force-terminate active runtime sessions** — re-surfaced in the window (filed 2026-04-10, still open). No `ecs list-tasks` equivalent for AgentCore microVMs; `list-sessions` lists *Memory* sessions only; `stop_runtime_session` does **not** kill the container. Reported incident: one runaway session ran ~58 min and burned **$72.67** — *state*: open — *relevance*: our DynamoDB lease + `cancelRequestedFor` are the only kill switch we actually have, and there is still no platform-level backstop. This is an argument *for* building the `$30` platform-ceiling piece of the quota-cooldown spec rather than waiting for AWS. +- **Standing issues — all four STILL OPEN, none fixed in 1.22.0.** #564 (metadata-filtered `ListEvents` misses marker events → false "new session", skips history restore; last touched **2026-07-20**, no AWS response in 7+ weeks) — https://github.com/aws/bedrock-agentcore-sdk-python/issues/564. #621 (`filter_restored_tool_context` vs extended thinking; 2026-08-08). #629 (TracerProvider never flushed before microVM freeze; 2026-08-10). **#646 — open but retitled** from a bug report to *"AgentCoreMemorySessionManager: allow excluding binary content from persisted messages"*, i.e. drifted to a feature request for an opt-out; the turn-killing behaviour is unchanged but the reframing makes it less likely to be prioritized as a defect (2026-08-20). +- **`bedrock-agentcore` 1.22.0 is not worth the bump.** Latest PyPI is 1.22.0 (2026-08-18), one minor ahead of our pin. The 1.21.0→1.22.0 delta is a **single feature PR** — `feat(payments): add MPP, x402 upto, and Quick Create support` (#643) — plus the release commit. No Runtime, Memory, Gateway, Identity, Browser or Code Interpreter changes, and **none of the four standing issues fixed**. https://github.com/aws/bedrock-agentcore-sdk-python/releases/tag/v1.22.0 +- **[Grep beats LSP? Why coding agents ignore your fancier tools]** — 97 points, 24 comments, 2026-09-04 — https://news.ycombinator.com/item?id=49560260 (article: https://www.agentconnect.md/blog/grep-beat-lsp-harness/) — *relevance*: a tool-design lens for our catalog, and the same failure mode as our tool-search/MCP token-bloat work — a richer tool the model won't select still costs full `toolConfig` prefix bytes on every turn for zero use. The comment thread partly disputes the article (it shows *that* grep wins without explaining *why* it gets chosen) and notes nobody has tried combining LSP + grep or `ast-grep`. One commenter's loop is cheap and directly applicable: run a task, watch which tools the agent actually reaches for, then **ask it what tooling would have helped**. +- **Filtered out deliberately, and worth saying why**: a large volume of same-day 1–4 point Show HNs advertising dramatic savings ("cutting token waste by up to 96%", "60–90%", "$50k+/month") with **no methodology and no discussion** — exactly the shape of claim our own cost work requires numbers for, and none supply any. **Reddit r/LocalLLaMA was not retrievable** (non-JSON interstitial); not retried, consistent with the 2026-05-18 decisions-log entry. **No Strands-specific HN stories appeared at all.** + +#### Cookbook / courses + +- **No notable items this week.** Both `anthropics/claude-cookbooks` and `anthropics/anthropic-cookbook` returned **zero commits** in the window (verified as an empty result, not an error, by re-running with a wider `since`). Newest activity sits one day before the window: `2026-09-03 — feat(claude_agent_sdk): add scheduled repository reviewer recipe (#860)` and `2026-09-02 — Retire claude-opus-4-1 across cookbooks (#807)`. ⚠️ A 7-day window will miss #860 next week too — it is the closest thing to an agent-loop/scheduling worked example the cookbooks have shipped recently, and we have our own scheduled-runs surface, so it is worth one out-of-cadence look. + +#### LibreChat + +- **No notable items this week.** Latest release remains **v0.8.8-rc2**, published **2026-09-03** — one day outside the window — https://github.com/danny-avila/LibreChat/releases/tag/v0.8.8-rc2. Nothing published since. Note that every LibreChat tag in the last 10 is marked *Pre-release*, including v0.8.6 and v0.8.7 — "no stable release" is their normal state, not a stalled cycle. + +#### Seasonal + +- **Out of window — none scanned this week.** re:Invent is late Nov / early Dec; no conference proceedings dropped. + +### Patterns worth considering + +- **Model lifecycle is becoming a catalog field, not a footnote.** A new Bedrock model-lifecycle policy took effect **for models launched on or after 2026-09-07** — inside the window — introducing a **45-day Legacy (EOL notice) period** alongside the existing 6-month one, and stating that once Legacy begins "new customers can't adopt the model, and **existing customers may lose access after 15 days of inactivity**." Our current fleet is not affected (Grok 4.6's card explicitly cites the pre-Sept-7 policy; Haiku 4.5 / Sonnet 4.6 / Fable 5.1 all predate the cutoff). + - **Where**: https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html + - **Fit**: a 45-day notice window is shorter than our curated-catalog → RBAC-grant → prod-rollout path, and the 15-days-of-inactivity clause is a live hazard for a model sitting in `curated-models.ts` behind a role grant nobody exercises — it could stop working with no deploy on our side. Cheap mitigation: read `modelLifecycle` from `ListFoundationModels` into the model catalog and surface Legacy on the admin models page. + - ⚠️ **single-source** (docs only), and the legacy-policy text was **not** diffed against the prior version — so "the 45-day tier and the 15-day clause are new" is unverified. Verify before acting. + - **Verdict**: Monitor, with one cheap build (surface `modelLifecycle`) worth pricing. +- **Branching is a session-metadata feature, not a snapshot feature.** The queued [2026-09-08] Strands-snapshots entry frames branch/regenerate as needing `take_snapshot`/`load_snapshot`. Claude Code's `/fork` handling suggests the *minimum viable* version is much cheaper: a `parentSessionId` + branch label on the session metadata row and a sidebar that renders forks as indented siblings — no new SSE event, no snapshot round-trip, no rebinding of `agent.messages` mid-life (which is the counter-gate that entry itself flags). + - **Where**: https://github.com/anthropics/claude-code/blob/main/CHANGELOG.md (2.1.268) + - **Fit**: replaces the *hard* half of a queued item with an easy one. Our session-metadata row already has a static SK + GSI4 (#175), so a parent pointer is additive. + - **Verdict**: Worth trying — and worth folding into the existing snapshots entry at review rather than queuing separately. +- **A denial should be a steer, not a dead end.** Claude Code now hands the model the *identity of the blocking rule* plus an instruction to try a safer method and finish unrelated work before stopping to ask. We have three interrupt surfaces (OAuth consent, tool approval, RBAC denial) and all of them currently terminate the model's plan rather than redirect it. + - **Where**: https://github.com/anthropics/claude-code/blob/main/CHANGELOG.md (2.1.268) + - **Fit**: each resume is a fresh model call against the cached prefix, so fewer round-trips is a cost item as well as a UX one. Lands on the tool-approval interrupt path and the RBAC denial message. + - **Verdict**: Worth trying — small, and on-thesis for cost. + +## Internal Audit + +> Every number below is produced by a command quoted with it, per the [2026-09-04] verification rule. + +### Activity (last 7 days) + +- **Commits on develop**: **133** — `git log origin/develop --since="2026-09-04" --oneline --no-merges | wc -l` +- **Files touched by area** (`git log origin/develop --since="2026-09-04" --no-merges --name-only --pretty=format: | grep -v '^$' | cut -d/ -f1 | sort | uniq -c | sort -rn`): frontend **714**, backend **500**, infrastructure **120**, docs 46, `.kiro` 34, tests 22, scripts 22, `.github` 15, tui 12, docs-site 12 +- **PRs opened**: **113** — **merged into develop**: **110** — **reverted**: **0** (`gh pr list --base develop --state all --limit 200 --json number,createdAt -q '[.[]|select(.createdAt>"2026-09-04T00:00:00Z")]|length'`; merged variant on `mergedAt`; reverts via `git log … | grep -icE "^[0-9a-f]+ revert"`) +- **Open PRs into develop**: **5** — #1043, #1042, #1041 (all opened this morning), #1025 (managed spot training), #1004 (WebMCP host spec). **All five have 0 comments and 0 reviews** — `gh pr view --json comments,reviews`. No repeated review feedback to mine this week. +- **Issues opened**: **0** — **closed**: **0** — `gh api "search/issues?q=repo:…+type:issue+created:>2026-09-04" -q '.total_count'` (and `closed:>…`). ⚠️ The `gh issue list --search` GraphQL route **timed out** on first attempt; the REST search route above is what produced the number. +- **Releases in window**: **1.19.1** (2026-09-07) and **1.20.0** (2026-09-09). +- **CI failures (workflow → count)**, `gh run list --status=failure --limit 30`: **Nightly Build & Test → 7** (2026-09-05 … 2026-09-11, consecutive), **CI (PR) → 3** (`feature/agentcore-browser-tool` ×1, `feature/branding` ×2 — all pre-merge, all on 09-04/09-09). No `develop`-push deploy failures in the window. + +### Repeated friction signals + +- **`Nightly Build & Test` has failed 7 consecutive nights** (2026-09-05 → 2026-09-11) after being green for at least the preceding five (`gh run list --workflow="Nightly Build & Test" --limit 12 --json conclusion,createdAt,databaseId`: failure ×7, then success on 09-04, 09-03, 09-02, 09-01, 08-31). **Two independent root causes**, neither a flake: + + **(a) Six filesystem-reading branding specs fail only under `--coverage`.** `gh run view 34582692457 --json jobs` shows `Test Frontend with Coverage` **failure** while `deploy / [deploy-develop] Test frontend` **succeeded in the same run on the same ref**. The two jobs differ in exactly one thing: nightly's `scripts/frontend/test.sh` runs `ng test --no-watch --coverage`, while `tests.yml` (PR CI *and* the deploy track) runs `npm run test:ci` = `ng test --watch=false`. + - Failing specs: `src/branding/rebranding-guide.doc-presence.spec.ts` (5/5), `src/app/surface-literal-guard.spec.ts`, `src/app/tailwind-theme-import-guard.spec.ts`, `src/branding/brand-theme-golden.spec.ts` (1/2), `src/branding/surface-theme-golden.spec.ts` (1/2), `src/branding/prestart-generator-parity.spec.ts` (**0 tests — fails at collection**). + - **Hypothesis** (strongly evidenced, mechanism not yet executed locally): under `--coverage`, `import.meta.url` / the `__dirname` shim resolves **two directory levels shallower** than the spec's source directory. Three pieces of evidence converge. (1) `rebranding-guide.doc-presence.spec.ts` resolves `resolve(SPEC_DIR, './README.md')` and the assertion error shows it read `'# AiClient\n\nThis project was genera…'` — that is `frontend/ai.client/README.md`; the correct target `frontend/ai.client/src/branding/README.md` begins `# Rebranding Guide` **on both `main` and `develop`** (`git show origin/develop:frontend/ai.client/src/branding/README.md | head -3`). (2) `prestart-generator-parity.spec.ts` fails at *collection* at line 31, which is `readdirSync(resolve(SPEC_DIR, '../../scripts/branding'))` — an ENOENT, i.e. the resolved path does not exist. (3) The two guards *find violations* rather than finding nothing: `surface-literal-guard.spec.ts:44` does `path.join(__dirname)` and `tailwind-theme-import-guard.spec.ts:30` does `path.resolve(__dirname, '..')`, so a shallower base walks the whole package — including generated Tailwind CSS, which is full of literal hex backgrounds and a direct `@import "tailwindcss"`. + - **Fix candidate**: resolve these specs' roots from a stable anchor rather than the module URL — a `vite`/`vitest` `define`d repo-root constant, `process.cwd()` pinned by the Angular builder, or an `import.meta.env`-independent helper in `src/test-setup.ts` — and **add `--coverage` to a PR-CI job** (or make `test:ci` pass it), because today PR CI *structurally cannot* catch this class. That second half is the real fix: six specs whose whole purpose is repo hygiene are themselves environment-dependent, and the only gate that exercises them runs at 2am on a branch nobody watches. + - **Provenance**: these specs landed with **PR #933** (`Merge pull request #933 … feature/branding`); `feature/branding`'s own PR CI failed twice on 2026-09-04 ([33917746689](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/33917746689), [33915030623](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/33915030623)) before merging. Branding is on `develop` and **not yet on `main`** (`git merge-base --is-ancestor af388588 origin/main` → false). + + **(b) `nightly-develop-PlatformStack` is wedged in `DELETE_FAILED`.** `gh run view 34582692457 --log-failed` on the `Deploy PlatformStack` job: `Stack:arn:aws:cloudformation:us-west-2:490617140655:stack/nightly-develop-PlatformStack/41c08940-… is in DELETE_FAILED state and can not be updated.` → `exit 1`. Every nightly deploy will fail identically until the stack is cleared by hand; nothing self-heals. ⚠️ **single-source**: the CI log is the only evidence — this session's AWS credentials are account **029812070295**, not dev-ai **490617140655** (`aws sts get-caller-identity`), so `describe-stacks` returned `Stack … does not exist` and the stack's `StackStatusReason` and retained-resource list could **not** be read here. Someone with dev-ai access should read the reason before deleting, because a `DELETE_FAILED` ephemeral stack usually means a resource with a retention policy or a non-empty bucket, and that resource is **still billing**. + +- **The frontend is where the churn is, 1.4:1 over the backend** (714 vs 500 files), and nearly all of it is one theme: this week shipped condensed tool cards, a tool-detail pane, MCP prompts/resources surfacing, a tools drawer with search + categories + OAuth state, and a loading indicator that names the real model state — 16 commits across 2026-09-10 alone. Not a friction signal by itself, but it is the surface where the [2026-09-04] "browser-verify layout" lesson and the two "passed unit tests, broke in the browser" precedents both live, and the *only* gate that runs the branding guards protecting it has been red the whole time. + +### Version-pin lag + +Pins read from `git show origin/develop:{backend/pyproject.toml,frontend/ai.client/package.json,infrastructure/package.json}` and `backend/uv.lock`. Latest versions from `curl -s https://pypi.org/pypi//json | jq -r '.info.version'` and `curl -s https://registry.npmjs.org/ | jq -r '.["dist-tags"].latest'`, all on 2026-09-11. + +| Dep | Pinned | Latest | Lag | Notes | +|---|---|---|---|---| +| `strands-agents` | 1.55.0 | **1.55.1** (2026-09-09) | 1 patch / 2 days | **Safe bump.** Patch-only, no breaking changes. Two fixes on our surfaces: #4199 (malformed immutable snapshot IDs) and #4228 (context-manager parity). #4154 (orphaned `toolResults` at any index) worth diffing against `_repair_tool_pairing`. | +| `strands-agents-tools` | 0.8.8 | 0.8.8 (2026-09-04) | **current** | Calculator sandbox escape already adopted (#1011). | +| `bedrock-agentcore` | 1.21.0 | 1.22.0 (2026-08-18) | 1 minor / 24 days | **Do not bump.** Delta is a single payments feature (#643); fixes **none** of the four standing issues we carry. Reference repo is on 1.22.0, which is not a reason. | +| `boto3` | 1.43.68 | 1.43.92 (2026-09-10) | 24 releases / 1-day-old latest | `botocore` resolves to 1.43.68 too. AgentCore service-model additions ride `botocore`, so this pin gates access to newer API shapes (e.g. the `filesystemConfigurations` family the queued workspaces entry depends on). | +| `botocore` (resolved) | 1.43.68 | 1.43.92 | 24 releases | Transitive; moves with `boto3`. | +| `fastapi` | 0.136.1 | 0.141.1 (2026-07-29) | 5 minors / ~44 days | No in-window change. | +| `pydantic` (resolved) | 2.12.5 | 2.13.5 (2026-08-28) | 1 minor | Transitive, not directly pinned. | +| `mcp` (resolved) | 1.28.1 | **2.2.0** (2026-09-07) | **1 major** | ⚠️ **New this week.** Strands relaxed its floor to `<2.2` ([#4151](https://github.com/strands-agents/harness-sdk/releases/tag/python%2Fv1.55.1)), so a transitive resolve can now pull **mcp 2.x** under us. Our `ClientSession` monkeypatch must be verified against **both** major lines before any resolve is refreshed. | +| `@angular/core` | 21.2.19 | 21.2.23 (`v21-lts`) / 22.1.6 (`latest`) | 4 patches in-line; 1 major behind `latest` | Stay on the 21 line. Angular 22 exists; not a candidate this cycle. | +| `vitest` | 4.1.5 | 4.1.11 (4.x) / **5.0.0** (2026-09-03) | 6 patches in-line; 1 major behind | ⚠️ The reference repo took vitest **5.0.0** in the window ([PR #289](https://github.com/aws-samples/sample-strands-agent-with-agentcore/pull/289)) with no config changes needed. **Do not lead with the major here** — our SPA has a documented history of vitest isolation/JIT flakes (`isolate: false`, `src/test-setup.ts`), and the coverage-only failure above means the suite is not currently a trustworthy baseline to migrate from. Fix the nightly first. Also note an in-window deprecation warning in our own logs: *"Promise returned by `expect(actual).rejects.toThrow(expected)` was not awaited … will cause the test to fail in the next Vitest major."* | +| `@analogjs/vite-plugin-angular` | 3.0.0-alpha.53 | 3.0.0-alpha.87 (pre) / 2.7.2 (stable) | 34 alphas on the pre-release line | We are **ahead** of stable on a pre-release line — not "lag" in the usual sense. Relevant because this package owns the test transform implicated in root cause (a) above. | +| `@analogjs/vitest-angular` | 3.0.0-alpha.30 | same line | — | Same. | +| `aws-cdk-lib` | 2.265.0 | 2.269.0 (2026-09-10) | 4 minors / 1 day | Bundles its own deps — npm overrides cannot patch a bundled CVE. | +| `constructs` | 10.6.0 | 10.8.1 (2026-08-03) | 2 minors | | +| `aws-cdk` (CLI) | 2.1128.0 | 2.1141.0 (2026-09-09) | 13 releases | Dev-only. | + +**Skipped**: none. All tracked deps resolved. + +### Retirement candidates + +- **`nightly-develop-PlatformStack` in dev-ai (490617140655)** — a `DELETE_FAILED` ephemeral stack that has blocked every nightly deploy for 7 days and whose retained resources are still billing. This is the highest-value deletion available and it is not code. +- **`scripts/sourcedir.tar.gz`** in the fine-tuning S3 prefix — orphaned by 1.20.0's move to per-family `sourcedir-{text,vision,vlm}.tar.gz`. The CHANGELOG says it plainly: *"The old object is orphaned, not read, and can be deleted by hand."* +- **`.claude/skills/angualar-best-practices/`** — **the directory name is misspelled** ("angualar"), and the skill's last commit is **2026-04-27, 137 days ago** (`git log origin/develop -1 --format=%ad --date=short -- `). The typo is the finding: it is the skill most likely to be wanted on a repo that touched 714 frontend files this week, and its name is wrong. Rename, don't retire. +- **`.claude/skills/cors-deployment/SKILL.md`** and **`.claude/skills/frontend-design/SKILL.md`** — both last committed 2026-04-27 (137 days), neither referenced in any PR in the window. `frontend-design` additionally duplicates a plugin skill of the same name available in-session. Candidates for retirement or consolidation. +- **`.claude/skills/kaizen-review-prep/SKILL.md`** — unmodified since **2026-05-10** (124 days) across five reviews that each proposed editing it. Already tracked by the open [2026-09-04] queue entry; noted here only so the streak is on the record. +- **Possibly `bedrock_cache_points_supported()`** — the gate last week's entry called "load-bearing, not redundant" because upstream #4168 was live. #4168 **closed 2026-09-04** and is fixed in our 1.55.x pin. ⚠️ Verify against a real non-Anthropic model id before deleting; do not infer it from the issue closing. + +### Risks introduced this week + +- **A red gate is worse than no gate.** Seven consecutive nightly failures with zero issues filed (0 opened, 0 closed in the window) means the nightly's signal is now being discarded wholesale. The [2026-09-04] queue entry already established the rule "never resolve a *flaky* entry on a consecutive-green count"; this is the mirror failure — a deterministic, reproducible red that read as background noise. Both halves are cheap to fix and the fix includes *making PR CI able to see it*. +- **Copying the reference repo's GPT-6 Astra row would double our input cost on that model.** Their `maxInputTokens: 1000000` is exactly what our deliberate GPT-5.6 `272000` cap exists to prevent, and the model card does not state whether the tier is chosen by declared window or actual usage — https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html +- **`mcp` 2.x can now arrive transitively.** Strands relaxed its floor to `<2.2` while our lock sits at 1.28.1. The next `uv sync` that refreshes the resolve could move us across a major under a `ClientSession` monkeypatch that has never been tested against it. +- **Grok 4.6 now has real rates and no cache-write SKU.** Any cost projection that applies our 1.25×-write / 0.1×-read assumptions to Grok is wrong in both directions (nothing to place; reads at 0.25×). If a Grok row lands in `curated-models.ts` on the existing derivation helper it will be silently wrong — the same failure mode as Fable 5.1's 0.025×, which is the second counterexample in two weeks. +- **`CLAUDE.md` currently asserts a false provenance for our most cost-sensitive numbers.** Two independent scans now agree the Price List API carries no Claude SKU newer than Claude 3. Anyone following that instruction to verify a rate will find nothing and may conclude the rate is unverifiable rather than that the instruction is wrong. +- **Upstream is building a context-manager/offloading stack that will collide with `ContextOffloader`.** Not a risk this week, but the collision is now visible on `main` and the decisions log already forbids the naive "delete ours" proposal — so the risk is that it gets adopted *piecemeal* without the migration design that entry requires. + +## Ideas — Top 5 (ranked) + +| # | Idea | Surface | Effort | Impact | Subtracts? | Unlocks? | +|---|---|---|---|---|---|---| +| 1 | Fix the nightly, both halves — anchor the branding specs' paths, and put `--coverage` where PR CI can see it | CI / frontend | **L–M** | **H** | yes — a permanently-red gate, a wedged billing stack, and a class of environment-dependent spec | — | +| 2 | Retire the hand-placed system `cachePoint` — `CacheConfig.system_prompt_ttl` / `tools_ttl` are already in our pin | backend | **M** | **H** | yes — the hand-placed `SystemContentBlock`, its ~40-line comment asserting a now-false invariant, and possibly `bedrock_cache_points_supported()` | — | +| 3 | Add GPT-6 Astra at the 272K tier — and make the context-tier boundary a catalog field | backend / frontend | **L–M** | **H** | no — addition, but it *prevents* a 2× cost regression we would otherwise import | a 1M-token frontier tier; a tier-aware catalog that survives the next two-tier model | +| 4 | Correct the rate provenance in `CLAUDE.md` and make cache-read a per-model input — Grok 4.6 is the second ratio-breaker | docs / frontend | **L** | **M–H** | yes — a false sourcing instruction and a hardcoded `0.1×` ratio now falsified by two live models | — | +| 5 | Retire the `ClientSession` monkeypatch via `_meta` `serverInfo` — without waiting on `server/discover` | backend | **M** | **M** | yes — a symbol patch on a Strands internal that Strands is actively changing | a standards-blessed App-frame header that survives the MCP client moving under us | + +### 1. Fix the nightly, both halves — anchor the branding specs' paths, and put `--coverage` where PR CI can see it + +- **Source**: internal — runs [33957119927](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/33957119927) (2026-09-05, first red) through [34582692457](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/34582692457) (2026-09-11), 7 consecutive. Specs landed with PR #933. +- **Surface area**: `.github/workflows/tests.yml` (the `test-frontend` job), `frontend/ai.client/package.json` (`test:ci`), `scripts/frontend/test.sh`, and the six specs: `src/branding/rebranding-guide.doc-presence.spec.ts`, `src/branding/prestart-generator-parity.spec.ts`, `src/branding/brand-theme-golden.spec.ts`, `src/branding/surface-theme-golden.spec.ts`, `src/app/surface-literal-guard.spec.ts`, `src/app/tailwind-theme-import-guard.spec.ts`. Plus one CloudFormation stack in dev-ai. +- **Change**: (a) resolve each spec's root from a stable anchor instead of `import.meta.url` / `__dirname` — a `define`d repo-root constant or a helper in `src/test-setup.ts` — so the path is identical with and without coverage instrumentation. (b) Make PR CI able to see this class at all: either pass `--coverage` in `test:ci`, or add a small PR job that runs the coverage invocation. (c) Separately, have someone with dev-ai access read `StackStatusReason` on `nightly-develop-PlatformStack`, delete the retained resource, and delete the stack. +- **Subtracts**: a gate that has produced only false signal for a week; a `DELETE_FAILED` stack whose retained resources are still billing; and the *category* of repo-hygiene spec that silently depends on how it was invoked. After (b), the nightly stops being the only place a whole class of failure can appear. +- **Effort × Impact**: **Low–Medium × High** +- **Verdict**: **Worth trying** — recommended #1. It is the cheapest item in the scan, the evidence is complete, and everything else in this doc is harder to trust while the suite's only full-fidelity run is red. ⚠️ Two cautions. The mechanism in (a) is strongly evidenced but **not yet reproduced locally** — run `cd frontend/ai.client && ./node_modules/.bin/ng test --no-watch --coverage` against `develop` and confirm the resolved paths before changing the specs; do not fix a mechanism you inferred from a log. And do **not** combine this with the vitest 4→5 major: the suite is not a trustworthy migration baseline until it is green. + +### 2. Retire the hand-placed system `cachePoint` — `CacheConfig.system_prompt_ttl` / `tools_ttl` are already in our pin + +- **Source**: https://github.com/strands-agents/harness-sdk/blob/python/v1.55.1/strands-py/src/strands/models/model.py and https://github.com/strands-agents/harness-sdk/issues/4168 (closed 2026-09-04). **Supersedes the [2026-09-04] "Gate the Strands 1.51 → 1.54 bump on the system-cachePoint collision" entry** — we are already on 1.55.0, so the bump is done and only the collision question remains. Merge them at review. +- **Surface area**: `backend/src/agents/main_agent/core/agent_factory.py:213-224` (the hand-placed system `cachePoint` and its now-false comment at :222), `core/model_config.py:375-400` (`strategy="auto"` and the `bedrock_cache_points_supported()` gate, plus the ~40-line cachePoint-budget comment at :380), `backend/pyproject.toml:59,74` (1.55.0 → 1.55.1). +- **Change**: (a) determine whether `CacheConfig(strategy="auto", system_prompt_ttl=…)` places the system point our hand-rolled block places, and delete the `SystemContentBlock` list if so. (b) Rewrite the comment at `agent_factory.py:222`, which asserts *"auto strategy strips only message-level cachePoints, never system ones"* — a 1.51-era fact. (c) Evaluate `tools_ttl` against the layered mixed-TTL technique the [2026-08-14] cookbook entry measured at 54% cheaper upstream, which was previously blocked and is now reachable. (d) Re-test whether `bedrock_cache_points_supported()` is still load-bearing now that #4168 is fixed. (e) Bump to 1.55.1 for #4199 and #4228. +- **Subtracts**: the hand-placed `SystemContentBlock` cachePoint, the ~40-line comment defending it, one comment asserting a false invariant, and possibly the `bedrock_cache_points_supported()` gate — the library-native subtraction this skill weights for, finally payable. +- **Effort × Impact**: **Medium × High** +- **Verdict**: **Worth trying.** The de-risking fact is upstream's own docstring: a hand-placed system cache point is *"honored rather than doubled"*, so an incremental migration will not double-write. ⚠️ **Never adopt a caching default on inspection alone** — this stack shipped one caching change whose premise was wrong and measured **57% more expensive** live (#954 → reverted by #956). `backend/scripts/probe_gpt56_cache_rates.py --mode both --grow-history` is the gate: beat the current arm, measured. ⚠️ The reference repo runs 1.54 green and **does not hand-place a system cachePoint at all** — their green build is not evidence for us, because they have nothing to collide. ⚠️ Also on `main` but unreleased: `cache_key` auto-derives `strands-` when a session manager is present, which will start transmitting on the next minor — that is our `build_prompt_cache_key()` retiring, but not yet. + +### 3. Add GPT-6 Astra at the 272K tier — and make the context-tier boundary a catalog field + +- **Source**: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html + https://aws.amazon.com/about-aws/whats-new/2026/09/openai-gpt-6-astra-on-amazon-bedrock/ (GA 2026-09-08) + https://github.com/aws-samples/sample-strands-agent-with-agentcore/pull/291 +- **Surface area**: `frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts`, the Mantle Responses path (`_create_mantle_model` / `bedrock_mantle_config`, per-model region routing), `apis/app_api/admin/services/model_access.py` for the role grant, and the model picker. +- **Change**: register `us.openai.gpt-6-astra` (or `global.` — prod has no SCP restriction; dev does) at `maxInputTokens: 272000`, with input $11.00 / cache-write $13.75 / cache-read $1.10 / output $55.00 per MTok on Geo CRIS, and $10.00 / $12.50 / $1.00 / $50.00 on Global. Then generalize: today `maxInputTokens` doubles as both a *capability* bound and a *pricing* cap, and Astra is the second model where those differ. Make the tier boundary explicit on the model row so the next two-tier model is a data change rather than a rediscovery. +- **Subtracts**: no — addition. Justified because it *prevents* a regression: the reference repo's row would import a 2× input cost, and we would not notice, because a 1M declared window looks like a capability win. +- **Unlocks**: + - **A genuine 1M-token frontier tier** alongside Fable 5.1 — the first model in the catalog where a whole corpus fits in one turn, which is a different product capability from anything the 200K-class models offer. + - **A tier-aware catalog.** Once the boundary is a field, the admin models page can show it, cost projections can respect it, and the quota math can account for Astra's **10× output-token quota burn** — which matters directly against 1.20.0's finding that the campus ceiling is the Bedrock TPM quota, not compute. +- **Effort × Impact**: **Low–Medium × High** — the Mantle Responses path already exists; this rides it. +- **Verdict**: **Worth trying**, with one blocking question answered first. ⚠️ **The model card does not state whether the price tier is selected by the declared window or by actual token count.** If it is the declared window, `272000` is mandatory; if it is actual usage, a 1M window is free until a call exceeds 272K and the cap is a *policy* choice instead. Answer that before registering. ⚠️ Other verified constraints: on `bedrock-runtime` Astra supports **no server-side tool use, no structured output, no CountTokens** (extending the known CountTokens gap from our context-attribution work), and Guardrails/application inference profiles are endpoint-split; `bedrock-mantle` is **us-west-2 only** (matches us). ⚠️ The ref repo's `rejectsTemperature: true` is **not on the model card** — verify, don't copy. ⚠️ Astra's cache ratios are conventional (0.1× read, 1.25× write) but its write SKU is labelled a **30-minute** TTL, not 5m/1h — do not conflate with `cache_ttl_seconds_for()`. + +### 4. Correct the rate provenance in `CLAUDE.md` and make cache-read a per-model input — Grok 4.6 is the second ratio-breaker + +- **Source**: a full diff of the republished offer files (`AmazonBedrock` `20260804165549` → `20260911124408`) + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html. **This closes the ⚠️ on the open [2026-09-04] "Finish the #914 rate correction" entry** — its provenance question is now answered, so the two should be merged and shipped together at review. +- **Surface area**: the prompt-cache contract bullet in `CLAUDE.md`; `frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts:96-97` (the `cacheRead = input × 0.1` helper); and the six stale `$2.50/MTok` sites the [2026-09-04] entry already enumerates, headed by `backend/src/agents/main_agent/core/model_config.py:380`. +- **Change**: (a) `CLAUDE.md` currently says rates "come from the AWS Price List API, not the pricing page." Replace with the verified reality: **per-model AWS model cards are authoritative for Claude, Nova and the OpenAI/Mantle family; the Price List API is authoritative for xAI, Google and AgentCore.** (b) Accept cache-read as an **input** on the model row with `0.1×` as a documented default, so a model that breaks the ratio is a data change. (c) Finish the six-site cleanup from the existing entry. +- **Subtracts**: a false sourcing instruction in the file every contributor reads before touching cost code; a hardcoded ratio now falsified by **two** live models; and six duplicated wrong constants. +- **Effort × Impact**: **Low × Medium–High** — cheapest high-leverage item after #1. +- **Verdict**: **Worth trying.** Three verified facts make it unambiguous. (1) The provenance claim **reproduces as false for the second consecutive week**: the entire offer file holds 10 Claude SKUs, none newer than Claude 3, and Haiku 4.5 / Sonnet 4.6 / Fable 5.1 / GPT-5.4 / Nova Micro have no us-west-2 SKUs at all. (2) The `0.1×` ratio already had one counterexample in **Fable 5.1 at 0.025×**; **Grok 4.6 is now the second, at 0.25×** — and Grok is worse than a wrong number, because it has **no cache-write SKU at all** and the card lists only *implicit* caching, so there is nothing for our contract's explicit `cachePoint` model to place. (3) The **1.100× Regional premium survives independently** — re-confirmed this week on the 18 newly-published us-west-2 Grok SKUs ($2.20 Geo vs $2.00 Global) and on both Astra tiers — so #914's *substance* is safe even though its *provenance* was not. ⚠️ Treat Grok as a **non-cached** model in cost projections until implicit caching is measured against `cacheStatus` on a real warm turn. + +### 5. Retire the `ClientSession` monkeypatch via `_meta` `serverInfo` — without waiting on `server/discover` + +- **Source**: https://github.com/modelcontextprotocol/ext-apps/releases/tag/v2.0.0 (2026-09-08) + the Strands MCP-client changes in https://github.com/strands-agents/harness-sdk/releases/tag/python%2Fv1.55.1. **Narrows the open [2026-09-04] "Migrate the MCP Apps host off `initialize` to `server/discover`" entry** — this is the half that is available now; keep the migration entry for the rest. +- **Surface area**: `backend/src/agents/main_agent/integrations/mcp_apps.py:23-32` (the `ClientSession` symbol patch) and `:673` (the `serverInfo` capture); the App-frame header fields (`serverName`, `icon`, `toolName`) on the `ui_resource` event. +- **Change**: read `io.modelcontextprotocol/serverInfo` from result `_meta` — now documented in `ext-apps` `schema.json` — instead of intercepting `initialize` through a patched Strands internal. Keep the patch as a fallback only for servers that do not emit it, and delete it once they all do. +- **Subtracts**: a monkeypatch on `strands.tools.mcp.mcp_client.ClientSession`, taken *because* the SDK offered no hook, on a class Strands is **actively changing** — this week alone: an httpx2 adapter (#4183), an `mcp` floor relaxed to `<2.2` (#4151, which can now pull mcp 2.x under us), SEP-2663 tasks (#4125), and changed `load_servers` defaults (#4177). Four independent reasons it breaks, none of which we control. +- **Unlocks**: an App-frame header that survives the MCP client moving under us, on a standards-blessed field rather than a private symbol — and it decouples that win from the much larger `server/discover` migration, which is still gated on two unanswered questions (does AgentCore Gateway speak it, does Strands expose it). +- **Effort × Impact**: **Medium × Medium** +- **Verdict**: **Worth trying**, second to #2 on the backend. ⚠️ **Verify the servers we actually call emit the `_meta` field before deleting anything** — the field being in `schema.json` is not evidence any of our Gateway targets or Lambda FastMCP servers populate it. ⚠️ In the same release, host-facing error codes changed (`-32002` → `-32602`, invalid params `-32603` → `-32602`) and the `MCP error N:` message prefix was removed — grep the App-error chain shipped in #1009/#1013 for any code or message-text matching, since so much of that chain was built on message text. ⚠️ The wire protocol is otherwise **unchanged** in v2.0.0, so nothing else in our host path needs touching. + +## Take + +The system is trending *toward* the ecosystem on the two axes it has spent months on: upstream finally shipped the provider-agnostic `CacheConfig` our hand-rolled caching code was waiting for, and closed the very issue our defensive gate exists to prevent — both already inside the version we pinned three days ago. The external week otherwise asks for restraint rather than adoption: the protocol layer stood still, and the one big arrival, GPT-6 Astra, comes with a two-tier price the reference repo has already registered wrong. The genuinely uncomfortable finding is internal and it is not a feature: **the nightly has been failing for seven nights, the cause is deterministic, and PR CI is structurally incapable of catching it** — six specs whose entire job is repo hygiene are themselves dependent on how they were invoked. Fix that first; it costs the least and everything else in this doc is easier to trust once the only full-fidelity run of the suite is green again. What Phil would notice first if shipped: a green nightly badge, and — on the next cost-anatomy read — a system prefix that Strands places for us instead of one we hand-place and defend in a forty-line comment. + +--- + +## Sources Scanned + +| # | Source | URL | Accessed | Items | +|---|---|---|---|---| +| 1 | AWS What's New (Bedrock/AgentCore filter) | https://aws.amazon.com/about-aws/whats-new/recent/feed/ | 2026-09-11 | 4 | +| 2 | AgentCore devguide release notes | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/release-notes.html | 2026-09-11 | 2 (both month-dated, neither confirmed in-window) | +| 3 | Strands harness-sdk releases | https://github.com/strands-agents/harness-sdk/releases/tag/python%2Fv1.55.1 | 2026-09-11 | 7 | +| 4 | Strands `CacheConfig` source (tag + main) | https://github.com/strands-agents/harness-sdk/blob/python/v1.55.1/strands-py/src/strands/models/model.py | 2026-09-11 | 2 | +| 5 | Strands harness-sdk `main` commits | https://github.com/strands-agents/harness-sdk/commits/main | 2026-09-11 | 6 | +| 6 | Strands PR #4193 (ours, `cache_write_tokens`) | https://github.com/strands-agents/harness-sdk/pull/4193 | 2026-09-11 | open, unreleased | +| 7 | Strands issue #3546 (`Usage` contract) | https://github.com/strands-agents/harness-sdk/issues/3546 | 2026-09-11 | open, touched 09-07 | +| 8 | Strands issue #4168 (cachePoint AccessDenied) | https://github.com/strands-agents/harness-sdk/issues/4168 | 2026-09-11 | **closed 09-04** | +| 9 | Reference repo commits + merged PRs | https://github.com/aws-samples/sample-strands-agent-with-agentcore/pull/290 | 2026-09-11 | 2 substantive, 6 Dependabot | +| 10 | Reference repo GPT-6 Astra catalog row | https://github.com/aws-samples/sample-strands-agent-with-agentcore/pull/291 | 2026-09-11 | 1 | +| 11 | MCP spec repo (merged PRs + commits) | https://github.com/modelcontextprotocol/modelcontextprotocol/pull/3282 | 2026-09-11 | 2 (no spec revision) | +| 12 | MCP blog | https://blog.modelcontextprotocol.io | 2026-09-11 | 0 in window | +| 13 | MCP ext-apps v2.0.0 | https://github.com/modelcontextprotocol/ext-apps/releases/tag/v2.0.0 | 2026-09-11 | 3 | +| 14 | MCP ext-apps new host adopter | https://github.com/modelcontextprotocol/ext-apps/pull/729 | 2026-09-11 | 1 | +| 15 | SEP-2549 residual open question | https://github.com/modelcontextprotocol/modelcontextprotocol/issues/2939 | 2026-09-11 | 1 | +| 16 | FastMCP releases (repo moved to PrefectHQ) | https://github.com/PrefectHQ/fastmcp/releases/tag/v4.0.3 | 2026-09-11 | 3 | +| 17 | FastMCP 4.0.0 (pre-window, load-bearing) | https://github.com/PrefectHQ/fastmcp/releases/tag/v4.0.0 | 2026-09-11 | 1 | +| 18 | FastMCP `UserSession`/`SessionId` | https://github.com/PrefectHQ/fastmcp/pull/4604 | 2026-09-11 | 1 | +| 19 | Cursor blog — Projects | https://www.cursor.com/blog/projects | 2026-09-11 | 1 | +| 20 | Claude Code CHANGELOG (2.1.268) | https://github.com/anthropics/claude-code/blob/main/CHANGELOG.md | 2026-09-11 | 6 | +| 21 | assistant-ui releases | https://github.com/Yonom/assistant-ui/releases | 2026-09-11 | 0 in window (latest 09-03) | +| 22 | NN/g AI topic | https://www.nngroup.com/topic/artificial-intelligence/ | 2026-09-11 | **not scanned — HTTP 404** | +| 23 | anthropic.com/news | https://www.anthropic.com/threat-intelligence-report-september-2026 | 2026-09-11 | 1 (not harness-relevant) | +| 24 | anthropic.com/engineering | https://www.anthropic.com/engineering | 2026-09-11 | 0 in window | +| 25 | GPT-6 Astra model card | https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html | 2026-09-11 | 4 | +| 26 | GPT-6 Astra What's New | https://aws.amazon.com/about-aws/whats-new/2026/09/openai-gpt-6-astra-on-amazon-bedrock/ | 2026-09-11 | 1 | +| 27 | xAI Grok 4.6 model card | https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html | 2026-09-11 | 1 | +| 28 | Bedrock model-lifecycle policy | https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html | 2026-09-11 | 1 (⚠️ single-source, text not diffed) | +| 29 | pydantic-ai releases (v2.40–v2.42) | https://github.com/pydantic/pydantic-ai/releases | 2026-09-11 | 4 | +| 30 | langchain 1.4.0 | https://github.com/langchain-ai/langchain/releases/tag/langchain%3D%3D1.4.0 | 2026-09-11 | 3 | +| 31 | opencode releases | https://github.com/anomalyco/opencode/releases/tag/v1.18.30 | 2026-09-11 | 3 | +| 32 | opencode v1.18.28 | https://github.com/anomalyco/opencode/releases/tag/v1.18.28 | 2026-09-11 | 1 | +| 33 | AgentCore SDK issue #659 (verified N/A) | https://github.com/aws/bedrock-agentcore-sdk-python/issues/659 | 2026-09-11 | 1 | +| 34 | AgentCore SDK issue #661 (`MemoryStore`) | https://github.com/aws/bedrock-agentcore-sdk-python/issues/661 | 2026-09-11 | 1 | +| 35 | Starter-toolkit issue #498 (no session kill) | https://github.com/aws/bedrock-agentcore-starter-toolkit/issues/498 | 2026-09-11 | 1 | +| 36 | AgentCore SDK standing issues #564/#621/#629/#646 | https://github.com/aws/bedrock-agentcore-sdk-python/issues/564 | 2026-09-11 | 4, all open | +| 37 | `bedrock-agentcore` v1.22.0 release | https://github.com/aws/bedrock-agentcore-sdk-python/releases/tag/v1.22.0 | 2026-09-11 | 1 | +| 38 | HN — grep vs LSP | https://news.ycombinator.com/item?id=49560260 | 2026-09-11 | 1 | +| 39 | Anthropic cookbooks (both repos) | https://github.com/anthropics/claude-cookbooks | 2026-09-11 | 0 in window | +| 40 | LibreChat releases | https://github.com/danny-avila/LibreChat/releases/tag/v0.8.8-rc2 | 2026-09-11 | 0 in window (latest 09-03) | +| 41 | r/LocalLLaMA | https://www.reddit.com/r/LocalLLaMA/ | 2026-09-11 | **not scanned — non-JSON interstitial** (consistent with the 2026-05-18 decisions entry) | + +**Price List API — reproducible commands** (run 2026-09-11; both offer files were republished the same day, so the window is fully covered): + +``` +aws sts get-caller-identity # arn:aws:iam::029812070295:user/webdevs_local +curl -s 'https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/index.json' # version 20260911124408 supersedes 20260804165549 +curl -s 'https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockAgentCore/index.json' # version 20260911124408 supersedes 20260901164424 +# then diffed products+terms of .../AmazonBedrock/{20260804165549,20260911124408}/index.json +# and .../AmazonBedrockAgentCore/{20260901164424,20260911124408}/index.json +``` + +Results: `AmazonBedrock` 10,998 → 11,657 SKUs (659 added, 18 of them `USW2-xai.grok-4.6-mantle-*` where there were previously zero); 48 price changes, **all `UGW1-*` GovCloud**, filtering to non-GovCloud returns empty. Total Claude SKUs in the file: **10**, none newer than Claude 3. `AmazonBedrockAgentCore` 6,481 → 6,481, zero adds/removes/changes, `products` dicts byte-equal. + +**Internal-audit commands** are quoted inline in the Internal Audit section, per the [2026-09-04] verification rule. + +## Web Budget + +**Used: 15 / 50** requests (target). + +Breakdown of actual `WebFetch`/`WebSearch` calls: AWS What's New + devguide **3**; frontier models **3**; pricing **3**; agentic UI/UX + harness **5**; MCP + FastMCP **1**. Zero for: Strands (8 `gh`/`curl` calls, all free), reference repo (all `gh`), opencode (all `gh`), AgentCore SDK issues + community (all `gh`/`curl`), cookbook + LibreChat (all `gh`). All internal-audit and version-pin work ran through `git`, `gh` and `curl`. + +**Skipped (unreachable)**: +- NN/g AI topic page — HTTP 404 on https://www.nngroup.com/topic/artificial-intelligence/; not retried per budget. Worth finding the current URL before next week's run. +- r/LocalLLaMA JSON endpoint — returned a non-JSON interstitial. Not retried; consistent with the 2026-05-18 decisions-log entry that Reddit is blocked at the domain level. + +**Skipped (other)**: +- `nightly-develop-PlatformStack` `StackStatusReason` — this session's AWS credentials are account 029812070295, not dev-ai 490617140655, so `describe-stacks` could not read it. The CI log is a **single source** for that half of finding #1. +- Local reproduction of the `--coverage` path-resolution mechanism — not run, to avoid checking out `develop` over Phil's working branch (`feature/agentcore-browser-tool`). The exact command to confirm it is named in Idea #1. + +**Notes**: well under target and deliberately so — two subagent categories stalled on `WebFetch` in an earlier attempt and were re-run with tighter per-agent caps, which pushed most sources onto the free `gh`/`curl` routes. No source was dropped silently; every skip is listed above. The budget was not padded: six sources were genuinely empty in the window (MCP spec, MCP blog, assistant-ui, anthropic.com/engineering, both cookbook repos, LibreChat) and are reported as empty rather than filled. diff --git a/docs/kaizen/review-queue.md b/docs/kaizen/review-queue.md index 5fe8a8fbf..bbadbe4b7 100644 --- a/docs/kaizen/review-queue.md +++ b/docs/kaizen/review-queue.md @@ -5,12 +5,80 @@ Items added by `kaizen-research`, consumed by `kaizen-review-prep`. ## Open -> ✅ **Queue hygiene completed 2026-08-14** (at Phil's request, ahead of `kaizen-review-prep`). **Nine** stale entries were resolved: four `bedrock-agentcore` bump entries and two Strands bump entries (all **shipped** in #857 — `bedrock-agentcore` 1.9.1 → **1.21.0** at zero lag, `strands-agents` → **1.51.0**; #482 and #571 closed upstream), two nightly-CI entries (**green 12 consecutive days**), and two MCP Apps spec-prep entries (superseded now the 2026-07-28 spec is final). Genuine residue was carried forward, not dropped: **#564** (still open upstream) and the **un-adopted Strands capabilities** are now their own entries below. See `## Resolved` for the evidence trail. +### [2026-09-11] ✅ ANSWERED: `bedrock_cache_points_supported()` STAYS — and it is wrong in the opposite direction from the one we suspected +- **Source**: measured, not read — `backend/scripts/probe_bedrock_cache_point_support.py` (added by this entry) against the pinned strands-agents 1.55.0 and live `bedrock-runtime` in dev-ai/us-west-2, 2026-09-11. **Answers check (d)** of the [2026-09-11] "Retire the hand-placed system `cachePoint`" entry below, and **narrows (c)** of the [2026-09-11] `ClientSession` monkeypatch entry. Does **not** close either — (a), (b) and the `tools_ttl` mixed-TTL evaluation are untouched. +- **Surface**: backend — `core/model_config.py` (`bedrock_cache_points_supported`, the cachePoint-budget comment), `core/agent_factory.py` (the hand-placed system block), `backend/scripts/probe_bedrock_cache_point_support.py`. +- **Effort × Impact**: S × M — the code change is three comments; the value is that two future "obvious" subtractions are now pre-refuted with numbers. +- **Subtracts**: **no, and that is the finding.** The proposal was to delete the predicate. It must stay. The only true subtraction available is its `tools_ttl` half, worth ~1 line. +- **Status**: done for the comments; **open as a standing question about model coverage.** + - **The tools half is dead weight as of 1.55.0.** `_build_tools_cache_point` applies the identical `_cache_strategy != "anthropic"` test itself (`bedrock.py:579`). `tools_ttl=True` and `tools_ttl=False` produce **byte-identical requests** on a non-Anthropic model. This half is what upstream #4168 closed. + - **The system half is load-bearing and cannot be retired by any upstream fix.** `format_request` copies `system_prompt_content` verbatim (`bedrock.py:376`) and `_apply_system_cache_ttl` only ever rewrites a TTL — it **never removes a point**. So a hand-placed system cachePoint reaches Bedrock unfiltered, and Bedrock refuses it: `us.meta.llama3-3-70b`, `mistral.mistral-large-2407` and `us.deepseek.r1` all failed, while the identical request without the point succeeded on all three. **The rejection comes from Bedrock, not the SDK** — which is why "#4168 is fixed in our line" was never evidence for deleting this. + - **⚠️ CORRECTION — the exception name in our own code was wrong for months.** `model_config.py` claimed **`ValidationException`**. Live Bedrock returns **`AccessDeniedException`** — *"You invoked an unsupported model or your request did not allow prompt caching."* research/2026-09-11.md had the right exception and our source comment had the wrong one, which is likely why #4168 read as describing our gate. Fixed in this change. + - **⚠️ THE REAL FINDING — the predicate is OVER-broad, not over-cautious.** Its docstring asserted *"Anthropic models are the only Bedrock family with prompt-cache support."* **False.** Nova Micro **accepts a system cachePoint and honors it**: 7,203 input tokens → 2, with 7,201 cache-written. Our predicate refuses it, and so does upstream — `_should_cache_system` gates on the same `_cache_strategy == "anthropic"` string test. **Neither we nor Strands will ever place a point on a Bedrock family that can cache but isn't Claude**, and nothing in CI can notice, because the predicate is a string test that keeps returning the same answer while the platform moves underneath it. This is the standing §2a watch's *next* payout, and it is an **addition**, not a subtraction — which is why §2a's "subtraction by construction" framing needs the amendment made in the same change. + - **⚠️ TRAP, newly measured, relevant to ALL future caching work — Bedrock silently ignores a cache point below the model's minimum.** No error, no cache buckets, indistinguishable from "unsupported". On Haiku 4.5 in us-west-2: **2,351 and 3,911 tokens wrote nothing; 5,202 wrote in full** — bracketing the real floor at **4,096**, which is *twice* the 2,048 the first-party Anthropic docs publish for Haiku. The first cut of the probe sized its prefix from the documented figure and produced a false "does not cache" for our own production model. Any future A/B of a caching change must clear this floor or it measures nothing. + - **Follow-on — ✅ MEASURED 2026-09-11 against all 54,932 Claude `C#` rows in PROD, and the worry is REFUTED. Do not spend time here.** The hypothesis was that small-prefix agents silently pay full input because both static cachePoints fall under the floor. Prod says caching is working: of turns billing **≥4,096** tokens, the uncached share is **0.1% (Jul) / 1.2% (Aug) / 0.2% (Sep)**. Only **670 rows in the table's entire history** are under-floor uncached, and those are *correctly* uncached — Bedrock will not cache that little from anyone, so there is no saving to recover. **The instrument validated itself on the way:** 2026-04 and 2026-05 read **100.0% uncached**, which is exactly the pre-#471 era before caching was re-enabled, and 2026-06 reads 57.5% as it rolled out. ⚠️ **Do not read a raw uncached count without splitting by month** — the pre-#471 rows are 105M tokens of history and will masquerade as a live leak (they did, in the first cut of this query: $166 of "waste" that evaporated on the time split). + - **The whole live residual is 283 turns / 3.3M tokens over six weeks, and it is ~90% explained:** **(A) 45% is `/chat/api-converse`** — 206 turns across 3 sessions, the API-key path that CLAUDE.md documents as raw boto3 on the Bedrock branch with **no cachePoints by design**. **(B) 8% is single-turn sessions with a large input**, the exact signature of the documented strands≥1.48 behaviour where a cachePoint is *skipped entirely* when a non-PDF document is the first content block. **(C) 46% is 61 turns across 58 sessions** — ~1 turn each, same attachment shape, **~$5 total at Sonnet 4.6 rates over six weeks.** Nothing systemic. + - **The one genuine (small) opportunity this surfaced:** the API-key `/chat/api-converse` path bills full input on every turn, forever, because it hand-rolls boto3 and places no cache points. 1.49M tokens in six weeks from **three** sessions. Negligible today, but it scales linearly with API-key adoption and the handler already moved to app-api in #621. File as its own item if API-key traffic grows. + + +### [2026-09-11] Fix the nightly, both halves — anchor the branding specs' paths, and put `--coverage` where PR CI can see it +- **Source**: research/2026-09-11.md ▸ Top 5 #1 — internal. Runs [33957119927](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/33957119927) (2026-09-05, first red) → [34582692457](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/34582692457) (2026-09-11). Specs landed with PR #933. +- **Surface**: CI / frontend — `.github/workflows/tests.yml` (`test-frontend`), `frontend/ai.client/package.json` (`test:ci`), `scripts/frontend/test.sh`, and six specs: `src/branding/rebranding-guide.doc-presence.spec.ts`, `src/branding/prestart-generator-parity.spec.ts`, `src/branding/brand-theme-golden.spec.ts`, `src/branding/surface-theme-golden.spec.ts`, `src/app/surface-literal-guard.spec.ts`, `src/app/tailwind-theme-import-guard.spec.ts`. Plus one CloudFormation stack in dev-ai. +- **Effort × Impact**: L–M × H +- **Subtracts**: yes — a gate that has produced only false signal for a week; a `DELETE_FAILED` stack whose retained resources are still billing; and the *category* of repo-hygiene spec that silently depends on how it was invoked. +- **Status**: open — **recommended #1, cheapest item in the scan, evidence complete.** **Seven consecutive nightly failures** (green through 2026-09-04), **two independent root causes, neither a flake.** **(a)** `Test Frontend with Coverage` fails while `deploy / [deploy-develop] Test frontend` **succeeds in the same run on the same ref** — the jobs differ in exactly one thing: nightly runs `ng test --no-watch --coverage`, PR CI and the deploy track run `npm run test:ci` = `ng test --watch=false`. Under `--coverage`, `import.meta.url` / the `__dirname` shim resolves **two levels shallower**. Three converging proofs: `rebranding-guide.doc-presence.spec.ts` read `'# AiClient\n\nThis project was genera…'` (= `frontend/ai.client/README.md`) when `resolve(SPEC_DIR, './README.md')` should reach `src/branding/README.md`, which begins `# Rebranding Guide` on both branches; `prestart-generator-parity.spec.ts` fails at **collection** on line 31 = `readdirSync(resolve(SPEC_DIR, '../../scripts/branding'))`, an ENOENT; and both guards *find violations* rather than nothing (`surface-literal-guard.spec.ts:44` does `path.join(__dirname)`, `tailwind-theme-import-guard.spec.ts:30` does `path.resolve(__dirname, '..')` — a shallower base walks the whole package, incl. generated Tailwind CSS full of literal hex and a direct `@import "tailwindcss"`). **PR CI structurally cannot catch this class** — it never passes `--coverage`. Fix = anchor the roots to a stable constant *and* put the coverage invocation somewhere a PR can see. **(b)** `nightly-develop-PlatformStack` is wedged in `DELETE_FAILED`: `Stack:arn:aws:cloudformation:us-west-2:490617140655:stack/nightly-develop-PlatformStack/… is in DELETE_FAILED state and can not be updated.` → `exit 1`. Nothing self-heals. ⚠️ **Reproduce (a) before fixing it** — `cd frontend/ai.client && ./node_modules/.bin/ng test --no-watch --coverage` on `develop`; the mechanism is inferred from logs, not executed. ⚠️ ⚠️ **single-source** on (b): this session's AWS creds are account 029812070295, not dev-ai 490617140655, so `StackStatusReason` and the retained-resource list could not be read — read them before deleting, since a stuck ephemeral stack usually means a retained resource that is **still billing**. ⚠️ Do **not** combine with the vitest 4→5 major — the suite is not a trustworthy migration baseline until green. Mirror-image of the [2026-09-04] "never resolve a flaky entry on a green streak" rule: this is a deterministic red that read as noise for a week, with **0 issues filed**. -### [2026-09-08] AgentCore Runtime workspaces — give the agent a filesystem; start by mounting the one we already have +### [2026-09-11] Retire the hand-placed system `cachePoint` — `CacheConfig.system_prompt_ttl` / `tools_ttl` are already in our pin +- **Source**: research/2026-09-11.md ▸ Top 5 #2 — https://github.com/strands-agents/harness-sdk/blob/python/v1.55.1/strands-py/src/strands/models/model.py + https://github.com/strands-agents/harness-sdk/issues/4168 (**closed 2026-09-04**). **Supersedes the [2026-09-04] "Gate the Strands 1.51 → 1.54 bump on the system-cachePoint collision" entry** — we are already on 1.55.0, so the bump is done and only the collision question remains. Merge them at review. +- **Surface**: backend — `core/agent_factory.py:213-224` (the hand-placed system `cachePoint`; the now-false comment at :222), `core/model_config.py:375-400` (`strategy="auto"`, the `bedrock_cache_points_supported()` gate, the ~40-line cachePoint-budget comment at :380), `backend/pyproject.toml:59,74`. +- **Effort × Impact**: M × H +- **Subtracts**: yes — the hand-placed `SystemContentBlock`, the ~40-line comment defending it, a comment asserting a false invariant, and possibly `bedrock_cache_points_supported()`. The library-native subtraction this skill weights for, finally payable. +- **Status**: open — **the standing §2a caching watch finally paid out, and the payout is already inside the version we pinned three days ago.** `CacheConfig` at 1.55.0/1.55.1 carries `strategy`, `ttl`, **`system_prompt_ttl: bool | str = True`**, `cache_key`, **`tools_ttl: bool | str | None`**, and `strands/models/_openai_cache.py` exists. Four checks: (a) does `strategy="auto"` + `system_prompt_ttl` place the point our block places — delete ours if so; (b) rewrite `agent_factory.py:222`, which asserts *"auto strategy strips only message-level cachePoints, never system ones"*, a 1.51-era fact; (c) evaluate `tools_ttl` against the layered mixed-TTL technique the [2026-08-14] cookbook entry measured at **54% cheaper** upstream — previously blocked, now reachable; (d) ~~re-test whether `bedrock_cache_points_supported()` is still load-bearing now #4168 is fixed in our line~~ — **✅ ANSWERED 2026-09-11, measured: KEEP IT.** See the entry at the top of this queue. The tools half is redundant with upstream; the **system** half is not retirable by any upstream fix, because Bedrock — not the SDK — rejects the point with `AccessDeniedException`. Do not re-propose deleting it. **(a) is also now partly answered**: upstream's auto-injected system point only arms on `_cache_strategy == "anthropic"`, so deleting our block would silently drop the system cachePoint on any non-Claude Bedrock model we ever add. Also bump 1.55.0 → **1.55.1** (patch-only) for #4199 (malformed snapshot IDs) and #4228 (context-manager parity). **De-risking fact:** upstream's docstring says a hand-placed system cache point is *"honored rather than doubled"*, so an incremental migration will not double-write. ⚠️ **Never adopt a caching default on inspection alone** — #954 shipped on a wrong premise and measured **57% more expensive** live before #956 reverted it. `backend/scripts/probe_gpt56_cache_rates.py --mode both --grow-history` is the gate. ⚠️ The reference repo runs 1.54 green and **does not hand-place a system cachePoint at all** — their green build is not evidence for us; they have nothing to collide. ⚠️ On upstream `main` but unreleased: `cache_key` widens to `str | Literal[False] | None` and **auto-derives `strands-`** when a session manager is present — that is `build_prompt_cache_key()` retiring on the next minor, not this one. ⚠️ Also on `main`: a first-party context-manager/offloading stack (#4254, #4187, #4146, #4118, #4231) that will collide with `ContextOffloader` — the decisions log forbids a bare "delete ours" proposal, so watch it, don't adopt it piecemeal. + +### [2026-09-11] Add GPT-6 Astra at the 272K tier — and make the context-tier boundary a catalog field +- **Source**: research/2026-09-11.md ▸ Top 5 #3 — https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html + https://aws.amazon.com/about-aws/whats-new/2026/09/openai-gpt-6-astra-on-amazon-bedrock/ (GA 2026-09-08) + https://github.com/aws-samples/sample-strands-agent-with-agentcore/pull/291 — **tier question resolved 2026-09-11** against https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html + the `AmazonBedrock` Price List offer `20260911124408` + our own `docs/specs/gpt-5-6-prompt-caching.md` billing observation. +- **Surface**: backend / frontend — `curated-models.ts`, the Mantle Responses path (`_create_mantle_model` / `bedrock_mantle_config`, per-model region routing), `admin/services/model_access.py` for the role grant, the model picker. +- **Effort × Impact**: L–M × H — the Mantle Responses path already exists; this rides it. +- **Subtracts**: no — addition. Justified because it *prevents* a regression we would otherwise import. +- **Unlocks**: + - **A genuine 1M-token frontier tier** (1,050,000 context / 128,000 max output) alongside Fable 5.1 — the first catalog model where a whole corpus fits in one turn. + - **A tier-aware catalog.** Today `maxInputTokens` doubles as both a capability bound and a pricing cap; Astra is the second model where those differ. Making the boundary explicit lets the admin page show it, lets cost projections respect it, and lets quota math account for Astra's **10× output-token quota burn** — which matters directly against 1.20.0's finding that the campus ceiling is the Bedrock TPM quota, not compute. +- **Status**: open — **✅ BLOCKING QUESTION ANSWERED: the tier is selected by the ACTUAL token count of the request, not by the declared window. `272000` is therefore a *policy* choice, not a correctness requirement — but it should still ship, for a corrected reason (below).** AWS states the rule in prose **nowhere**; the answer rests on three independent strands that converge, two of them external. + - **(1) Price List API — SKU shape.** In offer `20260911124408`, GPT-5.6 Terra/Luna carry `-long-ctx` as a value of the **`tokenType` attribute** under an *identical* `model` attribute, orthogonally across all four token buckets and all three service tiers: `input_tokens_mantle` $2.64 vs `input-tokens-long-ctx` $5.28; `cache-write-tokens-30m` $3.30 vs `…-30m-long-ctx` $6.60; `Cache Read Input Tokens` $0.264 vs `cache-read-tokens-long-ctx` $0.528; `output_tokens_mantle` $15.84 vs `output-tokens-long-ctx` $23.76. A `tokenType` is a **per-token billing meter**, not a configuration or a profile — no `model`, `inferenceType` or profile attribute distinguishes the tiers. ⚠️ These are GovCloud (`UGW1-`) rows, the only region where the OpenAI family has long-ctx SKUs published, and **GPT-6 Astra has ZERO SKUs in the entire 11,657-SKU offer file** — so the Price List speaks to the *family convention*, not to Astra directly. + - **(2) The GPT-5.6 Terra model card refutes the "declared window" reading outright.** Its card states **Context window: 1M tokens** and lists exactly **one model ID per endpoint** (`openai.gpt-5.6-terra` / `us.` / `global.`) — there is no long-context variant ID or inference profile to declare. Yet it publishes *both* a Short Context (272K) and a Long Context (1M) price table. If the tier followed the declared window, every Terra call would bill Long and the Short table would be unreachable. Its GovCloud tables match the Price List SKUs **to the cent** ($2.64 / $3.30 / $0.264 / $15.84 short; $5.28 / $6.60 / $0.528 / $23.76 long) — two independent sources agreeing, which also proves `-long-ctx` *is* the "Long Context Window" table. + - **(3) Our own live billing corroborates it.** `docs/specs/gpt-5-6-prompt-caching.md` records that in dev-ai Cost Explorer *"every row observed is `_standard` and no `-long-ctx` usage type has ever appeared in this account"* — while we call the **1M-window** `us.openai.gpt-5.6-*` IDs with requests held under 272K. A 1M-declared model billing at short-context rates is only possible if the tier follows actual request size. + - **(4) There is no declaration channel at all.** `maxInputTokens` is *ours*, not AWS's: in `backend/src/` it is read only to populate `final_metadata["contextWindow"]` and the compaction threshold (`stream_coordinator.py:710-712`, `:2808-2818`) and to serialize the catalog. **It is never placed in a Bedrock request.** Bedrock cannot see it, so it cannot price on it. +- **Consequence — our own framing was wrong, and the correction matters.** "Copying their row opts the whole fleet into the expensive tier" is **false**. Registering Astra at `maxInputTokens: 1_000_000` does not double our input cost; it opens the 2× card only on turns that *actually* exceed 272K. **But ship `272_000` anyway**, for the reason already written at `curated-models.ts:363`: `CuratedModel` holds **one flat rate per bucket**, and the cap is what keeps that single rate arithmetically correct. Raising it does not over-charge the fleet — it silently **under-charges** every turn that crosses the boundary (input 2×, output 1.5×, and per the SKU shape cache-read/write 2× as well). Same action, different reason, and the difference is the whole point of the generalization: **the tier boundary is a *billing-model* field, not a capability field.** ⚠️ Also note the ref repo sets `maxInputTokens: 1000000` on Sol/Terra/Luna too — their Astra row is consistency with their own family rows, **not** an independent judgment about Astra's tiering, so it is not evidence either way. +- **Residual unknown (honest, and only matters if we ever raise the cap).** AWS nowhere publishes the **predicate**: whether the 272K test is on input tokens alone, input+output, or total including cache reads; nor whether crossing it reprices the whole request or only the excess. The SKU shape — all four buckets have long-ctx twins — implies **whole-request repricing**, but that is inference, not a published rule. What would settle it: a single measured call above 272K in dev-ai with a same-day Cost Explorer read (the `probe_gpt56_cache_rates.py` harness already does this shape), or an AWS support case. Not worth doing while the cap holds. +- **Rates (unchanged, re-verified on the card).** Geo CRIS Short (≤272K) **$11.00 in / $13.75 cache-write / $1.10 cache-read / $55.00 out** per MTok; Long (1.05M) **$22.00 / $27.50 / $2.20 / $82.50**. Global CRIS a flat 10% under Geo ($10.00/$12.50/$1.00/$50.00 and $20.00/$25.00/$2.00/$75.00). In-Region is priced identically to Geo but is **not offered** on `bedrock-runtime`. +- **Also confirmed while there:** + - **`rejectsTemperature: true` is REAL and family-wide — not an Astra-specific invention.** In the ref repo's catalog (`revision 2026-09-09.1`) it is `true` for Astra, **all three GPT-5.6 models**, Claude Opus 5, Sonnet 5 and Grok 4.6 — it tracks reasoning models generally, and we already run the GPT-5.6 family. **Neither Astra's nor Terra's card has a supported-inference-parameters section at all**, so AWS does not contradict it, merely omits it. ⚠️ **single-source** for Astra specifically (ref repo + PR #291's claimed live `direct Bedrock Responses invocation`). Per **#915**, ship it as an explicit **non-empty** `supportedParams` that omits temperature — **never an empty `supportedParams`**, which the guard reads as a bypass. + - **Harness gaps — all CONFIRMED verbatim on the Astra card.** `bedrock-runtime` Not Supported: **server-side tool use, intelligent prompt routing, count tokens, structured outputs**. **CRIS-only** — Programmatic Access lists In-Region "Not supported", leaving `us.openai.gpt-6-astra` / `global.openai.gpt-6-astra`. The endpoint split is real: Guardrails and application inference profiles are supported on `bedrock-runtime` **Converse API only**, and application inference profiles are **Not Supported** on `bedrock-mantle`. `bedrock-mantle` is **us-west-2 only**. + - ⚠️ **NEW, and not previously in this entry: prompt caching is NOT listed for Astra on `bedrock-runtime`.** Terra's card lists Implicit **and** Explicit Prompt Caching under *both* endpoints; **Astra's lists them under `bedrock-mantle` only (Responses API only)**, and caching appears in **neither column** of its `bedrock-runtime` feature table. The plan is `us.openai.gpt-6-astra` on `bedrock-runtime`, so **resolve this before banking any cache savings** — our entire prompt-cache contract assumes explicit cache points, and a 1M-window model with no caching is a very different cost profile. + - **Cache ratios 0.1× read / 1.25× write — confirmed twice.** Arithmetic on the card holds in both tiers ($11.00→$1.10/$13.75; $22.00→$2.20/$27.50), and AWS states the rule in prose: *"Cached input is billed at a 90% discount compared to uncached input tokens, and tokens written to cache are billed at 1.25 times the uncached input rate."* + - **The 30-minute TTL is a real SKU dimension, not card prose** — the Price List `tokenType` is literally `cache-write-tokens-30m`. **Do not route it through `cache_ttl_seconds_for()`** (5m/1h, Anthropic). + - **10× output TPM burn — CONFIRMED verbatim**, and note the scoping: *"On the `bedrock-runtime` endpoint, limits are managed as tokens per minute (TPM) with a 10x burndown rate, where 1 output token consumes 10 tokens."* Identical sentence on Terra's card. ⚠️ It is stated **for `bedrock-runtime`**; the cards say nothing about burndown on `bedrock-mantle`. This lands directly against 1.20.0's finding that the campus ceiling is the Bedrock TPM quota, not compute — at 128,000 max output, one saturating Astra turn burns **1.28M TPM**. + - ⚠️ Dev/prod prefix divergence still applies (`global.*` is SCP-denied in dev, fine in prod). + +### [2026-09-11] Correct the rate provenance in `CLAUDE.md` and make cache-read a per-model input — Grok 4.6 is the second ratio-breaker +- **Source**: research/2026-09-11.md ▸ Top 5 #4 — a full diff of the republished offer files (`AmazonBedrock` `20260804165549` → `20260911124408`) + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html. **This closes the ⚠️ on the open [2026-09-04] "Finish the #914 rate correction" entry** — its provenance question is now answered; merge and ship the two together at review. +- **Surface**: docs / frontend — the prompt-cache contract bullet in `CLAUDE.md`; `curated-models.ts:96-97` (the `cacheRead = input × 0.1` helper); and the six stale `$2.50/MTok` sites the [2026-09-04] entry enumerates, headed by `model_config.py:380`. +- **Effort × Impact**: L × M–H — cheapest high-leverage item after #1. +- **Subtracts**: yes — a false sourcing instruction in the file every contributor reads before touching cost code; a hardcoded ratio now falsified by **two** live models; six duplicated wrong constants. +- **Status**: open — three verified facts. **(1) The provenance claim reproduces as FALSE for the second consecutive week.** `CLAUDE.md` says rates "come from the AWS Price List API, not the pricing page"; the entire `AmazonBedrock` offer file contains **10 Claude SKUs, none newer than Claude 3**, and Haiku 4.5 / Sonnet 4.6 / Fable 5.1 / GPT-5.4 / Nova Micro have **no us-west-2 SKUs at all**. Correct text: **per-model AWS model cards are authoritative for Claude, Nova and the OpenAI/Mantle family; the Price List API is authoritative for xAI, Google and AgentCore.** Reproducible commands are in research/2026-09-11.md ▸ Sources. **(2) The `0.1×` cache-read ratio now has two counterexamples.** Fable 5.1 reads at **0.025×**; **Grok 4.6 reads at 0.25×** ($0.55 against $2.20 input) — and Grok is worse than a wrong number, because it has **no cache-write SKU at all** and its card lists only *implicit* caching, so there is nothing for our explicit-`cachePoint` contract to place. Fix = accept cache-read as an **input** on the model row with `0.1×` as a documented default. **(3) The 1.100× Regional premium survives independently** — re-confirmed on the 18 newly-published us-west-2 Grok SKUs ($2.20 Geo vs $2.00 Global) and on both Astra tiers — so #914's *substance* is safe even though its *provenance* was not. ⚠️ Treat Grok as a **non-cached** model in cost projections until implicit caching is measured against `cacheStatus` on a real warm turn. ⚠️ Also noted: all 48 price changes in the republish were **GovCloud-only** GPT-5.6 moves (Luna $1.32 → $0.264/MTok, Terra −20%) — no us-west-2 change, but AWS is repricing that family somewhere, so re-check Oregon next week. + +### [2026-09-11] Retire the `ClientSession` monkeypatch via `_meta` `serverInfo` — without waiting on `server/discover` +- **Source**: research/2026-09-11.md ▸ Top 5 #5 — https://github.com/modelcontextprotocol/ext-apps/releases/tag/v2.0.0 (2026-09-08) + the MCP-client changes in https://github.com/strands-agents/harness-sdk/releases/tag/python%2Fv1.55.1. **Narrows the open [2026-09-04] "Migrate the MCP Apps host off `initialize` to `server/discover`" entry** — this is the half available now; keep that entry for the rest. +- **Surface**: backend — `integrations/mcp_apps.py:23-32` (the `ClientSession` symbol patch) and `:673` (the `serverInfo` capture); the App-frame header fields (`serverName`, `icon`, `toolName`) on the `ui_resource` event. +- **Effort × Impact**: M × M +- **Subtracts**: yes — a monkeypatch on a Strands internal, taken *because* the SDK offered no hook, on a class Strands is **actively changing**: this week alone an httpx2 adapter (#4183), an `mcp` floor relaxed to `<2.2` (#4151), SEP-2663 tasks (#4125), and changed `load_servers` defaults (#4177). +- **Unlocks**: an App-frame header on a standards-blessed field rather than a private symbol, decoupled from the much larger `server/discover` migration (still gated on two unanswered questions: does AgentCore Gateway speak it, does Strands expose it). +- **Status**: open — `ext-apps` v2.0.0's `schema.json` now documents **`io.modelcontextprotocol/serverInfo` in result `_meta`**, which is exactly what we intercept `initialize` to get. ⚠️ **Verify the servers we actually call emit it before deleting anything** — presence in `schema.json` is not evidence any of our Gateway targets or Lambda FastMCP servers populate it; keep the patch as a fallback until they do. ⚠️ **Same release changed host-facing error codes**: a handler-thrown `-32002` now reaches the View as `-32602`, invalid params on `ui/*` move `-32603` → `-32602`, and the `MCP error N:` message prefix is **gone** — grep the app-tool-error chain shipped in #1009/#1013 for code or message-text matching, since so much of it was built on message text. ✅ The MCP Apps **wire protocol is unchanged** in v2.0.0 (bidirectional interop test against 1.7.5), so nothing else in our host path needs touching. ⚠️ Related and new: `mcp` resolves at **1.28.1** in `uv.lock` while latest is **2.2.0** and Strands' floor now permits `<2.2` — the next refreshed resolve could cross a major under this patch. + +### [2026-09-08] AgentCore Runtime workspaces — give the agent a filesystem; start by mounting the one we already have — ⛔ **S3 BRIDGE BLOCKED** (build attempted 2026-09-11, stopped before any code) - **Source**: conversation with Phil, 2026-09-08 — **not** from a research scan. Verified against the pinned `botocore` 1.43.68 service model, the AgentCore devguide, and https://aws.amazon.com/blogs/machine-learning/its-safe-to-close-your-laptop-now-hosting-coding-agents-on-amazon-bedrock-agentcore/. - **Surface**: infrastructure — `CreateAgentRuntime`'s `filesystemConfigurations` (nowhere in `infrastructure/lib` today; ✅ verified zero hits). Downstream: `apis/shared/files/workspace.py` and `agents/builtin_tools/workspace_tools.py` if the two surfaces converge. -- **Effort × Impact**: L–M × M for the S3-mount bridge; M–H × H for `sessionStorage` behind the gates below. +- **Effort × Impact**: ~~L–M × M for the S3-mount bridge~~ — **the L–M estimate was wrong and is withdrawn**; the bridge is gated behind a Runtime network-mode migration (Blocker 1 below), so price it as **H × M**. `sessionStorage` remains M–H × H behind the gates below. - **Subtracts**: **no — and that is not the point.** Filed under `Unlocks`. ⚠️ This is the same feature Phil corrected the framing on **2026-05-10** — AgentCore BYO filesystem was written up then as "replaces future filesystem-staging glue," which is what prompted the dual-lens rule now in both kaizen skills. It was framed subtraction-first *again* on 2026-09-08 and corrected again. Third time, rank it on `Unlocks` or don't file it. - **Unlocks** — the whole case. Any **one** of the first three justifies building; the fourth is available now and waits on none of them: - **Multi-turn work over an uploaded file.** A CSV uploaded once, then five follow-ups; today every turn re-fetches and re-parses. Directly on the cost thesis — attachment conversations are **11% of sessions and 31% of prod spend** ([[project_document_conversations_cost]]) — and a workspace is the only place a parsed intermediate could live between turns. @@ -18,9 +86,14 @@ Items added by `kaizen-research`, consumed by `kaizen-review-prep`. - **Long-running scheduled runs.** The best fit in the stack: session identity is already the resume key and no interactive user pays the remount latency. The moment a scheduled prompt is a multi-hour agentic task rather than one turn, suspend/resume stops being a nicety. - **The SNR307 "Model C" thesis, if it ever grows a coding-agent-shaped product.** `sessionStorage` + `InvokeAgentRuntimeCommandShell` *is* that substrate; hand-rolling it would be indefensible. - **Skill assets** — the brand deck builder already fights this (`template_name` is session-scoped, the template cannot persist as a file). -- **What the API actually is** (✅ verified locally against pinned botocore, not from docs prose). `CreateAgentRuntime` takes `filesystemConfigurations`, a list of four mount flavors: `sessionStorage {mountPath}` (per-session, private), `s3FilesAccessPoint {accessPointArn, mountPath}`, `efsAccessPoint {accessPointArn, mountPath}`, `capacityProviderVolume {mountPath, volumeName}`. Sibling `lifecycleConfiguration {idleRuntimeSessionTimeout, maxLifetime}` is the knob we already know from the idle-reaper work (#827). Compute stops at idle timeout; the filesystem stays and remounts on the same session id. Shell access is separate: **`InvokeAgentRuntimeCommand` is already in our pinned botocore** (one-shot, HTTP/2, 1–3,600s timeout, structured exit code); `InvokeAgentRuntimeCommandShell` is **not** a botocore op — it lives in the `bedrock_agentcore` 1.21.0 SDK at `runtime/shell/` (WebSocket, k8s-style channel framing, 1h max, reconnect by `shellId`). -- **Status**: open. Recommended: **Ship the bridge, Defer `sessionStorage` behind gates.** - - **The Ship candidate is #4 above** — mount `s3FilesAccessPoint` over the **existing** `user-files/{u}/{s}/…` layout. The agent gets POSIX reads of files it already owns; no new namespace, no second source of truth, DynamoDB stays authoritative, and the metadata-table-first rule that `workspace_files` was built on ([[project_session_workspace_tools_spec]]) survives intact. Infra-only change, no new API surface. +- **What the API actually is** (✅ verified locally against pinned botocore, not from docs prose). `CreateAgentRuntime` takes `filesystemConfigurations`, a list of four mount flavors: `sessionStorage {mountPath}` (per-session, private), `s3FilesAccessPoint {accessPointArn, mountPath}`, `efsAccessPoint {accessPointArn, mountPath}`, `capacityProviderVolume {mountPath, volumeName}`. Sibling `lifecycleConfiguration {idleRuntimeSessionTimeout, maxLifetime}` is the knob we already know from the idle-reaper work (#827). Compute stops at idle timeout; the filesystem stays and remounts on the same session id. Shell access is separate: **`InvokeAgentRuntimeCommand` is already in our pinned botocore** (one-shot, HTTP/2, 1–3,600s timeout, structured exit code); `InvokeAgentRuntimeCommandShell` is **not** a botocore op — it lives in the `bedrock_agentcore` 1.21.0 SDK at `runtime/shell/` (WebSocket, k8s-style channel framing, 1h max, reconnect by `shellId`). **Corrected 2026-09-11 while attempting the build:** ⚠️ `s3files` is **its own AWS service**, not an S3 bucket access point — ARNs are `arn:aws:s3files:::file-system/fs-…/access-point/fsap-…`, with a separate botocore model and separate `aws-cdk-lib/aws-s3files` L1s (`CfnFileSystem`, which fronts an **existing** bucket with an optional `prefix`, plus `CfnAccessPoint`, `CfnMountTarget`, `CfnFileSystemPolicy`). ✅ CDK support is present in our pin: `s3FilesAccessPoint` is on `CfnRuntime` in `aws-cdk-lib` **2.265.0** (`package-lock.json`). ⚠️ **Local trap**: the main checkout's `infrastructure/node_modules` is stale at **2.251.0**, where `FilesystemConfigurationProperty` exposes **only** `sessionStorage` — a local `tsc --noEmit` fails misleadingly. Trust the lockfile, which is what CI installs. Mount-path rules: under `/mnt/` with exactly one level, pattern `/mnt/[a-zA-Z0-9._-]+/?`, 6–200 chars, unique and non-nested; limits are 5 filesystem configs per runtime, of which at most **2** S3 Files. +- **Status**: open, but the **S3 bridge is ⛔ BLOCKED — stop re-proposing it as a cheap infra-only item.** A build was attempted **2026-09-11** (reviews/2026-09-11.md ▸ Proposal #5, scoped read-only with `sessionStorage` explicitly excluded) and **stopped before any code was written**. ✅ The pinned-boto3 gate this entry and the proposal both flagged is **NOT** the problem: `filesystemConfigurations` **is** in pinned botocore 1.43.68 and `s3FilesAccessPoint` **is** in the pinned `aws-cdk-lib` 2.265.0 — **no bump needed**. Two *different* blockers, both verified against the service model and the devguide, kill the scope as written. + - ~~**The Ship candidate is #4 above** — mount `s3FilesAccessPoint` over the existing `user-files/{u}/{s}/…` layout, infra-only, no new API surface.~~ **Withdrawn 2026-09-11 — not shippable as described.** The convergence story it promised (DynamoDB stays authoritative, metadata-table-first preserved, [[project_session_workspace_tools_spec]]) was never reached, because the mount itself does not hold up. + - ⛔ **BLOCKER 1 — `s3FilesAccessPoint` requires `networkMode: VPC`; our Runtime is `PUBLIC`.** `inference-agentcore-construct.ts:301` (Browser and Code Interpreter are PUBLIC too). Devguide, verbatim: *"Both S3 Files and EFS require VPC connectivity on the agent runtime."* It further needs S3 Files **mount targets in the same VPC and the same AZ** as the runtime subnets, TCP 2049 in/out between the runtime SG and the mount-target SG, and VPC DNS. The stack's VPC (`network-construct.ts` — 2 AZs, 1 NAT, `PRIVATE_WITH_EGRESS`) exists but the Runtime does not use it. **So there is no "infra-only" version: the real ship candidate is a network-mode migration of the production inference path.** ⚠️ And the failure mode is the worst one on offer — a failed mount returns **HTTP 424 on *every* invocation** (*"all configured file systems mount in parallel — a single failure causes the entire invocation to fail"*), so a misconfigured mount bricks the whole chat path rather than degrading a file feature. See [[reference_agentcore_424_means_container_500]]. + - ⛔ **BLOCKER 2 — the mount is static, so `user-files/{u}/{s}/…` is cross-tenant.** `filesystemConfigurations` is settable only on `CreateAgentRuntime`/`UpdateAgentRuntime`; ✅ verified against the pinned model that **`InvokeAgentRuntime` carries no filesystem override** (`runtimeUserId` exists, but it scopes no mount). One access point = one fixed `rootDirectory` and one fixed POSIX uid/gid, shared by every session — devguide: BYO file systems are *"Shared – multiple sessions and agents access the same data."* Mounting the `user-files/` prefix therefore gives **every session POSIX reads of every user's files** — the exact opposite of this entry's own premise ("the agent gets POSIX reads of files it already owns"), and a worse version of the `allowedAppRoles` failure mode the entry warns about below. Per-user access points cannot rescue it: the config is static and capped at **2** S3 Files access points per runtime. + - ✅ **Read-only is achievable — but only via IAM.** Recorded so nobody re-derives it: **no `readOnly` flag exists anywhere** — not on `S3FilesAccessPointConfiguration` (`accessPointArn` + `mountPath` only), not on the access point (`posixUser` / `rootDirectory` only), not on the file system. The devguide's own instruction is to grant `s3files:ClientMount` + `s3files:GetAccessPoint` under an `ArnEquals` condition on `s3files:AccessPointArn`, and **"Omit `ClientWrite` if your agent only needs read access."** ⚠️ S3 Files syncs **bidirectionally** with the backing bucket by default, so omitting `ClientWrite` is the actual enforcement lever, not belt-and-braces. + - **If this is revisited**, the only shape that clears Blocker 2 is a **shared, non-user-specific, read-only corpus** — the *Skill assets* unlock above (the brand-deck templates), which needs no per-user scoping. It still has to pay the VPC migration first, so rank it against that cost rather than as an infra one-liner. `sessionStorage` remains the only mount that needs **no VPC and no extra IAM** — and it is the one gated on the deletion path below. + - ⚠️ **Not verified in AWS**: the 2026-09-11 session resolved to account `029812070295`, not dev-ai `490617140655` ([[reference_dev_ai_aws_data]]). Every finding above comes from the pinned botocore/CDK models, the repo, and the AgentCore devguide — nothing from the console. - **Gates on `sessionStorage`, two of them unresolved.** ⚠️ **Deletion path**: a persistent filesystem holding user files is invisible to our takedown/delete machinery, which is DynamoDB+S3-aware — if a user deletes a conversation, *what deletes the workspace?* Governance blocker, not a detail. ⚠️ **Durability is single-source**: `/mnt/workspace` and 14-day-inactivity retention come from the AWS blog only; the curated devguide index has **no** filesystem-persistence page, consistent with preview. Don't design against those numbers. Also needed: cost per idle conversation (we pin runtime session ids, so conversation ≈ runtime session — multiply before, not after), and a decided convergence story with `workspace_files` (coexist or merge, settled up front, or we recreate the `allowedAppRoles` failure mode where two surfaces disagree). - ⚠️ **Naming collision, three ways.** "Workspace" means (a) this — an AgentCore Runtime mount; (b) our `workspace_files` tools, which are DynamoDB-backed and deliberately *not* a filesystem; (c) nothing at all in Strands. ✅ Verified there is **no** workspaces concept in Strands: zero refs in the installed `strands-agents==1.51.0`, zero of 2,482 files in `strands-agents/harness-sdk`, no page in the docs repo (the only hits are an Nx monorepo and a suggested evals folder name). @@ -41,14 +114,6 @@ Items added by `kaizen-research`, consumed by `kaizen-review-prep`. - **Agent version snapshots** (marketplace, #783–#801) — same word, different object: those version agent *configuration* for publish/rollback, not runtime state. - **Status**: open. Recommended: **Ship the branch/regenerate spike, Defer the redesign.** ⚠️ **Prompt-cache note, in the feature's favour:** `load_snapshot` restores messages by `copy.deepcopy`, so a round-trip is byte-stable *by construction* — arguably safer than re-deriving history from AgentCore Memory through the sanitizers and pairing repair, which is the divergence `_adopt_session_conversation`'s docstring already warns about. ⚠️ **Counter-gate:** that docstring also says the stale path is a cache **hit**, where nothing runs; a snapshot design must rebind `agent.messages` mid-life on every turn, which it names as what would silently break the alias — `test_second_cache_key_for_a_session_shares_the_conversation` has to be re-reasoned, not just kept green. ⚠️ Making a snapshot the store of record over AgentCore Memory was considered and **rejected**: it breaks LTM extraction wholesale. ⚠️ Upstream limits: `"session"` is the only preset, no schema migration off `"1.0"`, and messages restore verbatim — an untrusted snapshot reaches the model. -### [2026-09-05] ✅ RESOLVED — PROD `gpt-5.4` cache rate set -- **Source**: live measurement in dev, then verified and fixed in prod the same day. -- **Status**: **done.** Prod `openai.gpt-5.4` now carries `supportsCaching: true`, `cacheReadPricePerMillionTokens: 0.275` (0.1x its $2.75 input, the ratio confirmed across the GPT family in the Price List) and `cacheWritePricePerMillionTokens: 0` (that model has a cache-read SKU and **no** cache-write SKU). Verified on the record at 2026-09-05T17:41Z; name, prices and `enabled` untouched. -- **How**, since #963 is a frontend change that has not reached prod: the backend has always accepted these fields for Mantle, so it went through `PUT /api/admin/managed-models/{id}` from the browser console. ⚠️ That endpoint enforces double-submit CSRF — the `__Host-bff_csrf` cookie is JS-readable and must be echoed in `X-CSRF-Token`, or it 403s with `CSRF token missing or invalid`. -- ⚠️ **History is not corrected.** Pricing snapshots are captured per message at write time, so turns recorded before 17:41Z keep their $0.00 cache cost and their inflated savings credit. Any prod cost figure quoted for gpt-5.4 before that timestamp is wrong in both directions. A backfill would have to rewrite `C#` rows and is not proposed here. -- **Still open, smaller**: prod's `google.gemma-4-31b` is also `caching=False`. Probably correct — Gemma is open-weight and there is no evidence it caches — but the same hidden-control gap applied to it, so it deserves one warm-turn check. -- **Deleting the model was considered and rejected**: seven prod assistants (C.A.R.L., News Report, IPS 410 Crisis Management Simulator, ISSS Work Buddy, CTL data analysis, Buster Bot, Owliver) hard-bind it via `modelConfig: {modelId, provider}`, and the `staff` role grants it explicitly. - ### [2026-09-05] Carry the CRIS prefix per environment — `global.*` works in prod, is SCP-denied in dev - **Source**: live failure in dev while adding GPT-5.6 Luna, 2026-09-05; scope corrected by Phil the same day — **prod is not affected**. - **Surface**: whatever copies model rows between environments — seed scripts, curated catalog entries in `curated-models.ts`, runbooks. No runtime code. @@ -97,13 +162,6 @@ Items added by `kaizen-research`, consumed by `kaizen-review-prep`. - **Subtracts**: yes — the largest single subtraction available this cycle, and it simplifies the forum rather than the codebase. Retires (a) the POC-comment loop: the `POC findings` field, the “tested outranks untested” tiebreak, and the one-week-lag philosophy hanging off them — **five cycles, zero comments, tiebreak never fired** — while three items shipped as code in five days through a channel the skills do not describe. Also retires (c) the “resolve on a green streak” rule, still in force because #4b was never adopted, which produced a documented false negative within four days. - **Status**: open — three edits. **(a)** Replace the POC-comment loop with the outcome signal that demonstrably works: review-prep reads **merged PRs against the prior review's proposals**. **(b)** Every internal-audit number must be produced by a command **quoted in the doc**. Three reproduction failures in two cycles — the 2026-08-28 Price List API result, “zero CI failures in the window” (there was one), and `@angular/core 21.2.17` (the scanned tree read `21.2.19`) — against one section that already quotes its method (the version-pin table) and is the most reliable in the doc. A rate/price/capability figure destined for code needs two independent sources or an explicit `⚠️ single-source` marker. **(c)** Never resolve a *flaky* entry on a consecutive-green count — only on a root-cause fix or an explicit “accepted flake, N/month” note. **Deliberately does NOT re-propose** the ✅→tracked-issue layer: it failed to land twice and its premise is falsified — verified 2026-09-04 that **no `kaizen` label exists in this repo** and three items shipped anyway. ⚠️ `kaizen-review-prep/SKILL.md` is unmodified since 2026-05-10 across four reviews that each proposed editing it — if this is going to land, it rides the review PR. -### [2026-09-04] Finish the #914 rate correction — six stale sites, a per-model cache-read ratio, and a provenance claim that doesn't reproduce -- **Source**: research/2026-09-04.md ▸ Top 5 #1 — internal (PR #914, merged 2026-09-03) + https://www.anthropic.com/claude-fable-and-mythos-5-1 + a full AWS Price List API enumeration (`AmazonBedrock`, 11,621 `usagetype` values, all regions) -- **Surface**: docs / backend / frontend — `model_config.py:380`, `turn_based_session_manager.py:19`, `test_compaction_stability.py:8`, `test_prompt_cache_observability.py:464`, `docs/specs/compaction-over-threshold-cache-spiral.md:13,252`, `curated-models.ts:96-97`, and the prompt-cache contract bullet in `CLAUDE.md` -- **Effort × Impact**: L × H -- **Subtracts**: yes — six duplicated wrong constants, one hardcoded cache-read ratio that is already false for a GA model, and one unreproducible sourcing claim -- **Status**: open — **recommended #1, and the cheapest item in the scan.** Three verified facts. (1) PR #914 fixed `$2.50/MTok` in `CLAUDE.md` and left the same constant in **six** other places, including the 40-line cachePoint-budget comment at `model_config.py:380` — the exact text a reader consults when reasoning about cache cost. (2) #914 replaced it with a *derivation* helper hardcoding `cacheRead = input × 0.1`, and **Claude Fable 5.1 — GA on Bedrock 2026-09-01 — reads at $0.25/MTok against $10/MTok input, i.e. 0.025×**, a 75% cut Anthropic states explicitly; the helper would silently 4× overstate any Fable row an admin adds. Fix is to accept cache-read as an *input* with 0.1 as a documented default, so the next model that breaks the ratio is a data change. (3) ⚠️ **This scan could not find a single Claude 4.x/5.x SKU in the Price List API** (10 Claude SKUs total, none newer than Claude 3, none with cache or output dimensions) — which **contradicts last week's scan**, whose figures drove #914. `CLAUDE.md` now asserts rates "come from the AWS Price List API"; that claim does not reproduce. The 1.100× Regional premium survives independently — confirmed this week on nine Global/Regional pairs of newly-published xAI Grok 4.6 SKUs — so #914's *substance* is probably safe, but the *provenance* is not. Re-run last week's query verbatim before the next rate edit. - ### [2026-09-04] Migrate the MCP Apps host off `initialize` to `server/discover` — FastMCP 4.0 made it real - **Source**: research/2026-09-04.md ▸ Top 5 #2 — https://modelcontextprotocol.io/specification/2026-07-28/changelog (SEP-2567/2575) + https://github.com/jlowin/fastmcp/releases (4.0.0, 2026-08-31) + Strands 1.53.0 MCP-client changes. **Supersedes and upgrades the [2026-08-14] entry of the same name** — merge them at review. - **Surface**: backend — `integrations/mcp_apps.py` (the `ClientSession` symbol patch at lines 23–32; the `serverInfo` capture at line 673), `external_mcp_client.py`, `gateway_mcp_client.py`, the mcp-sandbox proxy origin, and the OAuth pre-flight path (PR #872) @@ -123,14 +181,6 @@ Items added by `kaizen-research`, consumed by `kaizen-review-prep`. - **Unlocks**: naming a cause we currently **cannot see at all**. An external MCP server that edits a tool's `inputSchema` between turns re-writes our cacheable prefix with no deploy of ours and no description change for a human to spot — measured in the wild at **17 schema/annotation-only changes across 248 servers in 27 hours**. Verified locally that `toolConfigHash` hashes `get_all_tool_specs()` (full specs *including* `inputSchema`), so we already **detect** this class; a `tools_changed` label on a row with no deploy is the first time we could **attribute** it. - **Status**: open — cheapest high-leverage item after #1. Labels available essentially for free: `tools_changed`, `system_prompt_changed`, `history_changed`, `agent_switched` (the `agentSwitched` flag already exists), `ttl_expired`, `cold_start`. Claude Code shipped exactly this in 2.1.260 ("a likely cause for prompt-cache misses"), which is corroboration the ergonomics are worth it. -### [2026-09-04] Gate the Strands 1.51 → 1.54 bump on the system-cachePoint collision -- **Source**: research/2026-09-04.md ▸ Top 5 #4 — https://github.com/strands-agents/sdk-python/releases (python/v1.52.0–v1.54.0) + https://github.com/strands-agents/sdk-python/issues/4168. **Sharpens the [2026-08-28] "instrumented cache experiment" entry with verified specifics** — merge them at review. -- **Surface**: backend — `core/model_config.py:375-400` (`strategy="auto"` + the `bedrock_cache_points_supported()` gate), `core/agent_factory.py:213-224` (the hand-placed system `cachePoint` and its now-false comment), `TurnBasedSessionManager` (against 1.52's tool-pair trimming), `backend/pyproject.toml:59,74` -- **Effort × Impact**: M × H -- **Subtracts**: yes — potentially our hand-placed system cachePoint and the ~40-line comment defending it (the library-native subtraction this skill weights for, *if* 1.53's placement proves equivalent). Definitely subtracts a comment that now asserts a false invariant. -- **Unlocks**: 1.54's **external cancellation-signal injection** (candidate to retire hand-rolled cancel plumbing around the anyio `CancelScope` drop path, PR #863) and **`Agent.session_id`** (the runtime session pin the G1 agent-cache read had to establish by hand). -- **Status**: open — four concrete checks. (a) Does 1.53's auto-placed system point **duplicate or replace** ours? Delete our `SystemContentBlock` list if it duplicates. (b) Rewrite `agent_factory.py:222`, which asserts *"auto strategy strips only message-level cachePoints, never system ones"* — a 1.51-era fact that 1.53 falsifies. (c) Is 1.52's "trim at complete tool pairs" even reachable given our custom session manager? Two trimmers choosing different boundaries is precisely the byte-instability the compaction redesign exists to prevent. (d) **Keep the `bedrock_cache_points_supported()` gate** — upstream issue #4168 (filed 2026-09-04) is a live report of the exact `AccessDeniedException` it prevents, so it is load-bearing, not redundant. ⚠️ The reference repo runs 1.54 without incident but **does not hand-place a system cachePoint** — do not read their green build as evidence for us. This is a cost regression that ships silently and looks like a routine dep bump. - ### [2026-09-04] Close the `ActiveSessionCount` alarm item — the metric now exists — and scope instance-based Runtime against W5 - **Source**: research/2026-09-04.md ▸ Top 5 #5 — AWS Price List API (`AmazonBedrockAgentCore` us-west-2: **889 new `Runtime:Instance-based::Management-Hours` SKUs** in the 2026-09-01 republish) + https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/release-notes.html. **Unblocks the [2026-07-10] `ActiveSessionCount` entry** — merge them at review. - **Surface**: infrastructure — `lib/constructs/observability/` (the `AlarmFactory` that #910 made the only sanctioned path), the AgentCore Runtime construct; plus W5 in `project_cost_effectiveness_roadmap.md` @@ -354,20 +404,6 @@ Items added by `kaizen-research`, consumed by `kaizen-review-prep`. - **Subtracts**: possibly — removes the manual-redeploy band-aid that's been the workaround - **Status**: open — surfaced in reviews/2026-05-15.md ▸ Proposal #10 (Ship — recommended ship-first); no decision logged yet. **Friction intensifying**: 6+ "Deploy Inference API" failures May 15–17; a new "Deploy App API" failure cluster (8× May 16–17) may share a root cause. -### [2026-05-10] Scope AgentCore Runtime BYO filesystem (S3 Files / EFS) for persistent agent workspaces -- **Source**: research/2026-05-10.md ▸ AWS Bedrock / AgentCore (re-evaluated 2026-05-10 via strategic-lens follow-up — original framing under-weighted the capability-unlock angle) -- **Surface**: backend (`inference-api` invocation handler reads/writes mount) + infrastructure (VPC config, IAM mount permissions, S3 Files or EFS access points, per-user prefix/access-point layout for RBAC); ADR-worthy -- **Effort × Impact**: H × H -- **Subtracts**: no — pure capability addition -- **Unlocks**: - - Code-interpreter / persistent agent workspace (artifacts survive turn and session boundaries) - - Cross-session file uploads — PDFs/spreadsheets persist between conversations instead of re-staging per session - - Shared skill/template/prompt hot-swap without redeploying the runtime container - - A2A multi-agent intermediate-result handoff via shared mount - - Persistent vector indexes / embedding caches — avoids cold-start rebuild -- **Open questions**: GA vs preview status (March 2026 managed session storage was preview; May 2026 BYO needs verification); VPC requirement is a new architectural surface for the runtime; multi-tenancy isolation strategy (per-user S3 prefix vs per-user EFS access point); RBAC mount-path layout; runtime data plane still only proxies `/invocations` + `/ping` so this doesn't unlock new HTTP routes -- **Status**: open — deferred 4 weeks in reviews/2026-05-15.md (revisit 2026-06-12). MCP Apps host renderer is the dominant strategic initiative this cycle; layering another ADR-worthy bet on top would double the open architectural surface. - ### [2026-05-10] Audit `BedrockModel.stream` cancellation path against Strands #2266 - **Source**: research/2026-05-10.md ▸ Top 6 #4 - **Surface**: backend @@ -391,6 +427,31 @@ Items added by `kaizen-research`, consumed by `kaizen-review-prep`. ## Resolved +### [2026-09-11] ✅ VERIFIED NOT APPLICABLE — AgentCore SDK #659 (silent SSE corruption on unserialisable events) → RESOLVED — **premise verified NOT APPLICABLE** +- **Decision**: resolved by outcome (a merged PR or a superseding entry), not by a review mark. +- **Reasoning**: Closed on inspection in research/2026-09-11.md and recorded so it is not re-proposed. Verified three ways: we never import `BedrockAgentCoreApp` (zero grep hits — we run our own FastAPI app in the Runtime container and hand-roll SSE); `stream_processor.py:117-121` base64-encodes `bytes` explicitly; and `_create_event()` at `:157` runs `_serialize_object(data)` on every emitted event, with `redactedContent` routed through it at `:910`. The scanning subagent ranked this "highest of the week for us" and it is not a finding — the negative result is. +- **Reviewed in**: reviews/2026-09-11.md + +### [2026-09-05] ✅ RESOLVED — PROD `gpt-5.4` cache rate set → RESOLVED — **DONE**, verified on the record +- **Decision**: resolved by outcome (a merged PR or a superseding entry), not by a review mark. +- **Reasoning**: Prod `openai.gpt-5.4` carries `supportsCaching: true`, `cacheReadPricePerMillionTokens: 0.275` and `cacheWritePricePerMillionTokens: 0` as of 2026-09-05T17:41Z. The entry was already self-marked resolved; moved here by `kaizen-review-prep` 2026-09-11. ⚠️ **History is not corrected** — pricing snapshots are captured per message at write time, so turns recorded before 17:41Z keep a $0.00 cache cost and an inflated savings credit; any prod gpt-5.4 cost figure quoted before that timestamp is wrong in both directions. **Residue carried, not dropped**: prod `google.gemma-4-31b` is also `caching=False` and deserves one warm-turn check — re-queue if it matters. +- **Reviewed in**: reviews/2026-09-11.md + +### [2026-09-04] Finish the #914 rate correction — six stale sites, a per-model cache-read ratio, and a provenance claim that doesn't reproduce → RESOLVED — **MERGED** into the [2026-09-11] provenance entry; the ⚠️ is settled +- **Decision**: resolved by outcome (a merged PR or a superseding entry), not by a review mark. +- **Reasoning**: The blocking ⚠️ on this entry — two consecutive scans disagreeing about whether the AWS Price List API carries Claude 4.x/5.x rates — is **settled, against the claim**. research/2026-09-11 re-ran the enumeration against the republished offer file (`20260804165549` → `20260911124408`, 10,998 → 11,657 SKUs) and reproduced last week exactly: **10 Claude SKUs, none newer than Claude 3**, and no us-west-2 SKU at all for Haiku 4.5 / Sonnet 4.6 / Fable 5.1 / GPT-5.4 / Nova Micro. Correct provenance: **per-model AWS model cards for Claude, Nova and the OpenAI/Mantle family; the Price List API for xAI, Google and AgentCore.** **The work itself is NOT done** — the six `$2.5/MTok` sites and the hardcoded `cacheRead = input × 0.1` helper are verified still on disk as of 2026-09-11, two reviews after first being listed. It continues in the [2026-09-11] "Correct the rate provenance in `CLAUDE.md`…" entry, which now also carries **Grok 4.6 as a second live ratio counterexample** (0.25× cache-read, and no cache-write SKU at all). +- **Reviewed in**: reviews/2026-09-11.md ▸ Proposal #2 + +### [2026-09-04] Gate the Strands 1.51 → 1.54 bump on the system-cachePoint collision → RESOLVED — **BUMP SHIPPED** (#1012); collision residue **MERGED** into the [2026-09-11] entry +- **Decision**: resolved by outcome (a merged PR or a superseding entry), not by a review mark. +- **Reasoning**: `strands-agents` 1.51.0 → **1.55.0** shipped in [#1012](https://github.com/Boise-State-Development/agentcore-public-stack/pull/1012) (release 1.20.0, 2026-09-09), past the 1.54 this entry was written against. It was taken as the instrumented experiment this entry specified, not a routine bump: two arms against real Bedrock on the same salted prompt minutes apart — 1.51 wrote 7,059 cache tokens, **1.55 read the entry 1.51 wrote** — so Bedrock itself certified the cacheable prefix byte-identical across the version boundary. Check (a) is answered: 1.55 adds `_should_cache_system()` guarded on `not any("cachePoint" in block …)`, and `AgentFactory.create_agent` always places ours first on an equivalent predicate, so **no path doubles up**; upstream's docstring independently says a hand-placed system point is "honored rather than doubled". #1012 also caught a rename the upstream release notes omitted (Nova Sonic moved packages; `voice_agent.py` swallows the `ImportError`, so Voice Mode would have switched off fleet-wide behind one INFO line). **Residue carried, not dropped** — checks (b) and (d) are unfinished and live in the [2026-09-11] "Retire the hand-placed system `cachePoint`" entry: the comment at `agent_factory.py:267` still asserts the false 1.51-era invariant, and `bedrock_cache_points_supported()` may now be redundant since upstream #4168 closed 2026-09-04. +- **Reviewed in**: reviews/2026-09-11.md ▸ Proposals #3 and #10 + +### [2026-05-10] Scope AgentCore Runtime BYO filesystem (S3 Files / EFS) for persistent agent workspaces → RESOLVED — **SUPERSEDED** by the [2026-09-08] AgentCore Runtime workspaces entry +- **Decision**: resolved by outcome (a merged PR or a superseding entry), not by a review mark. +- **Reasoning**: Same feature, re-written Unlocks-led with the API verified against our pinned botocore rather than docs prose: `CreateAgentRuntime.filesystemConfigurations` takes `sessionStorage` / `s3FilesAccessPoint` / `efsAccessPoint` / `capacityProviderVolume`, with `lifecycleConfiguration` as the sibling knob. The [2026-09-08] entry carries every open question this one raised (multi-tenancy layout, RBAC mount paths, GA-vs-preview) plus two it did not: the **deletion-path governance blocker** — a persistent filesystem holding user files is invisible to our takedown/delete machinery, so if a user deletes a conversation, nothing deletes the workspace — and the **three-way `workspace` naming collision**. This entry was deferred 2026-05-15 until 2026-06-12 and carried three months past it; closing the duplicate rather than carrying both. ⚠️ `boto3`/`botocore` are pinned at 1.43.68 (24 releases behind), which gates the `filesystemConfigurations` shapes the successor depends on. +- **Reviewed in**: reviews/2026-09-11.md ▸ Carried Over + ### [2026-08-28] Correct the cache-write premium and fix the Global/Regional rate tier → RESOLVED — **SHIPPED IN PART** (#914) - **Source**: research/2026-08-28.md ▸ Top 5 #1 — AWS **Price List API** (us-west-2, `AmazonBedrockFoundationModels`), cross-checked against https://platform.claude.com/docs/en/build-with-claude/prompt-caching. Verified locally at `frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts:135-141`. - **Surface**: docs / frontend / backend (`CLAUDE.md` — the "$2.50/MTok cache-write premium" appears twice, in the prompt-cache contract and the cost-effectiveness tenet; `curated-models.ts` Claude templates' four rate fields; the managed-models DynamoDB rows in dev/prod; the read path at `apis/shared/costs/pricing_config.py:75`) diff --git a/docs/kaizen/reviews/2026-09-11.md b/docs/kaizen/reviews/2026-09-11.md new file mode 100644 index 000000000..87805aa71 --- /dev/null +++ b/docs/kaizen/reviews/2026-09-11.md @@ -0,0 +1,257 @@ +# Kaizen Review — Friday, September 11, 2026 + +> Prepared 11:15am MT. Review window: **September 4 – September 11** (7 days). +> Source: research/2026-09-11.md (PR [#1045](https://github.com/Boise-State-Development/agentcore-public-stack/pull/1045), opened this morning) + review-queue.md (**51 open** before this pass, **46** after). +> Branched off `kaizen/research-2026-09-11` so it can consume today's research and its six queue additions, with `develop` merged in so the diff is kaizen-only. Same arrangement as the last two weeks. + +## Week in Review + +**This was the largest window this forum has ever reviewed, by a factor of three, and none of last week's cheap recommendations were part of it.** 110 PRs merged into `develop`, 133 non-merge commits, three releases (1.19.0, 1.19.1, 1.20.0) — against 31 PRs last week. Verified today by grep, not inferred: the six `$2.5/MTok` constants are all still on disk, `missCause` appears nowhere in `backend/src/`, the false invariant comment sits unchanged at `agent_factory.py:267`, `.claude/skills/angualar-best-practices/` still carries its typo, and `docs/kaizen/decisions.md` has not been written to since **2026-05-18**. Of ten proposals, the three marked **Ship** at Low effort converted at zero. + +**What did convert was the one item this forum told Phil to defer — and it converted well.** Proposal #7 (the Strands bump, recommended *Defer 2 weeks, conditional on #3 landing*) shipped as [#1012](https://github.com/Boise-State-Development/agentcore-public-stack/pull/1012), 1.51.0 → **1.55.0**, and it did the instrumented experiment the proposal specified rather than skipping it: two arms against real Bedrock on the same salted prompt, minutes apart — 1.51 wrote 7,059 cache tokens, 1.55 *read* the entry 1.51 wrote. Bedrock itself certified the prefix byte-identical across the version boundary. It also caught a rename the upstream release notes never mentioned (Nova Sonic moved packages, and `voice_agent.py` swallows the `ImportError`, so Voice Mode would have switched off fleet-wide behind one INFO line). That is a better verification than this document asked for. The condition it was deferred on — #3, the ten-minute comment fix — never happened, so the bump landed with the comment still asserting a 1.51-era invariant. **The trap did not fire, and the sign that would have warned the next person is still wrong.** + +The uncomfortable half is internal and new: **the nightly has been red for seven consecutive nights, the cause is deterministic, zero issues were filed, and PR CI is structurally incapable of catching it.** Six repo-hygiene specs resolve their own paths differently under `--coverage`, which only the nightly passes. A gate that has been red for a week is not a gate — and the forum that adopted "never resolve a *flaky* entry on a green streak" four weeks ago just watched the mirror failure run unremarked for the same length of time. + +## Friction — the week's signal + +### Repeated patterns (≥2 occurrences) + +- **Conversion tracks what is interesting to build, not what this forum ranks cheapest** (2 cycles, now with a controlled comparison). Last week: three Low-effort **Ship** items converted in five days. This week: the same three classes of item — Proposal #1 (Low × High, `$2.5/MTok` cleanup), #2 (Low–Med × High, `missCause`), #3 (Low × High, a comment rewrite estimated at ten minutes) — converted at **zero**, inside a window that absorbed 110 PRs. Meanwhile Proposal #7, the only **Defer** on the engineering half of the board and the highest-effort item marked Med, shipped inside five days. The two windows differ in what was *available to build*, not in the ranking. + - *Hypothesis*: `Effort × Impact` predicts **what is worth doing**; it does not predict **what gets done**. What got done both weeks was whatever sat on the critical path of a feature Phil was already building — #914/#915/#916 rode the cost and steering work; #1012 rode the MCP Apps and browser-tool work that needed 1.55. A Low-effort cleanup with no feature attached has now gone two cycles at 0/6. + - *Candidate fix*: stop treating a standalone-PR recommendation as the default shipping unit. Where an item can ride a PR that is already going to be written, **say which one** — the ✅ costs nothing then. Proposals #2, #3 and #7 below each name a carrier. See also Proposal #9, which is the same argument applied to this document's own edits. + +- **A deterministic red gate read as background noise for seven days, with zero issues filed** (7 occurrences, one root-cause pair). `Nightly Build & Test` failed 2026-09-05 → 2026-09-11 after five consecutive greens (`gh run list --workflow="Nightly Build & Test" --limit 12`). Two independent causes, neither a flake: six filesystem-reading branding specs that resolve `__dirname` two levels shallower under `--coverage` (which only the nightly passes), and `nightly-develop-PlatformStack` wedged in `DELETE_FAILED`. In the same window: **0 issues opened, 0 closed** across 110 merged PRs. + - *Hypothesis*: the nightly's signal has been discarded wholesale because nothing routes it to a human. The 77-alarm observability baseline shipped in #910 routes *AWS* alarms to an SNS topic; a red GitHub Actions run on `main` at 2am routes nowhere. + - *Candidate fix*: Proposal #1. And note the structural half is the important one — **PR CI never passes `--coverage`, so it cannot catch this class at all.** Fixing the six specs without fixing that leaves the next instance to the same 2am run. + +- **`decisions.md` has not been written to in four months, across four reviews that each queued declines into it** (4 cycles). `git log -1 -- docs/kaizen/decisions.md` → **2026-05-18**. Queued and unexecuted: `duration_ms` (now **tenth** cycle; DROP 07-03, Decline 08-14, Decline 08-28, Decline 09-04), `oauth_required` SSE audit (**seventh** surfacing, ~16 weeks past its revisit date), and docling #405 + Guardrails #480 as kaizen items (fifth carry). All four are still in `## Open`. + - *Hypothesis*: same as pattern 1 in a different coat. A decline produces no artifact anyone wants, so it never becomes a keystroke — and the forum keeps re-ranking items it has already decided. + - *Candidate fix*: Proposal #4 executes all four in this PR. A decline that needs its own PR has now failed four times; this is the last cycle it is worth asking. + +### One-offs worth watching + +- **Correcting last week's record: Proposal #6(1) was already half-shipped when it was proposed.** `ActiveSessionCount` alarms landed in [#910](https://github.com/Boise-State-Development/agentcore-public-stack/pull/910) on **2026-09-02**, two days *before* the proposal claimed the metric was newly available and needed wiring — `git log -S'ActiveSessionCount'` names that commit. What exists today: an alarm on `Service: AgentCore.CodeInterpreter` (`ai-path-alarms-construct.ts:311`, threshold 50) and a **dashboard widget only** on `Service: AgentCore.Runtime` (`inference-agentcore-construct.ts:636`). So the gap is real but much narrower than described — one alarm, on a metric object that is already constructed. See Proposal #7. +- **1.20.0 shipped a deliberate ~2× cost increase and it was never on a kaizen board.** App-api Fargate defaults moved 512 CPU / 1024 MiB → **1024 / 2048** at 2 tasks (#1020). It is correctly flagged **Breaking (cost)** in the changelog with an opt-out, and it is plainly justified by the load-testing work — but a forum whose first tenet is cost effectiveness reviewed 110 PRs and saw the one deliberate compute doubling only in the changelog. +- **The observability baseline this forum celebrated last week shipped a noise generator.** #1016 deleted `bedrock-tpm-quota-usage`, which had been above threshold for **195 of 197 datapoints over 24 hours** with **205 state transitions in six days** — it compared an absolute token count against a literal `80` as if it were a percentage, on an account-wide roll-up that has no single denominator. Fixed in seven days, which is good; worth recording that "77 alarms, all routed" was a count, not a quality measure. +- **The first kaizen skill edit in fourteen weeks landed** — `kaizen-research/SKILL.md`, 2026-09-05, standing up the prompt-caching convergence watch and fixing the dead Strands changelog URL. Both were this forum's recommendations. The verification rule from Proposal #5(b) is visibly in force: research/2026-09-11 opens its internal audit with *"Every number below is produced by a command quoted with it"* and every figure in it reproduced when re-run today. +- **`bedrock-agentcore` 1.22.0 is confirmed a no-op for the third consecutive scan** (single payments feature, fixes none of the four standing issues). It should stop occupying a row in the lag table. + +### Silence that matters + +- **Zero comments on the kaizen PRs — sixth consecutive cycle.** [#926](https://github.com/Boise-State-Development/agentcore-public-stack/pull/926) and [#929](https://github.com/Boise-State-Development/agentcore-public-stack/pull/929) both merged with 0 comments and 0 reviews (`gh pr view --json comments,reviews`). The POC-comment mechanism is still specified in `kaizen-research/SKILL.md` at lines 21 and 406 — last week's Proposal #5(a) recommended deleting it, the skill was edited four days later, and **that edit did not include the deletion**. Six cycles, zero comments, tiebreak never fired. +- **`kaizen-review-prep/SKILL.md` is unmodified since 2026-05-10** — 124 days, now **five** reviews that each proposed editing it. Its sibling moved this week; it did not. +- **`#629` (TracerProvider never flushed before microVM freeze) is owned by nothing, third cycle.** Research covered all four standing AgentCore issues properly this week — a fix from last review's gap — and confirmed #629 is still open with no AWS response. The consequence stands unowned: our cost telemetry systematically under-reports **the last turn of every session**, which is a number this forum quotes. +- **`#646` quietly changed shape.** Still open, but **retitled** from a bug report to *"allow excluding binary content from persisted messages"* — a feature request. The turn-killing behaviour is unchanged; the reframing makes it less likely to be prioritized as a defect. Its watch date was today. + +## Proposals — ranked + + + +### 1. Fix the nightly, both halves — anchor the specs' paths, and put `--coverage` where PR CI can see it + +- **Source**: research/2026-09-11.md ▸ Top 5 #1 | review-queue.md (open since 2026-09-11) | independently re-verified in this pass +- **Surface area**: CI / frontend — `.github/workflows/tests.yml` (`test-frontend`), `frontend/ai.client/package.json` (`test:ci`), `scripts/frontend/test.sh`, and six specs: `src/branding/rebranding-guide.doc-presence.spec.ts`, `src/branding/prestart-generator-parity.spec.ts`, `src/branding/brand-theme-golden.spec.ts`, `src/branding/surface-theme-golden.spec.ts`, `src/app/surface-literal-guard.spec.ts`, `src/app/tailwind-theme-import-guard.spec.ts`. Plus one CloudFormation stack in dev-ai. +- **Change**: **(a)** resolve each spec's root from a stable anchor — a `define`d repo-root constant or a helper in `src/test-setup.ts` — instead of `import.meta.url` / `__dirname`, so the path is identical with and without coverage instrumentation. **(b)** Make PR CI able to see the class at all: pass `--coverage` in `test:ci`, or add a small PR job running the coverage invocation. **(c)** Have someone with dev-ai access read `StackStatusReason` on `nightly-develop-PlatformStack`, clear the retained resource, delete the stack. +- **Subtracts**: a gate that has produced only false signal for seven nights; a `DELETE_FAILED` stack whose retained resources are still billing; and the *category* of repo-hygiene spec that silently depends on how it was invoked. After (b), the nightly stops being the only place a whole class of failure can appear. +- **Effort**: Low–Med · **Impact**: High +- **Evidence**: verified today. `gh run list --workflow="Nightly Build & Test" --limit 12` → failure ×7 (09-05…09-11), then success ×5. In run [34582692457](https://github.com/Boise-State-Development/agentcore-public-stack/actions/runs/34582692457), `Test Frontend with Coverage` **failed** while `deploy / [deploy-develop] Test frontend` **succeeded on the same ref** — confirmed here that the two jobs differ in exactly one thing: `scripts/frontend/test.sh` runs `ng test --no-watch --coverage`; `tests.yml` runs `npm run test:ci` = `ng test --watch=false`. Also confirmed here that PR CI (`ci.yml`) calls the same `tests.yml` with `run_frontend: true`, which is why #1034/#1038/#1040/#1043 all show `Tests / Test frontend pass` while the nightly is red. +- **Ship means**: one CI/frontend PR for (a) and (b); (c) is a console task for whoever holds dev-ai write. +- **Decline means**: the suite's only full-fidelity run stays red, six specs whose entire purpose is repo hygiene stay untrusted, every nightly deploy keeps failing on a wedged stack, and the retained resources keep billing. +- **Recommendation**: **Ship.** Cheapest item on the board and everything else in this document is harder to trust while it is red. ⚠️ **Reproduce before fixing** — `cd frontend/ai.client && ./node_modules/.bin/ng test --no-watch --coverage` on `develop`, and confirm the resolved paths. The mechanism is strongly evidenced from logs but has not been executed locally; do not fix a mechanism you inferred. ⚠️ Do **not** combine with the vitest 4→5 major — the suite is not a trustworthy migration baseline until it is green. + +### 2. Finish the rate correction — the provenance question is now settled, against us + +- **Source**: research/2026-09-11.md ▸ Top 5 #4 | review-queue.md ([2026-09-04] and [2026-09-11] entries — **merged this pass**, the newer one closes the older one's ⚠️) | reviews/2026-09-04.md ▸ Proposal #1 (**Ship**, unactioned) +- **Surface area**: the prompt-cache contract bullet in `CLAUDE.md`; `curated-models.ts:114` (the `cacheReadPricePerMillionTokens: round(input * 0.1)` helper); and six stale sites — `model_config.py:396`, `turn_based_session_manager.py:19`, `test_compaction_stability.py:8`, `test_prompt_cache_observability.py:467`, `docs/specs/compaction-over-threshold-cache-spiral.md:13,252` +- **Change**: **(a)** `CLAUDE.md` says rates *"come from the AWS Price List API, not the pricing page."* Replace with the verified reality: **per-model AWS model cards are authoritative for Claude, Nova and the OpenAI/Mantle family; the Price List API is authoritative for xAI, Google and AgentCore.** **(b)** Accept cache-read as an **input** on the model row with `0.1×` as a documented default, so a model that breaks the ratio is a data change. **(c)** Strike the six constants. +- **Subtracts**: a false sourcing instruction in the file every contributor reads before touching cost code, a hardcoded ratio now falsified by **two** live models, and six duplicated wrong constants. Verified present today by grep, after #914 merged eight days ago. +- **Effort**: Low · **Impact**: Med–High +- **Evidence**: **the ⚠️ that blocked this last week is gone.** Two independent scans, one week apart, now agree: the `AmazonBedrock` offer file contains **10 Claude SKUs, none newer than Claude 3**, and Haiku 4.5 / Sonnet 4.6 / Fable 5.1 / GPT-5.4 / Nova Micro have no us-west-2 SKUs at all. Last week this document said "do not encode `0.025` until it reproduces from a second source" — that caution still stands for Fable specifically, but the *structural* fix no longer depends on it: **Grok 4.6 is the second counterexample at 0.25×** ($0.55 against $2.20 input), published this week with two independent sources agreeing to the cent, and it has **no cache-write SKU at all**. The 1.100× Regional premium survives independently, re-confirmed on 18 new Grok SKUs and both Astra tiers. +- **Ship means**: one PR. The `CLAUDE.md` line and the six constants are a single sitting; (b) is a small helper-signature change. +- **Decline means**: the contract keeps instructing every contributor to verify rates against an API that demonstrably does not carry them — and the next person concludes the rate is unverifiable rather than that the instruction is wrong. +- **Recommendation**: **Ship.** Second week at the top of the cheap column, and the one reason to hold it last week no longer applies. ⚠️ Treat Grok as a **non-cached** model in cost projections until implicit caching is measured against `cacheStatus` on a real warm turn. + +### 3. Rewrite the false cachePoint invariant now — the bump shipped without it + +- **Source**: research/2026-09-11.md ▸ Top 5 #2, **split** | review-queue.md ([2026-09-05] **Standing watch — retire our hand-rolled prompt-caching code as Strands' `CacheConfig` converges**, which is the entry that called this) | reviews/2026-09-04.md ▸ Proposal #3 (**Ship**, unactioned) and ▸ Proposal #7 (**Defer**, *shipped anyway*) +- **Surface area**: backend — `core/agent_factory.py:260-275` and the cachePoint-budget comment at `core/model_config.py:375-400` +- **Change**: rewrite two comments. `agent_factory.py:267` still reads *"Strands' auto strategy strips only message-level cachePoints, never system ones"* — verified verbatim on disk today, **on a tree that now runs 1.55.0**. Replace it with the invariant #1012 actually established: 1.55 adds `_should_cache_system()` (`system_prompt_ttl` defaults `True`) whose guard is `not any("cachePoint" in block …)`, and `AgentFactory.create_agent` always places ours first on an equivalent predicate — **so no path doubles up**, and upstream's own docstring says a hand-placed system point is *"honored rather than doubled."* Same pass fixes the `$2.5/MTok` figure in the adjacent budget comment (shared with Proposal #2). +- **Subtracts**: a comment that actively misleads, in the most cost-sensitive file we own, on the exact question a reader consults it for — and which is now *more* wrong than when this document first flagged it, because the version it described is two minors behind the pin. This is the whole change; no behavior moves. +- **Effort**: Low · **Impact**: High *(as risk removal)* +- **Evidence**: verified today at `agent_factory.py:267`, unchanged through #1012. The replacement text does not need research — #1012's own audit already established it, including that there is exactly **one** `BedrockModel` construction site in the repo, which made that audit tractable. **Credit where it is due**: this is the [2026-09-05] standing caching watch paying out. That entry was filed on the thesis *"my hope is that the default caching config replaces any custom code we have added"*, predicted the payout would arrive as an upstream landing, and it did — `CacheConfig` at 1.55.0 now carries `system_prompt_ttl` and `tools_ttl`, and `strands/models/_openai_cache.py` exists. The watch cost one scan per week and named its own finish line; it should be read as the mechanism working, not as a new discovery. +- **Ship means**: a comment-only PR, ten minutes. **Carrier**: the 1.55.0 → 1.55.1 patch bump (see Proposal #9's rider) — same file family, same reviewer context. +- **Decline means**: the sign stays wrong through the *next* bump too. The trap did not fire this time because #1012 measured rather than trusted the comment; that is a property of who did the work, not of the code. +- **Recommendation**: **Ship.** Third consecutive review recommending it at ten minutes, and the intervening week is the argument: the bump it was meant to protect went past it untouched. + +### 4. Add GPT-6 Astra at the 272K tier — and make the context-tier boundary a catalog field + +- **Source**: research/2026-09-11.md ▸ Top 5 #3 | review-queue.md (open since 2026-09-11) | https://github.com/aws-samples/sample-strands-agent-with-agentcore/pull/291 +- **Surface area**: `curated-models.ts`, the Mantle Responses path (`_create_mantle_model` / `bedrock_mantle_config`, per-model region routing), `admin/services/model_access.py` for the role grant, the model picker +- **Change**: register `us.openai.gpt-6-astra` at `maxInputTokens: 272000` with Geo CRIS rates $11.00 in / $13.75 cache-write / $1.10 cache-read / $55.00 out per MTok (Global: $10.00 / $12.50 / $1.00 / $50.00). Then generalize: today `maxInputTokens` doubles as both a *capability* bound and a *pricing* cap, and Astra is the second model where those differ. Make the tier boundary explicit on the row so the next two-tier model is a data change. +- **Subtracts**: no — addition. Justified because it **prevents a regression we would otherwise import**: the reference repo registered it on 2026-09-09 as `maxInputTokens: 1000000`, and the Long Context tier is **exactly 2× on input**. A 1M declared window looks like a capability win, which is why it would not be caught. +- **Unlocks**: a genuine **1M-token frontier tier** (1,050,000 context / 128,000 max output) — the first catalog model where a whole corpus fits in one turn. And a **tier-aware catalog**: once the boundary is a field, the admin page can show it, cost projections can respect it, and quota math can account for Astra's **10× output-token quota burn** — which lands directly against 1.20.0's load-testing finding that the campus ceiling is the Bedrock TPM quota, not compute. +- **Effort**: Low–Med · **Impact**: High +- **Evidence**: not POCed. Rates from the AWS model card (two tiers, both quoted), GA 2026-09-08. ⚠️ The ref repo's `rejectsTemperature: true` is **not on the model card** — verify, do not copy; note #915's guard is bypassed by an *empty* `supportedParams`. +- **Ship means**: answer one blocking question first, then a catalog PR. ⚠️ **The model card does not state whether the price tier is selected by the declared window or by actual token count.** If declared, `272000` is mandatory; if actual, a 1M window is free until a call exceeds 272K and the cap becomes a policy choice. That question gates the whole design, and it is a support question or a single measured call. +- **Decline means**: no 1M tier, and the tier-vs-capability conflation in `maxInputTokens` stays latent until a third two-tier model rediscovers it. +- **Recommendation**: **Ship, after the blocking question.** The Mantle Responses path already exists and this rides it. ⚠️ Other verified constraints: on `bedrock-runtime` Astra is CRIS-only with **no server-side tool use, no structured output, no CountTokens**; `bedrock-mantle` is us-west-2 only (matches us); the cache-write SKU is a **30-minute** TTL, not 5m/1h — do not conflate with `cache_ttl_seconds_for()`. Dev/prod prefix divergence still applies (`global.*` is SCP-denied in dev). + +### 5. AgentCore Runtime workspaces — ship the S3 bridge, gate `sessionStorage` on the deletion path + +- **Source**: review-queue.md ([2026-09-08], **Phil-initiated — not from a research scan**), verified against the pinned `botocore` 1.43.68 service model rather than docs prose. **Supersedes** the [2026-05-10] BYO-filesystem entry, which is closed to `## Resolved` this pass. +- **Surface area**: infrastructure — `CreateAgentRuntime`'s `filesystemConfigurations` (✅ verified zero hits in `infrastructure/lib` today). Downstream: `apis/shared/files/workspace.py` and `agents/builtin_tools/workspace_tools.py` if the two surfaces converge. +- **Change**: mount `s3FilesAccessPoint` over the **existing** `user-files/{u}/{s}/…` layout. The agent gets POSIX reads of files it already owns — no new namespace, no second source of truth, DynamoDB stays authoritative, and the metadata-table-first rule `workspace_files` was built on survives intact. Infra-only, no new API surface. `sessionStorage` is a separate, later decision. +- **Subtracts**: **no — and that is deliberately not the point.** ⚠️ This is the third time this feature has been written up and the **third** time the subtraction-first framing had to be corrected (2026-05-10, then again 2026-09-08). It is ranked here on `Unlocks`, per the dual-lens rule those corrections produced. +- **Unlocks** — any one of the first three justifies building: + - **Multi-turn work over an uploaded file.** A CSV uploaded once, then five follow-ups; today every turn re-fetches and re-parses. Directly on the cost thesis — attachment conversations are **11% of sessions and 31% of prod spend** — and a workspace is the only place a parsed intermediate could live between turns. + - **Artifacts that need a build step.** `artifact-render` is a single-shot Lambda; multi-file bundles, generated assets and a real dependency tree have nowhere to exist. + - **Long-running scheduled runs** — the best fit in the stack: session identity is already the resume key, and no interactive user pays the remount latency. + - **Skill assets** — the brand deck builder already fights this (`template_name` is session-scoped; the template cannot persist as a file). +- **Effort**: Low–Med (the S3 bridge) · **Impact**: Med — rising to Med–High × High for `sessionStorage`, which is **not** what is being proposed here +- **Evidence**: the API shape is ✅ **verified locally against pinned botocore**, not read from a blog: `filesystemConfigurations` takes `sessionStorage {mountPath}`, `s3FilesAccessPoint {accessPointArn, mountPath}`, `efsAccessPoint`, `capacityProviderVolume`; sibling `lifecycleConfiguration {idleRuntimeSessionTimeout, maxLifetime}` is the knob we already know from the idle-reaper work (#827). `InvokeAgentRuntimeCommand` is already in our pinned botocore; `InvokeAgentRuntimeCommandShell` is **not** a botocore op at all — it lives in the `bedrock_agentcore` SDK. ⚠️ Note `boto3`/`botocore` are pinned 24 releases behind at 1.43.68, which is what gates access to these shapes. +- **Ship means**: one infra PR mounting the access point. **Do not** bundle `sessionStorage` — it has two unresolved gates. ⚠️ **Deletion path** is a governance blocker, not a detail: a persistent filesystem holding user files is invisible to our takedown/delete machinery, which is DynamoDB+S3-aware — if a user deletes a conversation, *what deletes the workspace?* ⚠️ **Durability is single-source** (an AWS blog; the curated devguide has no filesystem-persistence page, consistent with preview) — do not design against `/mnt/workspace` or 14-day retention. +- **Decline means**: every attachment conversation keeps re-fetching and re-parsing the same file on every turn, on the cost line that is already 31% of prod spend. +- **Recommendation**: **Ship the S3 bridge; Defer `sessionStorage` until the deletion path has an owner.** ⚠️ Mind the three-way **naming collision**: "workspace" means this Runtime mount, *and* our DynamoDB-backed `workspace_files` tools which are deliberately not a filesystem, *and* nothing at all in Strands (✅ verified: zero refs in the installed SDK). Settle the convergence story with `workspace_files` up front, or we recreate the `allowedAppRoles` failure mode where two surfaces disagree. + +### 6. Strands Snapshots — take branch/regenerate, and take the cheap route to it + +- **Source**: review-queue.md ([2026-09-08], **Phil-initiated — not from a research scan**), read against our session-persistence code the same day | **merges** research/2026-09-11.md ▸ Patterns ("branching is a session-metadata feature, not a snapshot feature") as the Phase-1 route +- **Surface area**: a new SPA affordance + the session-metadata row (static SK + GSI4, #175) for Phase 1; `apis/inference_api/chat/service.py` (`_adopt_session_conversation`) only for the later redesign. Explicitly **not** `PausedTurnSnapshot` or the compaction path. +- **Change**: two phases, deliberately different sizes. **Phase 1 (cheap)** — `parentSessionId` + a branch label on the session-metadata row, forks rendered in the sidebar as indented siblings. No new SSE event, no snapshot round-trip, no rebinding of `agent.messages`. **Phase 2 (the entry's own framing)** — `take_snapshot` before a turn + `load_snapshot` to rewind, which is what actually makes *regenerate* and *edit-and-resend* work rather than just *fork*. +- **Subtracts**: **no**, and the entry says so on evidence: a four-candidate subtraction audit found **zero** replaceable — `PausedTurnSnapshot` holds disjoint content, we configure no Strands `ConversationManager` at all, `PreviewSessionManager` would not survive a container hop either, and marketplace agent-version snapshots are a different object. That negative result is recorded so it is not re-run; it is **not** the reason to rank this. +- **Unlocks**: + - **Branch / regenerate / edit-and-resend — a product capability we do not have at all.** ✅ Verified in the entry: no regenerate, no edit-and-resend, no conversation fork anywhere in the SPA. It is the standard affordance every comparable chat product ships, and Claude Code's `/fork` handling this week shows the minimum viable version is the metadata route, not the snapshot one. + - **Checkpoint-and-rollback for multi-step agentic work** — the natural pairing with long-running scheduled runs and with Proposal #5. + - **A structural answer to the `CLAUDE.md` rule "never cache session state on an agent instance"** — the #741 / #751 shape, which has bitten twice. A session-scoped snapshot loaded at the head of *every* turn, hit or miss, is the cleaner form of the aliasing `_adopt_session_conversation` does today. +- **Effort**: Low (Phase 1 spike) / Med–High (Phase 2 redesign) · **Impact**: Med–High +- **Evidence**: **already in our pin, no bump needed** — `Agent.take_snapshot` / `load_snapshot`, with the `"session"` preset capturing `messages`, `state`, `conversation_manager_state`, `interrupt_state`, `model_state`; storage is ours and `app_data` is a bag Strands never reads. **New this week**: 1.55.1 ships `fix(session): filter malformed immutable snapshot IDs (#4199)` — a fix landing on exactly this API, in exactly the patch Proposal #9's rider takes. +- **Ship means**: fold the branching-as-metadata route into the [2026-09-08] entry as its Phase 1 and spike that — what does a fork copy, and where does the copied history come from (AgentCore Memory, or a snapshot after all)? +- **Decline means**: the one product capability this forum has verified we lack entirely stays unbuilt, and the entry keeps carrying only its expensive framing. +- **Recommendation**: **Ship the Phase-1 spike; Defer the `_adopt_session_conversation` redesign.** ⚠️ **In the feature's favour**: `load_snapshot` restores by `copy.deepcopy`, so a round-trip is byte-stable *by construction* — arguably safer than re-deriving history from AgentCore Memory through the sanitizers and pairing repair. ⚠️ **Counter-gate, load-bearing**: the stale path is a cache **hit**, where nothing runs — a snapshot design must rebind `agent.messages` mid-life on every turn, which is precisely what would silently break the alias; `test_second_cache_key_for_a_session_shares_the_conversation` has to be re-reasoned, not just kept green. ⚠️ Snapshot-as-store-of-record was considered and **rejected**: it breaks LTM extraction wholesale. + +### 7. Wire the Runtime `ActiveSessionCount` alarm — scope corrected, now nearly free + +- **Source**: review-queue.md ([2026-07-10] entry, merged into [2026-09-04]) | reviews/2026-09-04.md ▸ Proposal #6(1) — **whose premise was partly wrong; corrected here** +- **Surface area**: `infrastructure/lib/constructs/inference-api/inference-agentcore-construct.ts:636` (the metric object already exists) and the `AlarmFactory` in `lib/constructs/observability/` +- **Change**: add one alarm on the `AgentCore.Runtime` `ActiveSessionCount` metric, through `AlarmFactory` so it routes to `{prefix}-alarms` as a consequence of being created. Model the threshold on the sibling that already exists. +- **Subtracts**: the `/ping` access-log as our runtime-lifetime instrument — a proxy adopted only because no real metric existed. +- **Effort**: Low · **Impact**: Med +- **Evidence**: **verified today, and it corrects last week's record.** `git log -S'ActiveSessionCount' -- infrastructure/lib` names `b681b2f5` (#910, 2026-09-02) — *before* last week's proposal claimed the metric was newly available. What shipped: an alarm on `Service: AgentCore.CodeInterpreter` (`ai-path-alarms-construct.ts:311`, threshold 50, 3 evaluation periods) and a **dashboard widget only** on `Service: AgentCore.Runtime` (`inference-agentcore-construct.ts:636`, with the comment *"`Sessions` is a cumulative creation counter; this is the live gauge"*). So the metric is wired, the pattern exists, and the gap is one alarm. +- **Ship means**: a ~10-line infra PR mirroring the Code Interpreter alarm onto the Runtime dimension. +- **Decline means**: runtime session leak and quota exhaustion stay undetected until a 429, on the one dimension where a runaway session is known to cost real money — starter-toolkit [#498](https://github.com/aws/bedrock-agentcore-starter-toolkit/issues/498) reports a single runaway session burning **$72.67** in 58 minutes, and confirms AWS still offers **no** API to list or force-terminate active runtime sessions. +- **Recommendation**: **Ship.** Cheaper than last week thought, for a different reason than last week gave. + +### 8. The process writes — four standing declines and two half-done skill edits, in this PR + +- **Source**: direct observation — Friction ≥2 ×2 | Silence that matters | reviews/2026-07-03, 08-14, 08-28 and 09-04, all of which queued these and none of which produced a write | reviews/2026-09-04.md ▸ Proposal #5 (**Ship (a) and (b)**; (b) landed, (a) did not) and ▸ Proposal #8(a) (**Ship**, unactioned) +- **Surface area**: `docs/kaizen/decisions.md`, `.claude/skills/kaizen-research/SKILL.md` (lines 21 and 406), and `.claude/skills/angualar-best-practices/`. No code, no `CLAUDE.md`. +- **Change**: **Declines —** append four entries to `decisions.md` and move the matching queue rows to `## Resolved`. **(a)** `duration_ms` tool-timing — *"deferred indefinitely; repeatedly out-prioritized, and context attribution shipped without it."* **(b)** `oauth_required` SSE flow audit — decline the standalone; its MRTR-readiness half folds into Proposal #9. **(c)** docling #405 and **(d)** Guardrails #480 — *declined as kaizen items, retained as ordinary product backlog*; per the standing rule on this repo the GitHub issues stay **open**, with a comment on each saying which forum owns them. Log the dormant-skill **keep** verdict (reviews/2026-09-04 ▸ #8b) in the same pass so research stops listing `cors-deployment` and `frontend-design` a fifth time. **Skills —** **(e)** delete the POC-comment loop from `kaizen-research/SKILL.md`, replacing it with what demonstrably works and is already in use: review-prep reads merged PRs against the prior review's proposals. **(f)** `git mv .claude/skills/angualar-best-practices .claude/skills/angular-best-practices`. +- **Subtracts**: four queue entries and the category they created — *"things this forum agrees are worth doing and will never do"* — plus a specified mechanism with **six cycles and zero comments**, the `POC findings` field and the "tested outranks untested" tiebreak that hang off it (never once fired), and one typo that makes a skill harder to invoke than it should be. +- **Effort**: Low (they are file writes) · **Impact**: — (honesty, not throughput) +- **Evidence**: verified today. `git log -1 -- docs/kaizen/decisions.md` → **2026-05-18**, four months. All four decline targets are still in `## Open`; `duration_ms` is on its **tenth** cycle, `oauth_required` on its **seventh**, ~16 weeks past a revisit date that reviews/2026-08-14 said *"must not carry a fifth review."* `grep -c 'POC' kaizen-research/SKILL.md` → **3**, mechanism intact at lines 21 and 406, *after* the 2026-09-05 edit that adopted this forum's other two recommendations for the same file. `gh pr view 926/929 --json comments,reviews` → 0 and 0. `ls .claude/skills/angular-best-practices` → No such file or directory. +- **Ship means**: mark ✅ and it all lands in **this** PR. The decline text is already written above, and the 2026-09-05 edit proves the skill-file channel works when something else is already moving in that file. +- **Decline means**: say so explicitly in `decisions.md` — *"this forum does not execute process writes; stop queueing them"* — and future reviews stop spending a slot on it. That is a legitimate and cheaper answer than a fifth ask. +- **Recommendation**: **Ship, in this PR.** Four cycles of asking for a separate PR produced nothing; the only route by which a process item has ever landed here is riding the review itself. If it does not land this week, the honest move next week is the Decline branch, permanently. + +### 9. Retire the `ClientSession` monkeypatch — with the `mcp` floor pinned first, and the cachePoint migration measured + +- **Source**: research/2026-09-11.md ▸ Top 5 #5 and the residue of Top 5 #2 | review-queue.md ([2026-09-04] `server/discover` entry, **narrowed** by the [2026-09-11] entry; and [2026-09-05] **Standing watch — `CacheConfig` convergence**, whose §2a lens produced (c)) | reviews/2026-09-04.md ▸ Proposal #9 (**Defer**, revisit 2026-09-18) +- **Surface area**: backend — `integrations/mcp_apps.py:23-32` (the symbol patch) and `:673` (the `serverInfo` capture); `backend/pyproject.toml` / `uv.lock`; `core/model_config.py:375-400` for the cachePoint half +- **Change**: three pieces of ascending cost, and **the first is a Ship now.** **(a) Pin the `mcp` floor** — add an explicit `<2` bound in `backend/pyproject.toml` with a comment naming the reason. Strands relaxed its own floor to `<2.2` in 1.55.1 while `uv.lock` sits at **1.28.1** and latest is **2.2.0**, so the next routine `uv sync` that refreshes the resolve can cross a major under this very patch. One line. **(b)** Read `io.modelcontextprotocol/serverInfo` from result `_meta` — now documented in `ext-apps` v2.0.0's `schema.json` — instead of intercepting `initialize` through a patched Strands internal, keeping the patch as a fallback. **(c)** Evaluate `CacheConfig.tools_ttl` against the layered mixed-TTL technique the [2026-08-14] cookbook entry measured at 54% cheaper upstream, and re-test whether `bedrock_cache_points_supported()` is still load-bearing now that upstream #4168 is **closed** (2026-09-04) and fixed in our 1.55.x line. +- **Subtracts**: a monkeypatch on a Strands internal that Strands is **actively changing** — this week alone an httpx2 adapter (#4183), the `mcp` floor relaxation (#4151), SEP-2663 tasks (#4125), changed `load_servers` defaults (#4177). And possibly `bedrock_cache_points_supported()`, which last week's entry called "load-bearing, not redundant" on evidence that expired four days later. +- **Unlocks**: an App-frame header on a standards-blessed field rather than a private symbol, **decoupled from** the `server/discover` migration — which is still gated on two unanswered questions (does AgentCore Gateway speak it, does Strands expose it) and stays open as its own entry. +- **Effort**: Low (a) / Med (b) and (c) · **Impact**: Med–High (a, as risk removal) / Med (b, c) +- **Evidence**: not POCed. ⚠️ `ext-apps` v2.0.0's wire protocol is **unchanged** (bidirectional interop test against 1.7.5), so nothing else in our host path needs touching — but the same release changed host-facing error codes (`-32002` → `-32602`, invalid params `-32603` → `-32602`) and **removed the `MCP error N:` message prefix**. Grep the app-tool-error chain shipped in #1009/#1013 for code or message-text matching; much of that chain was built on message text. On (c), this is the [2026-09-05] standing watch paying out a second time — see Proposal #3. +- **Ship means**: **(a) now**, as a one-line rider. For (b), verify first — do the servers we actually call emit `_meta` `serverInfo`? Presence in `schema.json` is not evidence any Gateway target or Lambda FastMCP server populates it. For (c), the gate is `backend/scripts/probe_gpt56_cache_rates.py --mode both --grow-history`, measured. +- **Decline means**: the patch stays and breaks on someone else's schedule — and if (a) is declined too, a dependency refresh can cross an MCP major without anyone choosing to, presenting as App-frame headers going generic rather than as an import error. +- **Recommendation**: **Ship (a) now; Defer (b) and (c) 2 weeks (revisit 2026-09-25).** ⚠️ **Never adopt a caching default on inspection alone** — this stack shipped one caching change on a wrong premise that measured **57% more expensive** live (#954, reverted by #956). The rider worth taking now alongside (a) is the 1.55.0 → **1.55.1** patch bump (#4199 malformed snapshot IDs — which lands on Proposal #6's API — and #4228 context-manager parity), which also carries Proposal #3. + +## Carried Over From Prior Reviews + +- **`duration_ms` tool-timing** (carried since 2026-05-15; DROP 07-03, Decline 08-14, 08-28, 09-04) — **tenth cycle, four decline recommendations, zero keystrokes.** → Proposal #8(a). +- **`oauth_required` SSE flow audit** (deferred 2026-05-10 until 2026-05-24) — **seventh surfacing, ~16 weeks overdue.** reviews/2026-08-14 said it *"must not carry a fifth review."* → Proposal #8(b). +- **docling #405 and Guardrails #480** (fifth carry) — both issues still open; reviews/2026-09-04 recommended declining them **as kaizen items** while keeping the issues as product backlog. → Proposal #8(c)(d). +- **Audit whether derived execution contexts re-evaluate RBAC** (deferred 2026-08-28 until **2026-09-11**) — **now due.** Not re-proposed above, and that is a decision to make rather than a slot to spend: it has never had a concrete surface named. *Recommendation*: **Decline or re-scope.** If it stays, it needs one named entry point and one named failure, or it will carry indefinitely like the two above. +- **AgentCore Runtime BYO filesystem** (deferred 2026-05-15 until 2026-06-12, re-listed as due 2026-09-11) — **superseded**: the [2026-09-08] "AgentCore Runtime workspaces" entry is the same feature, written Unlocks-led with the API verified against pinned botocore. *Recommendation*: **merge the [2026-05-10] entry into [2026-09-08] and close the old one.** Note the boto3 pin (1.43.68, 24 releases behind) gates access to the `filesystemConfigurations` shapes that entry depends on. +- **W5 Runtime Instances, track 2** (due 2026-09-11) — **the arithmetic is now cheaper than it was.** Research confirms `AmazonBedrockAgentCore` republished 2026-09-11 **byte-equal** on the `products` dict — 6,481 → 6,481 SKUs, 0 changes — so the 889 `Runtime:Instance-based:*:Management-Hours` SKUs from 2026-09-01 are stable enough to model against without waiting for another republish. *Recommendation*: **hold as arithmetic, not a build** — a spreadsheet and a paragraph, which either becomes a proposal or closes W5's "no proposal against it" gap with a documented *no*. Fourth month with no proposal against the largest share of the bill. +- **`bedrock-agentcore` #629** (dropped end-of-invocation spans → our cost telemetry under-reports the last turn of every session) — **third cycle unowned.** Research confirmed it still open, untouched since 2026-08-10. *Recommendation*: **Accept explicitly in `decisions.md`, or queue a guard.** Silence is the option already tried. +- **`bedrock-agentcore` #646** (watch until 2026-09-11) — **due, and it changed shape**: still open but **retitled** from a defect to a feature request (*"allow excluding binary content from persisted messages"*). The turn-killing behaviour is unchanged; the reframing makes it less likely to be prioritized. Attachment conversations are 11% of sessions and **31% of prod spend**. *Recommendation*: **extend the watch to 2026-10-09** and note the retitle, or fold it into the [2026-08-28] context-overflow hardening entry, which is the nearest owner. +- **Named A2A participants** (deferred 2026-05-15 until 2026-06-12) — still blocked on an A2A server construct. *Recommendation*: **keep deferred, no revisit date** — it earns its keep when the construct lands, and re-ranking it monthly is storage. +- **Nightly `DELETE_FAILED`** (resolved 2026-08-14 prematurely, re-opened 2026-08-28, held open 2026-09-04 with an "accepted flake" note) — **no longer a flake.** It is a hard blocker on every nightly deploy and its retained resources are billing. → Proposal #1(c). + +## Retirement Candidates + +- **⭐ `nightly-develop-PlatformStack` in dev-ai (490617140655)** — a `DELETE_FAILED` ephemeral stack that has blocked every nightly deploy for seven days and whose retained resources are **still billing**. Highest-value deletion available this week, and it is not code. Proposal #1(c). ⚠️ Read `StackStatusReason` first — a stuck ephemeral stack usually means a retention policy or a non-empty bucket. +- **The six surviving `$2.5/MTok` constants** — verified on disk today, eight days after #914 merged and a second review after they were first listed. Proposal #2. +- **The `agent_factory.py:267` invariant comment** — asserts behavior that was false at 1.53 and is now two minors stale against our 1.55.0 pin. Proposal #3. +- **⭐ The POC-comment loop in `kaizen-research/SKILL.md`** (lines 21, 406) — six cycles, zero comments, tiebreak never fired, and it survived the 2026-09-05 edit that adopted this forum's other two recommendations for the same file. Proposal #8(e). +- **Four unexecuted declines and the `decisions.md` that has not been written to since 2026-05-18** — the file exists to stop research re-proposing, and it is four months stale while the queue carries items on their seventh and tenth cycles. Proposal #8. +- **The `ClientSession` symbol patch** in `mcp_apps.py` — four independent upstream changes to that class this week alone. Proposal #9(b). +- **Possibly `bedrock_cache_points_supported()`** — upstream #4168 **closed 2026-09-04** and is fixed in our 1.55.x line. ⚠️ Verify against a real non-Anthropic model id before deleting; do not infer it from the issue closing. Proposal #9(c). +- **`scripts/sourcedir.tar.gz`** in the fine-tuning S3 prefix — orphaned by 1.20.0's move to per-family `sourcedir-{text,vision,vlm}.tar.gz`. The changelog says it plainly: *"The old object is orphaned, not read, and can be deleted by hand."* +- **The `bedrock-agentcore` row in the version-pin lag table** — 1.22.0 confirmed a no-op for the third consecutive scan (single payments feature; fixes none of the four standing issues). It costs a row and informs nothing. +- **The dormant-skill flag itself** — `cors-deployment` and `frontend-design` at 137 days. reviews/2026-09-04 settled this as **keep, stop flagging**; the verdict was never written down, so research listed them a fifth time. Log the verdict (Proposal #8) rather than re-deciding it. + +## Risks Acknowledged But Not Acted On + +- **⭐ A red gate is worse than no gate** — seven consecutive nightly failures, two deterministic causes, **0 issues filed** in a 110-PR window. — *what breaks*: the nightly's signal is now discarded wholesale, and PR CI structurally cannot see the class. — **Address now**, Proposal #1. +- **⚠️ `mcp` 2.x can arrive transitively under the `ClientSession` monkeypatch** — https://github.com/strands-agents/harness-sdk/releases/tag/python%2Fv1.55.1 (#4151) — *what breaks*: the next `uv sync` that refreshes the resolve crosses a major under a symbol patch never tested against it; the failure presents as App-frame headers going generic, not as an import error. — **Address now**, Proposal #9(a). +- **⚠️ Copying the reference repo's GPT-6 Astra row would double our input cost on that model** — https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html — *what breaks*: a 2× input rate that looks like a capability win, which is why nobody would catch it. — **Address now**, Proposal #4, after the tier-selection question. +- **⚠️ Grok 4.6 has real us-west-2 rates, no cache-write SKU, and implicit-only caching** — *what breaks*: any Grok row added through the existing derivation helper is silently wrong in both directions (nothing to place; reads at 0.25×, not 0.1×). Second ratio counterexample in two weeks. — **Address now** via Proposal #2(b); treat Grok as non-cached in projections until measured. +- **⚠️ `CLAUDE.md` asserts a false provenance for our most cost-sensitive numbers** — two independent scans now agree the Price List API carries no Claude SKU newer than Claude 3. — **Address now**, Proposal #2(a). +- **1.20.0's app-api Fargate default doubled and the increase was never reviewed here** — correctly flagged **Breaking (cost)** with an opt-out (`CDK_APP_API_CPU=512`, `CDK_APP_API_MEMORY=1024`), and justified by load testing. — *what breaks*: nothing; recorded because a cost-effectiveness forum saw the one deliberate compute doubling of the quarter only in a changelog. — recommendation: **Accept**, and confirm dev/prod sizing was a decision rather than a default. +- **⚠️ Upstream is building a first-party context-manager / offloading stack** (#4254, #4187, #4146, #4118, #4231 on `main`) that will collide with `ContextOffloader`. — *what breaks*: nothing this week; the risk is piecemeal adoption. The decisions log already forbids a bare "adopt the built-in, delete ours" proposal without a migration design covering tool-content truncation, LTM summary retrieval, DynamoDB checkpoint persistence, and the `compaction` SSE-once invariant. — recommendation: **Watch until 2026-10-09.** +- **The #629 consequence is still unowned, third cycle** — our cost telemetry under-reports the last turn of every session. — recommendation: **Accept explicitly, or queue a guard.** + +## What Shipped This Week + +*(7-day window — **110 PRs** merged into `develop`, **133** non-merge commits, **three releases**: 1.19.0 and 1.19.1 on 09-06/09-07, 1.20.0 on 09-09. Zero reverts. Seven nightly failures.)* + +- **#1012 — `strands-agents` 1.51.0 → 1.55.0** — *last review's Proposal #7, which this document recommended deferring. It shipped in five days and did the measurement the proposal specified: 1.55 read the cache entry 1.51 wrote, so Bedrock certified the prefix byte-identical across the boundary. It also caught the Nova Sonic package rename that would have switched Voice Mode off fleet-wide behind a swallowed `ImportError` — not mentioned in the upstream release notes.* +- **#1010 — `browse_web`** — *a nine-action browser tool driving Chrome in the AgentCore Browser sandbox over a hand-rolled CDP client on a SigV4-signed WebSocket. No new dependency. Seeded **disabled by default**, per the standing rule that a new injected tool must not default on.* +- **#1000–#1003, #1009, #1013, #993, #994, #1001 — the MCP Apps reliability run, eight PRs** — *app-initiated `tools/call` now resolves its OAuth token before dispatch (it bypassed the agent loop entirely, so the consent hook never warmed the cache); errors cross the AgentCore boundary as HTTP 200 + an `appToolError` envelope because Runtime rewrites any non-2xx to a generic 424 and discards the body; frames survive leaving and returning to a conversation; a torn-down MCP session is revived for an inter-turn call.* +- **#1030–#1043 — the tools drawer, twelve PRs** — *condensed tool cards with agent narration, a tool-detail pane, MCP prompts and resources surfaced, search + category grouping, OAuth connection state per tool, and a loading indicator that names the real model state.* +- **#1014, #1015, #1024 — generative VLM fine-tuning** — *`image-text-to-text` LoRA over a 4-bit NF4 base, five catalog models, loss masked to the response span, plus checkpoint-and-resume.* +- **#997, #998, #1006, #1007, #1008, #1018, #1019, #1027 — managed KB, eight PRs** — *an 8,000-char engine-aware context cap (a shared 2,000 cap was silently turning `top_k=5` into `top_k=1` at the model and producing confidently wrong answers), a fail-closed status filter, the dead-letter document reconciler and its nightly schedule, and born-managed KBs.* +- **#1020 — load-testing harness** — *Locust with real Cognito login and client-side SSE instrumentation, because ALB `TargetResponseTime` cannot complete until a stream closes. Produced the finding that the campus ceiling is the Bedrock TPM quota, not compute. Also a `Test load suite (pytest)` PR job, so renaming an SSE event now fails CI.* +- **#1016 — per-model Bedrock TPM quota alarms** — *the account-wide alarm it replaces had been above threshold for 195 of 197 datapoints over 24 hours with 205 state transitions in six days, burying every genuine alert on the SNS topic. Seven days from ship to fix.* +- **#988, #990 — the false "Response interrupted" marker and `displayText` write timing** — *a never-released `AbortController` was marking completed turns interrupted; ~1,505 prod sessions carry stale markers, with a dry-run backfill script shipped alongside.* +- **#1011 — calculator sandbox escape** (`strands-agents-tools` 0.8.6 → 0.8.8) — *the allowlist checked only the positional argument and ignored keyword arguments, so `symbols('…', cls=N)` rerouted a string through `sympify`. Registered on the default agent and seeded `enabledByDefault: True`, so the exposure was live. No CVE issued.* +- **CI**: **seven** nightly failures (09-05 → 09-11, consecutive) and three pre-merge PR CI failures on feature branches. **No `develop`-push deploy failures** — last week's Proposal #4 race did not recur, though nothing was shipped to prevent it. + +## Take + +**The forum's ranking is sound and its shipping model is wrong.** Two weeks ago three Ship-recommended items converted in five days; this week the same three classes converted at zero inside a window three times the size, while the single item marked **Defer** shipped — and shipped better than the proposal asked, with a live cross-version cache-identity measurement that Bedrock itself adjudicated. The difference between the two weeks is not effort, impact, or Phil's attention. It is that #914/#915/#916 and #1012 all sat on the critical path of something already being built, and a standalone ten-minute comment fix never does. The fix is not to rank harder — it is to name the carrier. Proposals #3, #6 and #10's rider all ride one patch bump; #4 and #9 ride this PR or they do not happen. + +**The one genuinely new failure is that a red gate ran unremarked for a week in a window that merged 110 PRs.** Both causes are deterministic and one of them is billing money right now. The structural half matters more than the specs: PR CI never passes `--coverage`, so six specs whose entire job is repo hygiene are only ever exercised at 2am on a branch nobody watches. Four weeks ago this forum adopted "never resolve a *flaky* entry on a consecutive-green count"; this is the mirror, and it cost seven nights. + +**If Phil ships three: #1** (the nightly — nothing else in this document is trustworthy while the suite's only full-fidelity run is red), **#2** (the rate correction — the ⚠️ that held it last week is gone, two scans agree, and Grok is the second live counterexample to a ratio we still hardcode), and **#8 as a rider on this PR** (four months of unexecuted declines and a feedback loop the skill still describes after six silent cycles — it will never justify its own PR, which is exactly the point). **#3** is ten minutes and this is the third consecutive ask; the bump it was meant to protect has already gone past it. **#9(a)** is one line and should ride whatever lands first. + +**A correction to this document's own process, worth stating plainly.** The first draft built the Proposals list research-first and backfilled from the queue — the inverse of what the skill specifies, where every `## Open` entry is the primary source and research is one input. That systematically buried the entries Phil wrote himself on 09-05 and 09-08, which are also the ones carrying the most verification: **AgentCore Runtime workspaces got no slot at all** and was used only as an instrument to close an older duplicate, **Strands Snapshots** was demoted beneath a cheaper reframing of its own Phase 1, and the **[2026-09-05] standing caching watch** — which *paid out this week* — went uncredited in the two proposals its payout produced. All three are fixed above. The failure mode is worth naming because it is not random: a research doc arrives written and ranked, and a queue entry does not. + +--- + +## Review Protocol (for Phil) + +1. Read Friction (2 min). +2. Scan Proposals — mark ✅ Ship / ❌ Decline / ⏸ Defer on each (3-5 min). +3. Scan Retirement Candidates — same marks (1-2 min). +4. Resolve Carried Over — six are due today and four need a keystroke, not a deliberation (2-3 min). +5. Resolve the Risks block. +6. Pick 1-3 to ship this week. Decline or defer the rest with a reason. + +Target: 10-15 minutes. + +## Post-review (for Phil — separate PRs) + +- ✅ Ship items → individual feature PRs over the week, **or a named carrier**. Proposals #3, #6 and #9(a) ride the 1.55.1 patch bump; #8 rides *this* PR. +- ❌ Decline items → appended to `docs/kaizen/decisions.md`. Proposal #8 writes the four that have been queued since May; the dormant-skill *keep* verdict (reviews/2026-09-04 ▸ #8b) belongs there too. +- ⏸ Defer items → kept open in `review-queue.md` with a revisit date; they resurface when due. + +This skill produces the agenda. Implementation never happens here. diff --git a/docs/kaizen/scoping/2026-09-11-conversation-branching-spike.md b/docs/kaizen/scoping/2026-09-11-conversation-branching-spike.md new file mode 100644 index 000000000..6df8b9736 --- /dev/null +++ b/docs/kaizen/scoping/2026-09-11-conversation-branching-spike.md @@ -0,0 +1,272 @@ +# Conversation branching spike — findings (six questions answered) + +**Date:** 2026-09-11 · **Status:** spike complete (code read + local probe) · **Proposal:** `docs/kaizen/reviews/2026-09-11.md` ▸ Proposal #6 · **Queue entry:** `docs/kaizen/review-queue.md` ▸ `[2026-09-08] Strands Snapshots` +**Method:** read our session-persistence, cost and SPA code on `develop`; read the installed `strands` and `bedrock_agentcore` sources; ran a throwaway probe against a **real** `strands.Agent` (no model call, no network) to measure the two claims the proposal rests on — snapshot byte-stability and the alias break. Every line reference below was resolved against this checkout. +**Not built:** nothing. No product code, no dependency change. The probe lived in a scratch directory and is not in this PR. +**Not re-run:** the four-candidate subtraction audit recorded in the `[2026-09-08]` queue entry (`PausedTurnSnapshot`, compaction state, `PreviewSessionManager`, marketplace agent-version snapshots — zero replaceable). It is cited where relevant and was not re-investigated. Snapshot-as-store-of-record over AgentCore Memory stays rejected; §Q2 adds an independent reason it could not have been adopted anyway. + +--- + +## Verdict at a glance + +| # | Question | Finding | +|---|---|---| +| 1 | What does a fork actually copy? | **Full history copy, or nothing.** Pointer-only yields an empty conversation in *both* readers. Copy-on-read is not expressible — the model's history comes from a Strands `SessionManager` whose restore contract is one `(memory, actor, session)` triple. | +| 2 | Where does copied history come from? | **A snapshot, or it is not byte-stable.** AgentCore Memory is the only store of model-shaped messages, but every read passes through transforms the live list never sees. `GET /messages` is a lossy display projection and cannot be fed back. | +| 3 | Does Phase 1 reach the product goal? | **No.** As specified it ships a sidebar breadcrumb on an empty chat. Regenerate and edit-and-resend need *history truncation*, which no session-metadata field can express. The review over-credited the cheap route. | +| 4 | What breaks the alias? | **`load_snapshot` itself** — measured. `agent.py:1607` rebinds `self.messages` to a fresh `deepcopy`. The guarding test cannot see it: it exercises `get_agent` only, and a per-turn load sits downstream of `get_agent`. | +| 5 | Prompt-cache impact | Round-trip byte-stability **confirmed by measurement** — and it is stability *with respect to the snapshot*, not with respect to Memory. A design that mixes both sources flips `historyHash` on an arbitrary turn. | +| 6 | Storage and quota | A fork gets its **own `totalCost`, starting at zero** — which silences the session notice on the exact conversation shape it was built to catch. A copy also re-runs LTM extraction into an **actor**-scoped namespace. | + +**Net recommendation: re-cut the phases.** Ship **regenerate first**, in place, in the existing session — it is the affordance users ask for most *and* the cheapest, because it needs no new session, no copy and no hierarchy. Fork-to-a-new-session is the expensive one, not the cheap one. See [Recommendation](#recommendation). + +--- + +## Q1 — What does a fork actually copy? + +Three candidate semantics. Tracing each through the two readers that matter. + +**The two readers are independent code paths in different services.** + +1. **The model's history** — `TurnBasedSessionManager.initialize()` (`turn_based_session_manager.py:151`) calls `super().initialize()`, which restores through `AgentCoreMemorySessionManager.list_messages` against `(memory_id, actor_id=user_id, session_id)`. There is no other input. The docstring at `:203–212` is explicit that restore "carries no in-process state" — it is a pure function of that triple. +2. **The SPA's transcript** — `get_messages_from_cloud` (`apis/shared/sessions/messages.py:359`) builds its own `AgentCoreMemoryConfig` against the same triple, then joins four session-scoped side stores: `C#`/`D#` metadata rows, pending interrupts, MCP-App UI resources, and tool summaries. + +### (a) Pointer only — `parentSessionId`, display-only + +Neither reader knows about the pointer. The child session id is new, so `list_messages` returns `[]` in both. Result: **a blank conversation with a breadcrumb**. The model has no history; the transcript is empty. This is "New chat, with a link to where it came from" — it is not a fork, and it is what Proposal #6's Phase 1 literally specifies ("no snapshot round-trip, no rebinding of `agent.messages`"). + +### (b) Pointer plus copy-on-read + +Would require *both* readers to walk the parent chain and concatenate. The SPA side is a normal refactor. The model side is not: it lives inside a Strands `SessionManager` subclass whose contract is single-session, and the walk would have to happen before `super().initialize()` returns — i.e. a second, independent implementation of chain resolution, inside the one method the byte-stability contract is written about. + +It also breaks the metadata join outright. `D#` rows are keyed `D#{session_id}#{message_id}` (`metadata.py:171`) and joined **positionally** — `metadata_index.get(str(idx))` at `messages.py:480`, where `idx` is the message's index in the restored list. Artifacts use the same shape (`msg-{session_id}-{index}`, `artifacts/models.py:57`). A concatenated view re-indexes every message, so every child-session row would have to be written against an offset that changes whenever the parent grows. + +### (c) Full history copy at fork time + +Write the parent's messages as events into the child's AgentCore Memory session, and copy the five side stores (`C#`, `D#`, UI resources, tool summaries, artifacts) plus `preferences` (`SessionPreferences` carries `assistant_id`, `agent_type`, `last_model`, `enabled_tools` — a fork without them runs as a different agent). + +This works, and it is the only one that does. It is also not cheap, and it has a cost the proposal does not mention: **copied events re-run long-term-memory extraction, and the namespaces are actor-scoped, not session-scoped.** `session_factory.py:209` and `:218` build `/strategies/{id}/actors/{actorId}` for preferences and semantic facts; only the summary namespace (`:228`) carries `/sessions/{sessionId}`. So forking a conversation extracts the same facts a second time into the *same user's* long-term memory. The duplicate is not confined to the fork. + +**Answer to Q1: a fork copies everything or it copies nothing.** There is no useful middle. + +--- + +## Q2 — Where does the copied history come from? + +This is the crux, and it has a clean answer: there are exactly three possible sources and two of them are unusable for a resumable fork. + +### AgentCore Memory — the only store of model-shaped messages, and not byte-stable against the live list + +Restore is not a read; it is a read plus a pipeline. `initialize()` runs, unconditionally and in order: + +- `_strip_document_bytes` (`:233` → `:954`) — replaces any `document` block carrying inline `source.bytes` with `{"text": "[Document placeholder: name=…, format=…, original_size=… bytes]"}`. +- `_sanitize_restored_content_blocks` (`:246` → `:916`) — drops blocks without a recognized Bedrock discriminator, and drops messages left empty. +- compaction slicing / truncation (`_apply_compaction`, `:270`). +- `_repair_restored_history` → `_repair_tool_pairing` (`:1098`). + +Below that, the SDK's own converter runs `_filter_empty_text` on every restored message (`bedrock_converter.py:82`). + +The live, accumulated list goes through **none** of this — `append_message` (`:135`) filters on the *write* side only, and after construction the list is append-only. So for any session that ever carried an inline attachment, the Memory-derived form and the live form differ at that index, by construction and by design. That is the divergence `_adopt_session_conversation`'s docstring already names as the reason re-restoring on a stale hit was rejected (`service.py:163–168`). + +### `GET /messages` — a display projection, lossy by construction + +`_convert_content_block` (`messages.py:131`) is an `if/elif` chain on `"text" / "toolUse" / "toolResult" / "image" / "document" / "reasoningContent"`: **only the first recognized key on a block survives**. `_ensure_image_base64` and `_ensure_document_base64` reshape `{"source": {"bytes": …}}` into `{"format": …, "data": …}`. The output is `MessageResponse`, not a Bedrock `Message`. It cannot be fed back to the model. + +Worth naming because it is a near-miss: **we already ship a point-in-time conversation copy.** `shares/service.py:87` snapshots `get_messages()` output plus session metadata plus pinned artifacts into S3, and `models.py:101` states the point-in-time promise explicitly. It is a working precedent for copying a conversation — and it is deliberately *not resumable*, for exactly the reason above. + +### A Strands snapshot — the only byte-stable source + +`Agent.take_snapshot` deep-copies `agent.messages` verbatim (`agent.py:1570`); `load_snapshot` deep-copies them back (`:1607`). Measured in §Q5: identical `historyHash`, including through a full JSON transport. This is the proposal's "in the feature's favour" note, and it holds. + +> ⚠️ **A structural finding the queue entry does not have.** strands 1.55.0 also ships `strands/session/snapshot_session_manager.py` — a full `SnapshotSessionManager` with append-only immutable checkpoints, `list_snapshot_ids`, and `restore_snapshot(snapshot_id=…)` time travel. It is *not* an option for us, and the reason is structural rather than a judgement call: an `Agent` has exactly one `session_manager`, and ours is `AgentCoreMemorySessionManager`. Adopting `SnapshotSessionManager` **is** snapshot-as-store-of-record, which was already rejected for breaking LTM extraction. The manual `take_snapshot` / `load_snapshot` pair, used alongside our session manager with storage we own, is the only usable surface — which is what the queue entry said, now with an independent reason. + +**Answer to Q2: from a snapshot. Any other source is either lossy or not byte-stable against the live list** — and it is a prompt-cache problem before it is a correctness one. + +### On the 1.55.1 fix + +Proposal #6 notes `fix(session): filter malformed immutable snapshot IDs (#4199)` as "a fix landing on exactly this API". Checked: it lands on `SnapshotSessionManager.list_snapshot_ids`, whose 1.55.0 body is `sorted(match.group(1) for key in keys if (match := _SNAPSHOT_REGEX.search(key)))` with no `is_uuid7` filter — a class we cannot adopt. The manual pair is **byte-identical between 1.51.0 and 1.55.0** (diffed `types/_snapshot.py` and the `take_snapshot`/`load_snapshot` bodies from the uv cache: no differences). **The spike does not depend on the 1.55.1 fix, and no bump is needed.** (The same diff is what makes the probe below faithful to the pin — see §Q5.) + +--- + +## Q3 — Does Phase 1 actually reach the product goal? + +**No.** Three independent reasons, in increasing order of how hard they are to design around. + +### 1. Phase 1 as written produces an empty conversation + +Per §Q1(a). "`parentSessionId` + a branch label, no snapshot round-trip, no rebinding of `agent.messages`" describes a metadata row and nothing else. To make it a real fork you must add the full copy — which is the expensive work the phase was defined to avoid, and which drags in LTM double-extraction (§Q1), five side stores, and a positional re-index. + +### 2. Regenerate and edit-and-resend need a primitive Phase 1 has no way to express + +Both mean: **discard the tail of an existing conversation and continue from before it.** The invoke contract carries `session_id` and `message` and nothing else — `InvocationRequest` (`inference_api/chat/models.py:104`) has no history override and no start index. History is 100% server-derived from the session id. No field on the session-metadata row can say "turn 7 onward is dead", because the two readers never consult that row for history; they read AgentCore Memory. + +This is the honest core of the answer. **Phase 1 delivers fork and only fork**, and fork is the affordance users ask for *least* of the three. + +### 3. The sidebar cannot render hierarchy cheaply + +`session-list.ts:89` groups sessions into **recency buckets** (Today / Yesterday / Last 7 Days / Last 30 Days / Older) over a **paginated** list backed by GSI4 (`metadata.py:1078`, `GSI4_SK = {lastMessageAt}#{session_id}`). "Forks as indented siblings" fights both: a fork and its parent routinely land in different buckets, and after a few days of use the parent is frequently not on the loaded page at all. Making the indent correct means either loading the whole chain out of band or abandoning recency ordering for forked rows. + +A "forked from **" line in the conversation header is strictly cheaper and probably better; the indent is a design question this spike does not settle. + +### Why the Claude Code `/fork` analogy does not carry + +`/fork` is metadata-cheap there because the transcript is a **local JSONL file** that can be copied with the filesystem. Ours is a managed remote event log with four positional side stores hanging off it, an actor-scoped LTM extractor reading it, and a denormalized cost aggregate on a separate row. The cheapness is a property of the storage, not of the idea. + +--- + +## Q4 — What breaks the alias, and what does a correct design do? + +### Measured, not inferred + +`Agent.messages` is a plain instance attribute — there is no property or setter on the class. `load_snapshot` does: + +```python +if "messages" in data: + self.messages = copy.deepcopy(data["messages"]) # agent.py:1607 +``` + +A **rebind**. Probe result against a real `strands.Agent`: + +``` +load_snapshot rebinds messages: True +after rewind: A=5 B=4 alias_intact=False +``` + +Instance B — standing in for the second cached agent that adopted A's list by reference — stays at 4 messages while A appends its post-rewind turn. **The alias is broken by `load_snapshot` itself, by construction.** This is precisely the hazard `_adopt_session_conversation` warns about (`service.py:150–157`: "A future compaction that rebinds mid-life would silently break the alias") and precisely why `_drop_abandoned_turn_tail` (`stream_coordinator.py:105`) mutates in place and says so in its docstring. + +`load_snapshot` also rebinds `self.state` (`AgentState(data["state"])`), `self._interrupt_state` and `self._model_state`. Nothing aliases those today; a design that starts to would inherit the same class of bug. + +### The test cannot catch it — and that is structural, not stylistic + +`test_second_cache_key_for_a_session_shares_the_conversation` (`backend/tests/apis/inference_api/test_chat_service.py:292`) is well written and deliberately future-proofed: its docstring says "a fix may reuse one instance, hand the message list between instances, or re-restore on a stale hit, and this test should pass either way." + +But it patches `service.create_agent` and asserts on the return of `service.get_agent`. It exercises **the `get_agent` boundary only.** A snapshot design loads at the head of a turn — in `stream_coordinator` / `chat/routes.py`, *after* `get_agent` has returned. The test would stay green while the behaviour it protects is gone. + +**Re-reasoned version:** the assertion has to move to the turn boundary. Run two turns under two different cache keys *through the code path that would call `load_snapshot`*, and assert the second key's instance observes the first's appends. That test does not exist today, and **it should be written before any snapshot work**, not after — otherwise the regression it guards is unobservable in CI. + +### What a correct design does + +| Option | Mechanism | Verdict | +|---|---|---| +| **Splice, don't rebind** | Restore everything *except* messages via `load_snapshot` on a `Snapshot` whose `data` has `messages` removed, then `live[:] = snapshot.data["messages"]` on the existing list object. Verified expressible: `Snapshot(scope="agent", schema_version="1.0", data={k: v for k, v in snap.data.items() if k != "messages"}, app_data={})` loads cleanly. | **Recommended.** Preserves the alias; same discipline `_drop_abandoned_turn_tail` already uses. | +| **Retire the alias** | One session = one `Agent` instance regardless of configuration; move system prompt / tools / model off the cache key onto per-call parameters. | The "structural answer to the `CLAUDE.md` rule" the queue entry hopes for — and much larger than Phase 2, since the cache key is what makes a tool or skill edit take effect at all. Not a spike-sized change. | +| **Re-restore every turn** | Drop the alias, restore from Memory on every turn. | Rejected in the docstring on prompt-cache grounds; §Q5 measures why. | + +--- + +## Q5 — Prompt-cache impact of each route + +### The probe + +Run against a real `strands.Agent` with a `MagicMock` model (no network), on a four-message history containing a `toolUse`/`toolResult` pair. `historyHash` computed the way `PrefixFingerprintHook` does (`prefix_fingerprint.py:93` — canonical JSON over `agent.messages`). + +``` +snapshot data keys: ['conversation_manager_state', 'interrupt_state', 'messages', 'model_state', 'state'] +schema_version: 1.0 | scope: agent +byte-stable round-trip : True e3b4d926d02b4c7e +byte-stable via JSON transport: True +load_snapshot rebinds messages: True +rewound to 1 message(s): ['user'] +``` + +The local venv is strands **1.51.0** while the pin is **1.55.0**; this is faithful because the snapshot API is byte-identical across that boundary (§Q2). `app_data` survives the transport untouched, confirming the entry's "a bag Strands never reads". + +**The claim holds: a snapshot round-trip is byte-stable by construction.** + +### The half the review does not say + +It is byte-stable *with respect to the snapshot*, not with respect to AgentCore Memory. Per §Q2, restore rewrites document blocks to placeholders and filters content blocks; the live list does not. So a design that snapshots from the live agent but **falls back** to a Memory restore — cold container, snapshot miss, container hop, snapshot-store outage — alternates between two byte-forms of the same conversation. `historyHash` flips on an arbitrary turn, `toolConfigHash` and `systemPromptHash` hold, and the whole 35k–150k prefix is re-written at the cache-write premium. That is the same failure mode the byte-stability contract at `turn_based_session_manager.py:15–23` exists to prevent, arriving through a new door. + +The `C#`-row fingerprints are the right instrument and need no new work: `historyHash` diverging while the other two hold *is* the signature of a history-source flip. + +### Per route + +- **Fork with a full copy.** The child's first turn re-sends the whole copied prefix. Whether that is a write or a read is an open question with a real upside: Bedrock's cache is keyed on prefix **content**, not on our session id, so a byte-identical copy could *read* the parent's live entry. #1012 already proved a cache entry surviving a change on our side of the wire (a version boundary). It has never been measured across session ids, and TTL bounds the win — a fork minutes later could hit, a fork the next day cannot. **Only available if the copy comes from a snapshot**; a Memory re-derive is a different byte-form and guarantees a write. +- **Regenerate / edit-and-resend.** These *shorten* the prefix, so everything up to the cut is unchanged and should still read. Rewinding is the cache-friendly operation. The risk here is not the cache; it is the store of record (§Q1, §Q6). +- **Per-turn `load_snapshot` as the general fix for the agent-cache rule.** Byte-stable, but see Q4: it must splice rather than rebind, and it adds a storage round-trip to every turn — including cache hits, which today do no I/O at all. That trade needs measuring before it is called an improvement. + +--- + +## Q6 — Storage and quota + +### A forked session gets its own `totalCost`, starting at zero + +The session row is `PK=USER#{user_id}`, `SK=S#{session_id}` (`metadata.py:1072`), and `_bump_session_aggregates` (`metadata.py:1629`) does `ADD totalCost :c` against the row addressed by *that* session id. A new session id is a new row is a fresh zero. + +The quota **session notice** reads exactly that field: `QuotaChecker._resolve_session_notice` (`quota/checker.py:212–254`) fetches the session metadata and compares `metadata.total_cost` against `session_notice_threshold_usd(limit, tier)`. `TopSessionCost` (`admin/costs/models.py:182`) documents the same field as the session's **lifetime** cost and explains why: "a runaway conversation is usually a single long thread that spans period boundaries (the incident session opened 2026-07-30 and blocked a quota on 2026-08-04)". + +**So forking a runaway conversation resets the notice to zero while carrying the entire expensive prefix forward.** The user's monthly quota is unaffected — that is `user-cost-summary`, keyed on user — so this is not quota evasion. It is worse in a subtle way: it silences the one signal built specifically to catch a single runaway thread, on the exact conversation shape most likely to be forked. + +Mitigation, and this must be a deliberate decision rather than a default: + +- **Seed the child's `totalCost` with the parent's at fork time.** Recommended. The notice's dedupe is keyed `(user, "session_notice", session_id)` within 60 minutes (`event_recorder.py:130–141`), so a seeded child fires its own notice correctly and immediately. +- *Not* a chain-walk at notice time: that is a per-turn read amplification on the hot quota path for a signal that fires rarely. + +### Storage + +A copy duplicates: AgentCore Memory events, `C#` + `D#` rows, MCP-App UI resources (`ui_resource_store.py:190`, `GSI_PK=SESSION#{id}`), tool summaries, and artifact rows — plus a second LTM extraction pass into an **actor**-scoped namespace (§Q1). Admin cost attribution then double-counts by construction: `TopSessionCost` lists parent and child at their lifetime totals, and the fleet already has a known over-count in `AdminUsageAggregates`. Adding a legitimate duplication vector on top of that deserves an explicit decision. + +**Cheap mitigation worth naming:** copy lazily, on the fork's **first turn**, not at fork creation. A fork the user opens and abandons then costs one metadata row. + +--- + +## Recommendation + +**Do not ship Phase 1 as specified.** Re-cut the phases so the cheap thing is the thing users actually ask for. + +### Phase A — regenerate, in place, in the existing session · **Med, ~3–5 days** + +No new session, no copy, no sidebar hierarchy, no alias break. What it needs is one new primitive: **truncate a session's history by N trailing messages.** Three parts: + +1. **In memory** — pop the trailing assistant turn off `agent.messages` *in place*. `_drop_abandoned_turn_tail` (`stream_coordinator.py:105`) already does exactly this shape, in place, for a different reason. +2. **In AgentCore Memory** — delete the corresponding events. `gmdp_client.delete_event` is already used by `update_message` (`session_manager.py:710`), so the capability exists. ⚠️ See the unknown below — addressing them is the hard part. +3. **In DynamoDB** — delete the `C#`/`D#` rows (and artifact rows) at and above the cut, *before* the replacement turn writes. Because the `D#` join is positional (`D#{session_id}#{message_id}`), truncate-then-append **collides** new rows with orphans if you overwrite instead of deleting. + +Cache-friendly: it shortens the prefix, so everything before the cut still reads. + +### Phase B — edit-and-resend · **Low on top of A, ~1–2 days** + +The same primitive with the cut one message earlier, plus a composer affordance. Nothing new structurally. + +### Phase C — fork to a new session · **Med–High, 2+ weeks, gated** + +Worth doing only after A and B prove the truncation primitive. Uses a snapshot as the copy source (§Q2), copies lazily at the fork's first turn (§Q6), seeds `totalCost`, and needs an answer on suppressing LTM extraction for copied events. The sidebar question (indent vs. a header line) is a separate design decision, not a blocker. + +### Sequencing notes + +- **Write the turn-boundary alias test first** (§Q4). It is a precondition, not a follow-up. +- **Do not bump to 1.55.1 for this.** The #4199 fix lands on a class we cannot adopt; the API we would use is byte-identical across 1.51.0→1.55.0 (§Q2). `CLAUDE.md` forbids installing without explicit approval, and there is no reason to ask. +- **Do not revisit the subtraction audit.** Recorded in the `[2026-09-08]` queue entry, four candidates, zero replaceable. + +--- + +## What remains unknown + +1. **Can a single message be addressed for deletion in AgentCore Memory?** *The most important unknown, and it gates Phase A.* `batch_size` defaults to 1 and we never override it (`session_factory.py:28` says so, `config.py:93` confirms the default), so it is one `create_event` per message — 1:1, good. But `append_message` (`session_manager.py:832–843`) persists `SessionMessage.from_message(message, 0)` — **message id `0` in the payload** — and keeps the real `eventId` only in the in-process `_latest_agent_message`. On restore, `events_to_messages` (`bedrock_converter.py:81`) rebuilds each `SessionMessage` from that payload, so **restored messages do not carry their event ids**. Truncation therefore has to drop to `gmdp_client.list_events` and pair events to messages by order — feasible at `batch_size=1`, but the pairing must survive blob events and the `_filter_empty_text` drop on read, which can make the two counts disagree. **Needs a live read of a real dev-ai session's event list before Phase A is committed to.** If the pairing turns out to be unreliable, in-place regenerate is off the table and every affordance routes through a new session, which changes the whole recommendation. +2. **Does writing copied events into a new session trigger LTM extraction, and can it be suppressed per event?** Not answerable from our code or the pinned SDK. Needs a dev-ai probe against a memory with strategies configured. Gates Phase C's real cost. +3. **Does a Bedrock cache entry written under session A get read under session B with a byte-identical prefix?** Almost certainly yes — the cache is keyed on content — but never measured across session ids. One salted two-arm probe using the #1012 method would settle it, and it is the difference between a fork's first turn being free and being a full-prefix write. +4. **Should the sidebar show hierarchy at all?** Not settled here. The recency-bucket + pagination conflict (§Q3) is real; a header line inside the conversation may dominate the indent on both cost and clarity. + +--- + +### Refs + +| Thing | Where | +|---|---| +| The alias and why it is by-reference | `backend/src/apis/inference_api/chat/service.py:133` | +| Rebinding mid-life breaks it (in-place, deliberately) | `backend/src/agents/main_agent/streaming/stream_coordinator.py:105` | +| The guarding test | `backend/tests/apis/inference_api/test_chat_service.py:292` | +| Restore pipeline + byte-stability contract | `backend/src/agents/main_agent/session/turn_based_session_manager.py:15`, `:151`, `:916`, `:954` | +| SPA transcript read + positional metadata join | `backend/src/apis/shared/sessions/messages.py:359`, `:480` | +| Display projection is lossy | `backend/src/apis/shared/sessions/messages.py:131` | +| Existing point-in-time conversation copy | `backend/src/apis/app_api/shares/service.py:87` | +| Static SK + GSI4 recency keys | `backend/src/apis/shared/sessions/metadata.py:1072`, `:1078` | +| `totalCost` aggregation | `backend/src/apis/shared/sessions/metadata.py:1629` | +| Session-notice threshold read | `backend/src/agents/main_agent/quota/checker.py:212` | +| Invoke contract has no history override | `backend/src/apis/inference_api/chat/models.py:104` | +| Prefix fingerprints | `backend/src/agents/main_agent/session/hooks/prefix_fingerprint.py:93` | +| Sidebar recency grouping | `frontend/ai.client/src/app/components/sidenav/components/session-list/session-list.ts:89` | +| Message actions today (Copy + Continue only) | `frontend/ai.client/src/app/session/components/message-list/components/message-actions.component.ts` | +| `take_snapshot` / `load_snapshot` | `strands/agent/agent.py:1543`, `:1591` | +| `SnapshotSessionManager` (not adoptable) | `strands/session/snapshot_session_manager.py` | +| `append_message` persists message id `0` | `bedrock_agentcore/memory/integrations/strands/session_manager.py:832` | diff --git a/docs/specs/agent-marketplace.md b/docs/specs/agent-marketplace.md index 6d02ad3ba..5ded68ddd 100644 --- a/docs/specs/agent-marketplace.md +++ b/docs/specs/agent-marketplace.md @@ -998,7 +998,7 @@ rather than restating the checks. ([`chat/routes.py:1319`](../../backend/src/apis/inference_api/chat/routes.py)) resolves against the caller, and there is no author-side lookup anywhere on the invocation path. Cost rows land on the same person: `PK = USER#{user_id}` - ([`sessions/services/metadata.py:157`](../../backend/src/apis/app_api/sessions/services/metadata.py)). + ([`shared/sessions/metadata.py:269`](../../backend/src/apis/shared/sessions/metadata.py)). So spend and quota agree, and a published Agent does not bill its author for a stranger's run. - **Review SLA** — committed, and split by queue. See the table in D2: two business days for submissions, same day for an `inappropriate` report, weekly for the rest. diff --git a/docs/specs/canvas-rubric-agent.md b/docs/specs/canvas-rubric-agent.md new file mode 100644 index 000000000..9194b23fd --- /dev/null +++ b/docs/specs/canvas-rubric-agent.md @@ -0,0 +1,907 @@ +# Spec: Canvas Rubric Agent + +**Status:** Built and smoke-tested in dev 2026-09-10 (§10, §10a). All server fixes merged (mcp-servers#38, #39, #40). **All seven §8.1 acceptance criteria pass** (§10a). Remaining: institutional KB content (§10a), then prod cutover (§8.2). +**Audience:** A fresh implementation session with no prior context — this doc is self-contained. +**Owner:** Phil Merrell +**Last updated:** 2026-09-10 + +--- + +## 1. One-line summary + +A marketplace Agent that lets a Boise State instructor say *"make me a rubric for the Final +Project in BIOL 101"* and get a standards-aligned rubric **published directly into Canvas and +attached to the assignment** — asking as few questions as possible, because most of what a +rubric generator normally asks for can be retrieved instead. + +The origin is a Colab notebook (`manage_rubrics.py`) that faculty currently use: fill in a course +ID, hand-author a CSV, run a cell, import to Canvas. This replaces the whole loop. + +--- + +## 2. Decisions already made (do not re-litigate) + +| Decision | Choice | Why | +|---|---|---| +| **Surface** | A marketplace **Agent**, not a bare skill | It must bind a tool, a knowledge base, a curated model, and conversation starters. Skills on this platform are pure knowledge bundles and bind no tools. | +| **Canvas access** | The existing **`canvas_faculty` external MCP server** | Already deployed, already RBAC'd, already has `create_rubric` / `associate_rubric`. No new tool protocol. | +| **CSV** | Demoted to an **optional import/export**, never the deliverable | The notebook needed CSV because a script cannot read prose. The agent can. A CSV terminus would just be the notebook with a nicer front end. | +| **Wizard style** | **Retrieve first, propose second, ask last** | Faculty-first. A blank five-variable template is a form, not a guide. See §5. | +| **Pedagogy location** | A **skill**, not the system prompt | Skills are progressively disclosed — the body costs nothing until activated. The system prompt is in the cached prefix on every turn. | +| **Publish gate** | **Platform approval interrupt** on `create_rubric`, *plus* a readable table in chat | Verified in dev 2026-09-10. Two layers on purpose: the table is readable but is the model's *claim*; the approval card renders the exact `tool_input` that will be sent, so it is the ground truth. See §4.4 and §10. | + +### Non-goals + +- **No grading.** The bound tool exposes `grade_submission`, `grade_with_rubric`, and + `bulk_grade_submissions`; the system prompt fences them off. Grading is a different product + with a different risk profile. +- **No assignment or course authoring.** Same reasoning — `create_assignment`, `create_page`, + `create_module` etc. come along with the binding and must be fenced. +- **No new MCP server.** Everything lands in `mcp-servers/packages/canvas-faculty`. +- **No rubric analytics.** Out of scope for v1. + +--- + +## 3. Verified current state (audited 2026-09-10) + +> Re-verify before building; this is a point-in-time snapshot. + +### 3.1 The MCP server already does the job + +`mcp-servers/packages/canvas-faculty/app.py` implements every operation the notebook performs: + +| Notebook cell | MCP tool | +|---|---| +| `create_rubric()` + `read_criteria_from_csv()` | `create_rubric(course_id, title, criteria, assignment_id?)` | +| `update_assignment()` / `create_rubric_association` | `associate_rubric(course_id, rubric_id, assignment_id)` | +| "Print all Assignment IDs" | `list_assignments` | +| "Print all Rubric IDs" | `list_rubrics` | +| "look at the URL for your course ID" | `list_courses` | + +`create_rubric` with `assignment_id` set creates **and** attaches in a single call. + +### 3.2 Environment state + +| | dev-ai (`dev-boisestateai-v2`) | prod-ai (`boisestateai-v2`) | +|---|---|---| +| Server build | **42 tools**, all rubric tools present | **7 tools**, no rubric tools | +| OAuth provider | `canvas-faculty`, 8 scopes | `canvas-faculty`, 7 scopes | +| Rubric scopes | **absent** | **absent** | +| Canvas instance | `boisestatecanvas.test.instructure.com` | `boisestatecanvas.instructure.com` | +| Tool record | `TOOL#canvas_faculty`, `enabledByDefault=false`, `isPublic=false` | same, `isPublic=true` | +| Cached tool snapshot | 7 tools (stale) | 7 tools (stale) | +| RBAC | `faculty` grants `canvas_faculty` (bare); `isPublic: false` | `faculty`, `staff`, `student` grant it (bare) — **and `isPublic: true`, which grants it to every authenticated user regardless of role** (§9) | + +Verify the live tool surface with: + +```bash +curl -sS -X POST "$LAMBDA_URL/mcp" -H 'Content-Type: application/json' \ + -H 'Accept: application/json, text/event-stream' -H 'Authorization: Bearer x' \ + -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"p","version":"1"}}}' +``` + +then repeat with `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`. `tools/list` does not call +Canvas, so any bearer value works. + +### 3.3 Platform primitives (confirmed against code) + +- **All four binding kinds resolve at runtime** — `knowledge_base`, `tool`, `skill`, + `memory_space` (`inference_api/chat/agent_binding_resolver.py`). The + *"`tool` and `skill` are accepted and stored but inert until Phase 2/3"* comment in + `apis/shared/assistants/models.py` is **stale**; ignore it. +- **Tool bindings take bare catalog ids *or* scoped ones.** ~~`can_access_tool` exact-matched the + id, so binding `canvas_faculty` brought all 42 tools and trimming would need a platform + change.~~ **Superseded** — `binding.ref` now accepts `canvas_faculty::create_rubric`, and a + scoped ref is admitted by a grant on its base server. A bare ref still means the whole server, + so nothing about the shape above changed for an agent that wants all of it. Measured on this + agent in dev: bare = 44 tools and a 27,959-token prompt; the 7 scoped refs below = 9,981. +- **Tool bindings replace** the request's `enabled_tools`; a bound tool the invoker cannot access + raises `AgentBindingBlockedError` and blocks the turn with a message (no silent drop). +- **Skills bind no tools.** `ChatAgent`: *"Skills are pure knowledge bundles … the tool universe + comes solely from the Agent's bindings and RBAC-gated `enabled_tools`."* `allowed_tools` on a + skill record is frontmatter passthrough, not a grant. +- **Skills are progressively disclosed** — an `` catalog plus a `skills` + activation tool; the body loads on activation, `resources` load on demand via `read_skill_file`. +- Agents also carry `starters`, `emoji`, `tagline`, `description`, `visibility`, and a + marketplace `listing`. + +--- + +## 4. Blockers — these must ship before the agent is buildable + +### 4.1 Rating `long_description` is silently dropped — FIX BUILT, mcp-servers#38 + +`create_rubric`'s ratings loop sends only two fields: + +```python +data.append((f"{rprefix}[description]", rating.get("description") or "")) +data.append((f"{rprefix}[points]", str(rating.get("points", 0)))) +``` + +Canvas's API supports `rubric[criteria][x][ratings][y][long_description]`, and **that is where a +descriptor lives**. The descriptors are this agent's entire product. + +**Validated in dev 2026-09-10 — the failure mode is worse than "descriptors go missing."** Asked +for three named levels (Exemplary / Proficient / Developing) plus a descriptor in all six cells, +the model had nowhere to put a descriptor, so it **overloaded the rating `description` field with +the descriptor sentence and dropped the level names entirely**. `get_rubric` confirms what Canvas +stored: ratings carry no `long_description` at all, criterion `long_description` is `""`, and the +rating name for a 4-point level reads *"Code is clean, efficient, and follows best practices with +no redundancy or inefficiency."* The rubric renders with paragraphs where the level labels belong +and no level labels anywhere. + +This is a direct consequence of the unconstrained `criteria` schema (§4.7): the docstring offers +ratings only `{description, points}`, so a descriptor has exactly one place to go and it is the +wrong one. Fixing the schema and the passthrough together is what makes this correct — either +alone still produces a wrong rubric. + +Fix: pass `long_description` through on ratings. One line. **Nothing else in this spec matters +until this lands** — §7.2's field-mapping skill is inert without it. + +### 4.2 No `update_rubric` / `delete_rubric` — FIX BUILT, mcp-servers#38 + +The server exposes `list_rubrics`, `get_rubric`, `create_rubric`, `associate_rubric`, +`grade_with_rubric` — no update, no delete. The conversation can iterate freely right up to the +write and then cannot iterate at all; a correction means the instructor opens Canvas. For +something billed as a guided experience this is the most damaging gap after §4.1. + +Fix: add `update_rubric` and `delete_rubric`. Note Canvas restricts editing rubrics already used +for grading — surface that as a clean tool error rather than a raw 401/403. + +### 4.3 Prod is unusable + +Prod runs a 7-tool build **and** lacks all four rubric scopes: + +``` +url:GET|/api/v1/courses/:course_id/rubrics +url:GET|/api/v1/courses/:course_id/rubrics/:id +url:POST|/api/v1/courses/:course_id/rubrics +url:POST|/api/v1/courses/:course_id/rubric_associations +``` + +**mcp-servers#38 adds two more scopes** — `url:PUT|/api/v1/courses/:course_id/rubrics/:id` and +`url:DELETE|/api/v1/courses/:course_id/rubrics/:id` — for `update_rubric` / `delete_rubric`. +**Six scopes total now, not four**, and dev needs the two new ones as well before those tools +work there. + +Fix: redeploy the current `canvas-faculty` image to prod; have a Canvas admin add the six scopes +to the boisestate.ai developer key (required if *Enforce Scopes* is on — the fact that we send a +specific list implies it is); update the provider record in both environments. + +Two related notes: +- Prod is also missing `url:POST|/api/v1/conversations`, so `send_conversation` is presumably + already broken there. Unrelated to rubrics; worth fixing in the same pass. +- **Scope widening DOES force re-consent automatically — verified in dev 2026-09-10.** An + earlier draft of this spec claimed the opposite; it was wrong. AgentCore's token vault keys on + the requested scope set, so the first tool call after a scope edit returned + *"AUTHORIZATION NEEDED — Connect Canvas for Faculty"* rather than a Canvas 401. No + `scopesHash` drift detection is needed; that field being unread is fine, not a gap. + **Rollout implication instead:** the prompt fires on the first call to *any* tool on that + provider, not just one needing the new scope — it hit `list_courses`, whose scope was + unchanged. So adding the scopes in prod makes every connected faculty member reconnect on + their next Canvas use. One click, not a broken state, but announce it rather than shipping it + silently. + +### 4.4 Stale catalog snapshot — RESOLVED in dev 2026-09-10 + +`mcpConfig.tools` on `TOOL#canvas_faculty` was a 7-tool snapshot, which gated the admin per-tool +picker and the `needsApproval` flag. **Fixed in dev** with *Discover from server* on +Admin → Tools → Canvas for Faculty: the catalog now caches all 42 tools, and existing approval +flags survived the rediscovery (`send_conversation` kept its own). + +`create_rubric` is now flagged `needsApproval: true` in dev, and the interrupt was verified in +both directions: + +- **Approve** → `MCPExternalApprovalHook` raises the interrupt, the SPA renders + *"APPROVAL NEEDED — Approve `create_rubric` to let the assistant continue"* with a + **View arguments** expander showing the pretty-printed `tool_input`, and the call proceeds. +- **Decline** → the tool result becomes *"User declined to approve the 'create_rubric' tool + call; the agent should not invoke it."* and nothing is written to Canvas. + +**Still to do in prod:** run the same discovery and set the same flag after the prod deploy +(§4.3). + +**Decided 2026-09-10: `associate_rubric` is gated too.** Dev now flags both +(`gated: send_conversation, create_rubric, associate_rubric`). The extra prompt on the +less-common path is worth it because `associate_rubric` is the call that re-points the +assignment (§4.5) — the one side effect in this feature that touches student-visible grades. + +### 4.5 Attaching a rubric silently rewrites the assignment's points — MITIGATED, mcp-servers#38 + +`associate_rubric` with `use_for_grading: true` (the default) caused Canvas to change the +assignment's `points_possible` from **5.0 to 8.0** — the rubric's total. Nobody asked for that, +nothing warned, and in a live course it is a grade-affecting change to an assignment the +instructor did not think they were editing. + +This is Canvas behaviour, not a bug in our tool, but it must be surfaced. + +**The approval gate does not cover this on its own.** The approval card renders `tool_input` — +for `associate_rubric` that is `course_id`, `rubric_id`, `assignment_id`, `use_for_grading`. +Nothing in those four values tells the instructor that approving will change the assignment +from 5 points to 8. **The card shows the inputs, not the consequences.** So the gate is +necessary but not sufficient: the agent has to state the point change in the message *before* +the prompt, or the instructor approves a payload whose effect is invisible. + +Minimum: the publishing skill warns before attaching (§7.3) and the agent states the change +afterwards. +Better: `associate_rubric` and `create_rubric` return the assignment's before/after +`points_possible` so the agent can report it without a second call. Best: the agent reads +`points_possible` first and, when the rubric total differs, asks the instructor which number +should win before writing. + +### 4.6 `get_rubric` cannot confirm an attachment — MITIGATED, mcp-servers#38 + +After a successful `associate_rubric` (association id returned, attachment real), +`get_rubric` returned `"associations": []`. The attachment *was* live — +`get_assignment_details` on the assignment showed the rubric and the changed point value — so +`get_rubric`'s `associations` array is unreliable as a verification signal despite the tool +requesting `include[]=associations`. + +Consequence: the "call `get_rubric` and compare" verification step in §7.3 does not work as +written. Verify attachment via `get_assignment_details` instead; `get_rubric` remains correct +for criteria and ratings. Worth a follow-up to find out why the include is not populating. + +### 4.7 Recommended at the same time — DONE in mcp-servers#38 + +- **Give `criteria` a real schema.** The live tool definition is + `{"type":"array","items":{"type":"object","additionalProperties":true}}` — no properties, no + required fields. Every bit of structure lives in a prose docstring. A Pydantic model costs + nothing at runtime (tool-definition text is already in the cached prefix) and is the highest- + leverage reliability fix on the server. +- **Default criterion `points`** to `max(rating points)` instead of raising `ToolError`. The + notebook never sends criterion points and Canvas derives them; our tool hard-requires them. + Canvas treats criterion points as authoritative for the rubric total, so a bad inference + silently produces a rubric whose total disagrees with its own ratings. +- **Decide the no-assignment association case.** The notebook always sends + `association_type: 'Course'`; `create_rubric` sends a `rubric_association` block *only* when + `assignment_id` is given. Verify in a sandbox whether a rubric created with no association + appears under Course → Rubrics. If it does not, "make a rubric, I'll attach it later" produces + an invisible rubric. + +--- + +## 5. The wizard contract + +The agent is *wizard-capable, not wizard-obligated*. The rule is **retrieve, then propose, then +ask** — questions are the last resort, not the interface. + +| Slot | Fill it from | Ask? | +|---|---|---| +| Course | `list_courses` — use it outright if they teach exactly one | Only to disambiguate | +| Assignment | `list_assignments`, matched on the name the instructor used | Only to disambiguate | +| Task description | **`get_assignment_details`** — the assignment's own description | Almost never | +| Points total | the assignment's `points_possible` | Never | +| Standards / outcomes | **knowledge base** — program and department outcomes | Only if no match | +| Scoring scale | knowledge base house default | **Propose, don't ask** | +| Criteria | derive from task description + outcomes | **Propose, don't ask** | + +Target: *"Make a rubric for the Final Project in BIOL 101"* reaches a complete draft rubric with +**zero** questions. Where a question is unavoidable, ask for everything missing in one message — +never one question per turn. + +This is the substantive departure from the template-prompt approach that motivated this work. +That prompt treats `{{task description}}`, `{{standards outcomes}}`, `{{scoring scale}}` and +`{{desired criteria}}` as things the instructor types. Three of the four are retrievable. Asking +for them anyway is the difference between a guide and a form. + +--- + +## 6. Agent configuration + +| Primitive | Value | Notes | +|---|---|---| +| **name** | Rubric Builder | | +| **tagline** | Build a rubric and publish it to Canvas | ≤80 chars | +| **instructions** | §7.1 | Keep short — cached prefix, every turn | +| **modelConfig** | Sonnet (not the Haiku default) | The descriptors *are* the deliverable | +| **binding: tool** | `canvas_faculty` | Bare id → all 42 tools, ~11.7k tokens | +| **binding: skill** | `rubric_authoring`, `canvas_rubric_publishing` | §7.2, §7.3 | +| **binding: knowledge_base** | Rubric Design Library | §7.4 | +| **binding: memory_space** | *optional*, one max | Instructor house style; see §9 | +| **starters** | see below | | +| **visibility** | `PRIVATE` → `SHARED` for pilot → marketplace `published` | | + +**Starters:** +- "Create a rubric for one of my Canvas assignments" +- "Turn an existing rubric into a Canvas rubric" +- "Build a rubric aligned to my program's outcomes" + +**Token budget note.** After mcp-servers#38 this is ~13.2k (44 tools + the structured `criteria` schema). The `canvas_faculty` binding puts ~13.2k tokens of tool definitions in the +cacheable prefix on every turn of every session with this agent. It is deterministic and cached, +so it is a one-time write amortized across the session — acceptable, but it is the single largest +line item in this agent's cost and the reason §7.1 must stay short. The rubric workflow itself +needs only 7 of the 42 tools (~1.8k tokens); capturing that saving requires scoped `binding.ref` +support, deliberately deferred. + +--- + +## 7. Draft artifacts + +### 7.1 System instructions + +``` +You are a rubric design partner for Boise State University instructors. You help +faculty create evaluation rubrics and publish them directly into their Canvas courses. + +## How you work + +Reach a complete draft rubric with as few questions as possible. Retrieve what you +can. Propose what you can infer. Ask only where you are genuinely blocked. + +Before asking the instructor anything, try to fill these slots: + +| Slot | Fill it from | +|---|---| +| Course | `list_courses`. If they teach exactly one, use it. | +| Assignment | `list_assignments`, matched on the name they used. | +| Task description | `get_assignment_details` — use the assignment's own description. | +| Points total | The assignment's `points_possible`. | +| Standards / outcomes | Search your knowledge base for the program's or department's outcomes. | +| Scoring scale | Use the default scale in your knowledge base. | +| Criteria | Derive them from the task description and the outcomes. | + +Anything the instructor already told you is filled — do not re-ask it or re-derive it. +Ask only for slots you could not fill, and ask for all of them in a single message. +Two questions is a lot. Zero is the goal. + +## Producing the rubric + +Activate the `rubric_authoring` skill before you draft. It carries the quality +standards this work is judged on. + +Always show the complete rubric as a **markdown table written directly in your +reply** before writing anything to Canvas: scoring levels as column headings with +their points, criteria as rows, and a descriptor in every cell. Say which outcomes +it aligns to and what the points total is. + +Do not use a charting or visualization tool to render the rubric. A rubric is text +in a grid, not a data visualization. + +## Publishing to Canvas + +Never call `create_rubric` or `associate_rubric` until the instructor has seen the +table and approved it. "Looks good", "publish it", "yes" are approval. Silence, +a follow-up question, or an edit request are not. + +Activate the `canvas_rubric_publishing` skill before your first write. It carries +the Canvas field mapping and the failure modes — following it is what makes the +published rubric match the table you showed. + +Creating a rubric requires the instructor to approve the tool call — they will see +an approval prompt showing the exact data you are about to send. This is expected, +not an error. If they decline, treat it as a considered editorial decision: ask what +they want changed and revise the draft. Never describe a decline as a permissions +problem, a Canvas restriction, or something to retry — they meant it. + +After publishing, name the rubric, say which assignment it is attached to, and give +them the Canvas link. + +## Scope + +You create, revise, and publish rubrics. You do not grade student work, message +students, or create or edit assignments, pages, modules, or announcements — even +though you can see tools for those. If asked, say plainly that you build rubrics and +point them at the right place in Canvas. +``` + +### 7.2 Skill: `rubric_authoring` + +The pedagogy. Adapted from the instructor-authored rubric-generator prompt already in use, with +the quality rules that prompt implies but does not state. + +``` +# Rubric authoring + +Write as an experienced instructor in the discipline at hand. Use student-friendly +language — the rubric is read by students before they start work, not only by the +grader after. + +Every rubric has three parts: a scoring scale, criteria, and descriptors. + +## Scoring scale + +Three to five levels, each with a name and a point value. Prefer the four-level +scale in the knowledge base unless the instructor or the department specifies +otherwise. Level names describe attainment, not praise. + +## Criteria + +Three to six. More than six and students stop reading; fewer than three and the +rubric cannot discriminate. Each criterion names something observable in the +student's work and maps to at least one stated outcome. Never grade effort, +compliance, or formatting as a criterion unless the outcomes actually call for it. + +Criterion points sum to the assignment's total. + +## Descriptors — this is the part that matters + +One descriptor per (criterion, level) cell. No blanks. + +- Describe what the work *does*, not how good it is. "Cites five or more + peer-reviewed sources published within the last ten years" — not "Uses good + sources." +- Keep a row parallel. The same dimension varies across the levels; only the + degree changes. If two cells differ only by an adverb, the row is not finished. +- Do not write the bottom level as pure negation. "Cites fewer than three sources, + or relies primarily on non-scholarly sources" — not "Does not use good sources." +- Be specific to the outcomes you were given. A descriptor that would fit any + assignment in any discipline is not doing any work. +- Make the top level attainable and the bottom level survivable. Neither should + describe a student who does not exist. + +## Alignment + +State which outcome each criterion serves. If an outcome the instructor gave you +has no criterion, say so — that is a gap worth naming, not something to paper over. + +## Common failures to avoid + +- A quality ladder (Excellent / Good / Fair / Poor) with no substance underneath +- Descriptors that differ only by an adverb or a number with no referent +- Criteria that grade compliance rather than learning +- Points that do not sum to the assignment total +- More than six criteria +``` + +### 7.3 Skill: `canvas_rubric_publishing` + +The mechanical contract. Separate from §7.2 because it changes for different reasons, and because +an instructor who only wants a rubric *document* never needs it. + +``` +# Publishing a rubric to Canvas + +## Field mapping — get this right or the rubric arrives gutted + +| Rubric concept | create_rubric field | +|---|---| +| Scoring level name ("Proficient") | `criteria[].ratings[].description` | +| Level points | `criteria[].ratings[].points` | +| Criterion name | `criteria[].description` | +| Criterion explanation | `criteria[].long_description` | +| **Descriptor (the cell text)** | **`criteria[].ratings[].long_description`** | +| Criterion maximum | `criteria[].points` | + +The descriptor is the rubric's whole value, and it goes in the *rating's* +`long_description` — not the rating's `description`, which holds only the level +name. Getting this wrong produces a rubric that looks structurally correct in +Canvas and carries none of the content. + +## Criterion points + +`create_rubric` requires `points` on every criterion. Set it to the highest rating's +points for that criterion. Canvas treats it as authoritative for the rubric total, +so if it disagrees with the ratings the displayed total will be wrong. + +## Attaching to an assignment + +`create_rubric` with `assignment_id` creates and attaches in one call — prefer it +over `create_rubric` followed by `associate_rubric`. Use `associate_rubric` only to +attach a rubric that already exists. + +`use_for_grading: true` (the default) makes rubric scores post to the gradebook. +Leave it on unless the instructor says the rubric is for feedback only. + +## The graded-discussion trap + +A graded discussion has both a discussion id and an assignment id, and only the +assignment id can carry a rubric association. `list_discussion_topics` returns the +discussion id. **Only ever use ids from `list_assignments`.** Attaching to a +discussion id fails or attaches to the wrong object. + +## Before writing + +Call `list_rubrics` first. If a rubric with a similar name already exists, show it +to the instructor and ask whether to replace, attach the existing one, or create a +second — do not silently create a duplicate. `create_rubric` is not idempotent and +has no dry-run, so never retry it blind after an error; check with `list_rubrics`. + +## Attaching changes the assignment's point value + +Canvas sets the assignment's `points_possible` to the rubric's total when you attach +with `use_for_grading`. Call `get_assignment_details` BEFORE attaching. If the +assignment's points and the rubric's total differ, say so and ask which should win — +do not silently re-point an assignment students may already have seen. + +## The approval prompt + +Creating or attaching a rubric pauses for the instructor's approval. The card they +see lists the raw arguments, not the effects — so anything consequential must be in +your message BEFORE the prompt, in plain language. Above all: if attaching will +change the assignment's point value, say both numbers first. + +A decline means they want something different. Ask what to change. Never retry the +same call and never tell them it is a permissions problem. + +## After writing + +Verify with `get_assignment_details`, not `get_rubric`. `get_rubric` returns an empty +`associations` array even for a live attachment, so it cannot confirm the rubric is +attached; it is still correct for criteria and ratings. + +Compare what came back against the table you showed the instructor. If anything is +missing — especially descriptors — say so rather than reporting success. +``` + +### 7.4 Knowledge base: Rubric Design Library + +This is what lets the agent fill the standards/outcomes slot without asking, and it is the single +highest-value non-code item in this spec. Suggested contents: + +1. **Program and department learning outcomes** — the biggest win. Lets the agent propose + alignment instead of asking `{{standards outcomes}}`. +2. **Accreditation frameworks** relevant to BSU programs (ABET, AACSB, CAEP, etc.). +3. **University rubric guidance** from the Center for Teaching and Learning — including the house + default scoring scale the system prompt refers to. +4. **Exemplar rubrics** from strong courses, spanning disciplines and assignment genres (lab + report, studio critique, research paper, presentation, code project, clinical performance). +5. **The Canvas rubric CSV format**, for the import/export path. + +Without (1) and (3) the agent must ask for outcomes and scale on every rubric, which collapses +§5's zero-question target. Build the KB before piloting. + +**Backend note:** as of #1027 (`MANAGED_KB_NEW_DEFAULT`), a newly finalized agent is *born +managed* — its knowledge base is provisioned as a Bedrock Managed KB on first upload rather than +the legacy S3 vector index. Confirm the flag's state in the target environment before building +the library, and see `bedrock-managed-kb-evaluation.md` for the embedding-immutability and +score-inversion gotchas that come with it. + +--- + +## 8. Phasing + +1. **Phase 0 — Unblock the server.** §4.1 (rating `long_description`), §4.2 (`update_rubric` / + `delete_rubric`), §4.7 (`criteria` schema, criterion-points default). One PR in `mcp-servers`. + Deploy to dev. +2. **Phase 1 — Unblock dev config.** Add the four rubric scopes to the dev provider record and + the Canvas test developer key. Re-run tool discovery on `TOOL#canvas_faculty` (§4.4). Verify + `create_rubric` end-to-end against a sandbox course, confirming descriptors survive. +3. **Phase 2 — Build the KB.** §7.4. Start with the CTL guidance and one program's outcomes; + breadth can follow. +4. **Phase 3 — Build the Agent in dev.** §6 configuration, §7.1–7.3 artifacts. Dev already runs + the 42-tool server, so this is testable as soon as Phase 1 lands. +5. **Phase 4 — Faculty pilot.** `SHARED` visibility with a handful of instructors. The thing to + watch is §5: count the questions asked per rubric. If it is consistently more than one, the KB + is thin or the slot-filling instructions are not being followed. +6. **Phase 5 — Prod.** §4.3 — redeploy the server, add prod scopes, resolve the re-consent + question. Then publish to the marketplace. + +Phases 0 and 1 are prerequisites for everything. Phase 2 can run in parallel with 0/1. + +--- + +### 8.1 Acceptance criteria + +The agent is done when a faculty member who has never used it can do this unaided: + +1. **Zero-question path.** "Make a rubric for the Final Project in BIOL 101" produces a complete + draft rubric — criteria, levels, a descriptor in every cell, aligned to named outcomes — with + **no clarifying questions**. This is the headline criterion; if it needs questions, either the + KB is thin or the slot-filling instructions are not landing (§5). +2. **Questions are batched.** Where a question *is* unavoidable, everything missing is asked in + one message, not one question per turn. +3. **The table matches the payload.** The markdown table the agent shows and the `tool_input` on + the approval card describe the same rubric. A divergence is a correctness bug, not a cosmetic + one. +4. **Descriptors survive the round trip.** `get_rubric` after publishing returns a + `long_description` on every rating, matching the table cell. This fails today (§4.1) and is + the single check that proves the blocker fixed. +5. **Point changes are announced before the gate.** If attaching will re-point the assignment, + the agent says so *before* the approval prompt, with both numbers (§4.5). +6. **Decline is respected.** Declining the gate produces a revision conversation, not a retry or + a permissions diagnosis (§10). +7. **Fences hold.** Asked to grade, message students, or edit an assignment, the agent declines + and redirects — even though it can see those tools. + +### 8.2 Prod cutover checklist + +Ordered; each step has a different owner, which is why it is worth writing down. + +| # | Step | Owner | +|---|---|---| +| 1 | Merge and deploy the `mcp-servers` fix (§4.1, §4.2, §4.7) | eng | +| 2 | Redeploy `canvas-faculty` to prod; confirm `tools/list` returns **44** (42 + `update_rubric` + `delete_rubric`) | eng | +| 3 | Add the **6** rubric scopes (4 original + PUT/DELETE from mcp-servers#38) **and** `url:POST|/api/v1/conversations` to the **production** Canvas developer key | Canvas admin | +| 4 | Add the same scopes to the prod `canvas-faculty` provider record | connectors admin | +| 5 | **Announce the reconnect** before step 4 lands — every connected faculty member gets a consent prompt on their next Canvas use, including for tools whose scopes did not change (§4.3) | comms | +| 6 | Admin → Tools → Canvas for Faculty → *Discover from server* (refreshes 7 → 44) | tools admin | +| 7 | Flag `create_rubric`, `associate_rubric` **and `delete_rubric`** as **Needs approval**; save | tools admin | +| 8 | **Decide who should hold the 44-tool version** — see §9. Step 6 widens it from 7 tools to 44 for *every authenticated user*, because `canvas_faculty` is `isPublic: true` (a role grant alone does not gate it). At minimum flag the destructive tools `needsApproval` | tools admin + RBAC admin | +| 9 | Build the KB (§7.4) and the Agent (§6) | eng | +| 10 | Smoke-test §8.1 criteria 3–7 against a sandbox course before publishing the listing | eng | + +Steps 3 and 4 must not be separated by long — between them, rubric tools 401 with a message that +tells the user a Canvas admin must act, which will already be done. + +### 8.3 Residual risk: the prompt fence is not a control — FIX SHIPPED + +The Agent bound `canvas_faculty` as a bare id, so it held all 44 tools including +`grade_submission`, `bulk_grade_submissions`, `create_assignment` and `create_page`. Two of the +44 were gated by approval; the rest were held back **only by the system prompt** (§7.1 Scope). +That is a real fence for ordinary use and no fence at all against a determined prompt. + +The structural fix — scoped tool bindings — has since shipped, so `binding.ref` accepts +`canvas_faculty::create_rubric` and the runtime builds the MCP client restricted to the named +tools (§3.3). **The agent is not rebound yet**; doing so is a one-click change in the Agent +Designer (open the Tools chip's caret, leave on only the seven below). + +The seven it needs: `list_courses`, `list_assignments`, `get_assignment_details`, `list_rubrics`, +`get_rubric`, `create_rubric`, `associate_rubric`. + +Verified in dev on the real agent: with those seven bound, the runtime logs the client as +`(tools: associate_rubric, create_rubric, get_assignment_details, get_rubric, list_assignments, +list_courses, list_rubrics)`, the whole turn prompt drops from 27,959 tokens to 9,981, and asked +whether it can grade, the agent answers that it has no such tool — where the bare-bound run named +`grade_submission`, `grade_with_rubric` and `bulk_grade_submissions` and declined by policy. + +Flagging the destructive tools `needsApproval` is still worth doing as defence in depth, and +keeping the Agent's visibility limited during the pilot still bounds the population. + +## 9. Open questions + +- ~~Scope re-consent~~ — **settled 2026-09-10**, see §4.3. Automatic; no code needed. The + remaining question is comms, not engineering: when prod scopes change, every connected faculty + member gets a reconnect prompt on their next Canvas use. +- **Memory space.** Binding one would let an instructor's house style (preferred scale, tone, + standing outcomes) persist so the second rubric asks less than the first. v1 supports one Memory + Space per Agent. Worth a phase-4 decision once we see whether faculty repeat themselves. +- **Who sees the 44-tool `canvas_faculty` in prod.** *An earlier draft of this spec said to "fix + the prod `student` role grant." That advice was wrong twice over, and the correction matters + because it changes the action.* + + **First: dropping the role grant would change nothing.** `canvas_faculty` is + **`isPublic: true`** in prod, and `rbac/service.py` `_tool_grant_set` unions every public tool + into each user's grant set (`granted | set(await get_public_tool_ids())`). There are two + independent grants. Removing one leaves the other, and every authenticated user keeps the tool. + + **Second: student access is probably deliberate, not an oversight.** It is one of 18 public + tools in prod and the *only* Canvas tool there — `student_myboisestate` is the portal, not + Canvas. With today's 7 read-ish tools, a student connecting Canvas gets "what are my courses + and assignments", scoped by Canvas to their own enrollments. Someone chose that, twice. + + **The real issue is not who holds the tool — it is what the tool becomes.** Step 6 of §8.2 + takes that same population from 7 tools to 44, putting `grade_submission`, + `bulk_grade_submissions`, `create_assignment` and `delete_rubric` in their picker. This is + **not** privilege escalation: Canvas enforces per-enrollment and a student's token 403s on + teacher actions. Three costs remain, in order: + + 1. **Canvas permissions are per-enrollment, not per-person.** Someone with the `student` app + role who also TAs a lab holds teacher rights *in that course*, so `grade_submission` and + `delete_rubric` genuinely work for them there. Narrow, but it is grading. + 2. A student seeing "grade submissions" in their own tool list generates support tickets. + 3. ~13.2k tokens of tool definitions in the prefix for users who can use a fraction of them. + + **Options:** + + - **A — leave it.** Accept that TAs can grade through chat. May even be desirable. + - **B — set `isPublic: false` *and* drop the `student` grant.** Both, or nothing changes. + Restricts to faculty/staff/admin but breaks the student "what are my assignments" use. + - **C — split into two catalog records** against the same server: a read-only student one and + the full faculty one. Preserves both uses. ⚠️ **Verify this actually filters at runtime + before relying on it** — a bare-id grant loads whatever the live server returns, and a second + record with a narrower cached `tools` list may restrict only the picker, not the turn. + - **D — flag the destructive tools `needsApproval`** so even a TA gets a prompt before a grade + changes. Cheap, certain, and stacks with any of the above. + + Recommendation: **D regardless** — one checkbox per tool, and it closes the grading path. Then + choose between A and C on whether students should keep Canvas access. +- **Rubric preview as an MCP App.** A rendered rubric grid would be a far better confirmation step + than a markdown table, and the natural place to put the approve/publish control. The + `canvas-faculty` server serves no UI resources today. Post-v1. +- **Does a rubric created with no association appear in Course → Rubrics?** (§4.7.) Needs a + sandbox test; determines whether "I'll attach it later" is a supported path. + +--- + +## 10. Dev validation log — 2026-09-10 + +Run against dev (`boisestatecanvas.test.instructure.com`), course 50994 "Faculty Demo: Intro to +MCP", as `system_admin`, Haiku 4.5, `canvas_faculty` enabled in the tool picker. + +| Step | Result | +|---|---| +| Canvas developer key: 4 rubric scopes added | done by admin | +| Provider record `canvas-faculty`: 8 → 12 scopes | persisted; no AgentCore re-registration, no client-secret re-entry | +| First Canvas tool call after the scope edit | **consent re-prompt** (§4.3) — fired on `list_courses`, not a rubric tool | +| Reconnect; consent screen | listed the rubric scopes | +| `list_courses` | course 50994 returned | +| `list_rubrics` | `GET .../rubrics` scope works — course had no rubrics | +| `create_rubric` | `POST .../rubrics` scope works — rubric **256107**, 2 criteria, 8 points | +| `list_assignments` | assignment **1756044** "Syllabus Acknowledgment", 5.0 points | +| `associate_rubric` | `POST .../rubric_associations` scope works — association **519900** | +| `get_rubric` | ratings have **no** `long_description`; descriptors sit in `description`; level names lost (§4.1) | +| `get_rubric` associations | **`[]`** despite a live attachment (§4.6) | +| `get_assignment_details` | rubric attached; `points_possible` now **8.0**, was 5.0 (§4.5) | + +### Approval gate (same session) + +| Step | Result | +|---|---| +| Admin → Tools → Canvas for Faculty → *Discover from server* | catalog refreshed 7 → **42** tools; existing `needsApproval` flags preserved | +| `create_rubric` → **Needs approval** ✓, saved | persisted `needsApproval: true` | +| "show me the rubric as a table first, then create it" | rendered a **markdown table**, then called `create_rubric` | +| Approval interrupt | *"APPROVAL NEEDED — Approve `create_rubric` to let the assistant continue"*, with **View arguments** showing the full pretty-printed `tool_input` | +| Approve | rubric **256108** created | +| Decline | *"User declined to approve the 'create_rubric' tool call; the agent should not invoke it."* — nothing written | +| `associate_rubric` → **Needs approval** ✓, saved | dev now gates `create_rubric` **and** `associate_rubric` (plus the pre-existing `send_conversation`) | + +Two behaviours worth designing around, both folded into §7.1: + +- Asked to "show the rubric as a table", the model first reached for a **charting tool** and + drew a bar chart of the point values before producing the markdown table. The instruction has + to say *markdown table written in your reply*, and say not to visualize. +- On decline, the model guessed at the cause — *"This may be a safety check or approval gate in + your Canvas environment... do you need to verify permissions first?"* A decline is an + editorial decision by the instructor, not a permissions failure, and the agent must treat it + that way or it will push users toward "fixing" a gate that is working. + +All four rubric scopes are validated end to end, and the approval gate works in both directions. +The auth, transport and consent paths are proven; what remains blocking is content fidelity +(§4.1) and the two Canvas behaviours found here (§4.5, §4.6). + +### Round-trip verification — 2026-09-10, after mcp-servers#38 and #39 + +**§8.1 criterion 4 is proven.** A rubric created with descriptors and **no assignment**, read +straight back with `get_rubric`: + +```json +{"id": "_1326", "description": "Exemplary", + "long_description": "Student supports all claims with specific, relevant evidence…", + "points": 4.0} +``` + +Level names in `description`, descriptors in `long_description`, criterion points derived to 4.0 +from the highest rating, and the rubric is fetchable despite having no assignment. + +Notably, **the schema alone changed the model's behaviour** — twice, with different wording, and +with no prompt guidance. Before #38 the same request packed descriptors into `description` and +lost the level names. That is the argument for typed tool inputs over docstring instructions. + +| Check | Result | +|---|---| +| Six rubric scopes on the Canvas test key + provider record | ✅ | +| Server 44 tools, structured `$defs` | ✅ | +| Catalog rediscovered to 44, flags preserved | ✅ | +| `create_rubric` / `associate_rubric` / `delete_rubric` gated | ✅ | +| Descriptors survive create → `get_rubric` | ✅ | +| Course-bound rubric readable with no assignment (#39) | ✅ | +| `delete_rubric` on an associated rubric | ✅ (256107, 256110) | +| `delete_rubric` on a pre-#39 orphan | ❌ Canvas **500** — UI only | +| `update_assignment` to restore assignment points | ❌ scope not on the dev provider | + +Three things this surfaced, all folded into mcp-servers#40 or below: + +- `create_rubric` reported a **course** id under an `assignment_id` key for the Course fallback. +- `delete_rubric` returned an all-null digest, so a success read as a failure. +- **Two gates can stack on one tool call.** A scope change and an approval gate both fired on the + same `create_rubric`, so the user saw *"Connect Canvas for Faculty"* and *"Approve + create_rubric"* simultaneously, with nothing indicating which comes first. Harmless for someone + who knows the system; a faculty member would reasonably guess wrong. Worth sequencing in the + SPA, and worth a line in §7.1 if not. + +**Admin-UI bug blocking §8.2 step 4:** saving the connector form fails with *"Discovery config +can only be updated together with a credential rotation (client_id + client_secret)."* The SPA +sends `oauthDiscoveryUrl` on every save and `admin/oauth/routes.py` treats any non-None value as +a discovery change — so **a scopes-only edit is impossible through the admin UI** for any +provider that has a discovery URL. Worked around with a direct scopes-only `PATCH` +(`X-CSRF-Token` from the `__Host-bff_csrf` cookie). A connectors admin following §8.2 step 4 in +prod will hit this. + +**Course 50994 cleanup:** 256107 and 256110 deleted via the API. **256108 and 256109 cannot be +deleted at all.** Every route 500s or 404s — including `DELETE` from a full Canvas *admin* browser +session, and a rescue attempt that POSTs a Course `rubric_association` (both `purpose` values). +They are absent from the Canvas UI's Rubrics page under both Saved and Archived, so they are inert; +only the API index endpoint reveals them. Removing them needs Instructure support (the 500 bodies +carry `error_report_id`s) or the monthly reset of the test instance. **The 500 is not an OAuth +scope problem** — it reproduces for an admin — so do not debug it as one. Assignment 1756044 is +still at 8 points (was 5): restoring it needs +`url:PUT|/api/v1/courses/:course_id/assignments/:id`, which the dev provider does not grant. + +--- + +## 10a. Built in dev — 2026-09-10 + +**Agent `ast-9149ef191614` "Rubric Builder"**, owned by phil, `PRIVATE`. + +| Primitive | Value | +|---|---| +| Model | `us.anthropic.claude-sonnet-5` | +| Tool binding | `canvas_faculty` | +| Skill bindings | `rubric_authoring`, `canvas_rubric_publishing` | +| Knowledge base | 5 documents, 33 chunks, all `complete` | +| Starters | the three from §6 | + +Skills created as system skills in the dev catalog. Knowledge base documents: +`rubric-design-guide.md`, `scoring-scales.md`, `canvas-rubric-mechanics.md`, +`canvas-rubric-csv-format.md`, `exemplar-rubrics.md`. + +### Smoke test — §8.1 criterion 1 passes + +*"Make a rubric for the Syllabus Acknowledgment assignment in my Canvas course."* + +`list_courses` → `list_assignments` → `get_assignment_details` → activated +`rubric_authoring` → complete draft table. **Zero questions asked.** Points totalled 8 to match +the assignment. It stopped and asked before publishing, so the confirm gate held. + +Two behaviours worth noting because they are the difference between a KB that is read and a KB +that is obeyed: + +- It **deviated from the guide with a stated reason** — used 2 criteria rather than the guide's + 3–6, explaining that "stretching it to 3+ criteria would force compliance-flavored rows that + the design guide says to avoid." That is the guide being reasoned with, not pattern-matched. +- It **named the outcome gap unprompted**: "no course learning outcomes are attached to this + assignment", then said what it aligned to instead. + +Turn cost $0.12 on Sonnet 5, ~28.9k context. Note the binding replacement is visible in the UI — +"Tools 1 enabled" — confirming the Agent's bindings override the user's own tool selection. + +### Full §8.1 acceptance pass — 2026-09-10, all seven criteria + +Run against the agent itself (Sonnet 5), not plain chat. Nothing was written to Canvas: the one +write attempt was declined on purpose to exercise criterion 6. + +| # | Criterion | Result | +|---|---|---| +| 1 | Zero-question path | ✅ complete draft, no questions | +| 2 | Questions batched | ✅ retrieved first, then asked exactly two things in one message | +| 3 | Table matches payload | ✅ `tool_input` matched the table word-for-word | +| 4 | Descriptors survive the round trip | ✅ (§10) | +| 5 | Point changes announced before the gate | ✅ | +| 6 | Decline respected | ✅ | +| 7 | Fences hold | ✅ refused to grade, redirected | + +**Criterion 5** is the one that protects grades, and it behaved better than the spec asked. Told +to build a 20-point rubric for a 100-point assignment, it stopped before the gate with: + +> ⚠️ Point mismatch to flag: the assignment is currently worth 100 points; you asked for a +> 20-point rubric. Attaching this rubric with grading enabled will re-point the assignment from +> 100 → 20. Let me know if that's intended, or if you'd like me to scale the rubric to 100 +> instead. + +Both numbers, the consequence, and an alternative — before the approval card, which shows only +the arguments (§4.5). + +**Criterion 6** also exceeded the spec. It confirmed the no-op state rather than just accepting +the decline: "No changes were made — the assignment is still worth 100 points and no rubric was +attached", then offered four concrete revision directions. No retry, no permissions diagnosis. + +**Criterion 2** produced direct evidence for the KB gap below — the agent named it itself: "I +don't have a program outcomes list in my knowledge base for this course/program, so please paste +them or point me to where they're defined." + +Also observed: the agent follows `canvas_rubric_publishing` without being told to — it called +`list_rubrics` before drafting, as the skill's "Before writing" section instructs. Turn costs +ran $0.03–$0.13 on Sonnet 5. + +### The gap that remains: institutional content + +The knowledge base currently holds **craft** guidance only — rubric design, scoring-scale +conventions, Canvas mechanics, the CSV format, and exemplar rubrics written as phrasing models. +All of it is authored for this agent and none of it is institutional. + +**Not present, and deliberately not invented:** Boise State program and department learning +outcomes, the Center for Teaching and Learning's actual rubric guidance and house scoring scale, +and accreditation framework criteria (ABET, AACSB, CAEP, …). Those are real institutional +documents; fabricating plausible substitutes would produce an agent that aligns rubrics to +outcomes nobody adopted. + +This is the difference between §5's target and what the agent does today. The smoke test hit zero +questions because that assignment had no outcomes to align to — the agent said so and aligned to +the task's own purpose. **Given a real assignment in a real program, the outcomes slot will not +fill and the agent will have to ask.** Supplying (1) program outcomes and (2) the CTL guidance and +default scale is what closes it, and it is the single highest-value item remaining. It needs no +engineering. + +--- + +## 11. Reference + +- Origin notebook: `manage_rubrics.py` (Colab, shared read-only) — the workflow this replaces. +- MCP server: `mcp-servers/packages/canvas-faculty/app.py`, `README.md` (carries the full + Canvas OAuth scope list and the 401-scope-vs-401-token diagnosis table). +- Binding resolution: `backend/src/apis/inference_api/chat/agent_binding_resolver.py` +- Binding validation: `backend/src/apis/app_api/agent_designer/services/binding_validation.py` +- Agent model: `backend/src/apis/shared/assistants/models.py` +- Skills runtime: `backend/src/agents/main_agent/skills/strands_mapping.py` +- Related specs: `agent-designer.md`, `agent-marketplace.md`, `google-tasks-todo.md` (Canvas + OAuth provider precedent), `assistant-kb-sync.md` diff --git a/docs/specs/compaction-over-threshold-cache-spiral.md b/docs/specs/compaction-over-threshold-cache-spiral.md index e3b966a61..799d83068 100644 --- a/docs/specs/compaction-over-threshold-cache-spiral.md +++ b/docs/specs/compaction-over-threshold-cache-spiral.md @@ -9,8 +9,14 @@ and the harm is already multi-user. their $30/month quota in 5 days on a **single conversation** (session `c94a3172-e1fb-4a1d-b375-6e51a56c75ad`, an essay-editing session created 2026-07-30). August: 56 model calls, $30.45 total, of which **$27.39 (90%) was -Bedrock cache writes** — every turn re-wrote the full ~200k-token prefix at the -$2.50/MTok write premium while reading only the ~11k tools+system segment. +Bedrock cache writes** — every turn re-wrote the full ~200k-token prefix while +reading only the ~11k tools+system segment, at an effective **$2.50/MTok**. +⚠️ That figure is this incident's own *implied* write price ($27.39 ÷ 10.95M +write tokens — see §3 PR-5), preserved here because the §4.2 replay harness +reproduces against it. It is **not** a platform constant and must not be quoted +as one: the contract's rule is a multiplier — cache write is **1.25× the +model's own base input rate** (see the prompt-cache contract in `CLAUDE.md`), +which today is $1.375/MTok on Haiku 4.5 and $4.125 on Sonnet 4.6. Observability recorded `cacheStatus="hit"`, `wastedUsd=0` on all 56 calls. **Related:** `docs/specs/agent-cache-extra-tools-bypass.md` (**dependency** — the bypass is why this session restored every turn; see D3 and §3 sequencing), @@ -256,6 +262,15 @@ which is what makes the per-call figure $0.437 (190k × 2.30/1M) and the session $20.98. Any replay that assumes a standard Sonnet snapshot ($3.75/$0.30) prices the same incident at ~$31 and will look like a regression against §1. +⚠️ **These are a historical record, deliberately not updated.** They are what +this one August 2026 session's billing implied, and the replay harness is +pinned to them so its acceptance band stays meaningful. Do **not** read them as +the current cost model: Bedrock's cache-write premium is **1.25× the model's +own base input rate** and its cache read is ~0.1× of the same, with no flat +per-MTok figure for either — see the prompt-cache contract in `CLAUDE.md`, +which is the one place that rule is maintained. Anything reasoning about +*today's* cost must price against the model actually in play. + **No backfill.** Rows written before the deploy keep whatever status they were given, so the incident session's own 56 rows still read `hit`. §4.4's standing regression case works forward from the deploy, not backward, and the fleet diff --git a/docs/specs/customize-surface.md b/docs/specs/customize-surface.md new file mode 100644 index 000000000..344851747 --- /dev/null +++ b/docs/specs/customize-surface.md @@ -0,0 +1,558 @@ +# Customize — a browse surface for tools, skills and connectors + +**Status:** Step 1 (Customize shell: Tools + Skills) SHIPPED — PR #1072, validated on dev. +Step 3 (drop model + params from the drawer) SHIPPED — PR #1073, validated on dev. +Step 4 (agent-lock surfacing) SHIPPED — PR #1075, validated on dev. +Step 2 (Connectors tab) SHIPPED — PR #1076, validated on dev. +Step 5 (drawer deleted) SHIPPED — PR #1079, validated on dev. +**Epic COMPLETE.** Step 6 declined on evidence; step 7 declined by the owner. +**Supersedes:** the composer settings drawer (`components/model-settings/`) as the home for +tool and skill enablement. +**Related:** `docs/specs/skills-as-agent-primitive.md` (D6 opt-in), `docs/specs/agent-marketplace.md` (D1 one noun), +`docs/specs/per-tool-mcp-enablement.md` equivalent in `tool-search-token-bloat-strategy.md`. + +## Problem + +Tool and skill enablement is **global, durable, per-user state**: + +- `services/skill/skill.service.ts:32` — "preferences persist globally per user" +- `services/tool/tool.service.ts:414` — `savePreferences()` POSTs to the user preferences endpoint + +But it is presented in a drawer hanging off the composer, opened by a settings icon +inside the chat input. That container reads as *settings for this conversation*. It is not. +A user who enables a tool to get through one question has changed the `toolConfig` of every +future turn, in every future session, permanently — and nothing in the UI said so. + +That is the defect. The secondary problem is capacity: the drawer is a ~320px column with +collapsible sections, and the tool catalog has outgrown it. Per-tool MCP enablement means a +single server (Student MyBoiseState, 17+ tools; canvas_faculty, 44) can exceed the entire +drawer's comfortable length on its own. Browsing is not a thing the drawer can be made to do. + +## What Customize is + +A full page at `/customize` that owns the things a user *adds to* their assistant: + +| Tab | Contents | Today's home | +|-----|----------|--------------| +| **Tools** | The RBAC-granted tool catalog, per-tool and per-server enablement | Drawer § Tools | +| **Skills** | Accessible skills (catalog-granted ∪ authored), opt-in toggles | Drawer § Skills | +| **Connectors** | OAuth connection state for external MCP servers | `Settings → Connectors` (folded in, step 2) | + +It is deliberately **capabilities only**. See §"What Customize is not". + +## Decision summary + +| Question | Decision | +|----------|----------| +| Does the composer settings drawer survive? | **No.** Deleted in step 5, once its contents have homes | +| Does the settings icon survive? | **No.** `showSettingsControl` default flips to `false`, then the input is removed | +| Where does model selection live? | Composer, where it already is (`chat-input.component.html:274`) | +| Where do inference params live? | **Nowhere user-facing.** Effort subsumes them (step 3) | +| What happens to Conversation Modes? | **Retired as a migration to Agents** (step 6), gated on prod usage | +| Does the Agent Marketplace move into Customize? | **No.** See §"What Customize is not" | +| Does Customize honour the Agent binding lock? | **No — deliberately.** See §"The agent-lock seam" | + +## Why the drawer can die entirely + +The drawer holds four things. Three of them are already redundant or dead: + +**Model selection — redundant.** `` is already in the composer at +`session/components/chat-input/chat-input.component.html:274`. The drawer's model section is +a second copy of a control the user can already see. + +**Advanced params — superseded by effort, already lying, and partly duplicated.** Effort +lives in the model dropdown's submenu with the active level in the trigger +(`components/model-dropdown/model-dropdown.component.ts:41`). Meanwhile GPT-5.6 +**hard-rejects** `temperature` and `top_p` (measured; see `docs/specs/gpt-5-6-prompt-caching.md` +and the inference-params findings), so a per-model numeric param form already misrepresents +part of the catalog. Effort is the portable abstraction; the form is not. + +**Measured on dev before cutting** (all 9 enabled models, via the live picker + drawer): + +| Model | Advanced rows the drawer offered | Effort in the picker | +|-------|----------------------------------|----------------------| +| GPT-5.6 Sol / Luna | Max Output Tokens, **Reasoning Effort** | yes | +| Claude Sonnet 5, Opus 4.7 | Max Output Tokens, **Effort** | yes | +| Claude Sonnet 4.6 | Temperature, Top P, Max Output Tokens, **Effort** | yes | +| Claude Haiku 4.5 | Temperature, Top P, Max Output Tokens | **no** | +| Gemma 4 31B, GPT-5.4 | *(none — section already hidden)* | no | + +Two things that changes: + +1. **No enabled model exposes Extended Thinking to users.** The `thinking` param is declared + in `curated-models.ts` for Sonnet 4.6 and Haiku 4.5, but the *deployed* records don't + enable it, so the row never renders. The feared capability loss does not exist — but note + the trap: the curated template is not the catalog, and only the live records answer this. + Re-check before removing anything param-shaped in an environment other than dev. +2. **Effort was rendered twice** — in the picker AND as a row in the drawer's Advanced list. + So the Advanced section was not merely superseded; for five of nine models its headline + control was a literal duplicate of one three inches away. + +What is left once Effort is deduped is Temperature, Top P and Max Output Tokens — sampling +knobs and a truncation guard. + +Removing the form also retires the `max_tokens` ↔ extended-thinking coupling — Anthropic +requires `thinking budget < max_tokens`, which is the entire reason `model-settings.ts` +carried `unsatisfiable`, `clampNotices`, `disabledByConflict` and the post-edit re-check in +`reconcileThinkingAfterMaxTokens`. That machinery and its whole error-state vocabulary go +with it: ~410 lines of component and ~330 of template. + +⚠️ `max_tokens` is a **truncation guard**, not a tuning knob. Removing the user control means +the admin default applies — which is already what every untouched user gets (the drawer read +"Defaults" for them). Admin-locked params (`row.locked`, "locked by admin") are unaffected: +this removes the *user-facing form*, not the governance behind it. + +⚠️ **Stale overrides are the real hazard, not the missing form.** Overrides live in +`sessionStorage` under `inferenceParamOverrides`, so a tab open across the deploy still holds +whatever the user last typed, and it would keep riding every request with nothing in the UI +to show or reset it. `ModelService.dropRetiredOverrides` strips non-effort keys once, on load, +and rewrites storage. Effort is preserved explicitly — `setEffort` writes through this same +store, so a blanket purge would clear a control the user can still see and is still using. + +**Conversation Mode — a strictly weaker Agent.** An admin-authored system prompt attached to +a conversation, with no tools, no skills, no bindings, no icon and no `@`-mention. That is +precisely the relationship Assistants had to Agents, which Marketplace D1 ("there is one noun, +and it is Agent") resolved by migration. Untouched since the PR that introduced it (#411) apart +from the delegated-admin-scope sweep and a theming pass. + +Retiring it is bigger than deleting a drawer section — it has admin CRUD pages and routes, an +`admin.system_prompts` delegated scope, an admin nav entry, a user-facing `/system-prompts` +app_api route, a DynamoDB entity, and `system_prompt_id` on the invocation payload +(`apis/inference_api/chat/models.py:188`). ⚠️ **Gate on prod usage before writing any of it.** +"Dormant in git" is not "dormant in prod"; if a department is using a Mode, it has users and +the answer is a migration with redirects, not a deletion. + +**Skills and Tools — global state in a conversational container.** The actual problem. They +move to Customize. + +Strip the first three and nothing conversation-scoped remains. The drawer is not slimmed; it +is emptied, and then removed. + +## What Customize is not + +**Not the Agent Marketplace.** Customize is *capabilities you add to your assistant*. An Agent +is not a capability you toggle — it is a thing you talk to. Putting Discover under Customize +while My Agents stays in the sidenav splits one noun across two surfaces, which is the exact +failure D1 exists to prevent and the reason the whole `/assistants` deprecation was written. + +The marketplace's real problem is narrower than placement: **Discover is already the first tab +of `/agents`** (`agents/components/agents-tabs.component.ts:22`), but `/agents` resolves to +*My Agents*, which is empty for nearly every user. The emptiest tab is the landing tab. The fix +is a routing change — land `/agents` on Discover when the user has no agents — not a +relocation. It is tracked here as step 7 and is independent of everything else in this spec. + +If a future decision does move the marketplace into Customize, it must move **all** of +`/agents` and drop the sidenav entry. Splitting it is the one outcome to avoid. + +## The agent-lock seam + +⚠️ This is the sharp edge in PR-1, and it is a pre-existing defect the new surface exposes +rather than one it creates. + +`ToolService` and `SkillService` are `providedIn: 'root'` singletons. When a conversation is +bound to an Agent, `session.page.ts:806` calls `lockToAgentTools()` / `lockToAgentSkills()`, +which makes `agentLocked()` true, rewrites what `visibleTools()` returns, and causes +`toggleTool()` to **early-return without saving** (`tool.service.ts:272`). + +`ngOnDestroy` does **not** release these locks. They are cleared only when the session page +later loads an unbound conversation (`session.page.ts:341,791`). So a user who navigates from +an agent-bound chat straight to `/customize` arrives at a page where: + +1. the list shows the Agent's bound set instead of their own preferences, and +2. every toggle silently does nothing. + +The lock is conversation-scoped state that leaked into a global singleton. Customize is a +global surface and therefore **must not consult it**: + +- Read the user's true state (`tool.isEnabled` / `skill.isEnabled`), never the + display-shim (`isToolShownEnabled` / `isSkillShownEnabled`). +- Write through a lock-agnostic path. `toggleTool` / `toggleSkill` take an optional + `{ respectAgentLock }`, defaulting to `true` so drawer behaviour is byte-identical. + +Deliberately **not** fixed by clearing locks in `ngOnDestroy`: the `/` ↔ `/s/:id` transition +recreates the session component on a legitimate navigation (`session.page.ts:498`), so a +destroy-time clear would drop and re-apply the lock mid-flow. + +The proper resolution is step 4 — the lock is a fact about the *conversation*, so it belongs +on the assistant indicator pill +(`session/components/assistant-indicator/assistant-indicator.component.ts`), which is already +in the conversation and already has an actions menu. + +**Step 4's shape.** The indicator takes an `AgentGovernance` input (`modelName`, `toolCount`, +`skillCount`; `null` on a field means "the user's own setting applies"), renders a lock glyph +on the chip, and lists what is fixed in its menu — ending with the line that closes the loop: +*"Your own choices in Customize don't apply in this conversation."* + +⚠️ It is derived from the **Agent record** (`chat-container`'s `agent()` input), NOT from +`ToolService.agentLocked()` & friends. Those are the leaking singletons this section is about; +reading them here would reintroduce the same staleness on the surface whose whole job is to +tell the truth about the current conversation. Deriving from the record also makes the preview +surfaces correct for free: the Designer preview and the marketplace test-drive render the +indicator without passing `[agent]`, so they get `null` and say nothing — right, because a +draft being previewed is not a conversation anyone's saved settings apply to. + +## Cost consequence + +⚠️ **A surface designed to make enabling tools easy will raise enabled-tools-per-user, and +every enabled tool grows `toolConfig` on every turn of every session** — the cacheable prefix +the cost-effectiveness tenet exists to protect. This is the same pressure that made new +injected tools ship `enabledByDefault=False` (default-on was a fleet-wide cache bypass), with +a friendlier face. + +Two obligations, in the spec rather than discovered post-ship: + +1. **Ordering stays deterministic** regardless of the order the new UI writes preferences in. + Prompt-cache stability is exact-prefix-match; a set that reorders because the user toggled + from a grid instead of a list rewrites the whole prefix at the cache-write premium. +2. **Browse should not be cost-blind.** Surfacing per-tool prompt weight (description + + schema token count) is the honest version of a store that encourages adding things. Not in + PR-1; named here so it is a decision rather than an omission. + +## Sequencing + +Each step is independently shippable. 3 and 6 do not depend on Customize at all. + +| # | Step | Depends on | +|---|------|-----------| +| 1 | **Customize shell** — `/customize`, Tools + Skills tabs, nav entry. Drawer stays; both live — **shipped (#1072)** | — | +| 2 | Fold `Settings → Connectors` in as the Connectors tab — **shipped (#1076)** | 1 | +| 3 | Drop model + Advanced params from the drawer (pure dedup + param removal) — **shipped (#1073)** | — | +| 4 | Agent-lock surfacing moves to the assistant indicator — **shipped (#1075)** | — | +| 5 | Delete the drawer and the settings icon — **shipped (#1079)** | 1, 2, 3, 4 | +| 6 | Conversation Modes retired as an Agent migration — **DECLINED**, see below | prod-usage check | +| 7 | `/agents` lands on Discover for users with no agents — **DECLINED** by the owner, not pursued | — | + +Step 5 is last for a reason: pull the icon before Customize exists and you have removed the +only path to skills and tools. The end-state composer already renders today — +`showSettingsControl` is an existing input, set `false` in the agent preview +(`agents/agent-form/components/agent-preview.component.ts:121`) and the marketplace review +test-drive (`admin/marketplace/components/review-test-drive.component.ts:144`). Step 5 flips +the default and deletes the input. + +## Resolved questions + +**Does the composer keep a pointer to Customize?** No. The sidenav entry is the path, always +visible and one click away — the same shape Claude uses. Adding a composer affordance would +have reintroduced an icon to replace the one step 5 removes. + +**What happens to Conversation Mode?** ⚠️ The spec originally had it retired in step 6 as "a +strictly weaker Agent", on the premise it was dormant. **That premise was wrong.** Prod carries +one enabled mode — *Guided Learning*, a Socratic tutoring prompt — and its use is accelerating: +1 session in July, 20 in August, **60 in the first 12 days of September**. Measured against +`boisestateai-v2-system-prompts` and `sessions-metadata` in the prod account. + +So Mode is not dormant, and it is the one genuinely **per-conversation** control the drawer +held. It could not follow Skills and Tools to Customize without recreating the exact scope lie +this epic exists to fix, so it went the other way: into the **composer**, beside the model and +effort controls. Those three are the same question — how should *this* conversation run. + +That also reframes step 6: retiring Modes is much harder to justify against growing usage, and +the migration is not clean — a Mode applies to the conversation you are already in, whereas an +Agent is a separate thing you start a chat with. Step 6 is now "reconsider", not "execute". + +This is the second time the "git history says dormant" heuristic has misled on this epic (the +first was `thinking` in step 3, declared in `curated-models.ts` and absent from the deployed +records). **Check the data in the environment that matters.** + +### ⚠️ The composer picker is PARKED (2026-09-13) + +`ConversationModePickerComponent` has been **removed from the composer and deleted from the +tree**, on the owner's call: the placement works, but he wants feedback and thinking time before +committing a permanent composer slot to it. Alternatives under consideration are the conversation +title menu, folding it into the model dropdown, the new-chat empty state, a `/mode` inline command, +and generalising the assistant indicator into a conversation-context chip. + +Everything behind the control is untouched and still live: `SystemPromptsService`, the +`/system-prompts/` catalog, `selected_prompt_id` on `SessionPreferences`, the hydration fix in +step 5 notes, and the admin CRUD at `/admin/system-prompts`. **Only the control is gone**, so +restoring it is a revert, not a rebuild. + +⚠️ **RELEASE GATE — do not ship this to prod alongside the drawer deletion.** `origin/main` still +carries `components/model-settings/`, so prod users select Guided Learning through the old drawer +today. `develop` deletes that drawer (step 5) *and* now has no picker. The first release that +carries both to prod leaves **no way to select a mode at all**, silently killing a feature at +~60 sessions/month and growing. Before that release: either restore the picker, land a +replacement placement, or accept the regression deliberately and tell derrickfink@boisestate.edu. + +## Step 5 notes + +⚠️ **A latent bug surfaced while verifying the new picker, and is fixed here.** The session +page hydrates the active mode twice on load: once provisionally, before the session's metadata +arrives, and again with the real value. The provisional call CLAIMED the session id, so the +clobber guard in `hydrateFromSession` rejected the real hydration that followed. + +That was not cosmetic. `chat-request.service` sends `selected_prompt_id` from +`activePromptId()`, so **after any reload the mode silently stopped being applied to every +later turn**, while the stored session preference still said it was on. On prod that is every +Guided Learning user who reloaded mid-conversation. The provisional call now passes +`claim: false`; a deliberate "None" still claims, so stale metadata cannot undo it. + +It is fixed here rather than deferred because step 5 promotes this control to a first-class +composer affordance, and shipping it more prominently while knowing it silently drops would be +worse than leaving it where it was. + +## PR-1 scope + +**In:** + +- `/customize` → `/customize/tools`, `/customize/skills`, lazy-loaded, `authGuard` +- Tabs strip mirroring `AgentsTabsComponent` +- Browse idiom borrowed from `agents/discover`: search box + category chips + responsive grid +- Cards read the existing root services — no new endpoints, no new state. Because both + surfaces share the same singletons, a toggle in Customize updates the drawer live and + vice versa +- Sidenav entry (which also gave `/my-skills` a reachable home — see the standing + `sidenav.html:92` comment saying its navigation was undecided; step 8 removed + `/my-skills` entirely, so the entry is now the only path to either half) +- The lock-agnostic write path described in §"The agent-lock seam" + +**Out:** connectors tab (step 2), tool detail pane / per-sub-tool expansion, any drawer +deletion, prompt-weight display, marketplace changes. + +## Step 2 notes + +The page moved wholesale (`git mv`, so history follows it); only the shell changed — tabs +plus an `h1`, and the row/empty/error containers went to `rounded-2xl` so the three tabs read +as one surface. The Connect/Disconnect buttons and the `vendor-*` icon tokens were left +exactly as they were: both carry load-bearing contrast reasoning in their comments. + +⚠️ `/settings/connectors` stays as a **redirect**, declared BEFORE the `settings` route whose +`loadChildren` would otherwise swallow it and land the user on the settings shell with no +matching child. A test asserts that ordering, because the failure is silent. + +## Tool detail + +`/customize/tools/:toolId` — the drill-in the browse grid's cards link to. It carries what +a card cannot: the full description (summary, with the docstring's reference material behind +*Show reference*), an MCP server's tools one by one with their own switches, the prompts and +resources the server exposes, and the catalog facts (protocol, status, granting roles). + +⚠️ **Step 5 deleted the drawer, so this is now the only per-sub-tool surface in the app.** +Between #1079 and this page there is nowhere to say "3 of Canvas's 48 tools"; the gap is worth +closing promptly rather than queueing. + +It is a **new component, not a port of the drawer's `ToolDetailComponent`**, for the same +reason the list page is not a port of the drawer's list: the drawer was conversation-scoped +(`isToolShownEnabled()`, writes through the Agent lock) and this surface is global +(`tool.isEnabled` / `sub.enabled`, `respectAgentLock: false`). See §"The agent-lock seam". + +The drawer's **tab strip did not come with it.** Tabs existed there because the pane was 320px; +on a page, Tools / Prompts / Resources / About are stacked sections, so find-in-page reaches +all of them and nothing hides behind a tab the user has to guess at. The real problem tabs +were solving — a 48-tool server — is solved directly: above eight sub-tools the list grows +its own filter box. + +Prompts and resources stay **read-only**, and stay a read of the stored capability snapshot +rather than a live probe, for the reasons the drawer recorded: probing opens an MCP session +per server, a 3LO server cannot be reached without a consent token the browser does not hold, +and acting on an entry needs `prompts/get` / `resources/read`, which this surface has no +endpoint for. The snapshot is only fetched for `mcp_external` tools — nothing else has a +server that could have been asked. + +⚠️ The card's name is a link whose `after:absolute inset-0` makes the whole card the +navigation target; the switch is raised out of it with `z-10` rather than nested inside it. +A switch inside the link is a control the user cannot reach by keyboard without also +following the link. A test asserts the switch has no `` ancestor. + +⚠️ Colored text uses `text-primary-accessible dark:text-primary-accessible-dark`, never the +numbered ramp. With the brand primary at `#0033a0`, `dark:text-primary-400` measures **2.59:1** +against the dark page background (`gray-900`, `#101828`) — a WCAG AA failure at the small text +sizes involved. The accessible alias is generated to clear 4.5:1 against the resolved dark +surface and measures 4.52:1. See `src/branding/README.md` §7. + +**Known gap, not fixed here:** 16% of catalog sub-tools (19 of 116) have docstrings whose first +paragraph runs past 400 characters — `canvas_faculty/import_course_package` reaches 2,058 — +because the prose precedes any `Args:` heading, so `splitToolDescription` returns all of it as +`summary` and the row renders it unclamped. The *detail* behind "Show details" is by comparison +modest (median 344 chars). Clamping the summary is the fix; moving the detail behind a modal +would not touch it. + +## Skill detail + +`/customize/skills/:skillId` — the sibling drill-in, reached the same way: the card's name is +a link, the switch stays its sibling. It carries the SKILL.md body the skill actually injects, +its supporting files, any composed skills, the advisory `allowed-tools` frontmatter, and the +catalog facts. + +⚠️ **Unlike the tool page, this one needed backend work.** `GET /tools/` already returns the +whole `Tool` including `serverTools`, so `/customize/tools/:toolId` is pure frontend. +`GET /skills/` returns six thin fields — id, name, description, category, `userEnabled`, +`isEnabled` — and *everything* worth opening a page for lives on `SkillDefinition` and never +reaches the SPA. The only per-skill read that existed, `GET /skills/mine/{id}`, is +**owner-scoped**: a catalog skill granted to you 404s there. + +So this adds **`GET /skills/{id}`**, access-checked by `resolve_accessible_skill_ids` — the +same resolution that builds the picker and that the runtime uses to decide what a turn may +activate — plus **`GET /skills/{id}/resources/{filename}`**, the access-scoped read +counterpart of the owner route, so a granted user can open a catalog skill's reference files. + +⚠️ **Route registration order is load-bearing.** Both live at the BOTTOM of +`apis/app_api/skills/routes.py`, below every `/mine` route. Starlette matches in registration +order and `SKILL_ID_PATTERN` happily matches the literal string `mine` — declare `/{skill_id}` +first and `GET /skills/mine` becomes a lookup for a skill called "mine", which 404s for every +user in the product. A test asserts it. + +`GET /skills/` was **not** fattened instead. It is a first-load payload covering every granted +skill; a SKILL.md body per row would be paid on every load to render a list that shows neither +the body nor the files. + +**What the detail response deliberately omits.** `ownerId` — `isOwned` is the only part of +ownership this surface needs, and a raw owner id would name one user to another. And +`allowedAppRoles`, which is an admin-display projection of RBAC (see the RBAC §in CLAUDE.md) +and has no business on a page any granted user can open. A skill the caller cannot reach 404s +rather than 403s, so the endpoint never confirms the existence of a skill someone else holds; +a non-ACTIVE catalog skill 404s too, matching the ACTIVE filter `GET /skills/` already applies +— though an owner still reads their own draft, because ownership is its own grant. + +**Instructions render expanded**, not behind a disclosure like the tool page's `Args:` block. +They are not a secret from a user the skill is granted to: this is the text their own turns +load on dispatch, so the honest answer to "what does this skill do" is to show it. Rendered +through `ngx-markdown` **with sanitization on** — do not add `[disableSanitizer]`; a SKILL.md +body can be authored by a non-admin (Skills v2 PR-3 user tier), and the reasoning recorded on +`announcement-modal.component.ts` applies unchanged. + +**`allowedTools` is rendered with its advisory status stated in the copy**, not as a bare list. +Skills v2 D4: the platform never grants, mounts or folds a tool because a skill names it. A +bare list of tool names on a page about a skill you just enabled would read as a grant. + +**A skill the user authored links to `/customize/skills/{id}/edit`** rather than growing a +second editor here. One destination for every card; the read view stays useful for your own +skill. (This link read `/my-skills/{id}/edit` until step 8 folded that route in.) + +**This page closes no functional gap**, and that is the difference from the tool detail page. +That one had to exist the moment #1079 deleted the drawer, because per-sub-tool enablement had +nowhere else to live. A skill has no sub-unit — the only control here is the same on/off the +card already offers — so this page is informational, and was queued rather than rushed. + +⚠️ The switch stays **disabled until the picker list lands**. `SkillService.toggleSkill` +silently returns on a skill it has never loaded, so on a deep link a click before the list +arrived would look like a broken switch rather than a dead moment. The page warms +`loadSkills()` in its constructor and gates the control on `initialized()`. + +**Cost:** none against the model. Everything here is catalog data read for display; nothing +reaches the system prompt or `toolConfig`, so the cacheable prefix is untouched. The added +traffic is one `GET /skills/{id}` per drill-in, cached for the life of the page. + +The `settings/connectors/` **services** deliberately did not move. `UserConnectorsService` and +`ConnectorStatusService` have nine importers across the app (oauth-consent, export-dialog, +knowledge-base, the drawer's tool-detail, the Customize Tools tab…), so relocating them is a +wide, purely-mechanical diff that belongs on its own. Their real home is probably +`services/connectors/` — noted, not done here. + + +## Outcome + +Five of seven steps shipped; two were declined on their merits rather than dropped. + +| # | Step | Result | +|---|------|--------| +| 1 | Customize shell (Tools + Skills) | #1072 | +| 2 | Connectors folded in | #1076 | +| 3 | Model + params out of the drawer | #1073 | +| 4 | Agent governance on the indicator | #1075 | +| 5 | Mode to the composer, drawer deleted | #1079 | +| 6 | Retire Conversation Modes | **Declined** | +| 7 | `/agents` lands on Discover | **Declined** | + +The end state: + +- **Composer** — per-conversation: model, effort, conversation mode. +- **Customize** — global: tools, skills, connectors. +- **Assistant indicator** — says which of those an Agent has fixed, and that Customize + choices do not apply here. + +The original defect is closed: nothing global is presented as conversational any more. + +### Step 6 — declined + +The premise ("a Conversation Mode is a strictly weaker Agent, and they're dormant") did not +survive contact with the data. Prod carries one enabled mode, *Guided Learning*, used in 81 +sessions and accelerating — 1 in July, 20 in August, 60 in the first 12 days of September. And +the migration was never clean: a Mode applies to the conversation you are **already in**, +whereas an Agent is a separate thing you start a chat with. Converting one into the other is a +product change for its users, not a refactor. + +Modes now have a better home than the one the retirement was meant to escape, so the +motivation is gone too. + +### What this epic should be remembered for + +**Check the data in the environment that matters.** The plan was wrong twice, in opposite +directions, and both times the code was the misleading source: + +- Step 3: `curated-models.ts` declared `thinking` for two models; the deployed records did not + enable it. Reading the file would have blocked a safe removal. +- Step 6: git history said Conversation Modes were untouched since #411; prod said usage was + compounding. Reading the history would have deleted a live feature. + +**Verify in a browser before merging.** Every step but one had a defect that only the browser +found — `1 tools`, a menu wrapping "Claude Sonnet 5" across three lines, and a mode that +silently stopped applying after a reload. None were caught by 2800 passing tests. + +## Step 8 — one skills surface + +`/my-skills` is gone. It was a top-level route reachable only by a link-out from +`/customize/skills`, which meant the same noun lived in two places with two different answers +to "what skills do I have?" — one page listed what you *authored*, the other what you could +*turn on*, and neither showed the whole set. Both now live under `/customize/skills`, split by +a `scope` query param rather than by route: + +| Scope | Population | Idiom | +|---|---|---| +| **Yours** (default) | skills you authored, at any status, **plus** catalog skills you have turned on | dense rows, with edit/delete on the ones you own | +| **Discover** | catalog skills your roles grant that are still off | browse cards with a switch | + +Turning a skill on is this platform's analogue of "installing" one. There is no install step: +access is RBAC (`resolve_accessible_skill_ids`) and the only state a user owns is the +enablement preference. That is what makes the two-scope split meaningful here rather than an +imported metaphor. + +**No backend change.** The page reads two endpoints that already existed and merges them +client-side: + +- `GET /skills/` (`SkillService`) — the picker feed: accessible **and ACTIVE**, with the + enablement preference. +- `GET /skills/mine` (`MySkillService`) — the authored tier, at **every** status. + +⚠️ The merge is what keeps a DRAFT skill visible to its author. A non-active skill is filtered +out of `GET /skills/` by status, and widening that endpoint to carry drafts was considered and +rejected: it feeds the composer picker, so a draft would appear as activatable in chat while +the runtime's `_apply_enabled_skills_filter` refuses it. Two reads on one page is the cheaper +mistake. + +⚠️ A draft therefore has **no toggle at all** (`toggleable` on `SkillRowComponent`), not a +disabled one. `SkillService.toggleSkill` returns silently for a skill it never loaded, so the +button would have been a control that does nothing. + +**Routing.** `/customize/skills/new` and `/customize/skills/{id}/edit` now host the authoring +form (`git mv`, so history follows it). The three old paths stay as **redirects** — they are in +bookmarks, and the detail page linked to `/my-skills/{id}/edit` for its whole life. + +⚠️ `customize/skills/new` MUST stay declared **above** `customize/skills/:skillId`. The router +matches in declaration order, so the parameterised route would otherwise swallow it and the +create form would render "skill not found" for a skill named `new`. Same class of trap as the +`/settings/connectors` ordering in step 2, and as `/{skill_id}` vs `/mine` on the backend +router. + +⚠️ `setScope` calls `router.navigate([], { relativeTo: this.route, ... })`. **`relativeTo` is +load-bearing** — without it the empty command list resolves against the root, the navigation +lands on the same URL with the query params dropped, and the scope silently never changes. +The browser found this; the tests did not, because they drive the `scope` input directly (the +test router has no matched route to bind a query param back through). + +**Add ▾** replaces the old "New skill" button and the link-out, offering *Upload skill* +(`?import=1`, which re-titles the form and leads with the SKILL.md import block) and *Create a +skill*. It is gated on `accessible$() === true` — the same 404-from-`/skills/mine` signal that +used to hide the whole `/my-skills` page, because with `SKILLS_ENABLED` off the form could only +fail to save. The picker is **not** auto-clicked on `?import=1`: a programmatic `.click()` on a +file input without a user gesture is blocked or suppressed in several browsers, and a menu item +that silently does nothing is worse than one extra click. + +## Known gaps + +- **The assistant indicator does not render until a conversation has messages** + (`showChatTopnav` requires `!isEmptyState()`). So the first turn of a new agent-bound + conversation happens with no governance cue on screen. Raised during step 4, deliberately + not fixed: it predates this epic and the launch card already names the agent. +- **`settings/connectors/` services** live under a feature folder with no page. Nine importers; + their real home is `services/connectors/`. Mechanical, deferred (step 2 notes). diff --git a/docs/specs/gpt-5-6-prompt-caching.md b/docs/specs/gpt-5-6-prompt-caching.md index b5cc98054..a1512bc5a 100644 --- a/docs/specs/gpt-5-6-prompt-caching.md +++ b/docs/specs/gpt-5-6-prompt-caching.md @@ -728,7 +728,7 @@ default, it just becomes selectable). `Cost Analytics` computes `savings = cacheRead x (inputPrice - cacheReadPrice)` per message from the pricing snapshot -(`app_api/sessions/services/metadata.py:257`). With `cacheReadPricePerMtok` +(`shared/sessions/metadata.py:710`). With `cacheReadPricePerMtok` absent it reads as `0`, so the same missing rate produced two compounding errors on `gpt-5.4`: diff --git a/docs/specs/quota-cooldown-windows.md b/docs/specs/quota-cooldown-windows.md index 9bb57f13b..1a69c6916 100644 --- a/docs/specs/quota-cooldown-windows.md +++ b/docs/specs/quota-cooldown-windows.md @@ -84,7 +84,7 @@ predictable. | Tier model | `agents/main_agent/quota/models.py:26` (`QuotaTier`) | `monthly_cost_limit`, optional `daily_cost_limit`, `period_type: Literal["daily","monthly"]`, `soft_limit_percentage` (default 80), `action_on_limit: Literal["block","warn"]` | | Check logic | `agents/main_agent/quota/checker.py:28` (`check_quota`) | Resolves tier → reads monthly/daily aggregate → warn at soft % (90% **hardcoded** at `checker.py:103`) → block at 100% if `action_on_limit=="block"`. No-tier fails **closed** (`:47`); cost-read error fails **open** (`:78`) | | Usage store | `apis/shared/storage/dynamodb_storage.py:312` (`update_user_cost_summary`) | `user-cost-summary` table, `PK=USER#{id}` / `SK=PERIOD#{YYYY-MM}`, atomic `ADD`, `GSI2 PeriodCostIndex` (`PERIOD#` / `COST#{padded-cents}`). **No TTL configured on this table** (`cost-tracking-tables-construct.ts:86` — only `sessions-metadata` has `timeToLiveAttribute`, `:49`) | -| Usage write (per turn) | `apis/app_api/sessions/services/metadata.py:190` → `_update_cost_summary_async` (`:201`) | Derives `period` from message timestamp (`:293`), calls `update_user_cost_summary`. Also writes per-message `C#` records with 365-day TTL (`:151`) | +| Usage write (per turn) | `apis/shared/sessions/metadata.py:324` → `_update_cost_summary_async` (`:632`) | Derives `period` from message timestamp (`:725`), calls `update_user_cost_summary`. Also writes per-message `C#` records with 365-day TTL (`:264`) | | Usage read | `apis/shared/costs/aggregator.py` (`CostAggregator.get_user_cost_summary`) | 30s in-process cache keyed `user+period` (`:20-27`) | | Reset cadence | implicit | No reset job — the `PERIOD#` key rolls over at month boundary. Monthly cutoff lives in exactly two places: the storage key and `_get_current_period()` (`checker.py:188`) | | Chat enforcement | `apis/inference_api/chat/routes.py:1116-1154` | Gated on `is_quota_enforcement_enabled()` + not resume/continuation; exceeded → **conversational assistant message** (`stop_reason="quota_exceeded"`, `build_quota_exceeded_event`), not an HTTP error. Warning injected first-thing into the SSE stream by `stream_with_quota_warning()` (`:1729`) | diff --git a/docs/specs/skill-slash-commands.md b/docs/specs/skill-slash-commands.md new file mode 100644 index 000000000..03afe0681 --- /dev/null +++ b/docs/specs/skill-slash-commands.md @@ -0,0 +1,167 @@ +# Skill slash commands + +Typing `/web-research` in the composer invokes that skill for **that message**. It is the +sibling of the `@`-mention (Marketplace D11): same menu shape, same keyboard, same +"rides one turn, does not bind the conversation" semantics. + +The menu's last row is **Browse skills →**, which goes to Customize → Skills. + +## Scope: the skills the user has turned on + +The menu lists exactly the skills already switched on for the conversation — the same set +the turn already discloses to the model in ``. + +That is the decision the whole design rests on. Because the invoked skill is already in +`enabled_skills`, a slash command **changes nothing about the cacheable prefix**. The +system prompt, the `toolConfig` and the disclosure block are byte-identical whether or not +a command was used. The entire cost of the feature is one short directive appended to the +turn's user message. + +Offering a switched-off skill would have meant one of two bad things: + +- **Widening the disclosure on the fly** — a new `` entry mid-session + rewrites a 30k–150k-token prefix at the cache-write premium, triggered by a keystroke. +- **Showing a command that does nothing** — the directive would name a skill the model + cannot see, and the model would burn a tool call discovering that. + +Turning a skill on stays where it belongs: the Customize page, which the menu links to. + +An Agent-bound conversation resolves through `visibleSkills` / `isSkillShownEnabled` like +every other skill surface, so it offers that Agent's bound skills — again, exactly the set +the turn will disclose. + +## The text is the binding + +Unlike the `@` menu, there is **no remembered pick**. The invoked set is derived from the +composer text on every keystroke: + +``` +/web-research what is the top headline on npr.org? +└─ findSkillCommands() → ['web-research'] → ['web_research'] +``` + +A slug is a single unambiguous token (an Agent name is not — it contains spaces, which is +why the `@` menu has to remember what was picked). Deriving is therefore exact, and it buys +two things: + +- A hand-typed command works identically to a menu pick. +- The chip and what gets sent **cannot disagree**. The chip's `✕` removes the `/slug` from + the text, because that is the only place the binding lives. + +### The token rule + +`/` is ordinary punctuation, so the rule has to keep the menu shut far more often than it +opens it. A command must start a word **and** must not be followed by another `/`: + +| Input | Result | +|---|---| +| `/web-research …` | command | +| `use /web-research, then …` | command (ordinary punctuation after is fine) | +| `and/or`, `24/7` | prose — does not start a word | +| `https://x.com/docs`, `src/app/docx` | prose — same reason | +| `/usr/bin/env` | prose — *starts* a word, excluded by the trailing-slash half | +| `/not-a-skill` | prose — only slugs the user can invoke resolve | + +That last-but-one row is the one that matters: an absolute path starts a word exactly like +a command does, so without the trailing-slash clause a skill slugged `usr` would be invoked +silently. The same rule is implemented three times — the composer's caret-anchored token, +`findSkillCommands`, and the thread renderer's `splitSkillCommands` — and all three must +agree, or a message would render as something different from what it sent. + +## The slug is served, not derived + +`GET /skills/` returns a `slug` per skill, computed with `slugify_skill_name` — the same +function that produces the `Skill.name` the `AgentSkills` plugin injects and the key its +`skills` tool accepts. The SPA never re-implements the rule; a client that drifted would +write a command the model cannot resolve. + +`slug` is **optional** in the SPA's `UserSkill`. The SPA and the backend deploy +independently and in no enforced order, so a client that lands first must degrade to "no +slash commands", not to a menu of `/undefined`. + +## Wire format + +The SPA sends `invoked_skills` alongside `enabled_skills`: + +```jsonc +{ + "message": "/web-research what is the top headline on npr.org?", + "enabled_skills": ["docx", "rubric_authoring", "web_research"], + "invoked_skills": ["web_research"] // always a subset +} +``` + +`_resolve_invoked_skill_slugs` intersects it against the turn's **effective** skill set — +the same narrow-never-grant rule `_apply_enabled_skills_filter` applies, re-run because an +Agent's skill bindings can still replace that set afterwards. The result is ordered by the +effective set rather than by the request, so two turns naming the same skills produce +byte-identical text. + +## Why a directive and not a pre-load + +The only activation path is the plugin's own `skills` tool, which the *model* calls. So +"explicit" is expressed as an instruction: + +``` +[The user invoked the `web-research` skill with a slash command. Activate it with the +`skills` tool before answering, and follow the loaded instructions for this message.] +``` + +Pre-loading the instructions server-side would duplicate the plugin's response formatting +and bypass its activation-state tracking, for the sake of saving one tool call. + +The note is appended **last**, after every prepended note (interruption, attachment +recovery, app context), so it sits closest to the model's first token. It rides +`original_message`, so the thread shows the user only what they typed — the literal +`/slug` — while the note stays an honest part of persisted history. It costs one line as +input this turn and as cached history thereafter; the disclosure block it points at is in +the prefix either way. + +## Not compatible with mid-turn steering + +A steer lands as a text block on the *tool-result* message of a turn whose skills were +already resolved, so its directive would have nothing to attach to. A queued follow-up +carrying a slash command is therefore never armed — it flushes as a normal turn, the same +way a follow-up with an attachment or an `@`-mention does. + +## Gating + +None of its own. It rides `SKILLS_ENABLED`: with skills off, `GET /skills/` 404s, the +command list is empty, and the menu never opens. The two embedded previews (Agent Designer, +marketplace test drive) pass `[showSkillCommands]="false"` for the same reason they pass +`[showAgentMentions]="false"` — those panes exercise one Agent whose skills the Agent +dictates. + +## Contrast, and why the chip is neutral + +**Do not use `bg-primary-50` / `-100` / `-200` as a tint.** The `primary` scale is generated +from `#0033a0` by lightness offset alone — `oklch(from #0033a0 calc(l + 0.4) c h)` — so it +keeps the full chroma of Boise State blue at every step. `primary-50` resolves to +**rgb(118, 179, 255)**: a saturated mid-blue, not the pale wash its name implies. Used as a +chip fill it reads as a blue blob behind small text. The `state-*` scales *are* real tints +(`state-success-50` is `rgb(240, 253, 244)`); `primary` is the exception, and the naming +hides it. + +So the chip and the menu's icon tile use **neutral surfaces with the brand blue in the +text**: white / `gray-100` in light, `gray-700` in dark, label `primary-accessible` +(`#0033a0`) / `primary-50`. + +Measured, composited against the real page background: + +| Element | Light | Dark | Bar | +|---|---|---|---| +| Chip label (12px, 500) | 10.60 | 4.74 | 4.5 (AA normal text) | +| Chip `✕` glyph | 4.84 | 3.96 | 3.0 (UI component) | +| Chip border | 1.47 | 2.13 | — (decorative; the label carries the meaning) | +| Menu icon tile | 9.63 | 4.74 | 3.0 (decorative, `aria-hidden`) | +| Menu `/slug` | 16.13 | 10.30 | 4.5 | + +Two traps worth keeping written down: + +- On `gray-700`, `primary-200` measures **3.42** and `primary-100` **4.02** — both fail. + `primary-50` (4.74) is the only step that clears AA on that surface. On `gray-800` the + whole range passes, but a `gray-800` chip disappears into the composer, which is also + `gray-800`. +- Verify light mode with **both** levers — remove `dark` from `` *and* emulate + `prefers-color-scheme: light`. The class alone leaves the `dark:` variants applying, and a + "light-mode" screenshot silently shows dark. diff --git a/frontend/ai.client/.claude/CLAUDE.md b/frontend/ai.client/.claude/CLAUDE.md index cbc446b55..e7867c55e 100644 --- a/frontend/ai.client/.claude/CLAUDE.md +++ b/frontend/ai.client/.claude/CLAUDE.md @@ -62,5 +62,5 @@ You are an expert in TypeScript, Angular, and scalable web application developme - Export a `…DialogData` type for inputs and a `…DialogResult` type for the return value alongside the component. `undefined` from `dialogRef.closed` MUST mean "cancelled"; any concrete value means "confirmed." - Inject `DialogRef` and `DIALOG_DATA` in the dialog component. Close via `this.dialogRef.close(value)`. - Parent opens the dialog via `inject(Dialog).open(Component, { data })` and awaits the result with `firstValueFrom(dialogRef.closed)`. Bind `(keydown.escape)` in the dialog's `host` to call the cancel path so Escape, backdrop click, and the explicit Cancel button all converge. -- **Design tokens for dialogs match the host page's list-page idiom**, NOT the legacy form idiom: `rounded-2xl` (not `rounded-md` / `rounded-sm`), `text-sm/6` (not `text-sm`), `bg-blue-600` for the primary action (not indigo), with the panel built as `rounded-2xl border border-gray-200 bg-white … dark:border-gray-700 dark:bg-gray-800`. Older dialogs in the codebase (e.g. `admin/tools/components/tool-role-dialog.component.ts`) use the pre-redesign tokens — match their **structure** (backdrop div + centered panel + DialogRef wiring), not their **styling**. +- **Design tokens for dialogs match the host page's list-page idiom**, NOT the legacy form idiom: `rounded-2xl` (not `rounded-md` / `rounded-sm`), `text-sm/6` (not `text-sm`), `bg-primary-accessible` for the primary action (not `bg-blue-600`, not indigo), with the panel built as `rounded-2xl border border-gray-200 bg-white … dark:border-gray-700 dark:bg-gray-800`. Older dialogs in the codebase (e.g. `admin/tools/components/tool-role-dialog.component.ts`) use the pre-redesign tokens — match their **structure** (backdrop div + centered panel + DialogRef wiring), not their **styling**. - Canonical example: `admin/manage-models/components/add-curated-model-dialog.component.ts`. diff --git a/frontend/ai.client/package-lock.json b/frontend/ai.client/package-lock.json index 980829165..1de6380db 100644 --- a/frontend/ai.client/package-lock.json +++ b/frontend/ai.client/package-lock.json @@ -1,12 +1,12 @@ { "name": "ai.client", - "version": "1.20.0", + "version": "1.21.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ai.client", - "version": "1.20.0", + "version": "1.21.0", "dependencies": { "@angular/cdk": "21.2.14", "@angular/common": "21.2.19", diff --git a/frontend/ai.client/package.json b/frontend/ai.client/package.json index dbb48b818..c75284bdc 100644 --- a/frontend/ai.client/package.json +++ b/frontend/ai.client/package.json @@ -1,6 +1,6 @@ { "name": "ai.client", - "version": "1.20.0", + "version": "1.21.0", "scripts": { "ng": "ng", "prestart": "tsx scripts/branding/generate-brand-theme.ts && tsx scripts/branding/generate-surface-theme.ts && tsx scripts/branding/generate-surface-colors.ts && tsx scripts/branding/generate-favicons.ts", diff --git a/frontend/ai.client/public/favicon/android-chrome-192x192.png b/frontend/ai.client/public/favicon/android-chrome-192x192.png index b06f836e5..4225e732f 100644 Binary files a/frontend/ai.client/public/favicon/android-chrome-192x192.png and b/frontend/ai.client/public/favicon/android-chrome-192x192.png differ diff --git a/frontend/ai.client/public/favicon/android-chrome-512x512.png b/frontend/ai.client/public/favicon/android-chrome-512x512.png index 17225e809..c1b67c928 100644 Binary files a/frontend/ai.client/public/favicon/android-chrome-512x512.png and b/frontend/ai.client/public/favicon/android-chrome-512x512.png differ diff --git a/frontend/ai.client/public/favicon/apple-touch-icon.png b/frontend/ai.client/public/favicon/apple-touch-icon.png index 47381720e..4539f7b7e 100644 Binary files a/frontend/ai.client/public/favicon/apple-touch-icon.png and b/frontend/ai.client/public/favicon/apple-touch-icon.png differ diff --git a/frontend/ai.client/public/favicon/favicon-16x16.png b/frontend/ai.client/public/favicon/favicon-16x16.png index 5ea9916c2..e13c26440 100644 Binary files a/frontend/ai.client/public/favicon/favicon-16x16.png and b/frontend/ai.client/public/favicon/favicon-16x16.png differ diff --git a/frontend/ai.client/public/favicon/favicon-32x32.png b/frontend/ai.client/public/favicon/favicon-32x32.png index 1a696605d..2e82b2892 100644 Binary files a/frontend/ai.client/public/favicon/favicon-32x32.png and b/frontend/ai.client/public/favicon/favicon-32x32.png differ diff --git a/frontend/ai.client/public/img/provider-logos/openai/dark.svg b/frontend/ai.client/public/img/provider-logos/openai/dark.svg index ba36fc2aa..adb130a0e 100644 --- a/frontend/ai.client/public/img/provider-logos/openai/dark.svg +++ b/frontend/ai.client/public/img/provider-logos/openai/dark.svg @@ -1,4 +1,4 @@ - + diff --git a/frontend/ai.client/public/img/provider-logos/openai/light.svg b/frontend/ai.client/public/img/provider-logos/openai/light.svg index 832fa6a5f..179a85c8b 100644 --- a/frontend/ai.client/public/img/provider-logos/openai/light.svg +++ b/frontend/ai.client/public/img/provider-logos/openai/light.svg @@ -1,4 +1,4 @@ - + diff --git a/frontend/ai.client/src/app/admin/connectors/pages/connector-form.page.ts b/frontend/ai.client/src/app/admin/connectors/pages/connector-form.page.ts index 2055138fe..db908ae5b 100644 --- a/frontend/ai.client/src/app/admin/connectors/pages/connector-form.page.ts +++ b/frontend/ai.client/src/app/admin/connectors/pages/connector-form.page.ts @@ -176,7 +176,7 @@ const ICON_ACCEPTED_MIME_TYPES = [
Add Connector @@ -326,7 +326,7 @@ import { SpinnerComponent } from '../../../components/spinner/spinner.component'

Add Connector diff --git a/frontend/ai.client/src/app/admin/costs/admin-costs.page.ts b/frontend/ai.client/src/app/admin/costs/admin-costs.page.ts index 0d7dff973..80f947da9 100644 --- a/frontend/ai.client/src/app/admin/costs/admin-costs.page.ts +++ b/frontend/ai.client/src/app/admin/costs/admin-costs.page.ts @@ -66,7 +66,7 @@ import { SpinnerComponent } from '../../components/spinner/spinner.component'; type="button" (click)="onExport()" [disabled]="loading()" - class="inline-flex items-center gap-2 px-4 py-2 bg-white border border-gray-300 rounded-sm text-sm font-medium text-gray-700 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-primary-500 disabled:opacity-50 disabled:cursor-not-allowed dark:bg-gray-800 dark:border-gray-600 dark:text-gray-200 dark:hover:bg-gray-700 transition-colors" + class="inline-flex items-center gap-2 px-4 py-2 bg-white border border-gray-300 rounded-2xl text-sm font-medium text-gray-700 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-primary-500 disabled:opacity-50 disabled:cursor-not-allowed dark:bg-gray-800 dark:border-gray-600 dark:text-gray-200 dark:hover:bg-gray-700 transition-colors" > Export diff --git a/frontend/ai.client/src/app/admin/costs/components/model-breakdown.component.ts b/frontend/ai.client/src/app/admin/costs/components/model-breakdown.component.ts index 851f3aa47..1d3966cf8 100644 --- a/frontend/ai.client/src/app/admin/costs/components/model-breakdown.component.ts +++ b/frontend/ai.client/src/app/admin/costs/components/model-breakdown.component.ts @@ -41,10 +41,10 @@ type ChartView = 'pie' | 'bar'; type="button" (click)="setChartView('pie')" class="px-3 py-1 text-sm font-medium rounded-md transition-colors" - [class.bg-primary-100]="chartView() === 'pie'" + [class.bg-gray-100]="chartView() === 'pie'" [class.text-primary-accessible]="chartView() === 'pie'" - [class.dark:bg-primary-900/30]="chartView() === 'pie'" - [class.dark:text-primary-accessible-dark]="chartView() === 'pie'" + [class.dark:bg-gray-700]="chartView() === 'pie'" + [class.dark:text-primary-50]="chartView() === 'pie'" [class.text-gray-600]="chartView() !== 'pie'" [class.dark:text-gray-400]="chartView() !== 'pie'" [class.hover:text-gray-900]="chartView() !== 'pie'" @@ -56,10 +56,10 @@ type ChartView = 'pie' | 'bar'; type="button" (click)="setChartView('bar')" class="px-3 py-1 text-sm font-medium rounded-md transition-colors" - [class.bg-primary-100]="chartView() === 'bar'" + [class.bg-gray-100]="chartView() === 'bar'" [class.text-primary-accessible]="chartView() === 'bar'" - [class.dark:bg-primary-900/30]="chartView() === 'bar'" - [class.dark:text-primary-accessible-dark]="chartView() === 'bar'" + [class.dark:bg-gray-700]="chartView() === 'bar'" + [class.dark:text-primary-50]="chartView() === 'bar'" [class.text-gray-600]="chartView() !== 'bar'" [class.dark:text-gray-400]="chartView() !== 'bar'" [class.hover:text-gray-900]="chartView() !== 'bar'" diff --git a/frontend/ai.client/src/app/admin/costs/components/top-users-table.component.ts b/frontend/ai.client/src/app/admin/costs/components/top-users-table.component.ts index 35839dc28..ace838042 100644 --- a/frontend/ai.client/src/app/admin/costs/components/top-users-table.component.ts +++ b/frontend/ai.client/src/app/admin/costs/components/top-users-table.component.ts @@ -190,10 +190,10 @@ type SortDirection = 'asc' | 'desc';
{{ getAvatarInitial(user) }} diff --git a/frontend/ai.client/src/app/admin/fine-tuning-access/fine-tuning-access.page.html b/frontend/ai.client/src/app/admin/fine-tuning-access/fine-tuning-access.page.html index c6e201ed7..c10fa534a 100644 --- a/frontend/ai.client/src/app/admin/fine-tuning-access/fine-tuning-access.page.html +++ b/frontend/ai.client/src/app/admin/fine-tuning-access/fine-tuning-access.page.html @@ -9,7 +9,7 @@

Fine-Tuning Acces

@@ -187,7 +187,7 @@

No diff --git a/frontend/ai.client/src/app/admin/manage-announcements/announcement-form.page.ts b/frontend/ai.client/src/app/admin/manage-announcements/announcement-form.page.ts index 66ac5a7ad..eacff4590 100644 --- a/frontend/ai.client/src/app/admin/manage-announcements/announcement-form.page.ts +++ b/frontend/ai.client/src/app/admin/manage-announcements/announcement-form.page.ts @@ -325,7 +325,7 @@ function byteLength(value: string): number { type="button" (click)="toggleRole(role.roleId)" [attr.aria-pressed]="isRoleSelected(role.roleId)" - class="rounded-sm border px-2.5 py-1 text-xs/5 font-medium focus:outline-none focus:ring-2 focus:ring-primary-500" + class="rounded-2xl border px-2.5 py-1 text-xs/5 font-medium focus:outline-none focus:ring-2 focus:ring-primary-500" [class]="roleChipClass(role.roleId)" > {{ role.displayName || role.roleId }} @@ -417,14 +417,14 @@ function byteLength(value: string): number {
Cancel
New announcement @@ -171,7 +171,7 @@ import { parseIso } from '../../utils/date'; type="button" (click)="onPublish(item.announcement)" [disabled]="busyId() !== null" - class="inline-flex items-center gap-1 rounded-sm border border-gray-300 bg-white px-2.5 py-1.5 text-sm/6 font-medium text-gray-700 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-gray-500 disabled:cursor-not-allowed disabled:opacity-50 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600" + class="inline-flex items-center gap-1 rounded-2xl border border-gray-300 bg-white px-2.5 py-1.5 text-sm/6 font-medium text-gray-700 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-gray-500 disabled:cursor-not-allowed disabled:opacity-50 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600" [attr.aria-label]="'Publish ' + item.announcement.title" > @@ -184,7 +184,7 @@ import { parseIso } from '../../utils/date'; type="button" (click)="onRevise(item.announcement)" [disabled]="busyId() !== null" - class="inline-flex items-center gap-1 rounded-sm border border-gray-300 bg-white px-2.5 py-1.5 text-sm/6 font-medium text-gray-700 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-gray-500 disabled:cursor-not-allowed disabled:opacity-50 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600" + class="inline-flex items-center gap-1 rounded-2xl border border-gray-300 bg-white px-2.5 py-1.5 text-sm/6 font-medium text-gray-700 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-gray-500 disabled:cursor-not-allowed disabled:opacity-50 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600" title="Bump the revision so everyone sees this again" [attr.aria-label]="'Show ' + item.announcement.title + ' again'" > @@ -195,7 +195,7 @@ import { parseIso } from '../../utils/date'; type="button" (click)="onArchive(item.announcement)" [disabled]="busyId() !== null" - class="inline-flex items-center gap-1 rounded-sm border border-gray-300 bg-white px-2.5 py-1.5 text-sm/6 font-medium text-gray-700 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-gray-500 disabled:cursor-not-allowed disabled:opacity-50 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600" + class="inline-flex items-center gap-1 rounded-2xl border border-gray-300 bg-white px-2.5 py-1.5 text-sm/6 font-medium text-gray-700 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-gray-500 disabled:cursor-not-allowed disabled:opacity-50 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600" [attr.aria-label]="'Archive ' + item.announcement.title" > @@ -205,7 +205,7 @@ import { parseIso } from '../../utils/date'; @@ -216,7 +216,7 @@ import { parseIso } from '../../utils/date'; type="button" (click)="onDelete(item.announcement)" [disabled]="busyId() !== null" - class="inline-flex items-center gap-1 rounded-sm border border-state-danger-300 bg-white px-2.5 py-1.5 text-sm/6 font-medium text-state-danger-700 hover:bg-state-danger-50 focus:outline-none focus:ring-2 focus:ring-state-danger-500 disabled:cursor-not-allowed disabled:opacity-50 dark:border-state-danger-500 dark:bg-gray-700 dark:text-state-danger-400 dark:hover:bg-state-danger-900/20" + class="inline-flex items-center gap-1 rounded-2xl border border-state-danger-300 bg-white px-2.5 py-1.5 text-sm/6 font-medium text-state-danger-700 hover:bg-state-danger-50 focus:outline-none focus:ring-2 focus:ring-state-danger-500 disabled:cursor-not-allowed disabled:opacity-50 dark:border-state-danger-500 dark:bg-gray-700 dark:text-state-danger-400 dark:hover:bg-state-danger-900/20" [attr.aria-label]="'Delete ' + item.announcement.title" > diff --git a/frontend/ai.client/src/app/admin/manage-models/manage-models.page.html b/frontend/ai.client/src/app/admin/manage-models/manage-models.page.html index af314044b..b4b4e888a 100644 --- a/frontend/ai.client/src/app/admin/manage-models/manage-models.page.html +++ b/frontend/ai.client/src/app/admin/manage-models/manage-models.page.html @@ -135,10 +135,18 @@

Manage Models

+ + +
- + {{ model.modelName }} @if (model.isDefault) { @@ -149,7 +157,10 @@

Manage Models

}
-

+

{{ model.modelId }}

diff --git a/frontend/ai.client/src/app/admin/manage-models/manage-models.page.ts b/frontend/ai.client/src/app/admin/manage-models/manage-models.page.ts index 28ab19202..e3c180e8b 100644 --- a/frontend/ai.client/src/app/admin/manage-models/manage-models.page.ts +++ b/frontend/ai.client/src/app/admin/manage-models/manage-models.page.ts @@ -16,6 +16,7 @@ import { ManagedModelsService } from './services/managed-models.service'; import { AppRolesService } from '../roles/services/app-roles.service'; import type { ManagedModel } from './models/managed-model.model'; import { SpinnerComponent } from '../../components/spinner/spinner.component'; +import { ModelIconComponent } from '../../components/model-icon/model-icon.component'; import { DeleteModelDialogComponent, DeleteModelDialogData, @@ -24,7 +25,7 @@ import { @Component({ selector: 'app-manage-models-page', - imports: [RouterLink, FormsModule, NgIcon, SpinnerComponent], + imports: [RouterLink, FormsModule, NgIcon, ModelIconComponent, SpinnerComponent], providers: [ provideIcons({ heroPlus, diff --git a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts index aa83ecc83..3965a0999 100644 --- a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts +++ b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts @@ -265,7 +265,7 @@ describe('ModelCatalogPage', () => { }); }); - describe('curated bedrock-responses (GPT-5.6) entries', () => { + describe('curated bedrock-responses (OpenAI family) entries', () => { it('renders them on their own tab', () => { const page = createComponent(); page.selectTab('bedrock-responses'); @@ -302,12 +302,70 @@ describe('ModelCatalogPage', () => { } }); - it('declares no supportedParams rather than an invented one', () => { - // AWS publishes no parameter table for GPT-5.6. A declared spec flips the - // #915 guard from permissive to restrictive, so a guessed one would - // silently block params the model actually accepts. + it('declares only the MEASURED supportedParams, never a guessed one', () => { + // Supersedes an earlier invariant that required NO spec at all. The bar + // was never "no spec" — it was "no invented spec": a declared spec flips + // the #915 guard from permissive to restrictive, so a guess silently + // blocks params the model really accepts. AWS still publishes no + // parameter table, so this spec comes from probing all four ids in + // us-west-2 on 2026-09-12 (see the block comment on the array). + // + // If a future sibling is added without re-probing, this fails — which is + // the point. for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { - expect(model.template.supportedParams ?? null).toBeNull(); + const params = model.template.supportedParams?.params; + expect(params, `${model.key} must declare a measured spec`).toBeTruthy(); + + // The endpoint's own 400 enumerates exactly these, identically on all four. + expect(`${model.key}:${params!['reasoning_effort'].allowed?.join(',')}`).toBe( + `${model.key}:none,low,medium,high,xhigh,max`, + ); + + // Measured 400: "Unsupported parameter: 'temperature' is not supported + // with this model." Declared false so the request never carries them. + expect(`${model.key}:${params!['temperature'].supported}`).toBe(`${model.key}:false`); + expect(`${model.key}:${params!['top_p'].supported}`).toBe(`${model.key}:false`); + + expect(`${model.key}:${params!['max_tokens'].supported}`).toBe(`${model.key}:true`); + + // `medium` pins what the provider was already doing implicitly — + // measured at ~376 reasoning tokens unset vs ~308 for medium, so this + // is cost-neutral-to-cheaper rather than an increase. A default must + // stay a member of `allowed` or the backend drops it. + expect(`${model.key}:${params!['reasoning_effort'].default}`).toBe( + `${model.key}:medium`, + ); + expect(params!['reasoning_effort'].allowed).toContain( + params!['reasoning_effort'].default, + ); + } + }); + + it('curates GPT-6 Astra on the Short Context rate card', () => { + // The 272K cap is covered by the family loop above; this pins the rates + // that cap keeps correct. Geo CRIS Short Context is $11.00 / $55.00 — + // Long Context (1.05M) is $22.00 / $82.50, and the tier is chosen by the + // request's actual token count, so nothing but the cap keeps a single + // flat rate per bucket true. + const astra = CURATED_BEDROCK_RESPONSES_MODELS.find(m => m.key === 'gpt-6-astra'); + + expect(astra?.template.modelId).toBe('us.openai.gpt-6-astra'); + expect(astra?.template.inputPricePerMillionTokens).toBeCloseTo(11.0, 6); + expect(astra?.template.outputPricePerMillionTokens).toBeCloseTo(55.0, 6); + }); + + it('declares Astra\'s published output cap and cutoff, which its siblings lack', () => { + // Astra's card publishes `Max output tokens: 128,000` and an April 30, + // 2026 cutoff; every GPT-5.6 card states neither, which is why the + // family default is null for both. Inheriting the default here would + // discard two numbers AWS actually publishes. + const astra = CURATED_BEDROCK_RESPONSES_MODELS.find(m => m.key === 'gpt-6-astra'); + + expect(astra?.template.maxOutputTokens).toBe(128_000); + expect(astra?.template.knowledgeCutoffDate).toBe('2026-04-30'); + + for (const sibling of CURATED_BEDROCK_RESPONSES_MODELS.filter(m => m.key !== 'gpt-6-astra')) { + expect(`${sibling.key}:${sibling.template.maxOutputTokens}`).toBe(`${sibling.key}:null`); } }); }); @@ -323,4 +381,65 @@ describe('ModelCatalogPage', () => { expect(gpt54?.template.cacheReadPricePerMillionTokens).toBeCloseTo(0.275, 6); expect(gpt54?.template.cacheWritePricePerMillionTokens).toBe(0); }); + + describe('curated picker placement', () => { + const ALL = [ + ...CURATED_BEDROCK_MODELS, + ...CURATED_MANTLE_MODELS, + ...CURATED_BEDROCK_RESPONSES_MODELS, + ]; + + // Demoted = superseded by a newer sibling ON THE SAME PROVIDER SURFACE, or + // specialist enough that it isn't a general chat default. Everything else + // stays at the picker's top level. This list is a change-detector: adding a + // model or re-ranking one should be a deliberate edit here, not a drift. + // + // "Same surface" is load-bearing. GPT-5.4 (mantle) looks superseded by the + // GPT-5.6 family until you notice those are bedrock-responses — a different + // provider an install may not use at all. Demoting it left Mantle with no + // featured model but a specialist coding one, which the family check below + // now catches. A template default cannot assume what else gets added. + const DEMOTED = ['claude-sonnet-4-6', 'qwen3-coder-30b', 'gpt-5-6-luna']; + + it('demotes exactly the superseded and specialist models', () => { + const demoted = ALL.filter(m => m.template.isFeatured === false) + .map(m => m.key) + .sort(); + expect(demoted).toEqual([...DEMOTED].sort()); + }); + + it('leaves featured models undeclared so they inherit the backend default', () => { + // `isFeatured` defaults true server-side. Featured rows say nothing + // rather than `true`, so the default stays in exactly one place. + for (const model of ALL) { + if (DEMOTED.includes(model.key)) continue; + expect( + model.template.isFeatured, + `${model.key} should not declare isFeatured`, + ).toBeUndefined(); + } + }); + + it('keeps a featured model in every provider family', () => { + // A catalog tab whose every entry is demoted would put an entire + // provider behind the submenu, which is never the intent. + for (const [label, group] of [ + ['bedrock', CURATED_BEDROCK_MODELS], + ['mantle', CURATED_MANTLE_MODELS], + ['bedrock-responses', CURATED_BEDROCK_RESPONSES_MODELS], + ] as const) { + const featured = group.filter(m => m.template.isFeatured !== false); + expect(featured.length, `${label} must keep a featured model`).toBeGreaterThan(0); + } + }); + + it('does not let two featured models both claim to be the most capable', () => { + // GPT-6 Astra outranks (and out-prices) GPT-5.6 Sol in the same catalog, + // so Sol's copy must not say "most capable". + const featuredCopy = ALL.filter(m => m.template.isFeatured !== false) + .map(m => m.template.shortDescription ?? ''); + const superlatives = featuredCopy.filter(d => /most capable/i.test(d)); + expect(superlatives).toEqual([]); + }); + }); }); diff --git a/frontend/ai.client/src/app/admin/manage-models/model-form.page.html b/frontend/ai.client/src/app/admin/manage-models/model-form.page.html index b84d8dbcc..530058ba3 100644 --- a/frontend/ai.client/src/app/admin/manage-models/model-form.page.html +++ b/frontend/ai.client/src/app/admin/manage-models/model-form.page.html @@ -82,6 +82,149 @@

Basic inform }

+ +
+ + +

+ Shown under the model name in the chat model picker. Keep it to a + fragment, not a sentence — the picker truncates. Leave blank to + show the provider name instead. +

+ @if (modelForm.controls.shortDescription.invalid && modelForm.controls.shortDescription.touched) { +

+ Keep the short description to 80 characters or fewer +

+ } +
+ + +
+ Icon +

+ Shown beside the model name in the chat model picker. Pick a built-in + logo where we ship one — it stays a crisp, theme-aware vector at every + size. Upload an image only for a model that maps to no vendor here. +

+ +
+ +
+ + {{ iconPreviewSource() }} +
+ + +
+ @for (slug of builtinIcons; track slug) { + + } + +
+
+ + +
+ @if (isEditMode()) { +
+ + @if (uploadedIconUrl()) { + + } + @if (isUploadingIcon()) { + Working… + } +
+

+ Square PNG or JPEG, at least 256×256 and at most 400 KB. Stored + re-encoded at 512×512, which also strips any camera metadata. + An uploaded image takes precedence over the built-in logo. +

+ } @else { +

+ Save the model first to upload a custom image — the stored file is + keyed to the saved record. +

+ } + @if (iconError(); as error) { +

{{ error }}

+ } +
+
+
@@ -396,6 +539,24 @@

Access contr Only one model can be the default. Setting this will unset any previous default.

+ + +
+ +

+ Unfeatured models move into the picker's "More models" submenu. They + stay fully available — this only changes where users find them. +

+
@@ -823,7 +984,7 @@

Custom paramet diff --git a/frontend/ai.client/src/app/admin/manage-models/model-form.page.ts b/frontend/ai.client/src/app/admin/manage-models/model-form.page.ts index f2486b1c1..31342f8fc 100644 --- a/frontend/ai.client/src/app/admin/manage-models/model-form.page.ts +++ b/frontend/ai.client/src/app/admin/manage-models/model-form.page.ts @@ -1,4 +1,5 @@ import { Component, ChangeDetectionStrategy, inject, signal, computed, OnInit } from '@angular/core'; +import { toSignal } from '@angular/core/rxjs-interop'; import { Router, ActivatedRoute, RouterLink } from '@angular/router'; import { AbstractControl, @@ -30,6 +31,13 @@ import { ModelProvider, SupportedParams, } from './models/managed-model.model'; +import { + BUILTIN_MODEL_ICONS, + BUILTIN_MODEL_ICON_LABELS, + BuiltinModelIcon, + resolveModelIcon, +} from './models/model-icons'; +import { ModelIconComponent } from '../../components/model-icon/model-icon.component'; import { ManagedModelsService } from './services/managed-models.service'; import { CuratedModelPrefillService } from './services/curated-model-prefill.service'; import { AppRolesService } from '../roles/services/app-roles.service'; @@ -223,6 +231,8 @@ function knownParamKeyControl(fb: FormBuilder, key: string): FormControl interface ModelFormGroup { modelId: FormControl; modelName: FormControl; + shortDescription: FormControl; + iconSlug: FormControl; provider: FormControl; providerName: FormControl; inputModalities: FormControl; @@ -233,6 +243,7 @@ interface ModelFormGroup { availableToRoles: FormControl; enabled: FormControl; isDefault: FormControl; + isFeatured: FormControl; inputPricePerMillionTokens: FormControl; outputPricePerMillionTokens: FormControl; cacheWritePricePerMillionTokens: FormControl; @@ -247,7 +258,7 @@ interface ModelFormGroup { @Component({ selector: 'app-model-form-page', - imports: [ReactiveFormsModule, RouterLink, NgIcon, SpinnerComponent], + imports: [ReactiveFormsModule, RouterLink, NgIcon, ModelIconComponent, SpinnerComponent], providers: [provideIcons({ heroArrowLeft, heroChevronDown, heroChevronRight })], templateUrl: './model-form.page.html', styleUrl: './model-form.page.css', @@ -341,6 +352,14 @@ export class ModelFormPage implements OnInit { readonly modelForm: FormGroup = this.fb.group({ modelId: this.fb.control('', { nonNullable: true, validators: [Validators.required] }), modelName: this.fb.control('', { nonNullable: true, validators: [Validators.required] }), + shortDescription: this.fb.control('', { + nonNullable: true, + validators: [Validators.maxLength(80)], + }), + // '' is a real value here, not "unset": the update path drops null fields + // (`exclude_none`), so null could never clear a slug once set. Same rule as + // `shortDescription`. + iconSlug: this.fb.control('', { nonNullable: true }), provider: this.fb.control('bedrock', { nonNullable: true, validators: [Validators.required] }), providerName: this.fb.control('', { nonNullable: true, validators: [Validators.required] }), inputModalities: this.fb.control([], { nonNullable: true, validators: [Validators.required] }), @@ -354,6 +373,7 @@ export class ModelFormPage implements OnInit { availableToRoles: this.fb.control([], { nonNullable: true }), enabled: this.fb.control(true, { nonNullable: true }), isDefault: this.fb.control(false, { nonNullable: true }), + isFeatured: this.fb.control(true, { nonNullable: true }), inputPricePerMillionTokens: this.fb.control(0, { nonNullable: true, validators: [Validators.required, Validators.min(0)] }), outputPricePerMillionTokens: this.fb.control(0, { nonNullable: true, validators: [Validators.required, Validators.min(0)] }), cacheWritePricePerMillionTokens: this.fb.control(null, { validators: [Validators.min(0)] }), @@ -867,6 +887,10 @@ export class ModelFormPage implements OnInit { try { const model = await this.managedModelsService.getModel(id); + // The uploaded icon is not a form field — it is written by its own request + // against the saved record — so it is held beside the form rather than in it. + this.uploadedIconUrl.set(model.iconUrl ?? null); + // Roles that reach this model via a wildcard grant or inheritance. Held // outside the form: they're server-derived and not editable here. this.inheritedAppRoles.set(model.inheritedAppRoles ?? []); @@ -875,6 +899,8 @@ export class ModelFormPage implements OnInit { this.modelForm.patchValue({ modelId: model.modelId, modelName: model.modelName, + shortDescription: model.shortDescription ?? '', + iconSlug: model.iconSlug ?? '', provider: model.provider as ModelProvider, providerName: model.providerName, inputModalities: model.inputModalities.map(m => m.toUpperCase()), @@ -885,6 +911,9 @@ export class ModelFormPage implements OnInit { availableToRoles: model.availableToRoles ?? [], enabled: model.enabled, isDefault: model.isDefault ?? false, + // Absent on records written before the field existed, and those + // models are featured today — mirror the backend default. + isFeatured: model.isFeatured ?? true, inputPricePerMillionTokens: model.inputPricePerMillionTokens, outputPricePerMillionTokens: model.outputPricePerMillionTokens, cacheWritePricePerMillionTokens: model.cacheWritePricePerMillionTokens ?? null, @@ -918,6 +947,8 @@ export class ModelFormPage implements OnInit { this.modelForm.patchValue({ modelId: template.modelId, modelName: template.modelName, + shortDescription: template.shortDescription ?? '', + iconSlug: template.iconSlug ?? '', provider: template.provider, providerName: template.providerName, inputModalities: template.inputModalities.map(m => m.toUpperCase()), @@ -928,6 +959,7 @@ export class ModelFormPage implements OnInit { availableToRoles: template.availableToRoles ?? [], enabled: template.enabled, isDefault: template.isDefault, + isFeatured: template.isFeatured ?? true, inputPricePerMillionTokens: template.inputPricePerMillionTokens, outputPricePerMillionTokens: template.outputPricePerMillionTokens, cacheWritePricePerMillionTokens: template.cacheWritePricePerMillionTokens ?? null, @@ -949,6 +981,7 @@ export class ModelFormPage implements OnInit { this.modelForm.patchValue({ modelId: params['modelId'] || '', modelName: params['modelName'] || '', + shortDescription: params['shortDescription'] || '', provider: params['provider'] || 'bedrock', providerName: params['providerName'] || '', inputModalities: params['inputModalities'] ? params['inputModalities'].split(',') : [], @@ -999,6 +1032,126 @@ export class ModelFormPage implements OnInit { return control.value?.includes(value) ?? false; } + // ── icon ─────────────────────────────────────────────────────────────────── + // Two independent controls that resolve to one avatar. The built-in slug is a + // plain form field saved with the rest of the model; the upload is its own + // request against an already-saved record, because the object key is derived + // from the record id. See `models/model-icons.ts` for the precedence. + + readonly builtinIcons = BUILTIN_MODEL_ICONS; + readonly builtinIconLabels = BUILTIN_MODEL_ICON_LABELS; + + /** The uploaded icon's serve path, or null when the model has none. */ + readonly uploadedIconUrl = signal(null); + readonly isUploadingIcon = signal(false); + /** Surfaced verbatim: the server's rejections name the limit that was broken. */ + readonly iconError = signal(null); + + // Mirrors the form control so the live preview and the radio group's checked + // state update as the admin clicks, without either of them reading + // `modelForm.value` during change detection. + private readonly iconSlugValue = toSignal(this.modelForm.controls.iconSlug.valueChanges, { + initialValue: this.modelForm.controls.iconSlug.value, + }); + + /** + * What the picker will actually draw for this model right now — the same + * resolution the chat picker runs, so the preview cannot promise one thing and + * the menu render another. + */ + readonly iconPreviewModel = computed(() => ({ + iconUrl: this.uploadedIconUrl(), + iconSlug: this.iconSlugValue(), + providerName: this.providerNameValue(), + modelName: this.modelNameValue(), + })); + + private readonly providerNameValue = toSignal( + this.modelForm.controls.providerName.valueChanges, + { initialValue: this.modelForm.controls.providerName.value }, + ); + private readonly modelNameValue = toSignal(this.modelForm.controls.modelName.valueChanges, { + initialValue: this.modelForm.controls.modelName.value, + }); + + /** + * Where the previewed icon actually came from. + * + * Read off the same resolution the picker runs, not off which control the + * admin last touched: with no slug and a provider we ship no logo for, "the + * provider name matched" is simply untrue, and the tile beside this caption + * is visibly a monogram. + */ + readonly iconPreviewSource = computed(() => { + const icon = resolveModelIcon(this.iconPreviewModel()); + if (icon.kind === 'upload') return 'Uploaded image'; + if (icon.kind === 'none') return 'No icon — showing the model\'s initial'; + return this.iconSlugValue() ? 'Built-in logo' : 'Matched from the provider name'; + }); + + /** Pick a built-in logo, or clear the selection by picking the active one again. */ + selectIconSlug(slug: BuiltinModelIcon | ''): void { + const control = this.modelForm.controls.iconSlug; + control.setValue(control.value === slug ? '' : slug); + control.markAsDirty(); + } + + isIconSlugSelected(slug: string): boolean { + return this.iconSlugValue() === slug; + } + + /** + * Upload the picked file and point the record at it. + * + * Only reachable in edit mode: the object key is `models/{record id}/icons/…`, + * so there is nothing to attach to until the model has been saved once. The + * input is reset afterwards so re-picking the same file still fires `change`. + */ + async onIconFileSelected(event: Event): Promise { + const input = event.target as HTMLInputElement; + const file = input.files?.[0]; + input.value = ''; + if (!file) return; + + const id = this.modelId(); + if (!id) return; + + this.iconError.set(null); + this.isUploadingIcon.set(true); + try { + const response = await this.managedModelsService.uploadIcon(id, file); + this.uploadedIconUrl.set(response.iconUrl ?? null); + } catch (error: any) { + // The server's message names the limit and the supplied value ("Icons must + // be square (this one is 512×256)"), which is the whole point of showing it + // rather than a generic failure. + this.iconError.set( + error?.error?.detail || error?.message || 'Failed to upload the icon. Please try again.', + ); + } finally { + this.isUploadingIcon.set(false); + } + } + + /** Remove the uploaded icon, falling back to the built-in slug (or the monogram). */ + async removeUploadedIcon(): Promise { + const id = this.modelId(); + if (!id) return; + + this.iconError.set(null); + this.isUploadingIcon.set(true); + try { + await this.managedModelsService.deleteIcon(id); + this.uploadedIconUrl.set(null); + } catch (error: any) { + this.iconError.set( + error?.error?.detail || error?.message || 'Failed to remove the icon. Please try again.', + ); + } finally { + this.isUploadingIcon.set(false); + } + } + /** * Submit the form */ @@ -1018,6 +1171,10 @@ export class ModelFormPage implements OnInit { const formData: ManagedModelFormData = { modelId: v.modelId, modelName: v.modelName, + // Empty string rather than null: the update path drops null fields + // (`exclude_none`), so null could never clear a description once set. + shortDescription: v.shortDescription.trim(), + iconSlug: v.iconSlug, provider: v.provider, providerName: v.providerName, inputModalities: v.inputModalities, @@ -1029,6 +1186,7 @@ export class ModelFormPage implements OnInit { availableToRoles: v.availableToRoles, enabled: v.enabled, isDefault: v.isDefault, + isFeatured: v.isFeatured, inputPricePerMillionTokens: v.inputPricePerMillionTokens, outputPricePerMillionTokens: v.outputPricePerMillionTokens, cacheWritePricePerMillionTokens: v.cacheWritePricePerMillionTokens, diff --git a/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts b/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts index 9739d2831..d2c5c28dd 100644 --- a/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts +++ b/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts @@ -1,4 +1,4 @@ -import { ManagedModelFormData, ModelProvider } from './managed-model.model'; +import { ManagedModelFormData, ModelProvider, SupportedParams } from './managed-model.model'; /** * A curated entry shown in the model catalog. Carries everything needed to @@ -17,9 +17,20 @@ import { ManagedModelFormData, ModelProvider } from './managed-model.model'; * covers — and reading that absence as "unpublished" put three rows into the * dev catalog at GovCloud prices, over-charging by 20%. * - * Claude rates below were read from the **AWS Price List API** - * (`AmazonBedrockFoundationModels`, us-west-2, published 2026-09-01), which - * does carry them. Re-verify there when bumping a model id: + * **Which source is authoritative depends on the vendor.** The per-model AWS + * model cards above are authoritative for **Claude, Nova and the + * OpenAI/Mantle family**; the Price List API is authoritative for **xAI, + * Google and AgentCore**. + * + * ⚠️ Do NOT re-derive Claude rates from the Price List API. A full + * enumeration of the `AmazonBedrock` offer file (run twice, a week apart, + * across a republish) returns **10 Claude SKUs, none newer than Claude 3**, + * and no us-west-2 SKU at all for Haiku 4.5, Sonnet 4.6, Fable 5.1, GPT-5.4 + * or Nova Micro. An earlier revision of this comment claimed the API "does + * carry them" and told you to re-verify there; it does not, and a lookup that + * comes back empty reads exactly like a model that is merely renamed. + * + * For the vendors the API does carry, re-verify with: * * aws pricing get-products --region us-east-1 \ * --service-code AmazonBedrockFoundationModels \ @@ -78,15 +89,28 @@ const claude4xDefaults = (): Pick< }); /** - * Bedrock publishes cache rates as fixed multiples of a model's base input - * rate: cache write is **1.25x**, cache read is **0.1x** (the 1-hour Claude - * write we do not use is 2x). Deriving them removes the two fields most likely - * to drift — the ratios were the one thing the old table got right. + * The cache-read multiple that has held for every model currently in the + * table. It is a DEFAULT, not a law — see `ratesWithDerivedCache`. + */ +const DEFAULT_CACHE_READ_MULTIPLIER = 0.1; + +/** + * Bedrock publishes cache rates as multiples of a model's base input rate. + * Cache write is **1.25x** across every family we have checked (the 1-hour + * Claude write we do not use is 2x), and that one is stable enough to derive: + * the GPT-5.6 model cards publish the same multiplier (Sol 4.40 -> 5.50), and + * commercial Cost Explorer billing reproduces it to four decimals on every + * clean day. Two model families, two independent sources, same ratio. * - * Not Claude-specific: the GPT-5.6 model cards publish exactly the same two - * multipliers (Sol 4.40 -> 5.50 / 0.44), and commercial Cost Explorer billing - * reproduces them to four decimals on every clean day. Two model families, two - * independent sources, same ratios. + * **Cache read is NOT stable at 0.1x and must not be treated as a constant.** + * It is the default because it holds for every row below, but there are live + * counterexamples on Bedrock today — Claude Fable 5.1 reads at 0.025x (a 75% + * cut Anthropic states explicitly) and xAI Grok 4.6 at 0.25x. Both are + * deliberately absent from this table: encoding a specific rate needs a second + * independent source, and Grok publishes no cache-write SKU at all (implicit + * caching only), so there is nothing for our explicit-`cachePoint` contract to + * place. Pass `cacheReadMultiplier` when a model's card says otherwise, so the + * next model that breaks the ratio is a data change and not a code change. * * `input` and `output` are the only independently published numbers, and both * are TIER-SPECIFIC. Pass the rates for the tier the `modelId` names, and set @@ -96,6 +120,7 @@ const claude4xDefaults = (): Pick< const ratesWithDerivedCache = ( input: number, output: number, + cacheReadMultiplier: number = DEFAULT_CACHE_READ_MULTIPLIER, ): Pick< ManagedModelFormData, | 'inputPricePerMillionTokens' @@ -111,7 +136,7 @@ const ratesWithDerivedCache = ( inputPricePerMillionTokens: input, outputPricePerMillionTokens: output, cacheWritePricePerMillionTokens: round(input * 1.25), - cacheReadPricePerMillionTokens: round(input * 0.1), + cacheReadPricePerMillionTokens: round(input * cacheReadMultiplier), }; }; @@ -125,6 +150,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ ...claude4xDefaults(), modelId: 'us.anthropic.claude-opus-4-7', modelName: 'Claude Opus 4.7', + shortDescription: 'For your toughest challenges', maxOutputTokens: 64_000, // Regional (CRIS): $5.50 / $27.50. Global is $5.00 / $25.00. ...ratesWithDerivedCache(5.5, 27.5), @@ -150,6 +176,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ ...claude4xDefaults(), modelId: 'global.anthropic.claude-sonnet-5', modelName: 'Claude Sonnet 5', + shortDescription: 'Strong reasoning over very long context', maxInputTokens: 1_000_000, maxOutputTokens: 128_000, // Global: $2.00 / $10.00 — correct as declared, this id really is @@ -177,6 +204,9 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ ...claude4xDefaults(), modelId: 'us.anthropic.claude-sonnet-4-6', modelName: 'Claude Sonnet 4.6', + shortDescription: 'Balanced reasoning for everyday work', + // Superseded by Claude Sonnet 5 in this same catalog. + isFeatured: false, maxOutputTokens: 64_000, // Regional (CRIS): $3.30 / $16.50. Global is $3.00 / $15.00. ...ratesWithDerivedCache(3.3, 16.5), @@ -201,6 +231,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ ...claude4xDefaults(), modelId: 'us.anthropic.claude-haiku-4-5-20251001-v1:0', modelName: 'Claude Haiku 4.5', + shortDescription: 'Fastest for quick answers', maxOutputTokens: 64_000, // Regional (CRIS): $1.10 / $5.50. Global is $1.00 / $5.00. This is the // platform default model, so this is the row every cost number rides on. @@ -266,6 +297,7 @@ export const CURATED_MANTLE_MODELS: CuratedModel[] = [ ...mantleDefaults(), modelId: 'openai.gpt-5.4', modelName: 'GPT-5.4', + shortDescription: 'Multimodal reasoning', providerName: 'OpenAI', inputModalities: ['TEXT', 'IMAGE'], maxInputTokens: 272_000, @@ -295,6 +327,9 @@ export const CURATED_MANTLE_MODELS: CuratedModel[] = [ ...mantleDefaults(), modelId: 'qwen.qwen3-coder-30b-a3b-instruct', modelName: 'Qwen3 Coder 30B', + shortDescription: 'Long-context coding', + // Specialist coding model, not a general chat default. + isFeatured: false, providerName: 'Qwen', inputModalities: ['TEXT'], maxInputTokens: 256_000, @@ -376,20 +411,152 @@ const bedrockResponsesDefaults = (): Pick< }); /** - * GPT-5.6 on `bedrock-runtime` via the Responses API. + * The OpenAI family on `bedrock-runtime` via the Responses API. * * Rates are the **Geo CRIS, Short Context (272K)** row from each model card — * Geo CRIS is the tier the `us.*` inference profiles resolve to, and these - * models are inference-profile-only (no ON_DEMAND). Verified 2026-09-06. + * models are inference-profile-only (no ON_DEMAND). Verified 2026-09-06, + * re-verified for GPT-6 Astra 2026-09-11. + * + * **Every card in this family publishes two price tables, and the tier is + * selected by the ACTUAL token count of the request** — not by a declared + * window, and not by a separate model id. Settled 2026-09-11 (see + * `docs/kaizen/review-queue.md`): the Price List API encodes `-long-ctx` as a + * value of `tokenType` under an identical `model` attribute, GPT-5.6 Terra + * declares a 1M window against a single model id yet still publishes a + * reachable Short Context table, and our own Cost Explorer rows bill + * `_standard` and never `-long-ctx` while calling those 1M-window ids. There + * is no field in which a window could be declared: `maxInputTokens` below is + * ours, is read only for compaction and telemetry, and never reaches a + * Bedrock request. + * + * That is exactly why `maxInputTokens: 272_000` is load-bearing. It does not + * protect us from being over-charged — nothing here does. It keeps the single + * flat rate per bucket that `CuratedModel` can hold **arithmetically true**, + * because a request that crosses 272K bills input at 2x, output at 1.5x and + * both cache buckets at 2x. Raising it silently UNDER-charges every long turn. + * + * `supportedParams` was deliberately absent here until 2026-09-12, when it was + * MEASURED — the bar this comment has always set. AWS still publishes no + * parameter table for these models (`model-parameters-openai.html` documents + * only the open-weight gpt-oss family), so the evidence is the endpoint's own + * responses, probed against all four ids in us-west-2: * - * `supportedParams` is deliberately absent. AWS publishes no parameter table - * for these models (`model-parameters-openai.html` documents only the - * open-weight gpt-oss family), and an invented spec would be worse than none: - * a declared spec flips the #915 guard from permissive to restrictive, so a - * wrong entry silently blocks a parameter the model actually accepts. Add one - * only from published or measured evidence. + * - `reasoning.effort` — sending a deliberately invalid value returns a 400 + * that ENUMERATES the enum: "Supported values are: 'none', 'low', + * 'medium', 'high', 'xhigh', and 'max'." Identical on Sol, Terra, Luna and + * Astra, and it matches the launch blog ("They also support none, low, + * medium, high, xhigh, and max reasoning effort"). Two independent sources. + * - `temperature` and `top_p` — hard 400 on all four: "Unsupported + * parameter: 'temperature' is not supported with this model." + * - `max_output_tokens` — accepted. + * + * That last pair is why declaring a spec here is a FIX, not just an enabler. + * With no spec the #915 guard stays permissive, so a `temperature` reaching + * this family from any caller that can set one kills the turn with a 400 + * mid-stream. Declaring them `supported: false` drops them before the request + * instead — the same failure class the guard inversion was written to close on + * Claude Opus 4.7. + * + * `reasoning_effort` defaults to `medium`, which was also measured rather than + * assumed. Neither the cards nor the blog publish a default, so the question + * was what the provider does when the param is ABSENT — which is not the same + * as sending `none`. Sending nothing still reasons; `none` is an explicit + * "off". Three samples per level on Luna, in reasoning tokens: + * + * unset 285 / 516 / 327 (mean 376) + * none 0 / 0 / 0 + * low 221 / 222 / 230 (mean 224) + * medium 274 / 346 / 303 (mean 308) + * high 516 / 363 / 497 (mean 459) + * + * So the implicit default already sits around medium, and declaring `medium` + * is cost-neutral-to-slightly-cheaper (-18% reasoning tokens), NOT an increase. + * It is declared anyway because otherwise the provider can move its own + * default and our spend follows with no code change and no signal — and + * because a declared default is what lets the picker show the level in force + * instead of a blank row. Counts are noisy (unset spanned 285-516 across three + * identical calls), so treat the middle levels as roughly interchangeable. + * + * The level that actually moves the bill is `max`: ~2.3x the output tokens of + * unset on Terra, and reasoning bills as output. It stays in `allowed` + * deliberately — dropping a level from that list is the lever if the exposure + * is ever unwanted, since the picker and the backend both read it. + * + * GPT-6 Astra inherits this default by sharing the helper. Its effort ENUM was + * probed directly, but its token counts were not — medium there is an + * extrapolation from its GPT-5.6 siblings, not a measurement. + * + * The standing rule is unchanged — a declared spec flips the guard from + * permissive to restrictive, so a wrong entry silently blocks a parameter the + * model really accepts. Add or widen one only from published or measured + * evidence, and re-probe rather than assume when a new sibling ships. */ +/** + * Measured Responses-API parameter profile for the OpenAI family on + * `bedrock-runtime`. See the block comment on + * {@link CURATED_BEDROCK_RESPONSES_MODELS} for the probe and its evidence. + * + * `temperature` / `top_p` are declared `supported: false` rather than omitted: + * omission drops them too (an authoritative spec treats silence as + * unsupported), but an explicit false records the measured fact and logs the + * clearer "unsupported inference param" line when one is dropped. + */ +const openaiResponsesParams = (maxOutputTokens: number | null = null): SupportedParams => ({ + params: { + reasoning_effort: { + supported: true, + allowed: ['none', 'low', 'medium', 'high', 'xhigh', 'max'], + // Pins what the provider was already doing implicitly — see the block + // comment above for the measurement. NOT an increase in reasoning. + default: 'medium', + }, + max_tokens: { + supported: true, + min: 1, + ...(maxOutputTokens === null ? {} : { max: maxOutputTokens }), + }, + temperature: { supported: false }, + top_p: { supported: false }, + }, +}); + export const CURATED_BEDROCK_RESPONSES_MODELS: CuratedModel[] = [ + { + key: 'gpt-6-astra', + tagline: 'Frontier model for the hardest end-to-end work — reasoning, coding and research.', + capabilities: ['Reasoning', 'Vision', 'Long context', 'Prompt caching'], + pricingTier: 'regional', + template: { + ...bedrockResponsesDefaults(), + // UNVERIFIED, and the one thing here worth re-checking before anyone + // banks cache savings on this row: Astra's card lists Implicit and + // Explicit Prompt Caching under `bedrock-mantle` ONLY. Caching appears + // in neither column of its `bedrock-runtime` feature table, where every + // GPT-5.6 card lists it under both endpoints — and this row routes over + // `bedrock-runtime`. `supportsCaching` still inherits `true`, which is + // the safe stance either way: `false` would zero the cache-rate fields + // and price cached tokens at $0.00 while AWS bills them in full. If + // caching turns out not to fire here the cost is a stale capability + // chip, not a mispriced bill. + modelId: 'us.openai.gpt-6-astra', + modelName: 'GPT-6 Astra', + shortDescription: 'Frontier reasoning, coding and research', + // Geo CRIS Short Context: $11.00 / $55.00. Global CRIS is $10.00 / + // $50.00. Long Context (1.05M) would be $22.00 / $82.50 — unreachable + // while maxInputTokens stays pinned at the 272K boundary. + ...ratesWithDerivedCache(11.0, 55.0), + // Astra departs from its GPT-5.6 siblings here: its card publishes a + // real cap (`Max output tokens: 128,000`) where theirs say N/A, so this + // overrides the family default of `null`. Declaring the published + // number beats inheriting "unknown". + maxOutputTokens: 128_000, + // Published on the card as April 30, 2026 — again unlike the GPT-5.6 + // cards, which state none. + knowledgeCutoffDate: '2026-04-30', + supportedParams: openaiResponsesParams(128_000), + }, + }, { key: 'gpt-5-6-sol', tagline: 'OpenAI\'s most capable model — frontier reasoning and agentic work.', @@ -399,9 +566,11 @@ export const CURATED_BEDROCK_RESPONSES_MODELS: CuratedModel[] = [ ...bedrockResponsesDefaults(), modelId: 'us.openai.gpt-5.6-sol', modelName: 'GPT-5.6 Sol', + shortDescription: 'Strong reasoning and agentic work', // Geo CRIS: $4.40 / $22.00. Global CRIS is $4.00 / $20.00. ...ratesWithDerivedCache(4.4, 22.0), knowledgeCutoffDate: null, + supportedParams: openaiResponsesParams(), }, }, { @@ -413,9 +582,11 @@ export const CURATED_BEDROCK_RESPONSES_MODELS: CuratedModel[] = [ ...bedrockResponsesDefaults(), modelId: 'us.openai.gpt-5.6-terra', modelName: 'GPT-5.6 Terra', + shortDescription: 'Balanced performance per dollar', // Geo CRIS: $2.20 / $13.20. Global CRIS is $2.00 / $12.00. ...ratesWithDerivedCache(2.2, 13.2), knowledgeCutoffDate: null, + supportedParams: openaiResponsesParams(), }, }, { @@ -427,9 +598,14 @@ export const CURATED_BEDROCK_RESPONSES_MODELS: CuratedModel[] = [ ...bedrockResponsesDefaults(), modelId: 'us.openai.gpt-5.6-luna', modelName: 'GPT-5.6 Luna', + shortDescription: 'Fast and affordable, for high volume', + // Its own card pitches it for classification, routing and high + // volume — a batch workhorse rather than a chat default. + isFeatured: false, // Geo CRIS: $0.22 / $1.32. Global CRIS is $0.20 / $1.20. ...ratesWithDerivedCache(0.22, 1.32), knowledgeCutoffDate: null, + supportedParams: openaiResponsesParams(), }, }, ]; diff --git a/frontend/ai.client/src/app/admin/manage-models/models/effort-control.spec.ts b/frontend/ai.client/src/app/admin/manage-models/models/effort-control.spec.ts new file mode 100644 index 000000000..655087544 --- /dev/null +++ b/frontend/ai.client/src/app/admin/manage-models/models/effort-control.spec.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from 'vitest'; +import { + ManagedModel, + effortLevelLabel, + resolveEffortControl, +} from './managed-model.model'; + +/** + * `resolveEffortControl` decides whether the chat model picker offers an + * Effort submenu at all. Its contract is "only offer what the backend will + * actually honor" — `_merge_inference_params` keeps an enum override ONLY + * when it's a member of the admin-declared `allowed` set, and pins `locked` + * params to the admin default regardless of what the client sends. Every + * case below is one of those backend behaviours, read back from the UI side. + */ +function model(params: Record | null): ManagedModel { + return { + id: 'm1', + modelId: 'test-model', + modelName: 'Test Model', + provider: 'bedrock', + providerName: 'Anthropic', + inputModalities: ['TEXT'], + outputModalities: ['TEXT'], + maxInputTokens: 200000, + maxOutputTokens: 4096, + allowedAppRoles: [], + availableToRoles: [], + enabled: true, + inputPricePerMillionTokens: 1, + outputPricePerMillionTokens: 1, + knowledgeCutoffDate: null, + supportsCaching: true, + isDefault: false, + supportedParams: params === null ? null : { params: params as never }, + }; +} + +describe('resolveEffortControl', () => { + it('returns null when the model declares no params at all', () => { + expect(resolveEffortControl(model(null))).toBeNull(); + }); + + it('returns null for a null/undefined model', () => { + expect(resolveEffortControl(null)).toBeNull(); + expect(resolveEffortControl(undefined)).toBeNull(); + }); + + it('returns null when the model declares no effort param', () => { + expect(resolveEffortControl(model({ temperature: { supported: true } }))).toBeNull(); + }); + + it('returns null when effort is declared but unsupported', () => { + const control = resolveEffortControl( + model({ effort: { supported: false, allowed: ['low', 'high'] } }), + ); + expect(control).toBeNull(); + }); + + it('returns null when effort is supported but enumerates no levels', () => { + // The backend's enum branch needs `allowed` to keep an override; without + // it there is nothing safe to render. + expect(resolveEffortControl(model({ effort: { supported: true } }))).toBeNull(); + expect(resolveEffortControl(model({ effort: { supported: true, allowed: [] } }))).toBeNull(); + }); + + it('returns null when the admin locked the param', () => { + // Locked params are pinned to the admin default server-side and user + // overrides are dropped, so offering the choice would be a lie. + const control = resolveEffortControl( + model({ effort: { supported: true, locked: true, allowed: ['low', 'high'], default: 'low' } }), + ); + expect(control).toBeNull(); + }); + + it('resolves levels and default for a configured Bedrock effort param', () => { + const control = resolveEffortControl( + model({ + effort: { supported: true, allowed: ['low', 'medium', 'high'], default: 'medium' }, + }), + ); + expect(control).toEqual({ key: 'effort', levels: ['low', 'medium', 'high'], defaultLevel: 'medium' }); + }); + + it('reports no default when the admin declared none', () => { + const control = resolveEffortControl( + model({ effort: { supported: true, allowed: ['low', 'high'] } }), + ); + expect(control?.defaultLevel).toBeNull(); + }); + + it('ignores a default that is not one of the allowed levels', () => { + // Mirrors the backend, which would fall through to the provider default + // rather than send a value outside the declared set. + const control = resolveEffortControl( + model({ effort: { supported: true, allowed: ['low', 'high'], default: 'max' } }), + ); + expect(control?.defaultLevel).toBeNull(); + }); + + it('falls back to reasoning_effort on the OpenAI-compatible surfaces', () => { + const control = resolveEffortControl( + model({ reasoning_effort: { supported: true, allowed: ['low', 'high'], default: 'high' } }), + ); + expect(control).toEqual({ key: 'reasoning_effort', levels: ['low', 'high'], defaultLevel: 'high' }); + }); + + it('prefers effort over reasoning_effort when a model somehow declares both', () => { + const control = resolveEffortControl( + model({ + effort: { supported: true, allowed: ['low'] }, + reasoning_effort: { supported: true, allowed: ['high'] }, + }), + ); + expect(control?.key).toBe('effort'); + }); + + it('coerces non-string levels to strings', () => { + const control = resolveEffortControl( + model({ effort: { supported: true, allowed: [1, 2], default: 2 } }), + ); + expect(control).toEqual({ key: 'effort', levels: ['1', '2'], defaultLevel: '2' }); + }); +}); + +describe('effortLevelLabel', () => { + it('title-cases a plain level', () => { + expect(effortLevelLabel('low')).toBe('Low'); + expect(effortLevelLabel('medium')).toBe('Medium'); + }); + + it('spells out the xhigh shorthand', () => { + expect(effortLevelLabel('xhigh')).toBe('Extra high'); + }); +}); diff --git a/frontend/ai.client/src/app/admin/manage-models/models/managed-model.model.ts b/frontend/ai.client/src/app/admin/manage-models/models/managed-model.model.ts index d15ce04d1..64ca3a9a9 100644 --- a/frontend/ai.client/src/app/admin/manage-models/models/managed-model.model.ts +++ b/frontend/ai.client/src/app/admin/manage-models/models/managed-model.model.ts @@ -162,6 +162,29 @@ export interface ManagedModel { modelId: string; /** Human-readable name of the model */ modelName: string; + /** + * One-line reason a user would pick this model, shown under its name in the + * chat model picker. Deliberately shorter than the catalog card's `tagline` + * — the picker is a narrow menu, not a card, so this reads as a fragment + * ("For your toughest challenges"), not a sentence. + */ + shortDescription?: string | null; + /** + * Built-in vendor logo slug (e.g. 'anthropic'), resolved client-side to the + * light/dark SVG pair the SPA ships. See `model-icons.ts` for the precedence + * against `iconUrl`. + */ + iconSlug?: string | null; + /** + * S3 object key for an uploaded icon. Server-side detail — read `iconUrl`, + * which is derived from it and carries the cache-busting `?v=` digest. + */ + iconKey?: string | null; + /** + * Relative app-api path serving an uploaded icon (`/models/{id}/icon?v=…`), + * or absent when the model has none. Takes precedence over `iconSlug`. + */ + iconUrl?: string | null; /** Model provider (AWS, OpenAI, Google) */ provider: ModelProvider; /** Provider name (e.g., 'Anthropic', 'Amazon', 'Meta') */ @@ -211,6 +234,18 @@ export interface ManagedModel { supportsCaching: boolean; /** Whether this is the default model for new sessions */ isDefault: boolean; + /** + * Whether the model sits at the top level of the chat model picker. `false` + * collapses it into the picker's "More models" submenu — the model stays + * fully available, it just stops competing for the first glance. + * + * Defaults to `true` on the backend so a catalog nobody has curated keeps + * showing every model exactly where it always has. + * + * Optional because a record stored before this field existed genuinely has + * no value for it. Read it as `isFeatured !== false`, never `=== true`. + */ + isFeatured?: boolean; /** * OpenAI-compatible API surface: `chat` (OpenAI Chat Completions, the * default) or `responses` (OpenAI Responses API — required by models that @@ -246,6 +281,20 @@ export interface ManagedModelFormData { modelId: string; /** Human-readable name of the model */ modelName: string; + /** + * One-line reason a user would pick this model, shown under its name in the + * chat model picker. Deliberately shorter than the catalog card's `tagline` + * — the picker is a narrow menu, not a card, so this reads as a fragment + * ("For your toughest challenges"), not a sentence. + */ + shortDescription?: string | null; + /** + * Built-in vendor logo to show beside this model in the chat picker. Empty + * string clears it — not null: the update path drops null fields, so null + * could never remove a slug once set (same rule as `shortDescription`). + * The uploaded icon, when there is one, wins over this. + */ + iconSlug?: string | null; /** Model provider (AWS, OpenAI, Google) */ provider: ModelProvider; /** Provider name (e.g., 'Anthropic', 'Amazon', 'Meta') */ @@ -286,6 +335,15 @@ export interface ManagedModelFormData { supportsCaching?: boolean; /** Whether this is the default model for new sessions */ isDefault: boolean; + /** + * Whether the model sits at the top level of the chat model picker. `false` + * collapses it into the picker's "More models" submenu — the model stays + * fully available, it just stops competing for the first glance. + * + * Defaults to `true` on the backend so a catalog nobody has curated keeps + * showing every model exactly where it always has. + */ + isFeatured?: boolean; /** * OpenAI-compatible API surface: `chat` or `responses`. Selectable for * `mantle`; forced to `responses` for `bedrock-responses`. Inert for other @@ -429,8 +487,19 @@ export const KNOWN_PARAMS: KnownParamMeta[] = [ label: 'Reasoning Effort', description: 'Reasoning depth (OpenAI o-series and reasoning models on the ' + - 'OpenAI-compatible Bedrock surfaces).', - kind: 'number', + 'OpenAI-compatible Bedrock surfaces). Check the levels this model ' + + 'supports; pick a default.', + // A string enum, not a number — `kind: 'number'` here was simply wrong, + // and it was load-bearing: the admin form renders the `allowed` checklist + // only for `kind: 'select'`, so there was no way to declare the levels a + // model accepts, and anything that reads `allowed` (the chat picker's + // Effort submenu, the backend's enum branch) could never see one. + kind: 'select', + // The union across the OpenAI-compatible surfaces. Per-model subsets are + // declared in each model's `allowed`, which is what actually gates a + // request — `none` and `max` are real GPT-5.6 levels but absent from the + // older o-series, so this list is a superset by design. + options: ['none', 'low', 'medium', 'high', 'xhigh', 'max'], providers: ['openai', 'mantle', 'bedrock-responses'], }, ]; @@ -446,3 +515,63 @@ export const AVAILABLE_ROLES = [ 'User', 'Guest', ] as const; + +/** + * Canonical param keys that express reasoning/output effort, in the order the + * picker prefers them. Anthropic calls it `effort` (output_config.effort); + * the OpenAI-compatible surfaces call it `reasoning_effort`. A model declares + * at most one. + */ +export const EFFORT_PARAM_KEYS = ['effort', 'reasoning_effort'] as const; + +/** An effort control the chat model picker can render for a model. */ +export interface EffortControl { + /** Canonical param key to send in `inference_params`. */ + key: string; + /** Selectable levels, admin-declared, ordered low -> high. */ + levels: string[]; + /** The admin's default level, or null when they didn't pick one. */ + defaultLevel: string | null; +} + +/** + * Resolve the effort control for a model, or null when it has none. + * + * Deliberately strict: an effort param is only offered when the admin declared + * it `supported`, left it unlocked, AND enumerated the `allowed` levels. That + * mirrors `_merge_inference_params` on the backend, whose enum branch keeps a + * user override *only* if it's a member of `allowed` and otherwise silently + * falls back to the default. Rendering a level the backend would discard would + * show the user a choice that does nothing. + * + * `locked` is excluded for the same reason — the backend pins those to the + * admin default and drops overrides without erroring. + */ +export function resolveEffortControl(model: ManagedModel | null | undefined): EffortControl | null { + const spec = model?.supportedParams?.params; + if (!spec) return null; + + for (const key of EFFORT_PARAM_KEYS) { + const paramSpec = spec[key]; + if (!paramSpec?.supported || paramSpec.locked) continue; + const levels = (paramSpec.allowed ?? []).map(level => String(level)); + if (levels.length === 0) continue; + const declaredDefault = + paramSpec.default === null || paramSpec.default === undefined + ? null + : String(paramSpec.default); + return { + key, + levels, + defaultLevel: declaredDefault !== null && levels.includes(declaredDefault) ? declaredDefault : null, + }; + } + + return null; +} + +/** Title-case an effort level for display ('xhigh' -> 'Extra high'). */ +export function effortLevelLabel(level: string): string { + if (level === 'xhigh') return 'Extra high'; + return level.charAt(0).toUpperCase() + level.slice(1); +} diff --git a/frontend/ai.client/src/app/admin/manage-models/models/model-icons.ts b/frontend/ai.client/src/app/admin/manage-models/models/model-icons.ts new file mode 100644 index 000000000..a4991d991 --- /dev/null +++ b/frontend/ai.client/src/app/admin/manage-models/models/model-icons.ts @@ -0,0 +1,76 @@ +import { ManagedModel } from './managed-model.model'; + +/** + * Built-in vendor logos shipped with the SPA. + * + * Each slug has a `public/img/provider-logos/{slug}/{light,dark}.svg` pair. Adding + * a vendor means dropping the pair in, listing it here, and listing it in the + * backend's `BUILTIN_MODEL_ICONS` (`apis/shared/models/model_icons.py`), which + * validates the slug on write — a slug only one side knows is a tile that renders + * as nothing, with no error anywhere to say why. + */ +export const BUILTIN_MODEL_ICONS = ['anthropic', 'openai', 'amazon', 'meta'] as const; + +export type BuiltinModelIcon = (typeof BUILTIN_MODEL_ICONS)[number]; + +/** Display names for the admin form's icon picker. */ +export const BUILTIN_MODEL_ICON_LABELS: Record = { + anthropic: 'Anthropic', + openai: 'OpenAI', + amazon: 'Amazon', + meta: 'Meta', +}; + +/** + * `providerName` values that name a vendor we ship a logo for. + * + * The last-resort fallback, and deliberately last: it is a guess from a free-text + * field an admin typed. A model whose `providerName` is "Anthropic (via Bedrock)" + * gets nothing from this and needs an explicit `iconSlug` — which is exactly why + * the slug exists rather than this map being the whole feature. + */ +const PROVIDER_NAME_TO_ICON: Record = { + anthropic: 'anthropic', + openai: 'openai', + amazon: 'amazon', + aws: 'amazon', + meta: 'meta', +}; + +export function iconForProviderName(providerName: string | null | undefined): BuiltinModelIcon | null { + if (!providerName) return null; + return PROVIDER_NAME_TO_ICON[providerName.trim().toLowerCase()] ?? null; +} + +function isBuiltinIcon(slug: string | null | undefined): slug is BuiltinModelIcon { + return !!slug && (BUILTIN_MODEL_ICONS as readonly string[]).includes(slug); +} + +/** Path to one half of a built-in logo's light/dark pair. */ +export function builtinIconPath(slug: BuiltinModelIcon, theme: 'light' | 'dark'): string { + return `/img/provider-logos/${slug}/${theme}.svg`; +} + +/** + * What to draw for a model, in precedence order. + * + * `upload` first: an admin who uploaded a file after picking a built-in logo meant + * the file. `builtin` next, from the explicit `iconSlug` and only then from the + * provider-name guess — a shipped SVG stays crisp and theme-correct at any size, + * which a stored raster cannot. `none` is a real outcome, not a failure: the picker + * falls back to a monogram rather than an empty gap. + */ +export type ModelIconSource = + | { kind: 'upload'; url: string } + | { kind: 'builtin'; slug: BuiltinModelIcon } + | { kind: 'none' }; + +export function resolveModelIcon( + model: Pick | null | undefined, +): ModelIconSource { + if (!model) return { kind: 'none' }; + if (model.iconUrl) return { kind: 'upload', url: model.iconUrl }; + if (isBuiltinIcon(model.iconSlug)) return { kind: 'builtin', slug: model.iconSlug }; + const guessed = iconForProviderName(model.providerName); + return guessed ? { kind: 'builtin', slug: guessed } : { kind: 'none' }; +} diff --git a/frontend/ai.client/src/app/admin/manage-models/services/managed-models.service.ts b/frontend/ai.client/src/app/admin/manage-models/services/managed-models.service.ts index fcd301c19..e39c2ffb2 100644 --- a/frontend/ai.client/src/app/admin/manage-models/services/managed-models.service.ts +++ b/frontend/ai.client/src/app/admin/manage-models/services/managed-models.service.ts @@ -12,6 +12,18 @@ export interface ManagedModelsListResponse { totalCount: number; } +/** + * The result of uploading or clearing a model's icon. + * + * Both fields are absent after a remove, which is the signal the form needs to + * fall back to the model's `iconSlug` without re-reading the catalog. + */ +export interface ManagedModelIconResponse { + id: string; + iconKey?: string | null; + iconUrl?: string | null; +} + /** * A model on Bedrock Mantle's live roster (`GET /admin/mantle/models`). * Mirrors the OpenAI list-models shape the Mantle endpoint speaks. @@ -190,6 +202,53 @@ export class ManagedModelsService { } } + + /** + * Upload a custom icon for a model. + * + * Multipart, not JSON: the bytes go straight to S3 and only the key lands on + * the record. The server validates and re-encodes to 512×512 — which is also + * what strips EXIF — so a rejection here carries an admin-facing reason and + * should be surfaced verbatim rather than replaced with "upload failed". + * + * @param modelId - Model record identifier (the UUID, not the Bedrock model id) + * @param file - A square PNG or JPEG, at least 256×256 and at most 400 KB + * @returns Promise resolving to the stored key and the path that serves it + */ + async uploadIcon(modelId: string, file: File): Promise { + const body = new FormData(); + body.append('file', file); + + const response = await firstValueFrom( + this.http.post(`${this.baseUrl()}/${modelId}/icon`, body), + ); + + // The catalog's cached copy still carries the old iconUrl (or none). + this.modelsResource.reload(); + + return response; + } + + /** + * Remove a model's uploaded icon, falling back to its `iconSlug`. + * + * Distinct from clearing `iconSlug` through the form: the two are independent, + * and an admin who uploaded the wrong file should get their built-in logo back + * rather than a blank tile. + * + * @param modelId - Model record identifier + * @returns Promise resolving to the cleared icon fields + */ + async deleteIcon(modelId: string): Promise { + const response = await firstValueFrom( + this.http.delete(`${this.baseUrl()}/${modelId}/icon`), + ); + + this.modelsResource.reload(); + + return response; + } + /** * Delete an enabled model * diff --git a/frontend/ai.client/src/app/admin/manage-user-menu-links/manage-user-menu-links.page.ts b/frontend/ai.client/src/app/admin/manage-user-menu-links/manage-user-menu-links.page.ts index 196d9aa52..3b120e049 100644 --- a/frontend/ai.client/src/app/admin/manage-user-menu-links/manage-user-menu-links.page.ts +++ b/frontend/ai.client/src/app/admin/manage-user-menu-links/manage-user-menu-links.page.ts @@ -37,7 +37,7 @@ import { UserMenuLink } from './models/user-menu-link.model';

New link @@ -90,7 +90,7 @@ import { UserMenuLink } from './models/user-menu-link.model'; @@ -99,7 +99,7 @@ import { UserMenuLink } from './models/user-menu-link.model'; @@ -210,7 +210,7 @@

Created

@@ -219,7 +219,7 @@

Created

 diff --git a/frontend/ai.client/src/app/admin/quota-tiers/pages/override-detail/override-detail.component.html b/frontend/ai.client/src/app/admin/quota-tiers/pages/override-detail/override-detail.component.html index 9252c0697..a57746b11 100644 --- a/frontend/ai.client/src/app/admin/quota-tiers/pages/override-detail/override-detail.component.html +++ b/frontend/ai.client/src/app/admin/quota-tiers/pages/override-detail/override-detail.component.html @@ -96,7 +96,7 @@

Override @if (selectedType() === 'custom_limit') { -
+
- } - - - @if (skillService.hasSkills() || skillService.agentLocked()) { -
- - - @if (isSkillsOpen()) { -
- @if (skillService.agentLocked()) { -

-

- } - @if (skillService.loading()) { -
Loading skills...
- } @else if (skillService.error()) { - - } @else if (!skillService.hasSkills()) { -
- No skills are available to you yet. -
- } @else { - @if (!skillService.agentLocked()) { -

- Skills are off by default. Turn on the ones you want this - conversation to be able to use. -

- } -
- @for (skill of skillService.visibleSkills(); track skill.skillId) { -
-
- -

- {{ skill.description }} -

-
- -
- } -
- } -
- } -
- } - - -
- - - @if (isToolsOpen()) { -
- @if (toolService.agentLocked()) { -

-

- } - @if (toolService.loading()) { -
Loading tools...
- } @else if (toolService.error()) { - - } @else if (toolService.tools().length === 0) { -
No tools available
- } @else { -
- @for (tool of toolService.visibleTools(); track tool.toolId) { -
-
-
- @if (isMcpServer(tool)) { - - } -
- -

- {{ tool.description }} -

- @if (isMcpServer(tool) && partialServerLabel(tool); as label) { -

- {{ label }} -

- } -
-
- - -
- - - @if (isMcpServer(tool) && isServerExpanded(tool.toolId)) { -
- @if (tool.serverTools && tool.serverTools.length > 0) { - @for (sub of tool.serverTools; track sub.name) { -
-
- {{ sub.name }} - @if (sub.description) { - {{ sub.description }} - } -
- -
- } - } @else { -
- - @if (discoverError()[tool.toolId]) { - {{ - discoverError()[tool.toolId] - }} - } @else { - Discover this server’s tools to enable only the ones you - need. - } -
- } -
- } -
- } -
- } -
- } -
- - - @if (systemPromptsService.hasPrompts()) { -
-
-

- Conversation Mode -

-

- Apply a custom set of instructions to this conversation. -

-
- - @if (systemPromptsService.loading()) { -
Loading...
- } @else { -
- - - - @for (prompt of systemPromptsService.prompts(); track prompt.prompt_id) { - - } -
- } -
- } -
-
-

diff --git a/frontend/ai.client/src/app/components/model-settings/model-settings.spec.ts b/frontend/ai.client/src/app/components/model-settings/model-settings.spec.ts deleted file mode 100644 index 3774bfa81..000000000 --- a/frontend/ai.client/src/app/components/model-settings/model-settings.spec.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; -import { TestBed } from '@angular/core/testing'; -import { ElementRef, signal } from '@angular/core'; -import { ModelService } from '../../session/services/model/model.service'; -import { ToolService } from '../../services/tool/tool.service'; -import { SkillService } from '../../services/skill/skill.service'; -import { ManagedModel } from '../../admin/manage-models/models/managed-model.model'; - -describe('ModelSettings', () => { - let mockModelService: any; - let mockToolService: any; - - const mockModel: ManagedModel = { - id: 'test-id', - modelId: 'test-model', - modelName: 'Test Model', - provider: 'bedrock', - providerName: 'Anthropic', - inputModalities: ['TEXT'], - outputModalities: ['TEXT'], - maxInputTokens: 200000, - maxOutputTokens: 4096, - allowedAppRoles: [], - availableToRoles: [], - enabled: true, - inputPricePerMillionTokens: 1, - outputPricePerMillionTokens: 2, - knowledgeCutoffDate: null, - supportsCaching: true, - isDefault: false, - }; - - beforeEach(() => { - TestBed.resetTestingModule(); - mockModelService = { - availableModels: signal([mockModel]), - selectedModel: signal(mockModel), - setSelectedModel: vi.fn(), - }; - mockToolService = { - tools: signal([]), - enabledTools: signal([]), - toolsByCategory: signal(new Map()), - categories: signal([]), - toggleTool: vi.fn(), - }; - - TestBed.configureTestingModule({ - providers: [ - { provide: ModelService, useValue: mockModelService }, - { provide: ToolService, useValue: mockToolService }, - // Mock SkillService so the component doesn't hold a real one (which - // would fire /skills/ HTTP); its async error logs can otherwise land - // during worker teardown and fail the run with an unhandled rejection. - { - provide: SkillService, - useValue: { - skills: signal([]), - visibleSkills: signal([]), - enabledSkillIds: signal([]), - enabledCount: signal(0), - hasSkills: signal(false), - loading: signal(false), - error: signal(null), - agentLocked: signal(false), - toggleSkill: vi.fn(), - }, - }, - { provide: ElementRef, useValue: { nativeElement: document.createElement('div') } }, - ], - }); - }); - - afterEach(() => { - TestBed.resetTestingModule(); - }); - - async function createComponent() { - const { ModelSettings } = await import('./model-settings'); - return TestBed.runInInjectionContext(() => new ModelSettings()); - } - - it('should initialize with closed dropdown state', async () => { - const component = await createComponent(); - expect(component['isModelDropdownOpen']()).toBe(false); - expect(component['focusedOptionIndex']()).toBe(-1); - }); - - it('should toggle model dropdown', async () => { - const component = await createComponent(); - component.toggleModelDropdown(); - expect(component['isModelDropdownOpen']()).toBe(true); - component.toggleModelDropdown(); - expect(component['isModelDropdownOpen']()).toBe(false); - }); - - it('should select model and close dropdown', async () => { - const component = await createComponent(); - component.selectModel(mockModel); - expect(mockModelService.setSelectedModel).toHaveBeenCalledWith(mockModel); - expect(component['isModelDropdownOpen']()).toBe(false); - }); -}); diff --git a/frontend/ai.client/src/app/components/model-settings/model-settings.ts b/frontend/ai.client/src/app/components/model-settings/model-settings.ts deleted file mode 100644 index 8b298320e..000000000 --- a/frontend/ai.client/src/app/components/model-settings/model-settings.ts +++ /dev/null @@ -1,571 +0,0 @@ -import { Component, ChangeDetectionStrategy, inject, input, output, signal, computed, effect, ElementRef } from '@angular/core'; -import { NgIcon, provideIcons } from '@ng-icons/core'; -import { heroXMark, heroCheck, heroChevronDown, heroChevronRight, heroArrowPath, heroLockClosed } from '@ng-icons/heroicons/outline'; -import { ModelService } from '../../session/services/model/model.service'; -import { ToolService, Tool } from '../../services/tool/tool.service'; -import { SkillService } from '../../services/skill/skill.service'; -import { SystemPromptsService } from '../../services/system-prompts/system-prompts.service'; -import { - KNOWN_PARAMS, - KnownParamMeta, - ManagedModel, - ModelParamSpec, - ModelProvider, -} from '../../admin/manage-models/models/managed-model.model'; - -/** Resolved row the template renders for a single inference param. */ -interface AdvancedParamRow { - key: string; - meta: KnownParamMeta; - spec: ModelParamSpec; - /** Effective min/max after merging catalog defaults with the model's spec. */ - min: number | null; - max: number | null; - /** Current effective value: user override if set, else the admin default. */ - value: unknown; - /** True when the user has overridden the admin default. */ - isOverridden: boolean; - /** Disabled because another active param's `incompatibleWith` includes us. */ - disabledByConflict: boolean; - /** Locked by the admin — show the value but block edits. */ - locked: boolean; - /** - * Set when the row's effective bounds collapse to nothing — e.g. thinking's - * floor (1024) is above the current `max_tokens − 1` cap. Surfacing this as - * a separate flag (vs. just disabling) lets the template explain *why*. - */ - unsatisfiable?: { reason: string }; -} - -@Component({ - selector: 'app-model-settings', - changeDetection: ChangeDetectionStrategy.OnPush, - imports: [NgIcon], - providers: [provideIcons({ heroXMark, heroCheck, heroChevronDown, heroChevronRight, heroArrowPath, heroLockClosed })], - host: { - '(document:click)': 'onDocumentClick($event)', - }, - templateUrl: './model-settings.html', - styleUrl: './model-settings.css', -}) -export class ModelSettings { - private elementRef = inject(ElementRef); - protected modelService = inject(ModelService); - protected toolService = inject(ToolService); - protected skillService = inject(SkillService); - protected systemPromptsService = inject(SystemPromptsService); - - // Input to control visibility - isOpen = input(false); - - // Session ID needed to persist prompt selection - sessionId = input(null); - - // Track if panel has ever been opened to avoid initial animation - protected hasBeenOpened = signal(false); - - // Model dropdown state - protected isModelDropdownOpen = signal(false); - protected focusedOptionIndex = signal(-1); - - // Advanced section collapse state. Default closed so the panel doesn't - // grow taller for users who never touch inference params. - protected isAdvancedOpen = signal(false); - protected isToolsOpen = signal(false); - protected isSkillsOpen = signal(false); - - // Per-param transient "clamped to N" notice keyed by param key. Cleared - // ~3s after it's set or the moment the user edits the row again. - protected clampNotices = signal>({}); - private clampTimers = new Map>(); - private static readonly CLAMP_NOTICE_MS = 3000; - - // Output event when panel should close - closed = output(); - - /** Effective merged inference-param view for the current model. */ - protected readonly advancedRows = computed(() => { - const model = this.modelService.selectedModel(); - const spec = model?.supportedParams?.params ?? {}; - const overrides = this.modelService.selectedModelOverrides(); - - // Active = user override if set, else admin default. Drives the - // incompatibility gate (e.g. thinking suppresses sampling params). - const activeValues: Record = {}; - for (const [key, paramSpec] of Object.entries(spec)) { - if (!paramSpec.supported) continue; - const override = overrides[key]; - activeValues[key] = override !== undefined ? override : paramSpec.default; - } - - const conflictedKeys = new Set(); - for (const meta of KNOWN_PARAMS) { - if (!meta.incompatibleWith?.length) continue; - const value = activeValues[meta.key]; - if (!value) continue; - for (const conflict of meta.incompatibleWith) conflictedKeys.add(conflict); - } - - // Anthropic requires `thinking budget < max_tokens`. Compute the - // effective max_tokens (override > admin default > provider bounds) - // so the thinking row's max input never exceeds budget − 1. - const maxTokensSpec = spec['max_tokens']; - const maxTokensActive = activeValues['max_tokens']; - const maxTokensProviderBounds = - KNOWN_PARAMS.find((p) => p.key === 'max_tokens')?.defaults?.[ - (model?.provider ?? 'bedrock') as ModelProvider - ]; - const effectiveMaxTokens = - typeof maxTokensActive === 'number' - ? maxTokensActive - : typeof maxTokensSpec?.default === 'number' - ? maxTokensSpec.default - : (maxTokensSpec?.max ?? maxTokensProviderBounds?.max ?? null); - - const rows: AdvancedParamRow[] = []; - for (const meta of KNOWN_PARAMS) { - const paramSpec = spec[meta.key]; - if (!paramSpec || !paramSpec.supported) continue; - const provider = (model?.provider ?? 'bedrock') as ModelProvider; - if (!meta.providers.includes(provider)) continue; - const providerBounds = meta.defaults?.[provider]; - const min = - paramSpec.min ?? - providerBounds?.min ?? - meta.defaultMin ?? - null; - let max = - paramSpec.max ?? - providerBounds?.max ?? - meta.defaultMax ?? - null; - let unsatisfiable: { reason: string } | undefined; - // Tighten the thinking budget cap to max_tokens − 1 so the form can't - // produce a request the Bedrock validator will reject. If the resulting - // window collapses (cap < min), mark the row unsatisfiable so the - // template can disable the toggle and explain *why* — otherwise the - // user can set it to a value that's both inside the input's HTML - // bounds and rejected at request time. - if (meta.key === 'thinking' && effectiveMaxTokens !== null) { - const cap = effectiveMaxTokens - 1; - max = max === null ? cap : Math.min(max, cap); - if (min !== null && max !== null && max < min) { - unsatisfiable = { - reason: - `Set Max Output Tokens above ${min} to enable extended thinking ` + - `(currently ${effectiveMaxTokens}).`, - }; - } - } - const override = overrides[meta.key]; - const value = override !== undefined ? override : paramSpec.default ?? null; - rows.push({ - key: meta.key, - meta, - spec: paramSpec, - min, - max, - value, - isOverridden: override !== undefined, - disabledByConflict: conflictedKeys.has(meta.key), - locked: !!paramSpec.locked, - unsatisfiable, - }); - } - return rows; - }); - - protected readonly hasAdvancedParams = computed(() => this.advancedRows().length > 0); - - protected readonly overriddenCount = computed( - () => this.advancedRows().filter((row) => row.isOverridden).length, - ); - - constructor() { - // Track when panel is first opened and manage body scroll - effect(() => { - const isOpen = this.isOpen(); - - if (isOpen && !this.hasBeenOpened()) { - this.hasBeenOpened.set(true); - } - - // Load the skills picker lazily on first open. SkillService deliberately - // has no constructor load (unlike ToolService): skills are opt-in and the - // feature is off in every deployed env until PR-5, so a boot-time fetch - // would be a guaranteed 404 for every user. `initialized` is set even on - // that 404, so this probes at most once per session. - if (isOpen && !this.skillService.initialized()) { - void this.skillService.loadSkills(); - } - - // Prevent background scrolling when panel is open - if (isOpen) { - document.body.style.overflow = 'hidden'; - } else { - document.body.style.overflow = ''; - } - }); - } - - onDocumentClick(event: MouseEvent): void { - // Close dropdown if clicking outside - if (this.isModelDropdownOpen() && !this.elementRef.nativeElement.contains(event.target)) { - this.isModelDropdownOpen.set(false); - } - } - - close(): void { - this.closed.emit(); - } - - toggleModelDropdown(): void { - this.isModelDropdownOpen.update(open => !open); - if (this.isModelDropdownOpen()) { - // Set focus to currently selected model - const models = this.modelService.availableModels(); - const selectedModel = this.modelService.selectedModel(); - const selectedIndex = models.findIndex(m => m.modelId === selectedModel?.modelId); - this.focusedOptionIndex.set(selectedIndex >= 0 ? selectedIndex : 0); - } - } - - selectModel(model: ManagedModel): void { - this.modelService.setSelectedModel(model); - this.isModelDropdownOpen.set(false); - } - - isModelSelected(model: ManagedModel): boolean { - return this.modelService.selectedModel()?.modelId === model.modelId; - } - - onDropdownKeydown(event: KeyboardEvent): void { - const models = this.modelService.availableModels(); - const currentIndex = this.focusedOptionIndex(); - - switch (event.key) { - case 'ArrowDown': - event.preventDefault(); - if (!this.isModelDropdownOpen()) { - this.isModelDropdownOpen.set(true); - this.focusedOptionIndex.set(0); - } else { - this.focusedOptionIndex.set(Math.min(currentIndex + 1, models.length - 1)); - } - break; - case 'ArrowUp': - event.preventDefault(); - if (this.isModelDropdownOpen()) { - this.focusedOptionIndex.set(Math.max(currentIndex - 1, 0)); - } - break; - case 'Enter': - case ' ': - event.preventDefault(); - if (this.isModelDropdownOpen() && currentIndex >= 0 && currentIndex < models.length) { - this.selectModel(models[currentIndex]); - } else { - this.toggleModelDropdown(); - } - break; - case 'Escape': - event.preventDefault(); - this.isModelDropdownOpen.set(false); - break; - case 'Tab': - this.isModelDropdownOpen.set(false); - break; - } - } - - /** Which MCP server rows are expanded to show their per-tool toggles. */ - protected expandedServers = signal>(new Set()); - /** Servers with a live discovery request in flight. */ - protected discoveringServers = signal>(new Set()); - /** Per-server discovery error messages. */ - protected discoverError = signal>({}); - - toggleTool(toolId: string): void { - this.toolService.toggleTool(toolId); - } - - /** True when a tool is an MCP server that supports per-tool enablement. */ - isMcpServer(tool: Tool): boolean { - return tool.protocol === 'mcp' || tool.protocol === 'mcp_external'; - } - - isServerExpanded(toolId: string): boolean { - return this.expandedServers().has(toolId); - } - - /** "3 of 8 tools enabled" when a server is partially enabled, else null. */ - partialServerLabel(tool: Tool): string | null { - const subs = tool.serverTools ?? []; - if (subs.length === 0) return null; - const on = subs.filter((s) => s.enabled).length; - if (on === 0 || on === subs.length) return null; - return `${on} of ${subs.length} tools enabled`; - } - - toggleServerExpanded(toolId: string): void { - this.expandedServers.update((set) => { - const next = new Set(set); - if (next.has(toolId)) { - next.delete(toolId); - } else { - next.add(toolId); - } - return next; - }); - } - - toggleServerTool(toolId: string, name: string): void { - this.toolService.toggleServerTool(toolId, name); - } - - async discoverServerTools(tool: Tool): Promise { - this.discoveringServers.update((s) => new Set(s).add(tool.toolId)); - this.discoverError.update((m) => { - const next = { ...m }; - delete next[tool.toolId]; - return next; - }); - try { - await this.toolService.discoverServerTools(tool.toolId); - } catch { - this.discoverError.update((m) => ({ - ...m, - [tool.toolId]: 'Could not list this server’s tools.', - })); - } finally { - this.discoveringServers.update((s) => { - const next = new Set(s); - next.delete(tool.toolId); - return next; - }); - } - } - - selectPrompt(promptId: string | null): void { - const sid = this.sessionId(); - this.systemPromptsService.setActivePrompt(sid, promptId) - .catch(err => console.error('Failed to persist prompt selection:', err)); - } - - toggleAdvanced(): void { - this.isAdvancedOpen.update((open) => !open); - } - - toggleTools(): void { - this.isToolsOpen.update((open) => !open); - } - - toggleSkills(): void { - this.isSkillsOpen.update((open) => !open); - } - - toggleSkill(skillId: string): void { - this.skillService.toggleSkill(skillId) - .catch(err => console.error('Failed to toggle skill:', err)); - } - - /** - * Read a coerced value off a number/range input. Returns `null` when the - * field is empty so the override is cleared rather than stored as 0. - */ - protected readNumberInput(event: Event): number | null { - const target = event.target as HTMLInputElement | null; - if (!target || target.value === '') return null; - const parsed = Number(target.value); - return Number.isFinite(parsed) ? parsed : null; - } - - onParamNumberChange(row: AdvancedParamRow, event: Event): void { - if (row.locked || row.disabledByConflict) return; - const raw = this.readNumberInput(event); - if (raw === null) { - this.clearClampNotice(row.key); - this.modelService.setInferenceParamOverride(row.key, null); - this.maybeAdjustThinkingForMaxTokens(row.key); - return; - } - const clamped = this.applyBounds(raw, row.min, row.max); - if (clamped !== raw) { - this.flashClampNotice(row.key, this.formatClampMessage(row, clamped)); - // Reflect the clamped value back into the input so the visible value - // matches what we'll actually send. Browser number inputs already - // refuse out-of-range submissions, but typed values can survive blur. - const target = event.target as HTMLInputElement | null; - if (target) target.value = String(clamped); - } else { - this.clearClampNotice(row.key); - } - this.modelService.setInferenceParamOverride(row.key, clamped); - this.maybeAdjustThinkingForMaxTokens(row.key); - } - - onParamToggle(row: AdvancedParamRow): void { - if (row.locked || row.disabledByConflict) return; - const next = !row.value; - this.modelService.setInferenceParamOverride(row.key, next); - } - - /** - * Enum-select params (e.g. `effort`). The empty option clears the override - * (fall back to the admin default), mirroring how emptying a number input - * clears it. Any non-empty value is sent verbatim; the server gates it - * against the model's `allowed` set, so an out-of-domain value can't slip - * through even if the option list is momentarily stale. - */ - onParamSelectChange(row: AdvancedParamRow, event: Event): void { - if (row.locked || row.disabledByConflict) return; - const target = event.target as HTMLSelectElement | null; - const raw = target?.value ?? ''; - this.modelService.setInferenceParamOverride(row.key, raw === '' ? null : raw); - } - - /** - * Extended thinking enable/disable. The stored value is `null` (off) or an - * int budget (on). Default budget falls back to the admin default, then to - * the catalog `defaultMin` (1024 for thinking). - * - * Refuses to enable when ``row.unsatisfiable`` is set — i.e. when the - * effective max_tokens window can't accommodate the budget floor. The - * template also disables the toggle in that state; this guard is defense - * in depth for keyboard/programmatic toggles. - */ - onThinkingToggle(row: AdvancedParamRow): void { - if (row.locked || row.disabledByConflict) return; - if (row.value) { - this.modelService.setInferenceParamOverride(row.key, null); - return; - } - if (row.unsatisfiable) return; - const fallback = - (typeof row.spec.default === 'number' ? row.spec.default : null) ?? - row.meta.defaultMin ?? - 1024; - // Pin the seed budget to the row's effective range so we never store a - // value the input itself would reject. - const seeded = this.applyBounds(fallback, row.min, row.max); - this.modelService.setInferenceParamOverride(row.key, seeded); - } - - onThinkingBudgetChange(row: AdvancedParamRow, event: Event): void { - if (row.locked || row.disabledByConflict) return; - const raw = this.readNumberInput(event); - if (raw === null || raw <= 0) { - this.clearClampNotice(row.key); - this.modelService.setInferenceParamOverride(row.key, null); - return; - } - // Clamp to the row's effective bounds (which already incorporate the - // `max_tokens − 1` cap from the advancedRows computation). Defensive - // because number inputs accept out-of-range values via keyboard. - const floored = Math.floor(raw); - const clamped = this.applyBounds(floored, row.min, row.max); - if (clamped !== floored) { - this.flashClampNotice(row.key, this.formatClampMessage(row, clamped)); - const target = event.target as HTMLInputElement | null; - if (target) target.value = String(clamped); - } else { - this.clearClampNotice(row.key); - } - this.modelService.setInferenceParamOverride(row.key, clamped); - } - - resetParam(row: AdvancedParamRow): void { - if (row.locked) return; - this.modelService.setInferenceParamOverride(row.key, null); - } - - resetAllParams(): void { - this.modelService.clearInferenceParamOverrides(); - } - - /** Template helper: extended thinking is on when value is a positive number. */ - protected isThinkingEnabled(value: unknown): boolean { - return typeof value === 'number' && value > 0; - } - - private applyBounds(value: number, min: number | null, max: number | null): number { - if (min !== null && value < min) return min; - if (max !== null && value > max) return max; - return value; - } - - /** - * After a max_tokens edit, re-check the thinking row's invariant - * (budget < max_tokens) and clear the budget if the new ceiling can no - * longer accommodate it. Avoids the "user lowers max_tokens, thinking - * silently violates invariant, request 400s at Bedrock" failure mode. - * - * No-op for any other param edit. Kept lazy and side-effecting so the - * `advancedRows` computed stays read-only — mutating model overrides - * inside a computed would create a glitchy reactive cycle. - */ - private maybeAdjustThinkingForMaxTokens(changedKey: string): void { - if (changedKey !== 'max_tokens') return; - const thinking = this.advancedRows().find((r) => r.key === 'thinking'); - if (!thinking) return; - if (!this.isThinkingEnabled(thinking.value)) return; - // Row was just rebuilt off the latest max_tokens — `unsatisfiable` is set - // when the floor (1024) now exceeds max_tokens-1, and `max` is the new - // cap otherwise. Either way, drop the override and tell the user. - if (thinking.unsatisfiable) { - this.modelService.setInferenceParamOverride(thinking.key, null); - this.flashClampNotice( - thinking.key, - 'Extended thinking turned off — Max Output Tokens is below the 1024 budget floor.', - ); - return; - } - if (typeof thinking.value === 'number' && thinking.max !== null && thinking.value > thinking.max) { - const reduced = thinking.max; - this.modelService.setInferenceParamOverride(thinking.key, reduced); - this.flashClampNotice( - thinking.key, - `Reduced to ${reduced} to stay below Max Output Tokens.`, - ); - } - } - - /** - * Phrase the clamp notice based on which bound was hit and which row it was - * — the thinking row mentions the max_tokens coupling explicitly because - * that's the most likely surprise for the user. - */ - private formatClampMessage(row: AdvancedParamRow, clampedTo: number): string { - if (row.key === 'thinking' && row.max !== null && clampedTo === row.max) { - return `Reduced to ${clampedTo} to stay below Max Output Tokens.`; - } - if (row.min !== null && clampedTo === row.min) { - return `Raised to ${clampedTo} (minimum allowed by this model).`; - } - return `Reduced to ${clampedTo} (maximum allowed by this model).`; - } - - private flashClampNotice(key: string, message: string): void { - this.clampNotices.update((current) => ({ ...current, [key]: message })); - const existing = this.clampTimers.get(key); - if (existing) clearTimeout(existing); - const handle = setTimeout(() => { - this.clearClampNotice(key); - }, ModelSettings.CLAMP_NOTICE_MS); - this.clampTimers.set(key, handle); - } - - private clearClampNotice(key: string): void { - const existing = this.clampTimers.get(key); - if (existing) { - clearTimeout(existing); - this.clampTimers.delete(key); - } - this.clampNotices.update((current) => { - if (!(key in current)) return current; - const next = { ...current }; - delete next[key]; - return next; - }); - } -} diff --git a/frontend/ai.client/src/app/components/pulsating-loader.component.spec.ts b/frontend/ai.client/src/app/components/pulsating-loader.component.spec.ts index a2eda3901..871db4f8e 100644 --- a/frontend/ai.client/src/app/components/pulsating-loader.component.spec.ts +++ b/frontend/ai.client/src/app/components/pulsating-loader.component.spec.ts @@ -1,59 +1,233 @@ -import { Component } from '@angular/core'; -import { TestBed } from '@angular/core/testing'; -import { describe, it, expect } from 'vitest'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { PulsatingLoaderComponent } from './pulsating-loader.component'; -@Component({ - imports: [PulsatingLoaderComponent], - template: ``, -}) -class HostComponent { - notice: string | null = null; +interface LoaderInputs { + notice: string | null; + status: string | null; + statusTool: string | null; + startedAt: number | null; } -function render(notice: string | null) { - const fixture = TestBed.createComponent(HostComponent); - fixture.componentInstance.notice = notice; - fixture.detectChanges(); +/** + * Driven through `setInput` on the component itself rather than a host + * wrapper: mutating host fields after the first change-detection pass trips + * NG0100 in dev mode, which is a harness artifact rather than anything the + * component does wrong. + */ +function render( + props: Partial = {}, +): ComponentFixture { + const fixture = TestBed.createComponent(PulsatingLoaderComponent); + setInputs(fixture, { notice: null, status: null, statusTool: null, startedAt: null, ...props }); return fixture; } +function setInputs( + fixture: ComponentFixture, + props: Partial, +) { + for (const [key, value] of Object.entries(props)) { + fixture.componentRef.setInput(key, value); + } + fixture.detectChanges(); +} + +const textOf = (fixture: { nativeElement: HTMLElement }) => + (fixture.nativeElement.textContent ?? '').replace(/\s+/g, ' ').trim(); + +/** Just the state phrase, without the bullet separators or the timer. */ +const stateOf = (fixture: { nativeElement: HTMLElement }) => + (fixture.nativeElement.querySelector('.state')?.textContent ?? '') + .replace(/\s+/g, ' ') + .trim(); + describe('PulsatingLoaderComponent', () => { - it('cycles its own phrases when no notice is set', () => { - const fixture = render(null); - const loader = fixture.debugElement.children[0].componentInstance as PulsatingLoaderComponent; - // The typewriter starts empty and fills in; what matters is that it is not - // showing a caller-supplied string. - expect(loader.displayText()).not.toContain('Retrying'); + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + describe('what it says', () => { + it('shows the live state from agent_status', () => { + expect(stateOf(render({ status: 'Thinking' }))).toBe('Thinking\u2026'); + }); + + it('shows the running tool by its real name', () => { + // The identifier is the most accurate label available while a tool runs, + // and is the same one the tool rail and admin catalog use. + const fixture = render({ status: 'Running', statusTool: 'list_assignments' }); + expect(stateOf(fixture)).toBe('Running list_assignments\u2026'); + }); + + it('renders the tool name as an identifier, not prose', () => { + const fixture = render({ status: 'Running', statusTool: 'list_assignments' }); + const mono = fixture.nativeElement.querySelector('.font-mono'); + expect(mono?.textContent?.trim()).toBe('list_assignments'); + }); + + it('invents nothing when there is no state yet', () => { + // Regression guard: this component used to cycle twenty fabricated + // phrases ("Pondering", "Cross-referencing") that looked identical + // whether the model was generating, waiting on a tool, or hung. + // + // The fallback is "Thinking", not a vaguer hedge: on a cold start the + // gap before the first agent_status can run several seconds, and that + // gap is the only thing the user sees. + expect(stateOf(render())).toBe('Thinking\u2026'); + }); + + it('lets a notice outrank the state', () => { + // A retry in progress is the more important truth. + const fixture = render({ + notice: 'The model is busy. Retrying…', + status: 'Thinking', + }); + const text = textOf(fixture); + expect(text).toContain('The model is busy. Retrying'); + expect(text).not.toContain('Thinking'); + }); + + it('does not double the punctuation a notice already carries', () => { + // Every notice string ends in its own punctuation. + expect(stateOf(render({ notice: 'Still working\u2026' }))).toBe('Still working\u2026'); + expect(stateOf(render({ notice: 'The model is busy. Retrying \u2014 attempt 2.' }))).toBe( + 'The model is busy. Retrying \u2014 attempt 2.', + ); + }); }); - it('shows the notice verbatim instead of a loading phrase', () => { - const fixture = render('The model is busy. Retrying…'); - const text = (fixture.nativeElement as HTMLElement).textContent ?? ''; - expect(text).toContain('The model is busy. Retrying'); + describe('layout', () => { + it('orders the line pulse, timer, state', () => { + const fixture = render({ status: 'Thinking', startedAt: Date.now() }); + const row = fixture.nativeElement.querySelector('[role="status"]')!; + const kinds = [...row.children].map(el => + el.classList.contains('pulse-dot') + ? 'dot' + : el.classList.contains('sep') + ? 'sep' + : el.classList.contains('state') + ? 'state' + : 'timer', + ); + expect(kinds).toEqual(['dot', 'sep', 'timer', 'sep', 'state']); + }); + + it('drops the second bullet when there is no timer', () => { + // A dangling separator next to nothing reads as a rendering bug. + const fixture = render({ status: 'Thinking' }); + expect(fixture.nativeElement.querySelectorAll('.sep').length).toBe(1); + }); + + it('shimmers the state on the healthy path', () => { + const state = render({ status: 'Thinking' }).nativeElement.querySelector('.state'); + expect(state?.classList.contains('shimmer')).toBe(true); + }); + + it('does not shimmer a notice', () => { + // A warning that shimmers reads as decoration rather than a warning. + const state = render({ notice: 'Still working…' }).nativeElement.querySelector('.state'); + expect(state?.classList.contains('shimmer')).toBe(false); + expect(state?.classList.contains('is-notice')).toBe(true); + }); + + it('rebuilds the state node when the state changes, so it can animate', () => { + // `@for ... track` over the visible text is what makes the enter + // keyframe run; a plain interpolation would mutate text in place and + // never animate. + const fixture = render({ status: 'Thinking', startedAt: Date.now() }); + const first = fixture.nativeElement.querySelector('.state'); + + setInputs(fixture, { status: 'Running', statusTool: 'browse_web' }); + + expect(fixture.nativeElement.querySelector('.state')).not.toBe(first); + }); + + it('keeps the same state node while only the timer ticks', () => { + // The state must not re-animate once a second. + const fixture = render({ status: 'Thinking', startedAt: Date.now() }); + const first = fixture.nativeElement.querySelector('.state'); + + vi.advanceTimersByTime(3000); + fixture.detectChanges(); + + expect(fixture.nativeElement.querySelector('.state')).toBe(first); + }); }); - it('drops the typing cursor for a notice', () => { - // The cursor reads as "still typing" on text that is finished. - const withNotice = render('Still working…'); - expect((withNotice.nativeElement as HTMLElement).querySelector('.typing-cursor')).toBeNull(); + describe('elapsed timer', () => { + it('counts up in seconds', () => { + const fixture = render({ status: 'Thinking', startedAt: Date.now() }); + expect(textOf(fixture)).toContain('0s'); + + vi.advanceTimersByTime(3000); + fixture.detectChanges(); + expect(textOf(fixture)).toContain('3s'); + }); + + it('switches to minutes past sixty seconds', () => { + const fixture = render({ status: 'Thinking', startedAt: Date.now() }); + + vi.advanceTimersByTime(64_000); + fixture.detectChanges(); + expect(textOf(fixture)).toContain('1m 4s'); + }); - const withoutNotice = render(null); - expect( - (withoutNotice.nativeElement as HTMLElement).querySelector('.typing-cursor'), - ).not.toBeNull(); + it('is hidden when no start time is known', () => { + // A zero that never moves is worse than no timer. + expect(textOf(render({ status: 'Thinking' }))).not.toContain('0s'); + }); + + it('never counts backwards from a clock skew', () => { + const fixture = render({ status: 'Thinking', startedAt: Date.now() + 5000 }); + expect(textOf(fixture)).toContain('0s'); + }); + + it('stops ticking when destroyed', () => { + const fixture = render({ status: 'Thinking', startedAt: Date.now() }); + const clearSpy = vi.spyOn(globalThis, 'clearInterval'); + + fixture.destroy(); + + expect(clearSpy).toHaveBeenCalled(); + }); }); - it('marks the indicator dot so the change is visible peripherally', () => { - const fixture = render('Still working…'); - const dot = (fixture.nativeElement as HTMLElement).querySelector('.pulsing-circle'); - expect(dot?.classList.contains('is-notice')).toBe(true); + describe('the dot', () => { + it('is a plain pulse on the healthy path', () => { + const dot = render({ status: 'Thinking' }).nativeElement.querySelector('.pulse-dot'); + expect(dot).not.toBeNull(); + expect(dot?.classList.contains('is-notice')).toBe(false); + }); + + it('changes colour for a notice', () => { + // The dot is the part a user tracks peripherally; changing only the text + // leaves the indicator looking routine during an outage. + const dot = render({ notice: 'Still working…' }).nativeElement.querySelector('.pulse-dot'); + expect(dot?.classList.contains('is-notice')).toBe(true); + }); }); - it('announces a notice to assistive tech', () => { - const fixture = render('Still working…'); - const status = (fixture.nativeElement as HTMLElement).querySelector('[role="status"]'); - expect(status?.getAttribute('aria-live')).toBe('polite'); - expect(status?.getAttribute('aria-label')).toContain('Still working'); + describe('accessibility', () => { + it('announces the state politely', () => { + const status = render({ notice: 'Still working…' }).nativeElement.querySelector( + '[role="status"]', + ); + expect(status?.getAttribute('aria-live')).toBe('polite'); + expect(status?.getAttribute('aria-label')).toContain('Still working'); + }); + + it('includes the tool name in the accessible name', () => { + const status = render({ + status: 'Running', + statusTool: 'list_assignments', + }).nativeElement.querySelector('[role="status"]'); + expect(status?.getAttribute('aria-label')).toBe('Running list_assignments'); + }); + + it('keeps the ticking timer out of the announcement', () => { + // A per-second re-announcement would be noise for a screen reader. + const fixture = render({ status: 'Thinking', startedAt: Date.now() }); + const timer = fixture.nativeElement.querySelector('.tabular-nums'); + expect(timer?.getAttribute('aria-hidden')).toBe('true'); + }); }); }); diff --git a/frontend/ai.client/src/app/components/pulsating-loader.component.ts b/frontend/ai.client/src/app/components/pulsating-loader.component.ts index 47d01504f..2d4b4b75c 100644 --- a/frontend/ai.client/src/app/components/pulsating-loader.component.ts +++ b/frontend/ai.client/src/app/components/pulsating-loader.component.ts @@ -6,86 +6,79 @@ import { input, OnInit, OnDestroy, + inject, + PLATFORM_ID, } from '@angular/core'; - -/** - * University-themed loading phrases for the typewriter effect - */ -const LOADING_PHRASES = [ - 'Forming a hypothesis', - 'Calculating', - 'Inferring', - 'Deriving', - 'Researching', - 'Analyzing data', - 'Reviewing literature', - 'Running experiments', - 'Consulting the archives', - 'Checking citations', - 'Cross-referencing', - 'Synthesizing findings', - 'Examining variables', - 'Testing assumptions', - 'Evaluating evidence', - 'Compiling results', - 'Pondering', - 'Deliberating', - 'Theorizing', - 'Extrapolating', -]; +import { isPlatformBrowser } from '@angular/common'; /** * PulsatingLoaderComponent * - * A loading indicator featuring a pulsing circle with expanding ring effect - * and a typewriter-style text animation. The text cycles through university-themed - * loading phrases, typing in character by character, pausing, then deleting - * before showing the next phrase. + * The line shown while a turn is running: a small pulsing dot, what the agent + * is doing, and how long it has been doing it. * - * When `notice` is set, the playful cycling stops and the loader states a - * specific fact instead — used when the backend is retrying a failed model - * call. A retry is otherwise indistinguishable from a hang, and cheerful - * phrases like "Pondering..." during a provider outage actively mislead. + * WHAT THIS DELIBERATELY NO LONGER DOES + * ------------------------------------- + * It used to cycle twenty invented phrases — "Pondering", "Cross-referencing", + * "Consulting the archives" — typed out character by character. They were + * charming and they were fiction: identical whether the model was generating, + * waiting on a Canvas round trip, or hung. A user watching "Cross-referencing" + * for ninety seconds learned nothing, and two of those ninety-second turns got + * abandoned in prod. * - * @example - * ```html - * - * - * ``` + * Everything shown here is now a fact we actually hold: + * + * - `status` comes from the runtime's `agent_status` events — the event loop's + * own model-call and tool-call boundaries. + * - `statusTool` is the tool's real name. While a tool runs, its name is the + * most accurate label available and invents nothing. + * - the elapsed timer is measured from the moment the turn was sent. + * + * `notice` outranks both. It states a specific fact that is NOT the healthy + * path (the model is being retried), so it takes the warning colour and the + * amber dot — the dot matters because it is the part a user tracks + * peripherally, and changing only the text leaves the indicator looking + * routine during an outage. */ @Component({ selector: 'app-pulsating-loader', changeDetection: ChangeDetectionStrategy.OnPush, template: `
- - + + + - -
+ + @if (elapsedLabel(); as elapsed) { + + + } + + + @for (frame of stateFrames(); track frame) { - {{ displayText() }} + @if (statusTool(); as tool) { + {{ label() }} {{ tool }}{{ trailer() }} + } @else { + {{ label() }}{{ trailer() }} + } - @if (!notice()) { - - } -
+ }
`, }) export class PulsatingLoaderComponent implements OnInit, OnDestroy { + private platformId = inject(PLATFORM_ID); + /** - * Fixed message that replaces the cycling phrases, e.g. a retry in - * progress. Null (the default) keeps the normal typewriter behaviour. + * A specific fact that is not the healthy path — currently only "the model + * is being retried". Outranks `status`: a retry in progress is the more + * important truth. */ notice = input(null); - // Base timing constants (in milliseconds) - private readonly TYPE_SPEED_BASE = 45; - private readonly TYPE_SPEED_VARIANCE = 35; - private readonly DELETE_SPEED_BASE = 15; - private readonly DELETE_SPEED_VARIANCE = 10; - private readonly PAUSE_AFTER_TYPING = 1200; - private readonly PAUSE_AFTER_DELETING = 300; - - // Characters that cause slight hesitation (less common, harder to reach) - private readonly SLOW_CHARS = new Set(['z', 'x', 'q', 'j', 'k', 'v', 'b', 'p', 'y', 'w']); - // Characters that flow quickly (home row, common) - private readonly FAST_CHARS = new Set(['a', 's', 'd', 'f', 'e', 'r', 't', 'i', 'o', 'n', ' ']); - - // State signals - private currentPhraseIndex = signal(0); - private currentCharIndex = signal(0); - private isDeleting = signal(false); - private isPaused = signal(false); - - // Timer reference for cleanup - private animationTimer: ReturnType | null = null; - - // Computed display text with ellipsis. A notice wins outright; the - // typewriter keeps ticking underneath so it resumes cleanly when the - // notice clears mid-turn. - displayText = computed(() => { - const notice = this.notice(); - if (notice) { - return notice; - } - const phrase = LOADING_PHRASES[this.currentPhraseIndex()] + '...'; - return phrase.substring(0, this.currentCharIndex()); - }); - - ngOnInit(): void { - this.startAnimation(); - } - - ngOnDestroy(): void { - if (this.animationTimer) { - clearTimeout(this.animationTimer); - } - } - - private startAnimation(): void { - this.tick(); - } - - private tick(): void { - const currentPhrase = LOADING_PHRASES[this.currentPhraseIndex()] + '...'; - const charIndex = this.currentCharIndex(); - const deleting = this.isDeleting(); - - if (this.isPaused()) { - return; // Wait for pause to complete - } + /** + * What the agent is doing, from `agent_status`. Null falls back to the + * generic waiting label. + */ + status = input(null); - if (!deleting) { - // Typing mode - if (charIndex < currentPhrase.length) { - // Type next character - this.currentCharIndex.update((v) => v + 1); - const nextChar = currentPhrase[charIndex] || ''; - this.scheduleNextTick(this.getTypingDelay(nextChar)); - } else { - // Finished typing, pause then start deleting - this.isPaused.set(true); - this.animationTimer = setTimeout(() => { - this.isPaused.set(false); - this.isDeleting.set(true); - this.tick(); - }, this.PAUSE_AFTER_TYPING); - } - } else { - // Deleting mode (faster, less variance - like holding backspace) - if (charIndex > 0) { - // Delete previous character - this.currentCharIndex.update((v) => v - 1); - this.scheduleNextTick(this.getDeletingDelay()); - } else { - // Finished deleting, pause then move to next phrase - this.isPaused.set(true); - this.animationTimer = setTimeout(() => { - this.isPaused.set(false); - this.isDeleting.set(false); - // Move to next phrase (random selection) - this.selectNextPhrase(); - this.tick(); - }, this.PAUSE_AFTER_DELETING); - } - } - } + /** + * The running tool's own name, rendered in mono beside `status`. Separate + * from `status` so the identifier is visibly an identifier. + */ + statusTool = input(null); /** - * Calculate typing delay based on character difficulty and randomness + * Epoch ms the turn started. Null hides the timer entirely rather than + * showing a zero that never moves. */ - private getTypingDelay(char: string): number { - const lowerChar = char.toLowerCase(); - let baseDelay = this.TYPE_SPEED_BASE; - - // Adjust base delay based on character - if (this.FAST_CHARS.has(lowerChar)) { - baseDelay *= 0.7; // Faster for common/easy chars - } else if (this.SLOW_CHARS.has(lowerChar)) { - baseDelay *= 1.4; // Slower for uncommon/harder chars - } + startedAt = input(null); - // Add pause after spaces (natural word break) - if (char === ' ') { - baseDelay += Math.random() * 60; - } + /** Ticks once a second so the elapsed readout recomputes. */ + private readonly now = signal(Date.now()); + private timer: ReturnType | null = null; - // Random variance for organic feel - const variance = (Math.random() - 0.5) * 2 * this.TYPE_SPEED_VARIANCE; + /** + * Falls back to "Thinking" rather than a vaguer word. + * + * Before the first `agent_status` arrives there is a real gap — the request + * in flight, the session loading, the agent building — and we cannot tell + * those apart. But from the user's side every one of them is the same fact: + * the assistant has the turn and has not answered yet. "Thinking" states + * that; "Working" was a hedge that said less and, on a cold start, was the + * only thing shown for the first several seconds. + */ + protected readonly label = computed( + () => this.notice() ?? this.status() ?? 'Thinking', + ); - // Occasional micro-pause (5% chance) - simulates brief hesitation - const microPause = Math.random() < 0.05 ? 80 : 0; + /** + * The trailing ellipsis on the live state — "Thinking…", "Running + * list_assignments…" — which reads as the ongoing action it is. + * + * A notice gets none: every notice string already ends in its own + * punctuation ("Still working…", "…attempt 2."), so appending here would + * double it. + */ + protected readonly trailer = computed(() => (this.notice() ? '' : '…')); + + protected readonly elapsedLabel = computed(() => { + const started = this.startedAt(); + if (!started) return null; + const seconds = Math.max(0, Math.floor((this.now() - started) / 1000)); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + return `${minutes}m ${seconds % 60}s`; + }); - return Math.max(15, baseDelay + variance + microPause); - } + /** + * A single frame keyed by the visible text. + * + * `@for ... track frame` over this is what animates the state change: when + * the text differs the old node is destroyed and a new one created, so the + * enter keyframe runs. A plain interpolation would mutate the text in place + * and never animate. The timer stays outside this loop deliberately — it + * changes every second and must not re-animate. + */ + protected readonly stateFrames = computed(() => { + const tool = this.statusTool(); + return [tool ? `${this.label()} ${tool}` : this.label()]; + }); /** - * Calculate deletion delay - faster and more consistent (like holding backspace) + * The timer is `aria-hidden` and re-announced text would be noise, so the + * accessible name carries the state only. */ - private getDeletingDelay(): number { - const variance = (Math.random() - 0.5) * 2 * this.DELETE_SPEED_VARIANCE; - return Math.max(10, this.DELETE_SPEED_BASE + variance); - } + protected readonly ariaLabel = computed(() => { + const tool = this.statusTool(); + return tool ? `${this.label()} ${tool}` : this.label(); + }); - private scheduleNextTick(delay: number): void { - this.animationTimer = setTimeout(() => this.tick(), delay); + ngOnInit(): void { + // No interval during SSR: it would never fire and would keep the platform + // from stabilising. + if (!isPlatformBrowser(this.platformId)) return; + this.timer = setInterval(() => this.now.set(Date.now()), 1000); } - private selectNextPhrase(): void { - // Select a random phrase different from the current one - let nextIndex: number; - do { - nextIndex = Math.floor(Math.random() * LOADING_PHRASES.length); - } while (nextIndex === this.currentPhraseIndex() && LOADING_PHRASES.length > 1); - - this.currentPhraseIndex.set(nextIndex); + ngOnDestroy(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } } } diff --git a/frontend/ai.client/src/app/components/sidenav/sidenav.branding.spec.ts b/frontend/ai.client/src/app/components/sidenav/sidenav.branding.spec.ts index c8512ee7b..7dcafd836 100644 --- a/frontend/ai.client/src/app/components/sidenav/sidenav.branding.spec.ts +++ b/frontend/ai.client/src/app/components/sidenav/sidenav.branding.spec.ts @@ -7,7 +7,7 @@ // `chat-container.component.branding.spec.ts`. import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { TestBed } from '@angular/core/testing'; -import { Router } from '@angular/router'; +import { provideRouter } from '@angular/router'; import { Component, input, output, signal } from '@angular/core'; import fc from 'fast-check'; @@ -15,9 +15,6 @@ import { SessionService } from '../../session/services/session/session.service'; import { UserService } from '../../auth/user.service'; import { SessionService as BffSessionService } from '../../auth/session.service'; import { SidenavService } from '../../services/sidenav/sidenav.service'; -import { MemorySpaceService } from '../../memory-spaces/services/memory-space.service'; -import { AgentService } from '../../agents/services/agent.service'; -import { LEGACY_MIGRATION_HOST } from '../../shared/utils/legacy-migration-host'; import { BrandingService } from '../../../branding/branding.service'; /** @@ -76,7 +73,10 @@ describe('Sidenav — Property 9: Logo alt text equals normalized app name', () TestBed.configureTestingModule({ providers: [ - { provide: Router, useValue: { navigate: vi.fn() } }, + // A real (empty-config) router, not a `navigate` stub: the nav's + // `routerLink` entries instantiate `RouterLink`, which resolves + // `ActivatedRoute` and builds hrefs through the router itself. + provideRouter([]), { provide: SessionService, useValue: { @@ -109,15 +109,6 @@ describe('Sidenav — Property 9: Logo alt text equals normalized app name', () canAccessAdmin: signal(false), }, }, - { - provide: MemorySpaceService, - useValue: { accessible$: signal(false), loadSpaces: vi.fn().mockResolvedValue(undefined) }, - }, - { - provide: AgentService, - useValue: { accessible$: signal(false), loadAgents: vi.fn().mockResolvedValue(undefined) }, - }, - { provide: LEGACY_MIGRATION_HOST, useValue: false }, { provide: BrandingService, useValue: mockBranding }, ], }); diff --git a/frontend/ai.client/src/app/components/sidenav/sidenav.error-handling.spec.ts b/frontend/ai.client/src/app/components/sidenav/sidenav.error-handling.spec.ts index a5e0eb55b..ede4bffdf 100644 --- a/frontend/ai.client/src/app/components/sidenav/sidenav.error-handling.spec.ts +++ b/frontend/ai.client/src/app/components/sidenav/sidenav.error-handling.spec.ts @@ -7,9 +7,6 @@ import { SessionService } from '../../session/services/session/session.service'; import { UserService } from '../../auth/user.service'; import { SessionService as BffSessionService } from '../../auth/session.service'; import { SidenavService } from '../../services/sidenav/sidenav.service'; -import { MemorySpaceService } from '../../memory-spaces/services/memory-space.service'; -import { AgentService } from '../../agents/services/agent.service'; -import { LEGACY_MIGRATION_HOST } from '../../shared/utils/legacy-migration-host'; /** * Feature: branding-customization @@ -76,15 +73,6 @@ describe('Sidenav — branding logo theme swap and error handling', () => { canAccessAdmin: signal(false), }, }, - { - provide: MemorySpaceService, - useValue: { accessible$: signal(false), loadSpaces: vi.fn().mockResolvedValue(undefined) }, - }, - { - provide: AgentService, - useValue: { accessible$: signal(false), loadAgents: vi.fn().mockResolvedValue(undefined) }, - }, - { provide: LEGACY_MIGRATION_HOST, useFactory: () => false }, ], }); }); diff --git a/frontend/ai.client/src/app/components/sidenav/sidenav.html b/frontend/ai.client/src/app/components/sidenav/sidenav.html index 2caf9584b..e153e337b 100644 --- a/frontend/ai.client/src/app/components/sidenav/sidenav.html +++ b/frontend/ai.client/src/app/components/sidenav/sidenav.html @@ -43,106 +43,75 @@ - @if (showAgents()) { - - @if (showAssistantsSignpost) { - -
- - - -
- Assistants - - } - - -
- - - -
- Agents - + +
+ + + +
+ Artifacts + - ⚠️ White on 500 measures exactly 4.50:1 — it clears the AA floor for - 10px text with nothing to spare. Any change that darkens the text, - lightens the fill, or puts this treatment on smaller type needs the - contrast re-measured, not assumed. + - New - - } + `/my-skills` no longer exists as its own route — Customize → Skills + absorbed it, so authoring and enablement are one surface reached from + this one entry, without a fourth top-level item competing with the + three that matter. - + +
+ + + +
+ Customize +