diff --git a/.gitignore b/.gitignore index b643784..94b21e9 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,11 @@ agent/gitleaks-report.json *.bkp Plans/ +# draw.io editor artifacts + any rendered screenshots kept locally for review +.$*.drawio.bkp +docs/diagrams/*.png +docs/diagrams/screenshots/ + # ────────────────────────────────────────────── # Explicit keeps (override ignores above) # ────────────────────────────────────────────── diff --git a/AGENTS.md b/AGENTS.md index b254c35..eaeb599 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,7 @@ Use this routing before editing so the right package and tests get updated: | Shared API request/response shapes | `cdk/src/handlers/shared/types.ts` | **`cli/src/types.ts`** (must stay in sync) | | `bgagent` CLI commands and HTTP client | `cli/src/`, `cli/test/` | `cli/src/types.ts` if API types change | | Agent runtime (clone, tools, prompts, container) | `agent/src/` (`pipeline.py`, `runner.py`, `config.py`, `hooks.py`, `policy.py`, `prompts/`, Dockerfile, etc.) | `agent/tests/`, `agent/README.md` for env/PAT | +| Agent progress events (written to `TaskEventsTable` from the MicroVM; read by `bgagent watch`) | `agent/src/progress_writer.py`, `agent/src/pipeline.py` and `agent/src/runner.py` (integration points) | `agent/tests/test_progress_writer.py`; `cli/src/commands/watch.ts` for the consumer side | | User-facing or design prose | `docs/guides/`, `docs/design/` | Run **`mise //docs:sync`** or **`mise //docs:build`** (do not edit `docs/src/content/docs/` by hand) | | Monorepo tasks, CI glue | Root `mise.toml`, `scripts/`, `.github/workflows/` | — | diff --git a/agent/README.md b/agent/README.md index 37c782f..453d73e 100644 --- a/agent/README.md +++ b/agent/README.md @@ -98,6 +98,7 @@ The `run.sh` script overrides the container's default CMD to run `python /app/sr | `MAX_BUDGET_USD` | No | | **Local batch only** (shell env when running `entrypoint.py` directly). Range 0.01–100; agent stops when the budget is reached. For deployed AgentCore **server** mode and production tasks, set **`max_budget_usd`** on task creation (REST API, CLI `--max-budget`, or Blueprint default); the orchestrator sends it in the `/invocations` JSON body — server mode does not read `MAX_BUDGET_USD` from the environment. | | `DRY_RUN` | No | | Set to `1` to validate config and print the prompt without running the agent | | `ANTHROPIC_DEFAULT_HAIKU_MODEL` | No | `anthropic.claude-haiku-4-5-20251001-v1:0` | Bedrock model ID for the pre-flight safety check (see below) | +| `NUDGES_TABLE_NAME` | No | | **Phase 2.** DynamoDB table for mid-task user nudges (`` XML blocks injected between turns). If unset, the agent runs without nudge support — `nudge_reader.read_pending()` returns `[]` and logs a WARN once. Set automatically by the CDK stack on both AgentCore runtimes. | **Pre-flight check model**: Claude Code runs a quick safety verification using a small Haiku model before executing each tool command. On Bedrock, the default Haiku model ID may not be enabled in your account, causing the check to time out with *"Pre-flight check is taking longer than expected"* warnings. The agent sets `ANTHROPIC_DEFAULT_HAIKU_MODEL` to a known-available Bedrock Haiku model ID to avoid this. If you see pre-flight timeout warnings, verify that this model is enabled in your Bedrock model access settings. diff --git a/agent/docker-compose.yml b/agent/docker-compose.yml new file mode 100644 index 0000000..0d00f8d --- /dev/null +++ b/agent/docker-compose.yml @@ -0,0 +1,24 @@ +# Local development services for agent testing. +# +# Usage: +# docker compose up -d # Start DynamoDB Local +# docker compose down # Stop and clean up +# +# The agent container (run via run.sh --local-events) connects to +# the "agent-local" network to reach DynamoDB Local at +# http://dynamodb-local:8000. + +services: + dynamodb-local: + image: amazon/dynamodb-local:latest + container_name: dynamodb-local + ports: + - "8000:8000" + command: ["-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb"] + networks: + - agent-local + +networks: + agent-local: + name: agent-local + driver: bridge diff --git a/agent/mise.toml b/agent/mise.toml index 43da38e..8beb605 100644 --- a/agent/mise.toml +++ b/agent/mise.toml @@ -72,3 +72,38 @@ run = [ { task = "security:bandit" }, { task = "security:image" }, ] + +# LOCAL DEVELOPMENT (DynamoDB Local for progress events) + +[tasks."local:up"] +description = "Start DynamoDB Local and create tables for local agent testing" +run = [ + "docker compose up -d", + "bash scripts/create-local-tables.sh", +] + +[tasks."local:down"] +description = "Stop DynamoDB Local (all data is ephemeral)" +run = "docker compose down" + +[tasks."local:events"] +description = "Query progress events from DynamoDB Local" +run = """ +aws dynamodb scan \ + --table-name TaskEventsTable \ + --endpoint-url http://localhost:8000 \ + --region us-east-1 \ + --no-cli-pager \ + --output table 2>/dev/null || echo "No events found (is DynamoDB Local running?)" +""" + +[tasks."local:events:json"] +description = "Query progress events from DynamoDB Local (JSON)" +run = """ +aws dynamodb scan \ + --table-name TaskEventsTable \ + --endpoint-url http://localhost:8000 \ + --region us-east-1 \ + --no-cli-pager \ + --output json 2>/dev/null || echo "{}" +""" diff --git a/agent/run.sh b/agent/run.sh index ac61028..da5fb57 100755 --- a/agent/run.sh +++ b/agent/run.sh @@ -8,12 +8,16 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # --------------------------------------------------------------------------- usage() { cat <<'EOF' -Usage: ./agent/run.sh [--server] [args...] +Usage: ./agent/run.sh [--server] [--local-events] [args...] Modes: (default) Local batch mode — runs the agent, then exits --server Server mode — starts FastAPI on port 8080 (/invocations + /ping) +Flags: + --local-events Connect to DynamoDB Local (port 8000) for progress events. + Requires: docker compose up -d && ./agent/scripts/create-local-tables.sh + The second argument (after flags) is auto-detected: - If numeric, treated as a GitHub issue number - Otherwise, treated as a task description @@ -47,6 +51,9 @@ Examples: # Local mode — dry run (print prompt, don't invoke agent) DRY_RUN=1 ./agent/run.sh "myorg/myrepo" 42 + # Local mode with progress events to DynamoDB Local + ./agent/run.sh --local-events "myorg/myrepo" 42 + # Server mode — start FastAPI, then invoke via curl ./agent/run.sh --server "myorg/myrepo" curl http://localhost:8080/ping @@ -61,6 +68,7 @@ EOF # Parse flags # --------------------------------------------------------------------------- MODE="local" +LOCAL_EVENTS=false while [[ $# -gt 0 ]]; do case "$1" in @@ -68,6 +76,10 @@ while [[ $# -gt 0 ]]; do MODE="server" shift ;; + --local-events) + LOCAL_EVENTS=true + shift + ;; --help|-h) usage ;; @@ -206,6 +218,24 @@ DOCKER_ARGS=( [[ -n "${MAX_TURNS:-}" ]] && DOCKER_ARGS+=(-e "MAX_TURNS=${MAX_TURNS}") [[ -n "${MAX_BUDGET_USD:-}" ]] && DOCKER_ARGS+=(-e "MAX_BUDGET_USD=${MAX_BUDGET_USD}") +# Local events mode: connect to DynamoDB Local via the agent-local network +if [[ "$LOCAL_EVENTS" == true ]]; then + # Verify DynamoDB Local is running + if ! docker inspect dynamodb-local >/dev/null 2>&1; then + echo "ERROR: DynamoDB Local is not running." >&2 + echo " Start it with: cd agent && docker compose up -d" >&2 + echo " Create tables: ./agent/scripts/create-local-tables.sh" >&2 + exit 1 + fi + DOCKER_ARGS+=( + --network agent-local + -e "TASK_EVENTS_TABLE_NAME=TaskEventsTable" + -e "TASK_TABLE_NAME=TaskTable" + -e "AWS_ENDPOINT_URL_DYNAMODB=http://dynamodb-local:8000" + ) + echo " Events: DynamoDB Local (http://localhost:8000)" +fi + # Server mode: expose port 8080 if [[ "$MODE" == "server" ]]; then DOCKER_ARGS+=(-p 8080:8080) @@ -236,6 +266,9 @@ echo "Monitor in another terminal:" echo " docker logs -f ${CONTAINER_NAME} # live output" echo " docker stats ${CONTAINER_NAME} # CPU, memory, disk I/O" echo " docker exec ${CONTAINER_NAME} du -sh /workspace # disk usage" +if [[ "$LOCAL_EVENTS" == true ]]; then +echo " mise run local:events # query progress events" +fi echo "" if [[ "$MODE" == "server" ]]; then diff --git a/agent/scripts/create-local-tables.sh b/agent/scripts/create-local-tables.sh new file mode 100755 index 0000000..592b719 --- /dev/null +++ b/agent/scripts/create-local-tables.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Create DynamoDB tables in DynamoDB Local for local agent testing. +# +# Prerequisites: +# docker compose up -d (starts DynamoDB Local on port 8000) +# AWS CLI installed +# +# Usage: +# ./agent/scripts/create-local-tables.sh + +set -euo pipefail + +ENDPOINT="http://localhost:8000" +REGION="us-east-1" + +# Common args for all commands +DDB_ARGS=(--endpoint-url "$ENDPOINT" --region "$REGION" --no-cli-pager) + +echo "Creating local DynamoDB tables..." + +# --------------------------------------------------------------------------- +# TaskEventsTable — matches cdk/src/constructs/task-events-table.ts +# PK: task_id (S), SK: event_id (S, ULID) +# TTL: ttl +# --------------------------------------------------------------------------- +if aws dynamodb describe-table --table-name TaskEventsTable "${DDB_ARGS[@]}" >/dev/null 2>&1; then + echo " TaskEventsTable already exists — skipping" +else + aws dynamodb create-table \ + --table-name TaskEventsTable \ + --attribute-definitions \ + AttributeName=task_id,AttributeType=S \ + AttributeName=event_id,AttributeType=S \ + --key-schema \ + AttributeName=task_id,KeyType=HASH \ + AttributeName=event_id,KeyType=RANGE \ + --billing-mode PAY_PER_REQUEST \ + "${DDB_ARGS[@]}" >/dev/null + echo " TaskEventsTable created" +fi + +# --------------------------------------------------------------------------- +# TaskTable — matches cdk/src/constructs/task-table.ts +# PK: task_id (S) +# TTL: ttl +# GSIs omitted (not needed for local agent testing) +# --------------------------------------------------------------------------- +if aws dynamodb describe-table --table-name TaskTable "${DDB_ARGS[@]}" >/dev/null 2>&1; then + echo " TaskTable already exists — skipping" +else + aws dynamodb create-table \ + --table-name TaskTable \ + --attribute-definitions \ + AttributeName=task_id,AttributeType=S \ + --key-schema \ + AttributeName=task_id,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + "${DDB_ARGS[@]}" >/dev/null + echo " TaskTable created" +fi + +echo "Done. Tables available at $ENDPOINT" diff --git a/agent/src/config.py b/agent/src/config.py index 4e26e2e..f3f07fc 100644 --- a/agent/src/config.py +++ b/agent/src/config.py @@ -52,6 +52,8 @@ def build_config( task_type: str = "new_task", branch_name: str = "", pr_number: str = "", + trace: bool = False, + user_id: str = "", ) -> TaskConfig: """Build and validate configuration from explicit parameters. @@ -102,6 +104,8 @@ def build_config( branch_name=branch_name, pr_number=pr_number, task_id=task_id or uuid.uuid4().hex[:12], + trace=trace, + user_id=user_id, ) @@ -118,6 +122,14 @@ def get_config() -> TaskConfig: max_budget_usd=float(os.environ.get("MAX_BUDGET_USD", "0")) or None, aws_region=os.environ.get("AWS_REGION", ""), dry_run=os.environ.get("DRY_RUN", "").lower() in ("1", "true", "yes"), + # Local-batch ``--trace`` parity (design §10.1). Without + # these env vars a developer running the agent outside + # AgentCore could never exercise the trace path. Both are + # opt-in; empty ``USER_ID`` with ``TRACE=1`` logs a skip + # warning (see ``pipeline.run_task``) rather than writing + # an unreachable ``traces//`` key. + trace=os.environ.get("TRACE", "").lower() in ("1", "true", "yes"), + user_id=os.environ.get("USER_ID", ""), ) except ValueError as e: print(f"ERROR: {e}", file=sys.stderr) diff --git a/agent/src/hooks.py b/agent/src/hooks.py index 1f9ca16..634b7f8 100644 --- a/agent/src/hooks.py +++ b/agent/src/hooks.py @@ -1,15 +1,27 @@ -"""PreToolUse and PostToolUse hook callbacks for policy enforcement. +"""PreToolUse, PostToolUse, and Stop hook callbacks. -Integrates the PolicyEngine (Cedar, pre-execution) and the output scanner -(regex, post-execution) with the Claude Agent SDK's hook system to enforce -tool-use policies at runtime. +- PreToolUse / PostToolUse: policy enforcement (Cedar policy engine and the + output scanner for secrets/PII). +- Stop: between-turns nudge injection (Phase 2). When the agent is about to + stop a turn we check the TaskNudgesTable for pending user nudges and, if + any are present, inject them as authoritative ```` blocks via + the SDK's ``decision: "block"`` / ``reason: ...`` mechanism, which tells + the CLI to continue with that text as the next user message. + +A module-level registry ``between_turns_hooks`` lets future phases (e.g. +Phase 3 approval gates) append additional synthetic-message producers +without touching the Stop hook callback itself. """ from __future__ import annotations +import asyncio import json +from collections.abc import Callable from typing import TYPE_CHECKING, Any +import nudge_reader +import task_state from output_scanner import scan_tool_output from shell import log @@ -148,9 +160,293 @@ async def post_tool_use_hook( return _PASS_THROUGH +# --------------------------------------------------------------------------- +# Between-turns hook registry (Phase 2 nudges, extensible for Phase 3) +# --------------------------------------------------------------------------- + +# A hook takes a context dict (currently ``{"task_id": str}``) and returns a +# list of synthetic user-message strings to inject before the agent's next +# turn. An empty list means "no injection — allow normal stop". +BetweenTurnsHook = Callable[[dict], list[str]] + + +# Process-lifetime dedup map: task_id -> set of nudge_ids already injected in +# this process. Guards against infinite re-injection if ``mark_consumed`` +# persistently fails (DDB throttling, IAM drift) — without this, the same +# nudge would be re-injected every Stop hook firing until ``max_turns`` is +# exhausted. Lives for the duration of the process (== task) so it doesn't +# leak across tasks in the same runtime. +_INJECTED_NUDGES: dict[str, set[str]] = {} + + +def _reset_injected_nudges_for_tests() -> None: + """Test-only helper to clear the in-process injected-nudge dedup set.""" + global _INJECTED_NUDGES + _INJECTED_NUDGES = {} + + +def _emit_nudge_milestone(ctx: dict, milestone: str, details: str) -> None: + """Emit ``agent_milestone`` to the progress writer. + + Best-effort — swallow errors so stream visibility failures never block + nudge injection itself. ``ctx`` may carry a ``progress`` ref stamped by + :func:`stop_hook`. Skips (with a log line in each case) when: + + - no ``progress`` ref is stamped on ``ctx`` (tests, early-boot, or a + hook invoked outside :func:`stop_hook`'s dispatch) + - the progress writer's circuit breaker has tripped after repeated + DDB write failures (``ProgressWriter._disabled``) + - the underlying ``write_agent_milestone`` raises despite the writer's + own fail-open contract + + Surfacing these as log lines (instead of silent drops) lets + ``--trace`` mode and CloudWatch Logs show when an ack could not be + delivered to the durable event stream. + """ + progress = ctx.get("progress") + if progress is None: + log("DEBUG", f"nudge milestone {milestone!r} skipped: no progress writer in ctx") + return + # Only skip when ``_disabled`` is explicitly True on a real ProgressWriter. + # ``getattr(..., False)`` is not safe — ``MagicMock`` returns an auto-mock + # attribute for any access, which evaluates truthy. + if getattr(progress, "_disabled", False) is True: + log( + "WARN", + f"nudge milestone {milestone!r} skipped: progress writer circuit breaker open", + ) + return + try: + progress.write_agent_milestone(milestone=milestone, details=details) + except Exception as exc: # pragma: no cover — defensive, writers never raise + log("WARN", f"nudge milestone {milestone!r} progress write failed: {exc}") + + +def _nudge_between_turns_hook(ctx: dict) -> list[str]: + """Read pending nudges for the task and return them as XML user messages. + + Best-effort: marks each nudge consumed after formatting. If + ``mark_consumed`` fails we still inject (the conditional-update contract + means at-most-once delivery on success, at-least-once on mark failures — + better to over-steer than to drop a user instruction). + + Additionally, a process-lifetime dedup set (``_INJECTED_NUDGES``) + prevents infinite re-injection of the same nudge across turns if + ``mark_consumed`` repeatedly fails. + + Emits a ``nudge_acknowledged`` ``agent_milestone`` event **before** + returning the injected user-message list (combined-turn ack, see + ``INTERACTIVE_AGENTS.md`` §AD-5) so the durable event stream records + the ack in the same turn the nudge is consumed. Emission is + best-effort: if the progress writer's circuit breaker has tripped + (repeated DDB write failures) or no ``progress`` ref is stamped on + ``ctx``, the ack is logged but skipped and the injection still + proceeds — better to steer the agent than block on a flaky event + table. + """ + task_id = ctx.get("task_id") or "" + if not task_id: + return [] + + # Belt-and-braces second guard against the "cancel consumes nudges" hazard + # (krokoko PR #52 review finding #3). The primary guard is the loop-level + # break in :func:`stop_hook` which short-circuits the dispatcher as soon as + # any earlier hook sets ``_cancel_requested``. That assumes + # ``_cancel_between_turns_hook`` runs BEFORE this hook — true for the + # module-level ``between_turns_hooks`` registry today (line 340), but a + # future reorder (or a test that rebinds the list without preserving + # order) would silently reintroduce the bug: ``read_pending`` + + # ``mark_consumed`` would flip the DDB rows to consumed and stamp + # ``_INJECTED_NUDGES`` for a dying agent that will never see the text. + # Early-returning here makes the invariant structural — no nudges are + # ever consumed once cancel is flagged, regardless of hook ordering. + if ctx.get("_cancel_requested"): + return [] + + try: + pending = nudge_reader.read_pending(task_id) + except Exception as exc: + log("WARN", f"nudge read_pending raised: {type(exc).__name__}: {exc}") + return [] + + # Filter out any nudges already injected in this process (regardless of + # whether mark_consumed succeeded previously). + already = _INJECTED_NUDGES.get(task_id, set()) + pending = [n for n in pending if n.get("nudge_id") not in already] + + if not pending: + return [] + + try: + formatted = nudge_reader.format_as_user_message(pending) + except Exception as exc: + log("WARN", f"nudge format failed: {type(exc).__name__}: {exc}") + return [] + + # Record injection BEFORE mark_consumed so a persistent mark_consumed + # failure cannot cause re-injection on a later turn. + task_set = _INJECTED_NUDGES.setdefault(task_id, set()) + for n in pending: + nid = n.get("nudge_id") + if nid: + task_set.add(nid) + + # Mark-consumed is best-effort; log failures but do not block injection. + for n in pending: + try: + nudge_reader.mark_consumed(task_id, n["nudge_id"]) + except Exception as exc: + log("WARN", f"nudge mark_consumed raised: {type(exc).__name__}: {exc}") + + count = len(pending) + log("NUDGE", f"Injecting {count} nudge(s) for task {task_id}") + + # Short details string for the stream — preview the first nudge, total + # count, and the nudge IDs for traceability. Kept under ~120 chars so + # it fits on a single terminal line. + first_msg = (pending[0].get("message") or "")[:60] + ids = ",".join(str(n.get("nudge_id", ""))[-8:] for n in pending) + details = f"{count} nudge(s) acknowledged (ids=…{ids}): {first_msg}" + ( + "…" if count > 1 or len(first_msg) == 60 else "" + ) + # AD-5: emit the ack BEFORE returning the injection list. + _emit_nudge_milestone(ctx, "nudge_acknowledged", details) + + return [formatted] if formatted else [] + + +def _cancel_between_turns_hook(ctx: dict) -> list[str]: + """Detect user-initiated cancellation and signal the Stop hook to halt. + + Reads the task record from DynamoDB each turn. If ``status == "CANCELLED"`` + sets ``ctx["_cancel_requested"] = True`` so :func:`stop_hook` returns + ``continue_=False`` and the SDK tears the agent down cleanly. + + Fail-open: a ``TaskFetchError`` (transient DDB failure) is treated as + "no cancel detected" to avoid stranding running tasks on blips. This is + symmetric with ``_nudge_between_turns_hook`` (also fail-open for DDB). + Worst case a cancel is missed for one turn; the next turn will catch it. + + Returns ``[]`` always — the cancel signal flows via the ctx sentinel, not + via injected text. Injecting text would cause the SDK to continue the + conversation, which is the opposite of what cancel needs. + """ + task_id = ctx.get("task_id") or "" + if not task_id: + return [] + try: + record = task_state.get_task(task_id) + except task_state.TaskFetchError as exc: + log("WARN", f"cancel hook get_task raised: {type(exc).__name__}: {exc}") + return [] + if record and record.get("status") == "CANCELLED": + ctx["_cancel_requested"] = True + _emit_nudge_milestone( + ctx, + "cancel_detected", + "Task cancelled by user; stopping agent after this turn.", + ) + return [] + + +# Global list of between-turns hooks. Cancel MUST run first so it can +# short-circuit nudges on cancelled tasks (no point injecting nudges into a +# dying agent — worse, the nudge reader mutates DDB state that the agent will +# never act on; see krokoko PR #52 review finding #3). The :func:`stop_hook` +# dispatcher breaks out of the loop as soon as ``_cancel_requested`` is set, +# and :func:`_nudge_between_turns_hook` early-returns when the flag is already +# present — belt-and-braces in case a future ``append`` reorders this list. +# Phase 3 (approval gates) should ``append`` additional hooks AFTER the +# nudge reader to preserve cancel-wins semantics. +between_turns_hooks: list[BetweenTurnsHook] = [ + _cancel_between_turns_hook, + _nudge_between_turns_hook, +] + + +async def stop_hook( + hook_input: Any, + tool_use_id: str | None, + hook_context: Any, + *, + task_id: str, + progress: Any = None, +) -> dict: + """Stop hook: run registered between-turns hooks; block if they produce text. + + Returning ``{"decision": "block", "reason": ""}`` tells the SDK to + continue the conversation with *text* as the next user message rather + than actually stopping. If no hook produces text we return an empty + dict (allow stop). + + Each between-turns hook is invoked via ``asyncio.to_thread`` so that + sync boto3 calls inside the hook (DDB query + update) do not stall the + asyncio loop driving ``client.receive_response()``. + + ``progress`` is an optional writer ref threaded into each hook's ``ctx`` + so hooks can emit their own milestone / progress events without holding + a module-global reference to it. + """ + ctx = { + "task_id": task_id, + "progress": progress, + } + + # Cancel-before-nudge short-circuit (krokoko PR #52 review finding #3). + # Previously the loop ran ALL hooks before checking ``_cancel_requested``, + # which meant the nudge hook's ``read_pending`` + ``mark_consumed`` path + # executed even on cancelled tasks — flipping the DDB rows to consumed + # and stamping ``_INJECTED_NUDGES`` for a dying agent. The user saw a + # 202 Accepted for their nudge but the injection was discarded when we + # returned ``continue_=False`` below. Breaking out of the loop as soon + # as any hook sets ``_cancel_requested`` guarantees subsequent hooks + # (notably the nudge reader) never run, so DDB state is never mutated + # for work the agent will never do. The registry at line 340 keeps + # ``_cancel_between_turns_hook`` first so this break fires before the + # nudge hook gets a chance. ``_nudge_between_turns_hook`` also carries + # an internal cancel-check as belt-and-braces in case a future refactor + # reorders the registry. + chunks: list[str] = [] + for hook in between_turns_hooks: + try: + produced = await asyncio.to_thread(hook, ctx) + except Exception as exc: + log( + "WARN", + f"between-turns hook raised (task_id={task_id}): {type(exc).__name__}: {exc}", + ) + continue + if produced: + chunks.extend(produced) + if ctx.get("_cancel_requested"): + # Any text produced by earlier hooks in this same loop iteration + # is discarded below — the ``_cancel_requested`` branch returns + # ``continue_=False`` and never reads ``chunks``. This is + # intentional: cancel wins, and we would rather drop a + # simultaneous nudge than inject into a dying agent. + break + + # Cancel takes precedence over nudge injection. ``continue_: False`` tells + # the SDK to end the turn loop and return control to the caller, which + # lets the pipeline see the CANCELLED status and skip post-hooks. + if ctx.get("_cancel_requested"): + return { + "continue_": False, + "stopReason": "Task cancelled by user", + } + + if not chunks: + return {} + + reason = "\n\n".join(chunks) + return {"decision": "block", "reason": reason} + + def build_hook_matchers( engine: PolicyEngine, trajectory: _TrajectoryWriter | None = None, + task_id: str = "", + progress: Any = None, ) -> dict: """Build hook matchers dict for ClaudeAgentOptions. @@ -159,6 +455,10 @@ def build_hook_matchers( The SDK expects ``dict[HookEvent, list[HookMatcher]]`` where HookMatcher has ``matcher: str | None`` and ``hooks: list[HookCallback]``. + + ``progress`` is forwarded to the Stop hook so that between-turns hooks + can emit milestones (e.g. ``nudge_acknowledged``) that show up in the + durable progress stream as a visible marker of Phase 2 nudge activity. """ from claude_agent_sdk.types import ( HookContext, @@ -193,7 +493,31 @@ async def _post( } return SyncHookJSONOutput(hookSpecificOutput=fail_closed) + async def _stop( + hook_input: HookInput, tool_use_id: str | None, ctx: HookContext + ) -> HookJSONOutput: + # Capture task_id up-front so it can be included in any wrapper + # crash log for post-hoc correlation with user complaints. + stop_task_id = task_id + try: + result = await stop_hook( + hook_input, + tool_use_id, + ctx, + task_id=stop_task_id, + progress=progress, + ) + except Exception as exc: + log( + "ERROR", + f"Stop wrapper crashed (task_id={stop_task_id}): {type(exc).__name__}: {exc}", + ) + return SyncHookJSONOutput() + # Empty dict == allow stop. SyncHookJSONOutput(**{}) is fine. + return SyncHookJSONOutput(**result) + return { "PreToolUse": [HookMatcher(matcher=None, hooks=[_pre])], "PostToolUse": [HookMatcher(matcher=None, hooks=[_post])], + "Stop": [HookMatcher(matcher=None, hooks=[_stop])], } diff --git a/agent/src/models.py b/agent/src/models.py index 0e4f8f1..a08ae70 100644 --- a/agent/src/models.py +++ b/agent/src/models.py @@ -108,11 +108,47 @@ class TaskConfig(BaseModel): branch_name: str = "" pr_number: str = "" task_id: str = "" + # Platform user_id (Cognito ``sub``) threaded from the orchestrator + # payload. Required ONLY when ``trace`` is true — the agent writes + # the trajectory dump to ``traces//.jsonl.gz`` + # (design §10.1), and the ``get-trace-url`` handler's per-caller- + # prefix guard refuses to presign keys outside the caller's own + # ``traces//`` prefix. Empty-string default for local + # batch runs (no orchestrator in the loop; no trace upload). + user_id: str = "" + # Opt-in debug preview cap (design §10.1). Threaded to BOTH the + # pipeline.py milestone writer AND the runner.py turn/tool writer — + # the runner's writer is where thinking/tool_input/tool_result + # previews live, so dropping ``trace`` here silently no-ops the + # feature for the fields that matter. + trace: bool = False # Enriched mid-flight by pipeline.py: cedar_policies: list[str] = [] issue: GitHubIssue | None = None base_branch: str | None = None + @model_validator(mode="after") + def _validate_trace_requires_user_id(self) -> Self: + """Fail at construction when trace=True without a user_id. + + The trace trajectory is uploaded to + ``traces//.jsonl.gz`` (design §10.1). An empty + ``user_id`` produces ``traces//.jsonl.gz``, which the + ``get-trace-url`` handler's per-caller-prefix guard refuses. + Catching this at construction time surfaces the misconfiguration + locally / in CI instead of deferring to runtime S3 upload. + """ + if self.trace and not self.user_id: + raise ValueError( + "trace=True requires a non-empty user_id. Local/batch runs " + "without an orchestrator must either set trace=False (the " + "default) or supply user_id explicitly. The trace trajectory " + "is uploaded to traces//.jsonl.gz (design " + "§10.1), and the get-trace-url handler refuses keys outside " + "the caller's traces// prefix." + ) + return self + class RepoSetup(BaseModel): model_config = ConfigDict(frozen=True) @@ -153,7 +189,19 @@ class TaskResult(BaseModel): build_passed: bool = False lint_passed: bool = False cost_usd: float | None = None + # Rev-5 DATA-1: historically the `turns` field was set to the SDK's + # `ResultMessage.num_turns`, which INCLUDES the attempted turn that + # tripped a cap (so `max_turns=6` yields `turns=7` under + # `agent_status='error_max_turns'`). That confused operators. We + # now expose both fields explicitly: + # * `turns_attempted` — the SDK's authoritative counter (ex-`turns`). + # * `turns_completed` — clamped to max_turns when we know the cap + # fired; otherwise equals `turns_attempted`. + # The legacy `turns` field is retained (= `turns_attempted`) so + # existing DDB consumers keep working during the transition. turns: int | None = None + turns_attempted: int | None = None + turns_completed: int | None = None duration_s: float = 0.0 task_id: str = "" disk_before: str = "" @@ -167,3 +215,9 @@ class TaskResult(BaseModel): output_tokens: int | None = None cache_read_input_tokens: int | None = None cache_creation_input_tokens: int | None = None + # S3 URI of the uploaded --trace trajectory dump, or ``None`` when + # the task did not run with ``--trace`` / the upload was skipped or + # failed. Threaded into ``task_state.write_terminal`` so the + # TaskRecord's ``trace_s3_uri`` field is set atomically with the + # terminal-status transition (design §10.1). + trace_s3_uri: str | None = None diff --git a/agent/src/nudge_reader.py b/agent/src/nudge_reader.py new file mode 100644 index 0000000..5a14617 --- /dev/null +++ b/agent/src/nudge_reader.py @@ -0,0 +1,266 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +"""Read pending user "nudges" from the TaskNudgesTable between agent turns. + +Phase 2 Nudge — short authoritative steering messages written by the REST API +and injected as XML-tagged user messages into the agent's next turn. + +Design contract +--------------- +Table shape (owned by the REST API, not this module): + - PK ``task_id`` (STRING) + - SK ``nudge_id`` (STRING, ULID — lexicographic == chronological) + - ``message`` (STRING) + - ``created_at`` (STRING, ISO-8601) + - ``consumed`` (BOOL) + - ``consumed_at`` (STRING, optional, set when consumed) + - ``user_id`` (STRING) + - ``ttl`` (NUMBER, optional) + +Table name read from env var ``NUDGES_TABLE_NAME``. If unset the reader +silently returns ``[]`` and logs a single WARN (fail-open). + +Resilience +---------- +All DDB exceptions (network, throttling, validation) are caught and logged at +WARN. Callers receive ``[]`` or ``False`` — a nudge-table outage MUST NOT +break the agent turn loop. +""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime +from typing import Any, TypedDict + +from shell import log + + +class PendingNudge(TypedDict): + """Shape of a single pending nudge returned by ``read_pending``.""" + + nudge_id: str + message: str + created_at: str + + +# Max items returned from a paginated ``read_pending`` scan. The table's PK +# is per-task, rate-limited to 10/min and TTL-retained for 30 days, so a +# healthy task should not exceed this. Truncating is preferable to +# unbounded memory growth; a WARN log surfaces the condition. +_MAX_PENDING_ITEMS = 100 + +# Module-level cache for the boto3 Table resource — initialised lazily on +# first read, reused across calls for the lifetime of the process. +_TABLE_CACHE: Any = None +_TABLE_NAME_WARNED = False + + +def _get_table() -> Any | None: + """Return a cached boto3 DynamoDB Table resource, or None if unavailable. + + Reads ``NUDGES_TABLE_NAME`` from the environment. When unset, logs a + single WARN and returns None on every subsequent call. + """ + global _TABLE_CACHE, _TABLE_NAME_WARNED + + if _TABLE_CACHE is not None: + return _TABLE_CACHE + + table_name = os.environ.get("NUDGES_TABLE_NAME") + if not table_name: + if not _TABLE_NAME_WARNED: + log("WARN", "NUDGES_TABLE_NAME unset — nudge reader disabled") + _TABLE_NAME_WARNED = True + return None + + try: + import boto3 + + region = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") + dynamodb = boto3.resource("dynamodb", region_name=region) + _TABLE_CACHE = dynamodb.Table(table_name) + return _TABLE_CACHE + except Exception as exc: + log("WARN", f"Failed to init nudge DDB table: {type(exc).__name__}: {exc}") + return None + + +def _reset_cache_for_tests() -> None: + """Test-only helper to clear module-level caches between test cases.""" + global _TABLE_CACHE, _TABLE_NAME_WARNED + _TABLE_CACHE = None + _TABLE_NAME_WARNED = False + + +def read_pending(task_id: str, table: Any | None = None) -> list[PendingNudge]: + """Return unconsumed nudges for *task_id*, sorted by ``nudge_id`` ASC. + + ULIDs sort chronologically, so ASC ordering == oldest-first. Returns + ``[]`` on any error or if the nudges table is not configured. + + Paginates on ``LastEvaluatedKey`` — DDB Query returns at most 1 MB per + page, and ``FilterExpression`` is applied post-page, so a task with + many consumed rows could hide pending nudges behind the first page. + Caps total items at ``_MAX_PENDING_ITEMS`` and logs a WARN if hit. + + Each returned dict contains ``nudge_id``, ``message``, ``created_at``. + """ + tbl = table if table is not None else _get_table() + if tbl is None: + return [] + + try: + from boto3.dynamodb.conditions import Attr, Key + + items: list[dict[str, Any]] = [] + last_key: dict[str, Any] | None = None + truncated = False + while True: + kwargs: dict[str, Any] = { + "KeyConditionExpression": Key("task_id").eq(task_id), + "FilterExpression": Attr("consumed").eq(False), + } + if last_key is not None: + kwargs["ExclusiveStartKey"] = last_key + response = tbl.query(**kwargs) + page_items = response.get("Items", []) or [] + items.extend(page_items) + if len(items) >= _MAX_PENDING_ITEMS: + truncated = True + items = items[:_MAX_PENDING_ITEMS] + break + last_key = response.get("LastEvaluatedKey") + if not last_key: + break + except Exception as exc: + log("WARN", f"Nudge DDB query failed: {type(exc).__name__}: {exc}") + return [] + + if truncated: + log( + "WARN", + f"Nudge read_pending truncated at {_MAX_PENDING_ITEMS} items for " + f"task {task_id}; older pending nudges were dropped", + ) + + # Query with HASH key already returns items sorted by SK ASC, but filter + # expression is applied post-sort; be explicit so callers can rely on + # ordering regardless of how the table is queried. + items.sort(key=lambda it: it.get("nudge_id", "")) + + return [ + PendingNudge( + nudge_id=str(it.get("nudge_id", "")), + message=str(it.get("message", "")), + created_at=str(it.get("created_at", "")), + ) + for it in items + if it.get("nudge_id") + ] + + +def mark_consumed(task_id: str, nudge_id: str, table: Any | None = None) -> bool: + """Atomically mark a nudge as consumed. + + Uses a conditional update (``consumed = false``) for idempotency — if two + workers race, only one will succeed. Returns True on success, False if + already consumed or on any error. + """ + tbl = table if table is not None else _get_table() + if tbl is None: + return False + + now_iso = datetime.now(UTC).isoformat() + + # Lazy import so tests without boto3 installed still load the module. + ClientError: type[Exception] | None + try: + from botocore.exceptions import ClientError as _CE + + ClientError = _CE + except Exception: # pragma: no cover — boto3/botocore always present at runtime + ClientError = None + + try: + tbl.update_item( + Key={"task_id": task_id, "nudge_id": nudge_id}, + # ``consumed`` is a DDB reserved keyword — alias via #c. + UpdateExpression="SET #c = :true, consumed_at = :now", + ConditionExpression="#c = :false", + ExpressionAttributeNames={"#c": "consumed"}, + ExpressionAttributeValues={ + ":true": True, + ":false": False, + ":now": now_iso, + }, + ) + return True + except Exception as exc: + # Structured ClientError path: boto3 wraps the DDB error code in + # ``exc.response["Error"]["Code"]``. + if ClientError is not None and isinstance(exc, ClientError): + code = exc.response.get("Error", {}).get("Code") + if code == "ConditionalCheckFailedException": + log("DEBUG", f"Nudge {nudge_id} already consumed (conditional check)") + return False + log( + "WARN", + f"Nudge mark_consumed ClientError for {nudge_id}: {code}: {exc}", + ) + return False + # Fallback: some tests/mocks raise a bare exception subclass named + # ``ConditionalCheckFailedException`` rather than a real ClientError. + exc_name = type(exc).__name__ + if exc_name == "ConditionalCheckFailedException": + log("DEBUG", f"Nudge {nudge_id} already consumed (conditional check)") + return False + # Also handle fake ClientError duck-types carrying response["Error"]["Code"]. + response = getattr(exc, "response", None) + if isinstance(response, dict): + code = ( + response.get("Error", {}).get("Code") + if isinstance(response.get("Error"), dict) + else None + ) + if code == "ConditionalCheckFailedException": + log("DEBUG", f"Nudge {nudge_id} already consumed (conditional check)") + return False + log("WARN", f"Nudge mark_consumed failed for {nudge_id}: {exc_name}: {exc}") + return False + + +def _xml_escape(text: str) -> str: + """Escape XML predefined entities for safe inclusion in text/attributes. + + Prevents a user nudge from forging a closing ```` tag and + smuggling content out of the authoritative block. + + We escape ``& < > "`` — all five XML entities minus ``'``. Apostrophe + escaping (``'``) is only needed inside single-quoted attribute + values, and we always emit double-quoted attributes; pasted user text + containing ``don't`` etc. stays readable in logs. + """ + return ( + text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) + ) + + +def format_as_user_message(nudges: list[PendingNudge]) -> str: + """Render a list of nudge dicts as authoritative ```` XML blocks. + + Each block is on its own line; multiple blocks are joined with a single + newline separator. Attribute values and message body are XML-escaped so + a malicious nudge cannot escape the envelope. + """ + if not nudges: + return "" + + blocks = [] + for n in nudges: + ts = _xml_escape(str(n.get("created_at", ""))) + nid = _xml_escape(str(n.get("nudge_id", ""))) + body = _xml_escape(str(n.get("message", ""))) + blocks.append(f'\n{body}\n') + return "\n".join(blocks) diff --git a/agent/src/pipeline.py b/agent/src/pipeline.py index b1dcafc..81ecc59 100644 --- a/agent/src/pipeline.py +++ b/agent/src/pipeline.py @@ -24,11 +24,18 @@ verify_build, verify_lint, ) +from progress_writer import _ProgressWriter from prompt_builder import build_system_prompt, discover_project_config from runner import run_agent from shell import log from system_prompt import SYSTEM_PROMPT -from telemetry import format_bytes, get_disk_usage, print_metrics +from telemetry import ( + _TrajectoryWriter, + format_bytes, + get_disk_usage, + print_metrics, + upload_trace_to_s3, +) _SDK_NO_RESULT_MESSAGE = ( "Agent SDK stream ended without a ResultMessage (agent_status=unknown). " @@ -48,6 +55,69 @@ def _chain_prior_agent_error(agent_result: AgentResult | None, exc: BaseExceptio return tail +def _maybe_upload_trace( + config: TaskConfig, + trajectory, + progress, +) -> str | None: + """Run the --trace S3 upload if the task opted in and user_id is set. + + Returns the resulting ``s3://`` URI (or ``None`` on any skip/fail). + Fully fail-open: an exception here does NOT propagate. Called from + both the happy path (post-hooks complete) and the crash path + (top-level ``except``) so a crashing task still produces a + debuggable artifact — which is exactly when ``--trace`` is most + useful (K2 review Finding #1). + + Gates (K2 Stage 3 review Finding #1): + - ``config.trace`` must be true. + - ``config.user_id`` must be non-empty, else we would write to + ``traces//.jsonl.gz`` — an unreachable key that no + Cognito caller can download through ``bgagent trace download``. + """ + if not config.trace: + return None + if not config.user_id: + log( + "WARN", + "Trace was enabled but user_id is empty — skipping S3 " + "upload to avoid writing an unreachable artifact key. " + f"task_id={config.task_id}", + ) + return None + try: + artifact = trajectory.dump_gzipped_jsonl() + except Exception as e: + log("WARN", f"Trace dump_gzipped_jsonl failed: {type(e).__name__}: {e}") + return None + if not artifact: + log( + "INFO", + "Trace accumulator is empty (no trajectory events captured). Skipping S3 upload.", + ) + return None + trace_s3_uri = upload_trace_to_s3( + task_id=config.task_id, + user_id=config.user_id, + body=artifact, + ) + if trace_s3_uri: + try: + progress.write_agent_milestone("trajectory_uploaded", trace_s3_uri) + except Exception as e: + # Milestone write is best-effort; don't mask the upload. + log("WARN", f"trajectory_uploaded milestone emit failed: {type(e).__name__}: {e}") + log("TASK", f"Trace artifact uploaded: {trace_s3_uri}") + else: + log( + "WARN", + "Trace upload returned no URI — see [trace/upload] logs " + "above for the reason (skipped or failed). Task proceeds " + "to terminal without trace_s3_uri.", + ) + return trace_s3_uri + + def _resolve_overall_task_status( agent_result: AgentResult, *, @@ -80,6 +150,29 @@ def _resolve_overall_task_status( return "error", err +def _compute_turns_completed( + agent_status: str, + turns_attempted: int | None, + max_turns: int, +) -> int | None: + """Clamp ``turns_completed`` to ``max_turns`` when the SDK hit the limit. + + Rev-5 DATA-1 — the Claude Agent SDK reports ``num_turns = max_turns + 1`` + on ``error_max_turns`` because the aborted attempt is counted. Clamping + at the final write keeps ``turns_completed`` truthful ("how many turns + actually executed") while ``turns_attempted`` keeps the raw SDK value + for debugging. + + Returns ``None`` if ``turns_attempted`` is ``None``/falsy so callers can + round-trip a missing SDK count without inventing a fake zero. + """ + if not turns_attempted: + return turns_attempted + if agent_status == "error_max_turns": + return min(turns_attempted, max_turns) + return turns_attempted + + def _write_memory( config: TaskConfig, setup: RepoSetup, @@ -149,6 +242,8 @@ def run_task( branch_name: str = "", pr_number: str = "", cedar_policies: list[str] | None = None, + trace: bool = False, + user_id: str = "", ) -> dict: """Run the full agent pipeline and return a serialized result dict. @@ -179,6 +274,8 @@ def run_task( task_type=task_type, branch_name=branch_name, pr_number=pr_number, + trace=trace, + user_id=user_id, ) # Inject Cedar policies into config for the PolicyEngine in runner.py @@ -203,6 +300,30 @@ def run_task( task_state.write_heartbeat(config.task_id) agent_result: AgentResult | None = None + progress = _ProgressWriter(config.task_id, trace=trace) + # --trace accumulator (design §10.1): when the task opted into + # trace, ``_TrajectoryWriter`` keeps an in-memory copy of each + # event so the pipeline can gzip+upload the full trajectory to + # S3 on terminal. Owned by the pipeline rather than the runner + # so the accumulator outlives ``run_agent``'s scope. + trajectory = _TrajectoryWriter(config.task_id, accumulate=trace) + # K2 review Finding #3 — surface accumulator truncation to the + # user via a ``trace_truncated`` milestone on TaskEventsTable + # (visible in ``bgagent watch``). Fire-once by design: the + # downloaded artifact's header reports the final drop count. + if trace: + + def _on_trace_truncated(max_bytes: int, first_dropped: int) -> None: + progress.write_agent_milestone( + "trace_truncated", + f"Trace accumulator hit its {max_bytes}-byte cap after " + f"{first_dropped} event drop(s); the downloaded " + f"artifact will be truncated. See the " + f"TRAJECTORY_ARTIFACT_HEADER row for the final " + f"drop count.", + ) + + trajectory.set_truncation_callback(_on_trace_truncated) try: # Context hydration with task_span("task.context_hydration"): @@ -275,6 +396,10 @@ def run_task( with task_span("task.repo_setup") as setup_span: setup = setup_repo(config) setup_span.set_attribute("build.before", setup.build_before) + progress.write_agent_milestone( + "repo_setup_complete", + f"branch={setup.branch} build_before={setup.build_before}", + ) system_prompt = build_system_prompt(config, setup, hc, system_prompt_overrides) @@ -306,13 +431,76 @@ def run_task( with task_span("task.agent_execution") as agent_span: try: agent_result = asyncio.run( - run_agent(prompt, system_prompt, config, cwd=setup.repo_dir) + run_agent( + prompt, + system_prompt, + config, + cwd=setup.repo_dir, + trajectory=trajectory, + ) ) except Exception as e: log("ERROR", f"Agent failed: {e}") agent_span.set_status(StatusCode.ERROR, str(e)) agent_span.record_exception(e) agent_result = AgentResult(status="error", error=str(e)) + progress.write_agent_milestone( + "agent_execution_complete", + f"status={agent_result.status} turns={agent_result.turns}", + ) + + # Cancel short-circuit: the Stop hook signalled cancel by stopping + # the SDK early, but that only stops the agent loop — post-hooks + # (ensure_committed, ensure_pr) would still run and push/open a PR + # on a cancelled task. Re-check the task status here and exit the + # pipeline before any side-effect-producing post-hook runs. The + # terminal state is already CANCELLED (written by cancel-task.ts), + # so we do NOT call write_terminal — its ConditionExpression only + # allows RUNNING/HYDRATING/FINALIZING, which would fail silently, + # but leaving the cancel record intact makes the intent explicit. + try: + _current_record = task_state.get_task(config.task_id) + except task_state.TaskFetchError: + _current_record = None # fail-open: let normal path proceed + if _current_record and _current_record.get("status") == "CANCELLED": + log("TASK", f"Task {config.task_id} cancelled; skipping post-hooks") + progress.write_agent_milestone( + "task_cancelled_acknowledged", + "Post-hooks skipped; terminal state already CANCELLED.", + ) + # L4 item 1c: best-effort trace upload + conditional + # self-heal on the cancel path. ``write_terminal``'s + # ConditionExpression rejects CANCELLED, so we cannot + # persist ``trace_s3_uri`` atomically with the terminal + # write — use ``write_trace_uri_conditional`` instead, + # which is scoped to ``attribute_not_exists(trace_s3_uri)`` + # AND a terminal status. Fully fail-open: any exception + # (upload, DDB, serialization) must not prevent the + # cancel fast-path from returning. + if config.trace: + log( + "TASK", + "Task cancelled mid-run; attempting best-effort " + "--trace upload + conditional persist so the " + "trajectory captured before cancel is still " + "recoverable.", + ) + try: + trace_s3_uri = _maybe_upload_trace(config, trajectory, progress) + if trace_s3_uri: + task_state.write_trace_uri_conditional(config.task_id, trace_s3_uri) + except Exception as e: + log( + "WARN", + f"Cancel-path trace upload/persist failed " + f"(fail-open): {type(e).__name__}: {e}", + ) + return { + "status": "cancelled", + "task_id": config.task_id, + "turns": agent_result.turns, + "turns_attempted": agent_result.num_turns or agent_result.turns, + } # Post-hooks (agent_result is guaranteed set by the try/except above) with task_span("task.post_hooks") as post_span: @@ -331,6 +519,8 @@ def run_task( post_span.set_attribute("build.passed", build_passed) post_span.set_attribute("lint.passed", lint_passed) post_span.set_attribute("pr.url", pr_url or "") + if pr_url: + progress.write_agent_milestone("pr_created", pr_url) # Memory write — capture task episode and repo learnings memory_written = False @@ -375,8 +565,24 @@ def run_task( pr_url=pr_url, ) + # --trace trajectory S3 upload (design §10.1). Runs AFTER + # post-hooks but BEFORE ``write_terminal`` so the resulting + # ``trace_s3_uri`` can be persisted atomically with the + # terminal-status transition. Fail-open: an S3 error does + # NOT flip the task to FAILED — the trajectory is a debug + # artifact, not a correctness gate. The same helper is also + # invoked from the crash path below so a pipeline exception + # still produces a usable debug artifact. + trace_s3_uri = _maybe_upload_trace(config, trajectory, progress) + # Build TaskResult usage = agent_result.usage + turns_attempted = agent_result.num_turns or agent_result.turns + turns_completed = _compute_turns_completed( + agent_status=agent_status, + turns_attempted=turns_attempted, + max_turns=config.max_turns, + ) result = TaskResult( status=overall_status, agent_status=agent_status, @@ -384,7 +590,10 @@ def run_task( build_passed=build_passed, lint_passed=lint_passed, cost_usd=agent_result.cost_usd, - turns=agent_result.num_turns or agent_result.turns, + # Legacy field (= turns_attempted) kept for back-compat. + turns=turns_attempted, + turns_attempted=turns_attempted, + turns_completed=turns_completed, duration_s=round(duration, 1), task_id=config.task_id, disk_before=format_bytes(disk_before), @@ -398,6 +607,7 @@ def run_task( output_tokens=usage.output_tokens if usage else None, cache_read_input_tokens=usage.cache_read_input_tokens if usage else None, cache_creation_input_tokens=usage.cache_creation_input_tokens if usage else None, + trace_s3_uri=trace_s3_uri, ) result_dict = result.model_dump() @@ -438,6 +648,23 @@ def run_task( except Exception as e: # Ensure the task is marked FAILED in DynamoDB even if the pipeline # crashes before reaching the normal terminal-state write. + # + # K2 review Finding #1 — crash-path trace upload. The + # trajectory accumulator is exactly the artifact the user + # enabled ``--trace`` to capture the failure with; dropping + # it on the crash path is a silent regression against the + # design intent. Fully wrapped in its own try/except so a + # trace upload failure cannot mask or replace the real + # exception (we re-raise ``e`` at the end). + crash_trace_s3_uri: str | None = None + try: + crash_trace_s3_uri = _maybe_upload_trace(config, trajectory, progress) + except Exception as upload_exc: + log( + "WARN", + f"Crash-path trace upload failed: {type(upload_exc).__name__}: {upload_exc}", + ) + agent_for_chain = agent_result combined = _chain_prior_agent_error(agent_for_chain, e) crash_result = TaskResult( @@ -445,6 +672,7 @@ def run_task( error=combined, task_id=config.task_id, agent_status=agent_for_chain.status if agent_for_chain else "unknown", + trace_s3_uri=crash_trace_s3_uri, ) task_state.write_terminal(config.task_id, "FAILED", crash_result.model_dump()) raise @@ -510,6 +738,8 @@ def main(): max_budget_usd=config.max_budget_usd, aws_region=config.aws_region, system_prompt_overrides=config.system_prompt_overrides, + trace=config.trace, + user_id=config.user_id, ) # Exit with error if agent failed diff --git a/agent/src/progress_writer.py b/agent/src/progress_writer.py new file mode 100644 index 0000000..46b6a6c --- /dev/null +++ b/agent/src/progress_writer.py @@ -0,0 +1,610 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +"""Write structured progress events to DynamoDB TaskEventsTable. + +Follows the same patterns as ``_TrajectoryWriter`` in ``entrypoint.py``: + - Lazy boto3 client initialization + - Best-effort, fail-open (never crash the agent) + - Circuit breaker: disable after 3 consecutive *transient* DDB write + failures (krokoko PR #52 review finding #6 — permanent errors like + ``ValidationException`` no longer trip the breaker). + - Reads ``TASK_EVENTS_TABLE_NAME`` from environment (already set on AgentCore Runtime) + +Each event is a DDB item with: + - ``task_id`` (PK) + - ``event_id`` (SK, ULID-compatible — time-sortable unique ID) + - ``event_type`` + - ``metadata`` (Map) + - ``timestamp`` (ISO 8601) + - ``ttl`` (90-day, matching task retention) + +Circuit-breaker state is **shared across all writer instances for the same +task** (krokoko PR #52 review finding #8). Runner-level (turn/tool events) +and pipeline-level (milestones) writers are two ``_ProgressWriter`` +instances with the same ``task_id``; without shared state a throttling burst +on one would let the other keep writing, producing visible gaps in the +event stream. See :class:`_SharedCircuitBreaker` for the mechanism. +""" + +from __future__ import annotations + +import json +import os +import random +import threading +import time +from datetime import UTC, datetime +from decimal import Decimal +from typing import Literal + +# Preview field cap defaults (design §10.1): +# - 200 chars for normal tasks — small DDB rows, cheap watch-stream bytes. +# - 4096 chars (4 KB) for ``--trace`` opt-in tasks — full enough to +# capture the critical lines of a tool invocation / model response +# without blowing through DDB's per-item byte budget. +_PREVIEW_MAX_LEN = 200 +_PREVIEW_MAX_LEN_TRACE = 4096 + +# 90 days in seconds — matches task retention TTL +_TTL_SECONDS = 90 * 24 * 60 * 60 + +# Crockford's Base32 alphabet for ULID encoding +_CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + + +def _generate_ulid() -> str: + """Generate a ULID-compatible string using only the standard library. + + Format: 10-char timestamp (ms since epoch) + 16-char random, both in + Crockford's Base32. Lexicographically sortable by time. + """ + timestamp_ms = int(time.time() * 1000) + + # Encode 48-bit timestamp into 10 Base32 chars (big-endian) + t_chars = [] + t = timestamp_ms + for _ in range(10): + t_chars.append(_CROCKFORD[t & 0x1F]) + t >>= 5 + t_part = "".join(reversed(t_chars)) + + # 80 bits of randomness → 16 Base32 chars + r = random.getrandbits(80) + r_chars = [] + for _ in range(16): + r_chars.append(_CROCKFORD[r & 0x1F]) + r >>= 5 + r_part = "".join(reversed(r_chars)) + + return t_part + r_part + + +def _truncate_preview(value: str | None, max_len: int = _PREVIEW_MAX_LEN) -> str: + """Truncate a string to *max_len* chars for DDB preview fields.""" + if not value: + return "" + if len(value) <= max_len: + return value + return value[:max_len] + "..." + + +# --------------------------------------------------------------------------- +# Error classification (krokoko PR #52 review finding #6) +# --------------------------------------------------------------------------- + +# DDB error codes that are NOT recoverable by retry — retrying will keep +# failing the same way forever, so letting them increment ``_failure_count`` +# would eventually trip the breaker and silence the entire progress stream +# for this task. Examples seen in practice: +# - ``ValidationException`` — e.g. a trace-heavy event pushes the item +# past the 400 KB DDB limit. Subsequent lighter events would succeed, +# but today's bare ``except Exception`` counter already tripped. +# - ``ItemCollectionSizeLimitExceededException`` — local-index partition +# collection exceeded 10 GB; same story. +# - ``AccessDeniedException`` / ``UnauthorizedOperation`` — IAM misconfig +# at deploy time; retry is futile and the breaker should flip +# *immediately* so we don't waste three events worth of noise finding +# this out. +# - ``ResourceNotFoundException`` — table genuinely does not exist in +# this deploy. Log loudly and disable. +_PERMANENT_DDB_ERROR_CODES: frozenset[str] = frozenset( + { + "ValidationException", + "ItemCollectionSizeLimitExceededException", + "ResourceNotFoundException", + "AccessDeniedException", + "UnauthorizedOperation", + } +) + +# DDB error codes that ARE expected to self-heal (throughput throttling, +# control-plane blips, network-level timeouts). These feed the normal +# circuit-breaker counter — three in a row and we disable, matching the +# original design. +_TRANSIENT_DDB_ERROR_CODES: frozenset[str] = frozenset( + { + "ProvisionedThroughputExceededException", + "RequestLimitExceeded", + "ThrottlingException", + "ServiceUnavailable", + "InternalServerError", + } +) + +# Non-``ClientError`` transient types — matched on class name because +# ``botocore.exceptions`` may not be importable in every environment (the +# writer must not crash the agent if boto3 is missing, see ``ImportError`` +# handling in ``_put_event``). +_TRANSIENT_NETWORK_EXC_NAMES: frozenset[str] = frozenset( + { + "ConnectionError", + "EndpointConnectionError", + "ReadTimeoutError", + "ConnectTimeoutError", + "ConnectionClosedError", + # boto3's retry wrapper surface + "ClientConnectionError", + } +) + + +def _classify_ddb_error(exc: BaseException) -> Literal["permanent", "transient", "unknown"]: + """Classify a DDB-layer exception for circuit-breaker accounting. + + Rules (krokoko PR #52 review finding #6): + + - ``ClientError`` with an AWS error code we recognise as permanent + (schema/size/IAM/missing-table) → ``"permanent"``. Drop the + individual event, do NOT trip the breaker — the next event may be a + smaller payload that succeeds just fine. + - ``ClientError`` with a recognised transient code, OR a network-layer + exception class we know (``EndpointConnectionError`` et al.) → + ``"transient"``. Feed the normal counter. + - Anything else → ``"unknown"``. Default to transient-style accounting + (increment the counter) but log at a louder ERROR level so operators + can add the new code to the permanent/transient table next release. + This is intentionally conservative: instant-disable on unknown errors + would over-correct from today's bare-except behaviour and risk a new + botocore release silencing the stream on benign retryable codes. + + We identify ``ClientError`` structurally (duck-typed via + ``exc.response["Error"]["Code"]``) rather than ``isinstance`` so the + classifier remains importable in environments where ``botocore`` is + missing (e.g. pure-python unit tests). + """ + # ClientError: response is a dict with a nested Error.Code string. + response = getattr(exc, "response", None) + if isinstance(response, dict): + error_block = response.get("Error") or {} + code = error_block.get("Code") if isinstance(error_block, dict) else None + if isinstance(code, str): + if code in _PERMANENT_DDB_ERROR_CODES: + return "permanent" + if code in _TRANSIENT_DDB_ERROR_CODES: + return "transient" + # Unknown AWS error code — fall through to class-name match, + # then to "unknown" default. + + # Non-ClientError: match by class name for network-layer exceptions. + exc_name = type(exc).__name__ + if exc_name in _TRANSIENT_NETWORK_EXC_NAMES: + return "transient" + + return "unknown" + + +# --------------------------------------------------------------------------- +# Shared circuit-breaker state (krokoko PR #52 review finding #8) +# --------------------------------------------------------------------------- + + +class _SharedCircuitBreaker: + """Process-wide circuit-breaker state keyed by ``task_id``. + + Both the runner (turn/tool events) and pipeline (milestones) create + their own ``_ProgressWriter`` instance. Before this class each writer + kept its own ``_failure_count`` / ``_disabled`` flag, so a throttling + burst would trip one writer while the other kept firing — producing a + visibly half-alive event stream (e.g. milestones arriving after all + turn events went silent). + + The contract is now: **one task's stream is either healthy or + disabled, never half-alive.** Every writer for the same ``task_id`` + reads/writes the same counter, and the first one to hit + ``_MAX_FAILURES`` disables the stream for the whole task. + + State resets on fresh ``task_id`` (new task, new state). A test-only + reset helper (:func:`_reset_circuit_breakers`) clears every entry so + one test's tripped breaker does not leak into the next. + + The empty-string sentinel ``"unknown"`` that ``runner.py`` falls back + to (``config.task_id or "unknown"``) is treated as a plain key here — + writers with a real ``task_id`` never collide with it, and two + ``"unknown"`` writers running in the same process would legitimately + share state (they can't be distinguished anyway). + """ + + def __init__(self) -> None: + # task_id -> (failure_count, disabled) + self._state: dict[str, dict[str, int | bool]] = {} + # A single lock for the whole map is plenty — write contention on + # this path is bounded by the DDB write rate (handful per second), + # so coarse-grained locking has no measurable cost and keeps the + # invariants (read-modify-write of failure_count) simple. + self._lock = threading.Lock() + + def is_disabled(self, task_id: str) -> bool: + with self._lock: + return bool(self._state.get(task_id, {}).get("disabled", False)) + + def disable(self, task_id: str) -> None: + """Flip the breaker open for this ``task_id`` immediately. + + Used by the permanent-error fast path (``AccessDeniedException``, + ``ResourceNotFoundException``) where retry has zero chance of + helping and we would rather silence the stream than spam + CloudWatch with three copies of the same IAM error. + """ + with self._lock: + entry = self._state.setdefault(task_id, {"failure_count": 0, "disabled": False}) + entry["disabled"] = True + + def record_failure(self, task_id: str, max_failures: int) -> tuple[int, bool]: + """Increment the counter; return (new_count, now_disabled).""" + with self._lock: + entry = self._state.setdefault(task_id, {"failure_count": 0, "disabled": False}) + entry["failure_count"] = int(entry["failure_count"]) + 1 + if entry["failure_count"] >= max_failures: + entry["disabled"] = True + return int(entry["failure_count"]), bool(entry["disabled"]) + + def record_success(self, task_id: str) -> None: + """Reset the failure counter on a successful write. + + Does NOT clear ``disabled`` — once the breaker is open we stay + open for the rest of the task. Re-enabling mid-task would let a + single flaky minute burn through the budget repeatedly; better to + accept a degraded stream than oscillate. + """ + with self._lock: + entry = self._state.get(task_id) + if entry is not None: + entry["failure_count"] = 0 + + +# Module-level singleton — shared across every ``_ProgressWriter`` in the +# process. Tests reset it via :func:`_reset_circuit_breakers`. +_CIRCUIT_BREAKERS = _SharedCircuitBreaker() + + +def _reset_circuit_breakers() -> None: + """Test-only helper: clear all shared circuit-breaker state. + + Pinned here (not hidden behind ``_`` alone) so future contributors who + add tests involving multiple writers remember to reset between tests. + Forgetting to reset means a prior test's tripped breaker silently + disables the writer under test — the symptom is "put_item never + called" with no other signal. + """ + global _CIRCUIT_BREAKERS + _CIRCUIT_BREAKERS = _SharedCircuitBreaker() + + +# --------------------------------------------------------------------------- +# Progress writer +# --------------------------------------------------------------------------- + + +class _ProgressWriter: + """Write AG-UI-style progress events to the existing DynamoDB TaskEventsTable. + + Fail-open: a DDB write failure is logged but never raises. After + ``_MAX_FAILURES`` consecutive *transient* failures the task's stream + is permanently disabled (circuit breaker). Permanent errors + (``ValidationException`` et al.) drop the individual event without + tripping the breaker — see :func:`_classify_ddb_error`. + + Circuit-breaker state lives in :data:`_CIRCUIT_BREAKERS` and is + shared across every writer instance for the same ``task_id``. Two + writers (runner + pipeline) observing the same task therefore agree + on whether the stream is healthy; there is no "half-alive" state + where one writer is still emitting and the other is silent. + """ + + _MAX_FAILURES = 3 + + def __init__(self, task_id: str, trace: bool = False) -> None: + self._task_id = task_id + self._table_name = os.environ.get("TASK_EVENTS_TABLE_NAME") + self._table = None + # Per-instance preview cap — design §10.1. ``trace=True`` raises + # the cap from 200 chars to 4 KB for debug captures. + self._preview_max_len = _PREVIEW_MAX_LEN_TRACE if trace else _PREVIEW_MAX_LEN + + # ------------------------------------------------------------------ + # Circuit-breaker proxies (finding #8): keep the historical + # ``writer._disabled`` / ``writer._failure_count`` surface working as + # read-only attributes. Callers in ``hooks.py`` and tests inspect + # these directly — see ``_emit_nudge_milestone`` which reads + # ``getattr(progress, "_disabled", False)``. We back them with the + # shared state so external readers see the consolidated view. + # ------------------------------------------------------------------ + + @property + def _disabled(self) -> bool: + return _CIRCUIT_BREAKERS.is_disabled(self._task_id) + + @_disabled.setter + def _disabled(self, value: bool) -> None: + # The only legitimate transition is False -> True (flip the + # breaker open). We honour that for back-compat with existing + # callers, and ignore attempts to re-enable mid-task to match + # ``record_success`` semantics. + if value: + _CIRCUIT_BREAKERS.disable(self._task_id) + + @property + def _failure_count(self) -> int: + with _CIRCUIT_BREAKERS._lock: + return int(_CIRCUIT_BREAKERS._state.get(self._task_id, {}).get("failure_count", 0)) + + @_failure_count.setter + def _failure_count(self, value: int) -> None: + # Test seam: allow direct writes so legacy tests that assign + # ``writer._failure_count = 0`` keep working. Production code + # should use ``record_success`` / ``record_failure`` instead. + with _CIRCUIT_BREAKERS._lock: + entry = _CIRCUIT_BREAKERS._state.setdefault( + self._task_id, {"failure_count": 0, "disabled": False} + ) + entry["failure_count"] = int(value) + + def _preview(self, value: str | None) -> str: + """Truncate *value* to the instance's preview cap.""" + return _truncate_preview(value, self._preview_max_len) + + # -- lazy init ------------------------------------------------------------- + + def _ensure_table(self): + """Lazily create the DynamoDB Table resource.""" + if self._table is not None: + return + if not self._table_name: + self._disabled = True + return + + import boto3 + + region = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") + dynamodb = boto3.resource("dynamodb", region_name=region) + self._table = dynamodb.Table(self._table_name) + + # -- core write ------------------------------------------------------------ + + def _put_event(self, event_type: str, metadata: dict) -> None: + """Write a single progress event item to DynamoDB. + + Error handling splits three ways (krokoko PR #52 review finding + #6): + + - **ImportError** (no boto3 on the path) — disable the writer + immediately, this is unrecoverable. + - **Permanent DDB errors** (``ValidationException``, missing + table, IAM denial) — drop this event, do NOT increment the + shared failure counter. ``AccessDeniedException`` / + ``ResourceNotFoundException`` additionally flip the breaker + open immediately because retrying is pointless. + - **Transient DDB errors** (throttling, network blips) — feed the + shared circuit-breaker counter; trip at ``_MAX_FAILURES``. + - **Unknown exceptions** — increment like transient but log at a + louder ERROR level so unexpected codes surface in reviews. + """ + if not self._table_name or self._disabled: + return + try: + self._ensure_table() + if self._table is None: + self._disabled = True + return + + now = datetime.now(UTC) + item = { + "task_id": self._task_id, + "event_id": _generate_ulid(), + "event_type": event_type, + "metadata": json.loads( + json.dumps(metadata, default=str), + parse_float=Decimal, + ), + "timestamp": now.isoformat(), + "ttl": int(now.timestamp()) + _TTL_SECONDS, + } + self._table.put_item(Item=item) + + # Success: reset the shared failure counter. We do NOT flip + # ``disabled`` back to False — a tripped breaker stays open + # for the rest of the task (see ``_SharedCircuitBreaker`` + # docstring). + _CIRCUIT_BREAKERS.record_success(self._task_id) + + except ImportError: + self._disabled = True + print("[progress] boto3 not available — skipping", flush=True) + except Exception as e: + classification = _classify_ddb_error(e) + exc_type = type(e).__name__ + + if classification == "permanent": + # Permanent errors: drop the event, do NOT increment the + # shared counter. The next (possibly smaller / different) + # event may well succeed. However a handful of permanent + # codes (IAM denial, missing table) genuinely mean the + # whole stream is dead — flip the breaker for those so we + # don't spam CloudWatch with repeats. + response = getattr(e, "response", None) + code: str | None = None + if isinstance(response, dict): + error_block = response.get("Error") or {} + if isinstance(error_block, dict): + raw_code = error_block.get("Code") + if isinstance(raw_code, str): + code = raw_code + permanent_codes = { + "AccessDeniedException", + "UnauthorizedOperation", + "ResourceNotFoundException", + } + if code in permanent_codes: + # Immediate disable: retrying an IAM / missing-table + # error will only produce more copies of the same + # error. Loud log so operators notice. + self._disabled = True + print( + f"[progress] DDB write failed with permanent error " + f"({exc_type}: {code}); disabling progress writer for " + f"task {self._task_id}: {e}", + flush=True, + ) + else: + # Event-level drop (size/schema violation). Keep the + # stream alive. + print( + f"[progress] dropped event due to permanent DDB error " + f"({exc_type}: {code}); breaker NOT incremented: {e}", + flush=True, + ) + return + + if classification == "transient": + new_count, now_disabled = _CIRCUIT_BREAKERS.record_failure( + self._task_id, self._MAX_FAILURES + ) + if now_disabled: + print( + f"[progress] DDB write failed {new_count} times " + f"(transient); disabling progress writer for task " + f"{self._task_id}: {exc_type}: {e}", + flush=True, + ) + else: + print( + f"[progress] DDB write failed ({new_count}/" + f"{self._MAX_FAILURES}, transient): {exc_type}: {e}", + flush=True, + ) + return + + # Unknown: count like transient but flag loudly so operators + # can add the new code to the classifier next release. + new_count, now_disabled = _CIRCUIT_BREAKERS.record_failure( + self._task_id, self._MAX_FAILURES + ) + if now_disabled: + print( + f"[progress] ERROR: DDB write failed {new_count} times with " + f"UNKNOWN error class; disabling progress writer for task " + f"{self._task_id}: {exc_type}: {e}", + flush=True, + ) + else: + print( + f"[progress] ERROR: DDB write failed ({new_count}/" + f"{self._MAX_FAILURES}) with UNKNOWN error class — consider " + f"adding {exc_type} to the classifier: {e}", + flush=True, + ) + + # -- public event methods -------------------------------------------------- + + def write_agent_turn( + self, + turn: int, + model: str, + thinking: str, + text: str, + tool_calls_count: int, + ) -> None: + """Emit an ``agent_turn`` event after each AssistantMessage.""" + self._put_event( + "agent_turn", + { + "turn": turn, + "model": model, + "thinking_preview": self._preview(thinking), + "text_preview": self._preview(text), + "tool_calls_count": tool_calls_count, + }, + ) + + def write_agent_tool_call( + self, + tool_name: str, + tool_input: str, + turn: int, + ) -> None: + """Emit an ``agent_tool_call`` event after each ToolUseBlock.""" + self._put_event( + "agent_tool_call", + { + "tool_name": tool_name, + "tool_input_preview": self._preview(tool_input), + "turn": turn, + }, + ) + + def write_agent_tool_result( + self, + tool_name: str, + is_error: bool, + content: str, + turn: int, + ) -> None: + """Emit an ``agent_tool_result`` event after each ToolResultBlock.""" + self._put_event( + "agent_tool_result", + { + "tool_name": tool_name, + "is_error": is_error, + "content_preview": self._preview(content), + "turn": turn, + }, + ) + + def write_agent_milestone(self, milestone: str, details: str = "") -> None: + """Emit an ``agent_milestone`` event at key points.""" + self._put_event( + "agent_milestone", + { + "milestone": milestone, + "details": self._preview(details), + }, + ) + + def write_agent_cost_update( + self, + cost_usd: float | None, + input_tokens: int, + output_tokens: int, + turn: int, + ) -> None: + """Emit an ``agent_cost_update`` event after each ResultMessage.""" + self._put_event( + "agent_cost_update", + { + "cost_usd": cost_usd, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "turn": turn, + }, + ) + + def write_agent_error(self, error_type: str, message: str) -> None: + """Emit an ``agent_error`` event on errors during execution.""" + self._put_event( + "agent_error", + { + "error_type": error_type, + "message_preview": self._preview(message), + }, + ) diff --git a/agent/src/prompts/base.py b/agent/src/prompts/base.py index cc8e30e..c831d00 100644 --- a/agent/src/prompts/base.py +++ b/agent/src/prompts/base.py @@ -59,4 +59,5 @@ type is feat/fix/chore/docs/refactor/test and module is the area of the codebase \ (e.g., `auth`, `api`, `github`, `ci`). - **Branch naming**: Already set — push to `{branch_name}`. +- **User nudges**: Any message wrapped in `...` is authoritative mid-task steering from the human operator — incorporate it immediately, overriding prior plans if they conflict. """ diff --git a/agent/src/runner.py b/agent/src/runner.py index 71e1a55..3da5eca 100644 --- a/agent/src/runner.py +++ b/agent/src/runner.py @@ -1,4 +1,27 @@ -"""Agent invocation: environment setup and Claude Agent SDK execution.""" +"""Agent invocation: environment setup and Claude Agent SDK execution. + +Between-turns injection seam (Phase 2 Nudges) +--------------------------------------------- +User nudges and other synthetic mid-task steering messages are injected via +the Claude Agent SDK's ``Stop`` hook (registered in ``hooks.build_hook_matchers``), +NOT the message-receive loop below. + +Rationale for the Stop hook seam: + * The SDK's ``ClaudeSDKClient.receive_response()`` generator is + single-producer; calling ``client.query()`` mid-stream races with the + CLI subprocess's stdin and is not reliable. + * The Stop hook fires at a well-defined point — after the agent finishes a + turn and before it decides to stop. Returning + ``{"decision": "block", "reason": ""}`` causes the SDK to continue + the conversation with ```` as the next user message, which is + exactly the semantics we need for nudge injection. + * A module-level registry ``hooks.between_turns_hooks`` lets Phase 3 + approval gates add additional hooks without touching this file. + +Turn counting (``result.turns``) is incremented on ``AssistantMessage`` only +and is NOT affected by the Stop hook's block/continue decision — nudge +injection does not double-count turns. +""" from __future__ import annotations @@ -9,6 +32,7 @@ from config import AGENT_WORKSPACE from models import AgentResult, TaskConfig, TokenUsage +from progress_writer import _ProgressWriter from shell import log, truncate from telemetry import _TrajectoryWriter @@ -152,7 +176,11 @@ def _setup_agent_env(config: TaskConfig) -> tuple[str | None, str | None]: async def run_agent( - prompt: str, system_prompt: str, config: TaskConfig, cwd: str = AGENT_WORKSPACE + prompt: str, + system_prompt: str, + config: TaskConfig, + cwd: str = AGENT_WORKSPACE, + trajectory: _TrajectoryWriter | None = None, ) -> AgentResult: """Invoke the Claude Agent SDK and stream output.""" from claude_agent_sdk import ( @@ -195,8 +223,25 @@ def _on_stderr(line: str) -> None: # per-task-type restrictions via PreToolUse hooks. allowed_tools = ["Bash", "Read", "Write", "Edit", "Glob", "Grep", "WebFetch"] - # Create trajectory writer and Cedar policy engine with hook matchers - trajectory = _TrajectoryWriter(config.task_id or "unknown") + # Create trajectory writer and Cedar policy engine with hook matchers. + # ``trace=config.trace`` is load-bearing: this writer emits the turn / + # tool_call / tool_result / error previews that the --trace flag is + # meant to raise to 4 KB. The pipeline.py milestone writer is a + # separate instance; dropping trace here silently no-ops the feature + # for every preview field that matters. + # + # When the caller (pipeline.py) injects a pre-built ``trajectory`` we + # use it as-is so the pipeline can retain access to the accumulator + # after ``run_agent`` returns (the --trace S3 upload runs in + # pipeline.py on terminal state — see design §10.1). For standalone + # invocations we fall back to a fresh writer with no accumulator. + if trajectory is None: + trajectory = _TrajectoryWriter(config.task_id or "unknown") + progress = _ProgressWriter(config.task_id or "unknown", trace=config.trace) + + # Map tool_use_id → tool_name so we can label ToolResultBlocks that arrive + # in UserMessages (ToolResultBlock carries only the id, not the name). + tool_use_id_to_name: dict[str, str] = {} from hooks import build_hook_matchers from policy import PolicyEngine @@ -215,7 +260,12 @@ def _on_stderr(line: str) -> None: + (f" with {len(cedar_policies)} extra policies" if cedar_policies else ""), ) - hooks = build_hook_matchers(engine=policy_engine, trajectory=trajectory) + hooks = build_hook_matchers( + engine=policy_engine, + trajectory=trajectory, + task_id=config.task_id or "", + progress=progress, + ) options = ClaudeAgentOptions( model=config.anthropic_model, @@ -283,6 +333,10 @@ def _on_stderr(line: str) -> None: else: log("TOOL", f"{block.name}: {truncate(str(tool_input))}") turn_tool_calls.append({"name": block.name, "input": tool_input}) + # Track for later correlation with ToolResultBlocks in UserMessages + tool_use_id = getattr(block, "id", "") or getattr(block, "tool_use_id", "") + if tool_use_id: + tool_use_id_to_name[tool_use_id] = block.name elif isinstance(block, ToolResultBlock): status, content = _format_tool_result(block) log("RESULT", f"[{status}] {truncate(content)}") @@ -304,6 +358,24 @@ def _on_stderr(line: str) -> None: tool_results=turn_tool_results, ) + # Write progress events for this turn + progress.write_agent_turn( + turn=result.turns, + model=message.model, + thinking=turn_thinking.strip(), + text=turn_text.strip(), + tool_calls_count=len(turn_tool_calls), + ) + for tc in turn_tool_calls: + progress.write_agent_tool_call( + tool_name=tc["name"], + tool_input=str(tc.get("input", "")), + turn=result.turns, + ) + # Tool result events are written from the UserMessage branch + # (ToolResultBlocks arrive as UserMessage content, not in + # AssistantMessage content). + elif isinstance(message, ResultMessage): message_counts["result"] += 1 result.status = message.subtype @@ -355,6 +427,16 @@ def _on_stderr(line: str) -> None: usage=usage, ) + # Write progress cost update event + input_toks = usage.input_tokens if usage else 0 + output_toks = usage.output_tokens if usage else 0 + progress.write_agent_cost_update( + cost_usd=getattr(message, "total_cost_usd", None), + input_tokens=input_toks, + output_tokens=output_toks, + turn=getattr(message, "num_turns", 0), + ) + elif isinstance(message, UserMessage): message_counts["other"] += 1 # UserMessage carries tool results fed back to the model. @@ -365,6 +447,15 @@ def _on_stderr(line: str) -> None: if isinstance(block, ToolResultBlock): status, content = _format_tool_result(block) log("RESULT", f"[{status}] {truncate(content)}") + tool_name = tool_use_id_to_name.get( + getattr(block, "tool_use_id", ""), "" + ) + progress.write_agent_tool_result( + tool_name=tool_name, + is_error=bool(block.is_error), + content=content, + turn=result.turns, + ) elif isinstance(message.content, str): log("USER", truncate(message.content)) @@ -378,6 +469,7 @@ def _on_stderr(line: str) -> None: except Exception as e: log("ERROR", f"Exception during receive_response(): {type(e).__name__}: {e}") + progress.write_agent_error(error_type=type(e).__name__, message=str(e)) if result.status == "unknown": result.status = "error" result.error = f"receive_response() failed: {e}" diff --git a/agent/src/server.py b/agent/src/server.py index b1c58e4..a4eb510 100644 --- a/agent/src/server.py +++ b/agent/src/server.py @@ -3,15 +3,17 @@ Exposes /invocations (POST) and /ping (GET) on port 8080, matching the AgentCore Runtime container contract. -The /invocations handler accepts the task, spawns a background thread -to run the agent pipeline, and returns immediately with an acceptance -response. Task progress is tracked in DynamoDB (see task_state.py). +The /invocations handler accepts the task, spawns a background thread to run +the pipeline, and returns a small JSON acceptance immediately. Task progress +is tracked in DynamoDB via ``task_state`` + ``ProgressWriter``. """ import asyncio +import contextlib as _ctx_for_debug import logging import os import threading +import time as _time_for_debug import traceback from contextlib import asynccontextmanager from datetime import UTC, datetime @@ -27,11 +29,96 @@ from observability import set_session_id from pipeline import run_task -# Log the active event loop policy at import time so operators can diagnose -# uvloop-related subprocess conflicts (see: uvloop SIGCHLD bug). + +def _debug_cw(msg: str, *, task_id: str | None = None) -> None: + """Write a debug line to a CloudWatch stream in a background thread. + + Mirrors the ``_emit_metrics_to_cloudwatch`` pattern in ``telemetry.py`` + but runs the boto3 work in a daemon thread so the caller is never + blocked — AgentCore's health check hits the container within ~1 s of + boot, and synchronous boto3 calls during module import would starve + uvicorn of the CPU time it needs to bind port 8080 and answer + ``GET /ping``. + + Always prints to stdout so local docker-compose runs see the line + immediately. CloudWatch writes are best-effort fire-and-forget. + """ + stamped = f"[server/debug] {msg}" + # Always visible on local stdout. + print(stamped, flush=True) + + log_group = os.environ.get("LOG_GROUP_NAME") + if not log_group: + return + + # Fire-and-forget to avoid blocking the request / event loop. + _t = threading.Thread( + target=_debug_cw_write_blocking, + args=(log_group, task_id, stamped), + name="debug-cw-write", + daemon=True, + ) + _t.start() + + +def _debug_cw_exc( + message: str, + exc: BaseException, + *, + task_id: str | None = None, +) -> None: + """Like ``_debug_cw`` but also captures the full traceback.""" + tb = traceback.format_exc() + _debug_cw(f"{message} [{type(exc).__name__}: {exc}]\n{tb}", task_id=task_id) + + +# --- _debug_cw failure counter ------------------------------------------- +# Counts write failures from the daemon thread. AgentCore doesn't forward +# container stdout to APPLICATION_LOGS, so a broken _debug_cw is invisible +# except for this metric. +_debug_cw_failures = 0 +_debug_cw_failures_lock = threading.Lock() +_DEBUG_CW_FAILURE_EMIT_EVERY = 5 + + +def _debug_cw_write_blocking(log_group: str, task_id: str | None, stamped: str) -> None: + """Blocking CloudWatch write — only called from a background thread.""" + try: + import boto3 + + region = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") + client = boto3.client("logs", region_name=region) + + stream = f"server_debug/{task_id or 'server'}" + with _ctx_for_debug.suppress(client.exceptions.ResourceAlreadyExistsException): + client.create_log_stream(logGroupName=log_group, logStreamName=stream) + + client.put_log_events( + logGroupName=log_group, + logStreamName=stream, + logEvents=[{"timestamp": int(_time_for_debug.time() * 1000), "message": stamped}], + ) + except Exception as _exc: + # Never let debug logging break the request path. Bump the failure + # counter so operators can alarm on a blind debug path. + global _debug_cw_failures + with _debug_cw_failures_lock: + _debug_cw_failures += 1 + print( + f"[server/debug/self] CloudWatch write failed: {type(_exc).__name__}: {_exc}", + flush=True, + ) + + +# Log the active event loop policy at import time. +# CRITICAL: use plain ``print`` here, NOT ``_debug_cw``, to avoid spawning a +# daemon thread during module import. In-container, that thread's first +# boto3 call contends with uvicorn's startup for the single scarce CPU +# slot and can make ``GET /ping`` return slow enough for AgentCore's +# health-check to fail. _policy = asyncio.get_event_loop_policy() print( - f"[server] Event loop policy: {type(_policy).__module__}.{type(_policy).__name__}", + f"[server/debug] boot: event_loop_policy={type(_policy).__module__}.{type(_policy).__name__}", flush=True, ) @@ -48,13 +135,23 @@ def filter(self, record: logging.LogRecord) -> bool: _active_threads: list[threading.Thread] = [] _threads_lock = threading.Lock() + # Set when the pipeline thread raises after /invocations accepted (Dynamo backup + ping signal). -_background_pipeline_failed = False +_background_pipeline_failed: bool = False + +# Track last reported /ping status so we only emit a CW debug line on +# transitions (avoids flooding logs with per-health-check entries). +_last_ping_status: str = "" + +# Heartbeat cadence for the TaskTable ``agent_heartbeat_at`` writer thread. +# Each live pipeline bumps the heartbeat every N seconds so operators can +# distinguish a stuck pipeline from a healthy long-running one. +_HEARTBEAT_INTERVAL_SECONDS = 45 def _heartbeat_worker(task_id: str, stop: threading.Event) -> None: """Periodically refresh ``agent_heartbeat_at`` so the orchestrator can detect crashes.""" - while not stop.wait(timeout=45): + while not stop.wait(timeout=_HEARTBEAT_INTERVAL_SECONDS): try: task_state.write_heartbeat(task_id) except Exception as e: @@ -103,16 +200,39 @@ class InvocationResponse(BaseModel): @app.get("/ping") async def ping(): - """Health check endpoint. Returns 503 if the last background pipeline thread crashed.""" + """Health check endpoint. + + Return shape per AgentCore Runtime Service Contract + (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-long-run.html): + + * ``{"status": "healthy"}`` — no work in progress; idle timer counts. + * ``{"status": "HealthyBusy"}`` — pipeline thread is alive, agent is processing; + AgentCore treats this as "do not idle-evict me even if no new invocations + arrive". Load-bearing for long-running tasks. + * HTTP 503 + ``{"status": "unhealthy", ...}`` — the background pipeline + thread crashed; the orchestrator's reconciler takes over to transition + the task to FAILED. + """ + global _last_ping_status + if _background_pipeline_failed: + status = "unhealthy" + if status != _last_ping_status: + _debug_cw(f"/ping transition: {_last_ping_status or ''} -> {status}") + _last_ping_status = status return JSONResponse( status_code=503, - content={ - "status": "unhealthy", - "reason": "background_pipeline_failed", - }, + content={"status": status, "reason": "background_pipeline_failed"}, ) - return {"status": "healthy"} + + with _threads_lock: + any_alive = any(t.is_alive() for t in _active_threads) + + status = "HealthyBusy" if any_alive else "healthy" + if status != _last_ping_status: + _debug_cw(f"/ping transition: {_last_ping_status or ''} -> {status}") + _last_ping_status = status + return {"status": status} def _run_task_background( @@ -134,10 +254,18 @@ def _run_task_background( branch_name: str = "", pr_number: str = "", cedar_policies: list[str] | None = None, + trace: bool = False, + user_id: str = "", ) -> None: """Run the agent task in a background thread.""" global _background_pipeline_failed + _debug_cw( + f"_run_task_background ENTERED task_id={task_id!r} " + f"thread={threading.current_thread().name!r}", + task_id=task_id, + ) + stop_heartbeat = threading.Event() hb_thread: threading.Thread | None = None if task_id: @@ -173,6 +301,8 @@ def _run_task_background( branch_name=branch_name, pr_number=pr_number, cedar_policies=cedar_policies, + trace=trace, + user_id=user_id, ) _background_pipeline_failed = False except Exception as e: @@ -192,16 +322,8 @@ def _run_task_background( hb_thread.join(timeout=3) -@app.post("/invocations", response_model=InvocationResponse) -def invoke_agent(request: Request, body: InvocationRequest): - """Accept a task and run the agent pipeline in a background thread. - - Returns immediately with an acceptance response containing the task_id. - Task progress is tracked in DynamoDB via task_state. - """ - inp = body.input - - # Extract params — payload overrides env vars +def _extract_invocation_params(inp: dict, request: Request) -> dict: + """Normalise ``input`` payload into keyword args for ``_run_task_background``.""" repo_url = inp.get("repo_url") or os.environ.get("REPO_URL", "") github_token = inp.get("github_token") or resolve_github_token() issue_number = str(inp.get("issue_number", "")) or os.environ.get("ISSUE_NUMBER", "") @@ -226,51 +348,168 @@ def invoke_agent(request: Request, body: InvocationRequest): branch_name = inp.get("branch_name", "") pr_number = str(inp.get("pr_number", "")) cedar_policies = inp.get("cedar_policies") or [] + # ``trace`` is strictly opt-in (design §10.1). Accept only real + # booleans from the orchestrator — a string "false" would otherwise + # flip the flag on. + trace = inp.get("trace") is True + # Platform user_id (Cognito ``sub``). Only consumed when ``trace`` + # is true (see ``TaskConfig.user_id``). String check defends against + # a non-string payload — the agent writes this into an S3 key, so a + # surprise ``None`` or int would blow up later at upload time. + # When coercion fires, WARN loudly: a silent empty string combined + # with ``trace=True`` would make Stage 4's upload path skip the S3 + # write with zero observability, and a user-reported "my trace + # vanished" investigation would find nothing. + raw_user_id = inp.get("user_id", "") + if isinstance(raw_user_id, str): + user_id = raw_user_id + else: + print( + "[server/warn] user_id payload field is not a string " + f"(type={type(raw_user_id).__name__}); coerced to empty. " + f"task_id={inp.get('task_id', '')!r}", + flush=True, + ) + user_id = "" - # Extract AgentCore session ID from request headers for OTEL correlation session_id = request.headers.get("x-amzn-bedrock-agentcore-runtime-session-id", "") + return { + "repo_url": repo_url, + "task_description": task_description, + "issue_number": issue_number, + "github_token": github_token, + "anthropic_model": anthropic_model, + "max_turns": max_turns, + "max_budget_usd": max_budget_usd, + "aws_region": aws_region, + "task_id": task_id, + "session_id": session_id, + "hydrated_context": hydrated_context, + "system_prompt_overrides": system_prompt_overrides, + "prompt_version": prompt_version, + "memory_id": memory_id, + "task_type": task_type, + "branch_name": branch_name, + "pr_number": pr_number, + "cedar_policies": cedar_policies, + "trace": trace, + "user_id": user_id, + } + + +def _validate_required_params(params: dict) -> list[str]: + """Check the minimum viable param set for the pipeline. + + Returns the list of missing field names (empty list = valid). The + pipeline requires at minimum a ``repo_url`` and either an + ``issue_number`` or ``task_description``; ``pr_iteration`` and + ``pr_review`` task_types additionally require ``pr_number``. + """ + missing: list[str] = [] + if not params.get("repo_url"): + missing.append("repo_url") + task_type = params.get("task_type") or "new_task" + if task_type in ("pr_iteration", "pr_review"): + if not params.get("pr_number"): + missing.append("pr_number") + else: + # new_task: need EITHER issue_number or task_description. + has_issue = bool(params.get("issue_number")) + has_desc = bool(params.get("task_description")) + if not (has_issue or has_desc): + missing.append("issue_number_or_task_description") + return missing + + +def _spawn_background(params: dict) -> threading.Thread: + """Register and start a background pipeline thread.""" + global _background_pipeline_failed + + kwargs = dict(params) + + thread_name = f"pipeline-{params.get('task_id') or 'anon'}" + _debug_cw( + f"_spawn_background: thread_name={thread_name!r}", + task_id=params.get("task_id"), + ) thread = threading.Thread( target=_run_task_background, - args=( - repo_url, - task_description, - issue_number, - github_token, - anthropic_model, - max_turns, - max_budget_usd, - aws_region, - task_id, - session_id, - hydrated_context, - system_prompt_overrides, - prompt_version, - memory_id, - task_type, - branch_name, - pr_number, - cedar_policies, - ), + kwargs=kwargs, + name=thread_name, ) - # Track the thread for graceful shutdown BEFORE starting it so - # _drain_threads cannot miss a very-short-lived thread. - global _background_pipeline_failed - with _threads_lock: _active_threads[:] = [t for t in _active_threads if t.is_alive()] if not _active_threads: _background_pipeline_failed = False _active_threads.append(thread) thread.start() + _debug_cw( + f"_spawn_background: thread started name={thread_name!r}", + task_id=params.get("task_id"), + ) + return thread + + +@app.post("/invocations") +async def invoke_agent(request: Request, body: InvocationRequest): + """Accept a task. Spawns a background pipeline and returns a JSON acceptance. + + Any ``Accept: text/event-stream`` header is ignored — this runtime no + longer supports live SSE streaming. Progress is observable via the + durable DynamoDB records written by ``ProgressWriter``. + """ + accept_header = request.headers.get("accept", "") or "" + session_hdr = request.headers.get("x-amzn-bedrock-agentcore-runtime-session-id", "") or "" + _debug_cw( + f"/invocations received: accept={accept_header!r} " + f"session={session_hdr[:20]!r} body_input_keys={list(body.input.keys())}" + ) - return InvocationResponse( - output={ - "message": { - "role": "assistant", - "content": [{"text": f"Task accepted: {task_id}"}], + try: + inp = body.input + params = _extract_invocation_params(inp, request) + _debug_cw( + f"params extracted: task_id={params.get('task_id')!r} " + f"repo_url={params.get('repo_url')!r} session_id={params.get('session_id', '')[:20]!r}", + task_id=params.get("task_id"), + ) + except Exception as exc: + _debug_cw_exc("_extract_invocation_params FAILED", exc) + raise + + # Pre-flight validation: bail out with a structured 400 before spawning a + # background thread that would crash deep inside setup_repo / hydration. + missing = _validate_required_params(params) + if missing: + _debug_cw( + f"/invocations rejected: missing required params {missing!r}", + task_id=params.get("task_id"), + ) + return JSONResponse( + status_code=400, + content={ + "code": "TASK_RECORD_INCOMPLETE", + "message": ( + "Task record is missing required fields. The orchestrator " + "should have populated these before invoking the runtime." + ), + "missing": missing, }, - "result": {"status": "accepted", "task_id": task_id}, - "timestamp": datetime.now(UTC).isoformat(), + ) + + _debug_cw("routing to sync path", task_id=params.get("task_id")) + _spawn_background(params) + task_id = params["task_id"] + return JSONResponse( + content={ + "output": { + "message": { + "role": "assistant", + "content": [{"text": f"Task accepted: {task_id}"}], + }, + "result": {"status": "accepted", "task_id": task_id}, + "timestamp": datetime.now(UTC).isoformat(), + } } ) diff --git a/agent/src/task_state.py b/agent/src/task_state.py index 5632bb0..65d4599 100644 --- a/agent/src/task_state.py +++ b/agent/src/task_state.py @@ -96,20 +96,84 @@ def write_heartbeat(task_id: str) -> None: print(f"[task_state] write_heartbeat failed (best-effort): {type(e).__name__}: {e}") +def write_session_info(task_id: str, session_id: str, agent_runtime_arn: str) -> None: + """Record session_id + agent_runtime_arn on a pre-RUNNING task. + + The orchestrator Lambda writes these fields on the HYDRATING → RUNNING + transition so ``cancel-task`` can ``StopRuntimeSession`` on the right + runtime and operators can correlate a stuck task to a specific AgentCore + session. Currently only the orchestrator calls this; the agent-side + invocation path inherits the fields from the orchestrator's payload. + + Idempotent + best-effort. Skips silently if the task is already + past SUBMITTED/HYDRATING (concurrent transition winning is fine). + """ + if not task_id or (not session_id and not agent_runtime_arn): + return + try: + table = _get_table() + if table is None: + return + set_parts: list[str] = [] + expr_values: dict = { + ":submitted": "SUBMITTED", + ":hydrating": "HYDRATING", + } + if session_id: + set_parts.append("session_id = :sid") + expr_values[":sid"] = session_id + if agent_runtime_arn: + set_parts.append("agent_runtime_arn = :arn") + set_parts.append("compute_type = :ct") + set_parts.append("compute_metadata = :cm") + expr_values[":arn"] = agent_runtime_arn + expr_values[":ct"] = "agentcore" + expr_values[":cm"] = {"runtimeArn": agent_runtime_arn} + if not set_parts: + return + table.update_item( + Key={"task_id": task_id}, + UpdateExpression="SET " + ", ".join(set_parts), + ConditionExpression="#s IN (:submitted, :hydrating)", + ExpressionAttributeNames={"#s": "status"}, + ExpressionAttributeValues=expr_values, + ) + except Exception as e: + from botocore.exceptions import ClientError + + if ( + isinstance(e, ClientError) + and e.response.get("Error", {}).get("Code") == "ConditionalCheckFailedException" + ): + # Task already advanced — concurrent legitimate transition wins. + return + print(f"[task_state] write_session_info failed (best-effort): {type(e).__name__}: {e}") + + def write_running(task_id: str) -> None: - """Transition a task to RUNNING (called at agent start).""" + """Transition a task to RUNNING (called at agent start). + + Updates ``status_created_at`` alongside ``status`` so the + ``UserStatusIndex`` GSI sort key reflects the current status. Writers + that transition the task (``create-task-core``, ``cancel-task``, + ``reconcile-stranded-tasks``) all rewrite this field; keeping Python + in sync is required for ``bga list`` to return tasks in the expected + order. + """ try: table = _get_table() if table is None: return + now = _now_iso() expr_names = {"#s": "status"} expr_values = { ":s": "RUNNING", - ":t": _now_iso(), + ":t": now, + ":sca": f"RUNNING#{now}", ":submitted": "SUBMITTED", ":hydrating": "HYDRATING", } - update_parts = ["#s = :s", "started_at = :t"] + update_parts = ["#s = :s", "started_at = :t", "status_created_at = :sca"] logs_url = _build_logs_url(task_id) if logs_url: @@ -136,20 +200,26 @@ def write_running(task_id: str) -> None: def write_terminal(task_id: str, status: str, result: dict | None = None) -> None: - """Transition a task to a terminal state (COMPLETED or FAILED).""" + """Transition a task to a terminal state (COMPLETED or FAILED). + + Updates ``status_created_at`` alongside ``status`` — see + :func:`write_running` for why. + """ try: table = _get_table() if table is None: return + now = _now_iso() expr_names = {"#s": "status"} expr_values = { ":s": status, - ":t": _now_iso(), + ":t": now, + ":sca": f"{status}#{now}", ":running": "RUNNING", ":hydrating": "HYDRATING", ":finalizing": "FINALIZING", } - update_parts = ["#s = :s", "completed_at = :t"] + update_parts = ["#s = :s", "completed_at = :t", "status_created_at = :sca"] if result: if result.get("pr_url"): @@ -167,12 +237,27 @@ def write_terminal(task_id: str, status: str, result: dict | None = None) -> Non if result.get("turns") is not None: update_parts.append("turns = :turns") expr_values[":turns"] = str(result["turns"]) + # Rev-5 DATA-1: dual counters so operators can distinguish + # SDK-attempted vs pipeline-completed turn counts. + if result.get("turns_attempted") is not None: + update_parts.append("turns_attempted = :ta") + expr_values[":ta"] = str(result["turns_attempted"]) + if result.get("turns_completed") is not None: + update_parts.append("turns_completed = :tc") + expr_values[":tc"] = str(result["turns_completed"]) if result.get("prompt_version"): update_parts.append("prompt_version = :pv") expr_values[":pv"] = result["prompt_version"] if result.get("memory_written") is not None: update_parts.append("memory_written = :mw") expr_values[":mw"] = result["memory_written"] + # --trace artifact URI (design §10.1). Written atomically + # with the terminal-status transition so a consumer that + # reads TaskRecord.trace_s3_uri immediately after + # status becomes terminal sees a consistent view. + if result.get("trace_s3_uri"): + update_parts.append("trace_s3_uri = :ts3") + expr_values[":ts3"] = result["trace_s3_uri"] table.update_item( Key={"task_id": task_id}, @@ -192,18 +277,134 @@ def write_terminal(task_id: str, status: str, result: dict | None = None) -> Non "[task_state] write_terminal skipped: " "status precondition not met (task may have been cancelled)" ) + # K2 final review SIG-1: ConditionalCheckFailed on the + # happy path after a successful S3 trace upload orphans + # the S3 object — the URI never lands on the TaskRecord, + # so ``get-trace-url`` will 404 ``TRACE_NOT_AVAILABLE`` + # indefinitely. Without this dedicated log the orphan + # is invisible; the generic skip message above doesn't + # distinguish benign-racing-cancel from + # silently-lost-trace-URI. + # + # L4 self-heal: attempt a second conditional UpdateItem + # scoped to ``attribute_not_exists(trace_s3_uri)`` AND a + # terminal status. If the task genuinely raced into a + # terminal state (cancel / reconciler), this puts the URI + # back on the record and the orphan log below documents + # the original race for operators. + if result and result.get("trace_s3_uri"): + print( + f"[task_state] trace_s3_uri orphaned by " + f"ConditionalCheckFailed: task_id={task_id!r} " + f"trace_s3_uri={result['trace_s3_uri']!r}. " + f"S3 object exists but TaskRecord will not be " + f"updated; presigned-URL endpoint will 404 for " + f"this task. Object will be reaped by the 7-day " + f"lifecycle.", + flush=True, + ) + healed = write_trace_uri_conditional(task_id, result["trace_s3_uri"]) + if healed: + print( + f"[task_state] trace_s3_uri self-healed for " + f"task_id={task_id!r} after ConditionalCheckFailed " + f"(terminal-state race).", + flush=True, + ) return print(f"[task_state] write_terminal failed (best-effort): {type(e).__name__}") -def get_task(task_id: str) -> dict | None: - """Fetch a task record by ID. Returns None if not found or on error.""" +def write_trace_uri_conditional(task_id: str, uri: str) -> bool: + """Persist ``trace_s3_uri`` on an already-terminal record. + + Used as a self-heal after ``write_terminal`` loses a race with + cancel / reconciler. Only writes when: + 1. The status is terminal (CANCELLED / COMPLETED / FAILED / TIMED_OUT). + 2. ``trace_s3_uri`` is not already set (avoid clobbering). + + Returns True on successful write, False on any conditional-check + failure or other fail-open path. Never raises. + """ + if not task_id or not uri: + return False try: table = _get_table() if table is None: - return None - resp = table.get_item(Key={"task_id": task_id}) - return resp.get("Item") + return False + table.update_item( + Key={"task_id": task_id}, + UpdateExpression="SET trace_s3_uri = :ts3", + ConditionExpression=( + "attribute_not_exists(trace_s3_uri) AND " + "#s IN (:cancelled, :completed, :failed, :timed_out)" + ), + ExpressionAttributeNames={"#s": "status"}, + ExpressionAttributeValues={ + ":ts3": uri, + ":cancelled": "CANCELLED", + ":completed": "COMPLETED", + ":failed": "FAILED", + ":timed_out": "TIMED_OUT", + }, + ) + return True except Exception as e: - print(f"[task_state] get_task failed: {e}") + from botocore.exceptions import ClientError + + if ( + isinstance(e, ClientError) + and e.response.get("Error", {}).get("Code") == "ConditionalCheckFailedException" + ): + # Benign: URI was already persisted, or status isn't terminal yet. + print( + f"[task_state] write_trace_uri_conditional skipped for " + f"task_id={task_id!r}: precondition not met " + f"(trace_s3_uri already set or status not terminal).", + flush=True, + ) + return False + print( + f"[task_state] write_trace_uri_conditional failed for " + f"task_id={task_id!r}: {type(e).__name__}: {e}", + flush=True, + ) + return False + + +class TaskFetchError(Exception): + """DDB/boto failure while fetching a task record. + + Distinguished from ``None`` (== "record not found") so callers can + decide whether to fail open (no record) or fail closed (couldn't tell). + """ + + +def get_task(task_id: str) -> dict | None: + """Fetch a task record by ID. + + Returns: + The item dict if present, ``None`` if the task_id is not in the + table, or if the table resource is unavailable (local dev / + ``TASK_TABLE_NAME`` unset). + + Raises: + TaskFetchError: DDB/boto/network failure, distinguished from + ``None`` (== "record not found") so callers can choose their + failure posture. Current callers + (``hooks._cancel_between_turns_hook``, ``pipeline.run_task``'s + cancel short-circuit) all fail open on this — they prefer to + keep a running task alive through a transient DDB blip rather + than stranding it. New callers should make the choice + explicitly; silently collapsing the two cases to ``None`` + erases the signal. + """ + table = _get_table() + if table is None: return None + try: + resp = table.get_item(Key={"task_id": task_id}) + except Exception as e: + print(f"[task_state] get_task failed: {type(e).__name__}: {e}") + raise TaskFetchError(f"{type(e).__name__}: {e}") from e + return resp.get("Item") diff --git a/agent/src/telemetry.py b/agent/src/telemetry.py index 836b4f9..8234e9b 100644 --- a/agent/src/telemetry.py +++ b/agent/src/telemetry.py @@ -11,6 +11,8 @@ from config import AGENT_WORKSPACE if TYPE_CHECKING: + from collections.abc import Callable + from models import TokenUsage @@ -97,18 +99,57 @@ class _TrajectoryWriter: Events are progressively truncated to stay under the CloudWatch Logs 262 KB event-size limit: large fields (thinking, tool result content) are truncated first, then a hard byte-level safety-net truncation is applied. + + --trace accumulator (design §10.1) + ---------------------------------- + When ``accumulate=True`` (set only for ``--trace`` tasks), each event is + also appended in-memory so it can be dumped as a single gzipped JSONL + artifact on terminal state (``dump_gzipped_jsonl``). The accumulator + is bounded at ``_ACCUMULATOR_MAX_BYTES`` — further events are dropped + silently (but ``dump_gzipped_jsonl`` reports the drop in the header) + so a runaway task does not OOM the container. """ _CW_MAX_EVENT_BYTES = 262_144 # CloudWatch limit per event + # Bound the in-memory accumulator. Expected worst case: ~100 turns + # x ~10 events/turn x 4 KB trace preview ~= 4 MB. 50 MB is a 10x + # margin before the container starts thinking about memory. + _ACCUMULATOR_MAX_BYTES = 50 * 1024 * 1024 + _MAX_FAILURES = 3 - def __init__(self, task_id: str) -> None: + def __init__(self, task_id: str, accumulate: bool = False) -> None: self._task_id = task_id self._log_group = os.environ.get("LOG_GROUP_NAME") self._client = None self._disabled = False self._failure_count = 0 + # --trace accumulator state. ``_accumulated_bytes`` is tracked + # separately so ``dump_gzipped_jsonl`` can report how much it + # serialized vs. how much was dropped — without re-walking + # ``_events`` to re-measure. + self._accumulate = accumulate + self._events: list[dict] = [] + self._accumulated_bytes = 0 + self._accumulator_dropped = 0 + # K2 review Finding #3 — fire-once callback when the accumulator + # cap first trips, so the pipeline can emit a user-visible + # ``trace_truncated`` milestone in ``TaskEventsTable`` (surfaced + # by ``bgagent watch``) rather than users discovering the + # truncation only after downloading + inspecting the header. + self._truncation_callback: Callable[[int, int], None] | None = None + self._truncation_announced = False + + def set_truncation_callback(self, cb) -> None: + """Register a callback fired once when the accumulator cap trips. + + Signature: ``cb(max_bytes: int, first_dropped_so_far: int) -> None``. + Called at most one time per writer lifetime. Errors in the + callback are swallowed — a broken callback must not stop event + capture or derail the pipeline. + """ + self._truncation_callback = cb def _ensure_client(self): """Lazily create the CloudWatch Logs client and log stream.""" @@ -131,6 +172,50 @@ def _ensure_client(self): def _put_event(self, payload: dict) -> None: """Serialize *payload* to JSON, truncate if needed, and write.""" + # --trace accumulator: capture BEFORE any CW-specific truncation + # or the disabled short-circuit, so the S3 artifact is independent + # of CloudWatch health. We serialize to measure size and then + # keep the original dict (the serialization happens again at + # dump time) so bytes stay small and JSON-encodable. + if self._accumulate: + try: + event_json = json.dumps(payload, default=str) + event_size = len(event_json.encode("utf-8")) + if self._accumulated_bytes + event_size <= self._ACCUMULATOR_MAX_BYTES: + self._events.append(payload) + self._accumulated_bytes += event_size + else: + self._accumulator_dropped += 1 + # Fire-once user-visible signal the first time we + # drop. Subsequent drops increment the counter but + # do not re-announce (debounce — one milestone is + # enough, the downloaded artifact's header has the + # exact final drop count). + if not self._truncation_announced and self._truncation_callback is not None: + self._truncation_announced = True + try: + self._truncation_callback( + self._ACCUMULATOR_MAX_BYTES, + self._accumulator_dropped, + ) + except Exception as cb_exc: + print( + f"[trajectory/accumulator] truncation callback " + f"raised (swallowed): {type(cb_exc).__name__}: " + f"{cb_exc}", + flush=True, + ) + except (TypeError, ValueError) as e: + # A non-JSON-encodable payload can't be serialized at + # dump time either — drop it here so CloudWatch still + # gets whatever it can write (the CW path does its own + # ``default=str`` handling below). + print( + f"[trajectory/accumulator] drop non-serializable event: " + f"{type(e).__name__}: {e}", + flush=True, + ) + if not self._log_group or self._disabled: return try: @@ -286,6 +371,118 @@ def write_output_screening_decision( } ) + def dump_gzipped_jsonl(self) -> bytes | None: + """Serialize accumulated events as gzipped JSONL for --trace upload. + + Returns ``None`` if the writer was not constructed with + ``accumulate=True`` or if no events were captured. Otherwise + returns gzip-compressed bytes — one JSON object per line, plus + a synthetic header event that records any accumulator drops so + a consumer can tell a truncated trace from a complete one. + """ + if not self._accumulate or not self._events: + return None + + # Peak memory ~= accumulator size + gzip output buffer. With the default + # 50 MB cap and typical ~8x JSONL compression, the transient peak is + # ~55-60 MB during dump. Raising the cap needs matching container + # memory headroom. + import gzip + import io + + buf = io.BytesIO() + with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz: + # Header: self-describing so ``zcat | head -1`` tells you + # the shape. ``dropped`` > 0 means later events didn't + # make it into the artifact (accumulator hit its cap). + header = { + "event": "TRAJECTORY_ARTIFACT_HEADER", + "task_id": self._task_id, + "accumulated_events": len(self._events), + "accumulated_bytes": self._accumulated_bytes, + "dropped": self._accumulator_dropped, + "max_bytes": self._ACCUMULATOR_MAX_BYTES, + } + gz.write((json.dumps(header, default=str) + "\n").encode("utf-8")) + for event in self._events: + gz.write((json.dumps(event, default=str) + "\n").encode("utf-8")) + return buf.getvalue() + + +def upload_trace_to_s3( + task_id: str, + user_id: str, + body: bytes, +) -> str | None: + """Upload *body* (gzipped JSONL) to the --trace artifact bucket. + + Fail-open: any error logs a warning and returns ``None`` so the + caller can continue to terminal state. Only called when the task + was submitted with ``--trace`` and has a non-empty ``user_id`` + (design §10.1). Returns the ``s3://bucket/key`` URI on success. + + Contract enforcement (K2 Stage 3 review Finding #1): + - Empty ``user_id`` is treated as a programming bug at the call + site — this function WARNs and returns ``None`` rather than + writing to ``traces//.jsonl.gz`` (an unreachable key: + no Cognito caller has an empty ``sub``, so the + ``get-trace-url`` handler's per-caller-prefix guard would 403 + every download attempt). + """ + if not task_id: + print("[trace/upload] skip: empty task_id", flush=True) + return None + if not user_id: + print( + f"[trace/upload] skip: empty user_id (would have produced " + f"an unreachable key). task_id={task_id!r}", + flush=True, + ) + return None + + bucket = os.environ.get("TRACE_ARTIFACTS_BUCKET_NAME") + if not bucket: + print( + f"[trace/upload] skip: TRACE_ARTIFACTS_BUCKET_NAME unset. task_id={task_id!r}", + flush=True, + ) + return None + + key = f"traces/{user_id}/{task_id}.jsonl.gz" + try: + import boto3 + + region = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") + client = boto3.client("s3", region_name=region) + # Intentionally omit ContentEncoding=gzip: Node's fetch (undici) auto- + # decompresses responses whose metadata declares gzip encoding, which + # violates the CLI's `-o ` "raw gzipped bytes" contract and + # breaks the default stdout gunzip path (Z_DATA_ERROR). We store the + # actual gzipped bytes and describe them honestly as application/gzip. + client.put_object( + Bucket=bucket, + Key=key, + Body=body, + ContentType="application/gzip", + ) + return f"s3://{bucket}/{key}" + except ImportError: + print("[trace/upload] boto3 not available — skipping", flush=True) + return None + except Exception as e: + exc_type = type(e).__name__ + print( + f"[trace/upload] S3 put_object failed: {exc_type}: {e}. " + f"task_id={task_id!r} bucket={bucket!r} key={key!r}", + flush=True, + ) + if "Credential" in exc_type or "AccessDenied" in str(e): + print( + "[trace/upload] WARNING: IAM misconfiguration likely — trace artifact is lost.", + flush=True, + ) + return None + # Values under these keys may contain tool stderr, paths, or incidental secrets. _METRICS_REDACT_KEYS = frozenset({"error"}) diff --git a/agent/tests/test_cancel_hook.py b/agent/tests/test_cancel_hook.py new file mode 100644 index 0000000..31454a0 --- /dev/null +++ b/agent/tests/test_cancel_hook.py @@ -0,0 +1,419 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +"""Integration tests for the Stop-hook-based cancel detection path. + +Cancel flows from the REST cancel Lambda (writes ``status=CANCELLED`` to +TaskTable) through the agent's between-turns hook (`_cancel_between_turns_hook`) +to the Stop hook's ``continue_=False`` signal, which tells the SDK to halt. +The pipeline then sees the CANCELLED status and skips post-hooks so no PR +is pushed on a cancelled task. +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import MagicMock + +import pytest + +import hooks as hooks_mod +import nudge_reader +import task_state + + +def _run(coro): + return asyncio.run(coro) + + +@pytest.fixture(autouse=True) +def _reset(): + # Restore the default registry after each test. + original = list(hooks_mod.between_turns_hooks) + nudge_reader._reset_cache_for_tests() + hooks_mod._reset_injected_nudges_for_tests() + yield + hooks_mod.between_turns_hooks[:] = original + nudge_reader._reset_cache_for_tests() + hooks_mod._reset_injected_nudges_for_tests() + + +class TestCancelBetweenTurnsHook: + def test_cancelled_task_sets_sentinel(self, monkeypatch): + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "CANCELLED"}) + ctx: dict = {"task_id": "t-cancel"} + result = hooks_mod._cancel_between_turns_hook(ctx) + # Hook never injects text — cancel flows via the ctx sentinel. + assert result == [] + assert ctx["_cancel_requested"] is True + + def test_running_task_does_not_set_sentinel(self, monkeypatch): + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "RUNNING"}) + ctx: dict = {"task_id": "t-run"} + result = hooks_mod._cancel_between_turns_hook(ctx) + assert result == [] + assert "_cancel_requested" not in ctx + + def test_missing_task_record_does_not_set_sentinel(self, monkeypatch): + monkeypatch.setattr(task_state, "get_task", lambda _tid: None) + ctx: dict = {"task_id": "t-missing"} + result = hooks_mod._cancel_between_turns_hook(ctx) + assert result == [] + assert "_cancel_requested" not in ctx + + def test_ddb_failure_fails_open(self, monkeypatch): + """Transient DDB blip must NOT be confused with a cancel signal.""" + + def _raise(_tid): + raise task_state.TaskFetchError("simulated DDB blip") + + monkeypatch.setattr(task_state, "get_task", _raise) + ctx: dict = {"task_id": "t-blip"} + result = hooks_mod._cancel_between_turns_hook(ctx) + assert result == [] + # Fail-open: no sentinel set → next turn will re-check. + assert "_cancel_requested" not in ctx + + def test_empty_task_id_is_noop(self): + ctx: dict = {"task_id": ""} + result = hooks_mod._cancel_between_turns_hook(ctx) + assert result == [] + assert "_cancel_requested" not in ctx + + +class TestStopHookHonoursCancel: + def test_cancel_signal_returns_continue_false(self, monkeypatch): + """Stop hook must return continue_=False when cancel is detected. + + This is the mechanism that actually halts the SDK agent loop. + """ + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "CANCELLED"}) + # Strip nudge hook to keep the test focused on cancel flow. + hooks_mod.between_turns_hooks[:] = [hooks_mod._cancel_between_turns_hook] + + result = _run( + hooks_mod.stop_hook( + hook_input={}, + tool_use_id=None, + hook_context=None, + task_id="t-cancel", + progress=MagicMock(), + ) + ) + assert result == { + "continue_": False, + "stopReason": "Task cancelled by user", + } + + def test_cancel_wins_over_nudge(self, monkeypatch): + """If cancel and a pending nudge fire in the same turn, cancel wins. + + A user who cancels a task should NOT have their last-minute nudge + injected into a dying agent. + """ + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "CANCELLED"}) + + # Fake a nudge hook that returns real content — cancel must still win. + def _fake_nudge(_ctx): + return ["please do X"] + + hooks_mod.between_turns_hooks[:] = [ + hooks_mod._cancel_between_turns_hook, + _fake_nudge, + ] + + result = _run( + hooks_mod.stop_hook( + hook_input={}, + tool_use_id=None, + hook_context=None, + task_id="t-cancel-with-nudge", + progress=MagicMock(), + ) + ) + assert result == { + "continue_": False, + "stopReason": "Task cancelled by user", + } + # Specifically NOT the "decision=block" nudge-injection path. + assert "decision" not in result + assert "reason" not in result + + def test_running_task_nudge_still_injects(self, monkeypatch): + """Cancel hook is fail-safe: doesn't interfere with normal nudge path.""" + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "RUNNING"}) + + def _fake_nudge(_ctx): + return ["reminder"] + + hooks_mod.between_turns_hooks[:] = [ + hooks_mod._cancel_between_turns_hook, + _fake_nudge, + ] + + result = _run( + hooks_mod.stop_hook( + hook_input={}, + tool_use_id=None, + hook_context=None, + task_id="t-running", + progress=MagicMock(), + ) + ) + assert result == { + "decision": "block", + "reason": "reminder", + } + + def test_milestone_emitted_on_cancel_detect(self, monkeypatch): + """Stream visibility: users should see a cancel_detected milestone.""" + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "CANCELLED"}) + hooks_mod.between_turns_hooks[:] = [hooks_mod._cancel_between_turns_hook] + + progress = MagicMock() + + _run( + hooks_mod.stop_hook( + hook_input={}, + tool_use_id=None, + hook_context=None, + task_id="t-cancel-milestone", + progress=progress, + ) + ) + + progress.write_agent_milestone.assert_called_once() + call_kwargs = progress.write_agent_milestone.call_args.kwargs + assert call_kwargs["milestone"] == "cancel_detected" + + +class TestCancelShortCircuitsNudgeConsumption: + """Regression for krokoko PR #52 review finding #3. + + Before the fix, :func:`stop_hook` iterated ALL between-turns hooks BEFORE + checking ``_cancel_requested`` — so when cancel fired, the nudge hook had + already run, mutated DDB (``mark_consumed`` + stamped ``_INJECTED_NUDGES``), + and had its return value silently discarded by the cancel branch. Users + saw a 202 Accepted for their nudge but the instruction was never injected + into the (dying) agent. + + The fix is two-layered: + 1. ``stop_hook`` breaks out of the dispatcher loop as soon as any hook + sets ``_cancel_requested``, so the nudge hook never runs on a + cancelled turn. + 2. ``_nudge_between_turns_hook`` itself early-returns when + ``_cancel_requested`` is already present, as belt-and-braces in + case a future refactor reorders the registry. + """ + + def test_nudge_hook_not_invoked_when_cancel_fires_first(self, monkeypatch): + """Happy-path regression: cancel hook flips sentinel → nudge hook is + never called → DDB query never issued → injected-nudges set untouched. + """ + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "CANCELLED"}) + + nudge_calls = {"count": 0} + + def _spy_nudge(_ctx): + nudge_calls["count"] += 1 + return ["should never be injected"] + + hooks_mod.between_turns_hooks[:] = [ + hooks_mod._cancel_between_turns_hook, + _spy_nudge, + ] + + result = _run( + hooks_mod.stop_hook( + hook_input={}, + tool_use_id=None, + hook_context=None, + task_id="t-cancel-nudge-race", + progress=MagicMock(), + ) + ) + + # Cancel-wins semantics unchanged. + assert result == { + "continue_": False, + "stopReason": "Task cancelled by user", + } + # Critical invariant: the nudge hook was NEVER called. Before the + # fix, ``nudge_calls["count"]`` would have been 1 and the pending + # DDB row would have been marked consumed. + assert nudge_calls["count"] == 0 + # In-process dedup set must be untouched — the "task set" should not + # have been created because the nudge hook never ran. + assert "t-cancel-nudge-race" not in hooks_mod._INJECTED_NUDGES + + def test_real_nudge_reader_not_touched_on_cancel(self, monkeypatch): + """End-to-end regression: with the ACTUAL ``_nudge_between_turns_hook`` + registered alongside the cancel hook, a pending DDB row MUST NOT be + read or marked consumed when cancel fires in the same turn. + + This is the scenario the review was concerned about — a user submits + a nudge, then immediately cancels, and the nudge disappears silently + because it was consumed but never injected. + """ + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "CANCELLED"}) + + table = MagicMock() + # If the nudge hook runs, it would see this pending row. + table.query.return_value = { + "Items": [ + { + "task_id": "t-cancel-real", + "nudge_id": "01NUDGE", + "message": "please add logging", + "created_at": "2026-05-05T12:00:00Z", + "consumed": False, + } + ] + } + table.update_item.return_value = {} + nudge_reader._TABLE_CACHE = table + + # Default registry order: cancel first, nudge second. + hooks_mod.between_turns_hooks[:] = [ + hooks_mod._cancel_between_turns_hook, + hooks_mod._nudge_between_turns_hook, + ] + + result = _run( + hooks_mod.stop_hook( + hook_input={}, + tool_use_id=None, + hook_context=None, + task_id="t-cancel-real", + progress=MagicMock(), + ) + ) + + assert result["continue_"] is False + # DDB must not have been queried — the nudge hook never ran. + table.query.assert_not_called() + # And therefore no ``mark_consumed`` call either. + table.update_item.assert_not_called() + + def test_preloop_cancel_skips_all_hooks_via_internal_guard(self, monkeypatch): + """If cancel is already flagged on ``ctx`` entering the dispatcher + (e.g. a Phase 3 hook prepended to the registry sets it, or a future + code path stamps the flag before hook dispatch), the nudge hook's + own early-return covers it. + + Today ``stop_hook`` builds ``ctx`` fresh each call so the pre-loop + case is not reachable from the normal SDK entry point, but the + nudge hook's internal guard is tested here directly to document the + second line of defence. + """ + table = MagicMock() + table.query.return_value = { + "Items": [ + { + "task_id": "t-preloop", + "nudge_id": "01PRELOOP", + "message": "should not be consumed", + "created_at": "2026-05-05T12:00:00Z", + "consumed": False, + } + ] + } + table.update_item.return_value = {} + nudge_reader._TABLE_CACHE = table + + # Cancel sentinel already set on ctx entering the nudge hook. + ctx = {"task_id": "t-preloop", "_cancel_requested": True} + result = hooks_mod._nudge_between_turns_hook(ctx) + + assert result == [] + # Belt-and-braces check: the nudge hook returned before any DDB I/O. + table.query.assert_not_called() + table.update_item.assert_not_called() + # And the in-process dedup set was not stamped. + assert "t-preloop" not in hooks_mod._INJECTED_NUDGES + + def test_nudge_hook_internal_guard_fires_even_if_registry_reordered(self, monkeypatch): + """If a future refactor accidentally puts nudge before cancel in the + registry, the loop-level break no longer helps — but the nudge + hook's own ``_cancel_requested`` check still has to short-circuit. + + Simulate this by registering a synthetic "early cancel" hook that + flips the sentinel BEFORE the nudge hook, but keeping nudge second + as usual. The loop will break after the cancel hook (finding + already covered); here we verify the nudge hook's internal guard + by driving it directly with cancel already set in ctx and an + attached progress writer. + """ + table = MagicMock() + table.query.return_value = { + "Items": [ + { + "task_id": "t-guard", + "nudge_id": "01GUARD", + "message": "must not inject", + "created_at": "ts", + "consumed": False, + } + ] + } + table.update_item.return_value = {} + nudge_reader._TABLE_CACHE = table + + progress = MagicMock() + ctx = { + "task_id": "t-guard", + "progress": progress, + "_cancel_requested": True, + } + result = hooks_mod._nudge_between_turns_hook(ctx) + + assert result == [] + # The early-return happens before ``_emit_nudge_milestone`` — no + # ``nudge_acknowledged`` event should be written for a cancelled task. + progress.write_agent_milestone.assert_not_called() + table.query.assert_not_called() + table.update_item.assert_not_called() + + def test_running_task_nudge_still_consumed_normally(self, monkeypatch): + """Negative control: the guard must not regress the happy path. + + A RUNNING task with a pending nudge should still flow through: + cancel hook returns [] without setting the sentinel, nudge hook + reads + consumes + injects as before. + """ + monkeypatch.setattr(task_state, "get_task", lambda _tid: {"status": "RUNNING"}) + + table = MagicMock() + table.query.return_value = { + "Items": [ + { + "task_id": "t-live", + "nudge_id": "01LIVE", + "message": "add docs", + "created_at": "ts", + "consumed": False, + } + ] + } + table.update_item.return_value = {} + nudge_reader._TABLE_CACHE = table + + hooks_mod.between_turns_hooks[:] = [ + hooks_mod._cancel_between_turns_hook, + hooks_mod._nudge_between_turns_hook, + ] + + result = _run( + hooks_mod.stop_hook( + hook_input={}, + tool_use_id=None, + hook_context=None, + task_id="t-live", + progress=MagicMock(), + ) + ) + + assert result["decision"] == "block" + assert "add docs" in result["reason"] + table.query.assert_called_once() + table.update_item.assert_called_once() diff --git a/agent/tests/test_models.py b/agent/tests/test_models.py index f6de72d..9123611 100644 --- a/agent/tests/test_models.py +++ b/agent/tests/test_models.py @@ -280,6 +280,40 @@ def test_validate_assignment(self): config.max_turns = 50 assert config.max_turns == 50 + def test_trace_true_with_empty_user_id_raises_at_construction(self): + """trace=True + user_id='' must fail at construction, not at S3 upload.""" + with pytest.raises(ValidationError, match="trace=True requires a non-empty user_id"): + TaskConfig( + repo_url="owner/repo", + github_token="ghp_test", + aws_region="us-east-1", + trace=True, + # user_id omitted — defaults to "" + ) + + def test_trace_true_with_valid_user_id_constructs_cleanly(self): + """Happy path: trace=True with a non-empty user_id is accepted.""" + config = TaskConfig( + repo_url="owner/repo", + github_token="ghp_test", + aws_region="us-east-1", + trace=True, + user_id="cognito-sub-abc-123", + ) + assert config.trace is True + assert config.user_id == "cognito-sub-abc-123" + + def test_trace_false_allows_empty_user_id(self): + """Negative control: local batch runs (trace=False, user_id='') still work.""" + config = TaskConfig( + repo_url="owner/repo", + github_token="ghp_test", + aws_region="us-east-1", + # trace defaults to False; user_id defaults to "" + ) + assert config.trace is False + assert config.user_id == "" + class TestRepoSetup: def test_construction(self): diff --git a/agent/tests/test_nudge_hook.py b/agent/tests/test_nudge_hook.py new file mode 100644 index 0000000..a23a20c --- /dev/null +++ b/agent/tests/test_nudge_hook.py @@ -0,0 +1,484 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +"""Integration tests for the Stop-hook-based nudge injection path.""" + +from __future__ import annotations + +import asyncio +import threading +from unittest.mock import MagicMock + +import pytest + +import hooks as hooks_mod +import nudge_reader + + +def _run(coro): + return asyncio.run(coro) + + +@pytest.fixture(autouse=True) +def _reset(): + nudge_reader._reset_cache_for_tests() + hooks_mod._reset_injected_nudges_for_tests() + # Restore the default registry after each test. + original = list(hooks_mod.between_turns_hooks) + yield + hooks_mod.between_turns_hooks[:] = original + nudge_reader._reset_cache_for_tests() + hooks_mod._reset_injected_nudges_for_tests() + + +class TestNudgeBetweenTurnsHook: + def test_pending_nudge_produces_xml_injection(self, monkeypatch): + table = MagicMock() + table.query.return_value = { + "Items": [ + { + "task_id": "t-1", + "nudge_id": "01ABC", + "message": "please prioritise error handling", + "created_at": "2026-04-22T12:00:00Z", + "consumed": False, + } + ] + } + table.update_item.return_value = {} + nudge_reader._TABLE_CACHE = table + + result = hooks_mod._nudge_between_turns_hook({"task_id": "t-1"}) + assert len(result) == 1 + assert "steer"] + + hooks_mod.between_turns_hooks[:] = [fake] + result = _run(hooks_mod.stop_hook({}, None, {}, task_id="t-1")) + assert result["decision"] == "block" + assert "steer" in result["reason"] + + def test_hook_exception_swallowed(self): + def broken(_ctx): + raise RuntimeError("kaboom") + + def ok(_ctx): + return ["still ok"] + + hooks_mod.between_turns_hooks[:] = [broken, ok] + result = _run(hooks_mod.stop_hook({}, None, {}, task_id="t-1")) + assert result["decision"] == "block" + assert "still ok" in result["reason"] + + def test_multiple_hooks_joined(self): + hooks_mod.between_turns_hooks[:] = [ + lambda _ctx: ["one"], + lambda _ctx: ["two", "three"], + ] + result = _run(hooks_mod.stop_hook({}, None, {}, task_id="t-1")) + assert "one" in result["reason"] + assert "two" in result["reason"] + assert "three" in result["reason"] + + def test_registry_default_contains_cancel_then_nudge(self): + # Freshly-imported registry: cancel runs first so it short-circuits + # nudge injection on cancelled tasks; nudge second for running tasks. + import importlib + + importlib.reload(hooks_mod) + assert hooks_mod.between_turns_hooks[0] is hooks_mod._cancel_between_turns_hook + assert hooks_mod.between_turns_hooks[1] is hooks_mod._nudge_between_turns_hook + + +class TestInProcessDedup: + """Process-lifetime dedup guards against mark_consumed failures.""" + + def test_already_injected_nudge_not_reinjected_even_if_mark_consumed_failed(self): + """If mark_consumed persistently fails, read_pending keeps returning + the same row, but the in-process dedup set prevents re-injection.""" + table = MagicMock() + # Both reads return the SAME pending row (mark_consumed is failing). + pending_row = { + "task_id": "t-1", + "nudge_id": "01ABC", + "message": "persistent", + "created_at": "ts", + "consumed": False, + } + table.query.return_value = {"Items": [pending_row]} + # Simulate mark_consumed failing repeatedly. + table.update_item.side_effect = Exception("DDB throttled") + nudge_reader._TABLE_CACHE = table + + first = hooks_mod._nudge_between_turns_hook({"task_id": "t-1"}) + assert len(first) == 1 + + # Second call: same row still returned by read_pending, but dedup + # set suppresses re-injection. + second = hooks_mod._nudge_between_turns_hook({"task_id": "t-1"}) + assert second == [] + + def test_different_nudge_ids_on_same_task_all_injected(self): + table = MagicMock() + table.query.side_effect = [ + { + "Items": [ + { + "task_id": "t-1", + "nudge_id": "01A", + "message": "first", + "created_at": "t1", + "consumed": False, + } + ] + }, + { + "Items": [ + { + "task_id": "t-1", + "nudge_id": "01B", + "message": "second", + "created_at": "t2", + "consumed": False, + } + ] + }, + ] + table.update_item.return_value = {} + nudge_reader._TABLE_CACHE = table + + first = hooks_mod._nudge_between_turns_hook({"task_id": "t-1"}) + second = hooks_mod._nudge_between_turns_hook({"task_id": "t-1"}) + assert len(first) == 1 + assert "first" in first[0] + assert len(second) == 1 + assert "second" in second[0] + + def test_different_tasks_do_not_share_dedup(self): + table = MagicMock() + # Same nudge_id "01A" appears for both tasks — each should still inject. + table.query.side_effect = [ + { + "Items": [ + { + "task_id": "t-A", + "nudge_id": "01A", + "message": "for A", + "created_at": "t", + "consumed": False, + } + ] + }, + { + "Items": [ + { + "task_id": "t-B", + "nudge_id": "01A", + "message": "for B", + "created_at": "t", + "consumed": False, + } + ] + }, + ] + table.update_item.return_value = {} + nudge_reader._TABLE_CACHE = table + + a = hooks_mod._nudge_between_turns_hook({"task_id": "t-A"}) + b = hooks_mod._nudge_between_turns_hook({"task_id": "t-B"}) + assert len(a) == 1 and "for A" in a[0] + assert len(b) == 1 and "for B" in b[0] + + +class TestStopHookThreading: + """Fix 2: sync hooks must run off the asyncio loop via ``to_thread``.""" + + def test_sync_hook_is_run_in_a_thread(self): + main_thread_id = threading.get_ident() + captured: dict[str, int] = {} + + def sync_hook(_ctx) -> list[str]: + captured["tid"] = threading.get_ident() + return ["ok"] + + hooks_mod.between_turns_hooks[:] = [sync_hook] + result = _run(hooks_mod.stop_hook({}, None, {}, task_id="t-1")) + assert result["decision"] == "block" + # The sync hook must have executed on a worker thread, not the + # asyncio event-loop thread that test_main is driving. + assert captured["tid"] != main_thread_id + + +class TestStopWrapperTaskIdLogging: + """Fix 5: Stop wrapper crashes must include task_id at ERROR severity.""" + + def test_stop_wrapper_crash_logs_task_id_at_error(self, monkeypatch): + logs: list[tuple[str, str]] = [] + + def fake_log(prefix: str, text: str) -> None: + logs.append((prefix, text)) + + # Patch the ``log`` name imported into hooks_mod. + monkeypatch.setattr(hooks_mod, "log", fake_log) + + # Make stop_hook raise — the _stop wrapper should catch, log, and + # return an empty output. We rebuild matchers to get a fresh _stop + # closure bound to a distinct task_id. + async def broken_stop_hook(*_a, **_k): + raise RuntimeError("boom") + + monkeypatch.setattr(hooks_mod, "stop_hook", broken_stop_hook) + + matchers = hooks_mod.build_hook_matchers( + engine=MagicMock(), + trajectory=None, + task_id="task-XYZ", + ) + stop_matcher = matchers["Stop"][0] + stop_callback = stop_matcher.hooks[0] + + # Invoke the wrapped callback. + _run(stop_callback({}, None, {})) + + # Find the ERROR log entry and assert it mentions task_id. + error_entries = [(p, t) for (p, t) in logs if p == "ERROR"] + assert len(error_entries) >= 1 + assert any("task-XYZ" in t for (_p, t) in error_entries) diff --git a/agent/tests/test_nudge_reader.py b/agent/tests/test_nudge_reader.py new file mode 100644 index 0000000..ac924dc --- /dev/null +++ b/agent/tests/test_nudge_reader.py @@ -0,0 +1,327 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +"""Unit tests for nudge_reader.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +import nudge_reader +from nudge_reader import ( + _reset_cache_for_tests, + _xml_escape, + format_as_user_message, + mark_consumed, + read_pending, +) + + +@pytest.fixture(autouse=True) +def _clear_cache(): + """Reset module-level caches between tests.""" + _reset_cache_for_tests() + yield + _reset_cache_for_tests() + + +# --------------------------------------------------------------------------- +# read_pending +# --------------------------------------------------------------------------- + + +class TestReadPending: + def test_empty_table_returns_empty_list(self): + table = MagicMock() + table.query.return_value = {"Items": []} + assert read_pending("task-1", table=table) == [] + + def test_returns_items_sorted_by_nudge_id_ascending(self): + table = MagicMock() + # Return in reverse order — reader must sort ASC. + table.query.return_value = { + "Items": [ + { + "task_id": "t1", + "nudge_id": "01HZZ", + "message": "third", + "created_at": "2026-04-22T12:02:00Z", + "consumed": False, + }, + { + "task_id": "t1", + "nudge_id": "01HAA", + "message": "first", + "created_at": "2026-04-22T12:00:00Z", + "consumed": False, + }, + { + "task_id": "t1", + "nudge_id": "01HMM", + "message": "second", + "created_at": "2026-04-22T12:01:00Z", + "consumed": False, + }, + ] + } + result = read_pending("t1", table=table) + assert [n["message"] for n in result] == ["first", "second", "third"] + assert [n["nudge_id"] for n in result] == ["01HAA", "01HMM", "01HZZ"] + + def test_returns_empty_when_env_var_unset(self, monkeypatch): + monkeypatch.delenv("NUDGES_TABLE_NAME", raising=False) + # Passing table=None forces _get_table() which should return None. + assert read_pending("t1") == [] + + def test_returns_empty_on_ddb_error(self): + table = MagicMock() + table.query.side_effect = Exception("DDB on fire") + assert read_pending("t1", table=table) == [] + + def test_filters_items_missing_nudge_id(self): + table = MagicMock() + table.query.return_value = { + "Items": [ + {"nudge_id": "01A", "message": "ok", "created_at": "t"}, + {"message": "no id — dropped"}, # no nudge_id + ] + } + result = read_pending("t1", table=table) + assert len(result) == 1 + assert result[0]["nudge_id"] == "01A" + + def test_query_uses_task_id_pk(self): + table = MagicMock() + table.query.return_value = {"Items": []} + read_pending("task-xyz", table=table) + table.query.assert_called_once() + _, kwargs = table.query.call_args + # KeyConditionExpression and FilterExpression must be present. + assert "KeyConditionExpression" in kwargs + assert "FilterExpression" in kwargs + + def test_paginates_when_last_evaluated_key_is_returned(self): + """Two-page response: first page has LastEvaluatedKey, second does not.""" + table = MagicMock() + table.query.side_effect = [ + { + "Items": [ + { + "task_id": "t1", + "nudge_id": "01A", + "message": "one", + "created_at": "t1", + "consumed": False, + } + ], + "LastEvaluatedKey": {"task_id": "t1", "nudge_id": "01A"}, + }, + { + "Items": [ + { + "task_id": "t1", + "nudge_id": "01B", + "message": "two", + "created_at": "t2", + "consumed": False, + } + ] + }, + ] + result = read_pending("t1", table=table) + assert [n["nudge_id"] for n in result] == ["01A", "01B"] + # Must have queried twice. + assert table.query.call_count == 2 + # Second call must pass ExclusiveStartKey from the first response. + second_kwargs = table.query.call_args_list[1].kwargs + assert second_kwargs["ExclusiveStartKey"] == {"task_id": "t1", "nudge_id": "01A"} + + +# --------------------------------------------------------------------------- +# mark_consumed +# --------------------------------------------------------------------------- + + +class TestMarkConsumed: + def test_success_returns_true(self): + table = MagicMock() + table.update_item.return_value = {} + assert mark_consumed("t1", "01A", table=table) is True + table.update_item.assert_called_once() + _, kwargs = table.update_item.call_args + assert kwargs["Key"] == {"task_id": "t1", "nudge_id": "01A"} + assert "ConditionExpression" in kwargs + # ``consumed`` is a DDB reserved keyword — must be aliased via + # ExpressionAttributeNames, otherwise DDB rejects the whole update + # with ``ValidationException: reserved keyword: consumed``. + assert "ExpressionAttributeNames" in kwargs + names = kwargs["ExpressionAttributeNames"] + assert "consumed" in names.values(), f"Expected 'consumed' to be aliased; got {names!r}" + # The raw attribute name must NOT appear in the update/condition + # expressions (DDB will reject it). + update_expr = kwargs["UpdateExpression"] + cond_expr = kwargs["ConditionExpression"] + # Allow "consumed_at" but not bare "consumed" followed by space/=. + import re + + bare_consumed = re.compile(r"\bconsumed\b(?!_at)") + assert not bare_consumed.search(update_expr), ( + f"Raw 'consumed' keyword in UpdateExpression: {update_expr!r}" + ) + assert not bare_consumed.search(cond_expr), ( + f"Raw 'consumed' keyword in ConditionExpression: {cond_expr!r}" + ) + + def test_conditional_check_failure_returns_false(self): + table = MagicMock() + # Simulate via name-based detection path (ConditionalCheckFailedException). + exc_cls = type("ConditionalCheckFailedException", (Exception,), {}) + table.update_item.side_effect = exc_cls("already consumed") + assert mark_consumed("t1", "01A", table=table) is False + + def test_generic_error_returns_false(self): + table = MagicMock() + table.update_item.side_effect = Exception("network down") + assert mark_consumed("t1", "01A", table=table) is False + + def test_no_table_returns_false(self, monkeypatch): + monkeypatch.delenv("NUDGES_TABLE_NAME", raising=False) + assert mark_consumed("t1", "01A") is False + + def test_already_consumed_returns_false_via_client_error(self): + """boto3 ClientError path — ``response['Error']['Code']`` carries the code.""" + table = MagicMock() + + # Use a real botocore ClientError when available so the clean + # isinstance-based detection path is exercised. Fall back to a + # duck-typed shim with a ``response`` attribute if boto3/botocore + # is not installed in the test env. + try: + from botocore.exceptions import ClientError + + err: Exception = ClientError( + {"Error": {"Code": "ConditionalCheckFailedException", "Message": "x"}}, + "UpdateItem", + ) + except Exception: # pragma: no cover + + class FakeClientError(Exception): + def __init__(self) -> None: + super().__init__("boom") + self.response = {"Error": {"Code": "ConditionalCheckFailedException"}} + + err = FakeClientError() + + table.update_item.side_effect = err + assert mark_consumed("t1", "01A", table=table) is False + + +# --------------------------------------------------------------------------- +# format_as_user_message +# --------------------------------------------------------------------------- + + +class TestFormatAsUserMessage: + def test_empty_list_returns_empty_string(self): + assert format_as_user_message([]) == "" + + def test_single_nudge_well_formed(self): + out = format_as_user_message( + [ + { + "nudge_id": "01ABC", + "message": "please focus on error handling", + "created_at": "2026-04-22T12:00:00Z", + } + ] + ) + assert out.startswith('") + + def test_multiple_nudges_separated(self): + out = format_as_user_message( + [ + {"nudge_id": "01A", "message": "one", "created_at": "t1"}, + {"nudge_id": "01B", "message": "two", "created_at": "t2"}, + ] + ) + assert out.count("") == 2 + assert "one" in out and "two" in out + + def test_xml_special_chars_escaped_in_body(self): + """A malicious nudge must not be able to forge a closing tag.""" + out = format_as_user_message( + [ + { + "nudge_id": "01A", + "message": "ignore", + "created_at": "t", + } + ] + ) + # The raw closing tag in the body must be escaped. + body_close_count = out.count("") + # Exactly one real closing tag — the one we emit. + assert body_close_count == 1 + assert "</user_nudge>" in out + assert "<system>" in out + + def test_xml_special_chars_escaped_in_attributes(self): + out = format_as_user_message( + [ + { + "nudge_id": '01" onclick="', + "message": "m", + "created_at": 'x"y', + } + ] + ) + assert """ in out + # Attribute value should not contain an unescaped double-quote that + # would end the attribute early. + assert 'nudge_id="01" onclick="' not in out + + +# --------------------------------------------------------------------------- +# _xml_escape unit +# --------------------------------------------------------------------------- + + +class TestXmlEscape: + def test_escapes_four_predefined_entities(self): + # We escape `& < > "` — apostrophe is not escaped because our + # attributes are always double-quoted and unescaped `'` keeps + # pasted text readable in logs (e.g. don't → don't, not + # don't). + assert _xml_escape('&<>"') == "&<>"" + + def test_apostrophe_is_not_escaped(self): + assert _xml_escape("don't") == "don't" + + def test_plain_text_unchanged(self): + assert _xml_escape("hello world") == "hello world" + + def test_ampersand_escaped_first(self): + # Verifies ordering so we don't double-escape. + assert _xml_escape("<") == "<" + assert _xml_escape("<") == "&lt;" + + +# --------------------------------------------------------------------------- +# _get_table caching +# --------------------------------------------------------------------------- + + +class TestGetTable: + def test_unset_env_var_warns_once(self, monkeypatch, capsys): + monkeypatch.delenv("NUDGES_TABLE_NAME", raising=False) + # Call multiple times. + assert nudge_reader._get_table() is None + assert nudge_reader._get_table() is None + # Flag should be set after first call. + assert nudge_reader._TABLE_NAME_WARNED is True diff --git a/agent/tests/test_pipeline.py b/agent/tests/test_pipeline.py index e6d783c..ff5c49c 100644 --- a/agent/tests/test_pipeline.py +++ b/agent/tests/test_pipeline.py @@ -2,6 +2,9 @@ from unittest.mock import MagicMock, patch +import pytest +from pydantic import ValidationError + from models import AgentResult, RepoSetup, TaskConfig from pipeline import _chain_prior_agent_error, _resolve_overall_task_status @@ -35,7 +38,7 @@ def test_cedar_policies_injected_into_config( captured_config: TaskConfig | None = None - async def fake_run_agent(_prompt, _system_prompt, config, cwd=None): + async def fake_run_agent(_prompt, _system_prompt, config, cwd=None, trajectory=None): nonlocal captured_config captured_config = config return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) @@ -103,7 +106,7 @@ def test_cedar_policies_absent_when_not_passed( captured_config: TaskConfig | None = None - async def fake_run_agent(_prompt, _system_prompt, config, cwd=None): + async def fake_run_agent(_prompt, _system_prompt, config, cwd=None, trajectory=None): nonlocal captured_config captured_config = config return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) @@ -231,3 +234,966 @@ def test_success_preserves_existing_error(self): status, err = _resolve_overall_task_status(ar, build_ok=True, pr_url=None) assert status == "success" assert err == "non-fatal warning" + + +class TestCancelSkipsPostHooks: + """Cancel short-circuit: if task is CANCELLED when run_agent returns, the + pipeline must skip post-hooks so no PR is pushed on a cancelled task. + """ + + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + def test_cancelled_task_skips_post_hooks( + self, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + monkeypatch, + ): + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + return AgentResult(status="success", turns=2, cost_usd=0.01, num_turns=2) + + mock_run_agent.side_effect = fake_run_agent + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + # Simulate cancel-task.ts having already flipped the status. + mock_get_task = MagicMock(return_value={"status": "CANCELLED"}) + + mock_ensure_pr = MagicMock() + mock_ensure_committed = MagicMock() + + with ( + patch("pipeline.ensure_committed", mock_ensure_committed), + patch("pipeline.verify_build"), + patch("pipeline.verify_lint"), + patch("pipeline.ensure_pr", mock_ensure_pr), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + patch("pipeline.task_state") as mock_task_state_mod, + ): + # Route get_task through the mock; keep TaskFetchError importable. + mock_task_state_mod.get_task = mock_get_task + mock_task_state_mod.TaskFetchError = Exception # type: ignore[attr-defined] + + from pipeline import run_task + + result = run_task( + repo_url="o/r", + task_description="x", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-cancelled", + ) + + # CRITICAL: no PR push, no commit safety-net on cancelled task. + mock_ensure_pr.assert_not_called() + mock_ensure_committed.assert_not_called() + assert result["status"] == "cancelled" + assert result["task_id"] == "t-cancelled" + + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + def test_running_task_runs_post_hooks_normally( + self, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + monkeypatch, + ): + """Regression guard: a task that is NOT cancelled still runs post-hooks.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + return AgentResult(status="success", turns=2, cost_usd=0.01, num_turns=2) + + mock_run_agent.side_effect = fake_run_agent + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + mock_ensure_pr = MagicMock(return_value="https://github.com/o/r/pull/1") + + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", return_value=True), + patch("pipeline.verify_lint", return_value=True), + patch("pipeline.ensure_pr", mock_ensure_pr), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + patch("pipeline.task_state") as mock_task_state_mod, + ): + # Task is RUNNING (not cancelled) — normal path must execute. + mock_task_state_mod.get_task = MagicMock(return_value={"status": "RUNNING"}) + mock_task_state_mod.TaskFetchError = Exception # type: ignore[attr-defined] + + from pipeline import run_task + + run_task( + repo_url="o/r", + task_description="x", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-running", + ) + + mock_ensure_pr.assert_called_once() + + +# --------------------------------------------------------------------------- +# Chunk K1 — trace threading into TaskConfig (design §10.1) +# --------------------------------------------------------------------------- + + +class TestTraceThreading: + """run_task(trace=...) must land on ``TaskConfig.trace`` so the + runner.py _ProgressWriter picks it up. This is the exact junction a + reviewer caught as silently dropping the flag in review; lock it + with a dedicated test. + """ + + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + @patch("pipeline.task_state") + def test_run_task_trace_true_sets_config_trace_true( + self, + _mock_task_state, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + monkeypatch, + ): + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + captured_config: TaskConfig | None = None + + async def fake_run_agent(_prompt, _system_prompt, config, cwd=None, trajectory=None): + nonlocal captured_config + captured_config = config + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", return_value=True), + patch("pipeline.verify_lint", return_value=True), + patch( + "pipeline.ensure_pr", + return_value="https://github.com/org/repo/pull/1", + ), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + ): + from pipeline import run_task + + run_task( + repo_url="owner/repo", + task_description="deep debug", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-trace", + trace=True, + user_id="cognito-sub-trace-user", + ) + + assert captured_config is not None + # The config reaching run_agent carries trace=True so runner.py's + # _ProgressWriter(config.task_id, trace=config.trace) picks it up. + assert captured_config.trace is True + assert captured_config.user_id == "cognito-sub-trace-user" + + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + @patch("pipeline.task_state") + def test_run_task_trace_default_is_false( + self, + _mock_task_state, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + monkeypatch, + ): + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + captured_config: TaskConfig | None = None + + async def fake_run_agent(_prompt, _system_prompt, config, cwd=None, trajectory=None): + nonlocal captured_config + captured_config = config + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", return_value=True), + patch("pipeline.verify_lint", return_value=True), + patch( + "pipeline.ensure_pr", + return_value="https://github.com/org/repo/pull/1", + ), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + ): + from pipeline import run_task + + run_task( + repo_url="owner/repo", + task_description="normal task", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-notrace", + ) + + assert captured_config is not None + assert captured_config.trace is False + + +class TestTraceS3Upload: + """K2 Stage 4 — pipeline triggers the S3 trace upload only when + ``trace=True`` AND ``user_id`` is non-empty; threads the resulting + ``trace_s3_uri`` into ``task_state.write_terminal`` so the + TaskRecord update is atomic with terminal-status.""" + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + @patch("pipeline.task_state") + def test_upload_happens_when_trace_and_user_id( + self, + mock_task_state, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + # Simulate the runner accumulating one event so dump returns bytes. + if trajectory is not None: + trajectory._put_event({"event": "TURN", "turn": 1}) + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + mock_upload.return_value = "s3://b/traces/u-1/t-up.jsonl.gz" + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", return_value=True), + patch("pipeline.verify_lint", return_value=True), + patch("pipeline.ensure_pr", return_value=None), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + ): + from pipeline import run_task + + result = run_task( + repo_url="owner/repo", + task_description="debug it", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-up", + trace=True, + user_id="u-1", + ) + + # Upload was called with the expected identifiers. + assert mock_upload.called + call_kwargs = mock_upload.call_args.kwargs + assert call_kwargs["task_id"] == "t-up" + assert call_kwargs["user_id"] == "u-1" + assert isinstance(call_kwargs["body"], bytes) + + # trace_s3_uri was threaded into the terminal write. + assert result["trace_s3_uri"] == "s3://b/traces/u-1/t-up.jsonl.gz" + mock_task_state.write_terminal.assert_called() + terminal_result = mock_task_state.write_terminal.call_args.args[2] + assert terminal_result["trace_s3_uri"] == "s3://b/traces/u-1/t-up.jsonl.gz" + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + @patch("pipeline.task_state") + def test_upload_skipped_when_trace_false( + self, + _mock_task_state, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", return_value=True), + patch("pipeline.verify_lint", return_value=True), + patch("pipeline.ensure_pr", return_value=None), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + ): + from pipeline import run_task + + result = run_task( + repo_url="owner/repo", + task_description="normal", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-nt", + trace=False, + user_id="u-1", + ) + + assert not mock_upload.called + assert result["trace_s3_uri"] is None + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + @patch("pipeline.task_state") + def test_upload_skipped_when_user_id_empty_and_trace_true( + self, + _mock_task_state, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + """krokoko review Finding #11 — trace=True with empty user_id now + fails at ``TaskConfig`` construction time (pre-flight validation) + rather than silently skipping the upload and returning + ``trace_s3_uri=None``. + + Previously (rev-5) this was a best-effort defensive skip inside + ``pipeline.run_task``'s trace-upload block; shifting the check to + the Pydantic model means misconfigured callers surface the error + immediately, before any agent work runs. The upload mock is never + exercised because we never reach the upload path. + """ + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", return_value=True), + patch("pipeline.verify_lint", return_value=True), + patch("pipeline.ensure_pr", return_value=None), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + ): + from pipeline import run_task + + with pytest.raises(ValidationError, match="trace=True requires a non-empty user_id"): + run_task( + repo_url="owner/repo", + task_description="trace without user", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-no-uid", + trace=True, + user_id="", # empty — now rejected at TaskConfig construction + ) + + assert not mock_upload.called + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + @patch("pipeline.task_state") + def test_upload_fail_open_does_not_fail_task( + self, + _mock_task_state, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + """A failed S3 upload (fail-open returns None) must NOT flip + the task to FAILED — the trajectory is a debug artifact.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + if trajectory is not None: + trajectory._put_event({"event": "TURN", "turn": 1}) + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + mock_upload.return_value = None # simulate S3 failure + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", return_value=True), + patch("pipeline.verify_lint", return_value=True), + patch("pipeline.ensure_pr", return_value=None), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + ): + from pipeline import run_task + + result = run_task( + repo_url="owner/repo", + task_description="trace fail", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-fail", + trace=True, + user_id="u-1", + ) + + assert mock_upload.called + # Fail-open: task is still success, trace_s3_uri just absent. + assert result["status"] == "success" + assert result["trace_s3_uri"] is None + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + def test_cancel_path_does_not_upload_trace_when_trace_false( + self, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + """Cancel path must NOT attempt an S3 upload when ``trace=False``. + + L4 flipped the previous blanket "no upload on cancel" rule: the + cancel path now best-effort uploads and self-heals when + ``config.trace=True`` (so users can debug cancelled-mid-run + tasks). This test pins the negative side — without ``--trace``, + there is still no upload on the cancel path. Post-hooks must + still be skipped in both cases.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + if trajectory is not None: + trajectory._put_event({"event": "TURN", "turn": 1}) + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + mock_get_task = MagicMock(return_value={"status": "CANCELLED"}) + + with ( + patch("pipeline.ensure_committed") as mock_ensure_committed, + patch("pipeline.verify_build"), + patch("pipeline.verify_lint"), + patch("pipeline.ensure_pr") as mock_ensure_pr, + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + patch("pipeline.task_state") as mock_task_state_mod, + ): + mock_task_state_mod.get_task = mock_get_task + mock_task_state_mod.TaskFetchError = Exception # type: ignore[attr-defined] + + from pipeline import run_task + + result = run_task( + repo_url="owner/repo", + task_description="mid-run cancel no trace", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-cancelled-no-trace", + trace=False, # no --trace → no upload even on cancel + user_id="u-1", + ) + + mock_upload.assert_not_called() + mock_ensure_committed.assert_not_called() + mock_ensure_pr.assert_not_called() + assert result["status"] == "cancelled" + assert result["task_id"] == "t-cancelled-no-trace" + assert "trace_s3_uri" not in result + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + def test_cancel_path_uploads_and_self_heals_when_trace( + self, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + """L4 item 1c — cancel path with ``trace=True`` best-effort + uploads to S3 and calls ``write_trace_uri_conditional`` so the + trajectory captured before cancel stays recoverable. + + ``write_terminal`` cannot persist ``trace_s3_uri`` atomically on + this path because its ConditionExpression rejects CANCELLED — + the conditional-self-heal helper (scoped to + ``attribute_not_exists(trace_s3_uri) AND status IN (...)``) + handles the persistence instead.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + # Seed the accumulator so dump_gzipped_jsonl returns bytes. + if trajectory is not None: + trajectory._put_event({"event": "TURN", "turn": 1}) + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + mock_upload.return_value = "s3://bucket/traces/u-1/t-cancelled-trace.jsonl.gz" + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + mock_get_task = MagicMock(return_value={"status": "CANCELLED"}) + + with ( + patch("pipeline.ensure_committed") as mock_ensure_committed, + patch("pipeline.verify_build"), + patch("pipeline.verify_lint"), + patch("pipeline.ensure_pr") as mock_ensure_pr, + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + patch("pipeline.task_state") as mock_task_state_mod, + ): + mock_task_state_mod.get_task = mock_get_task + mock_task_state_mod.TaskFetchError = Exception # type: ignore[attr-defined] + mock_task_state_mod.write_trace_uri_conditional = MagicMock(return_value=True) + + from pipeline import run_task + + result = run_task( + repo_url="owner/repo", + task_description="mid-run cancel with trace", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-cancelled-trace", + trace=True, + user_id="u-1", + ) + + # Upload was attempted. + mock_upload.assert_called_once() + # Self-heal was invoked with the resulting URI. + mock_task_state_mod.write_trace_uri_conditional.assert_called_once_with( + "t-cancelled-trace", + "s3://bucket/traces/u-1/t-cancelled-trace.jsonl.gz", + ) + # write_terminal is NOT called on the cancel path (its + # ConditionExpression would reject CANCELLED). + mock_task_state_mod.write_terminal.assert_not_called() + + # Post-hooks still skipped (cancel short-circuit). + mock_ensure_committed.assert_not_called() + mock_ensure_pr.assert_not_called() + # Cancel-shaped return payload. + assert result["status"] == "cancelled" + assert result["task_id"] == "t-cancelled-trace" + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + def test_cancel_path_heal_failure_is_fail_open( + self, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + """L4 item 1c — when the self-heal helper raises on the cancel + path, the cancel fast-path must still return cleanly; an + upload/persist error must not propagate and turn a valid cancel + into a pipeline crash.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + if trajectory is not None: + trajectory._put_event({"event": "TURN", "turn": 1}) + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + mock_upload.return_value = "s3://bucket/traces/u-1/t-cancelled-crash.jsonl.gz" + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + mock_get_task = MagicMock(return_value={"status": "CANCELLED"}) + + with ( + patch("pipeline.ensure_committed"), + patch("pipeline.verify_build"), + patch("pipeline.verify_lint"), + patch("pipeline.ensure_pr"), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + patch("pipeline.task_state") as mock_task_state_mod, + ): + mock_task_state_mod.get_task = mock_get_task + mock_task_state_mod.TaskFetchError = Exception # type: ignore[attr-defined] + # Self-heal raises — cancel path must swallow it. + mock_task_state_mod.write_trace_uri_conditional = MagicMock( + side_effect=RuntimeError("ddb boom") + ) + + from pipeline import run_task + + # No exception should escape — fail-open contract. + result = run_task( + repo_url="owner/repo", + task_description="cancel with heal failure", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-cancelled-crash", + trace=True, + user_id="u-1", + ) + + # Upload was attempted; heal raised but was swallowed. + mock_upload.assert_called_once() + assert result["status"] == "cancelled" + assert result["task_id"] == "t-cancelled-crash" + + +class TestTraceCrashPath: + """K2 review Finding #1 — a pipeline crash (exception after the + agent loop) must still attempt the trace upload so the user can + debug the failure. The upload is fully fail-open under the crash + path too: an S3 error must not mask or replace the underlying + pipeline exception.""" + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + @patch("pipeline.task_state") + def test_crash_path_uploads_trace_and_threads_uri( + self, + mock_task_state, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + # Accumulate something so dump has bytes, then later cause + # the pipeline to crash post-hooks. + if trajectory is not None: + trajectory._put_event({"event": "TURN", "turn": 1}) + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + mock_upload.return_value = "s3://b/traces/u-1/t-crash.jsonl.gz" + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + # Force a crash after agent completes but before terminal write: + # ``verify_build`` raises, which escapes to the outer except. + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", side_effect=RuntimeError("build verify boom")), + patch("pipeline.verify_lint", return_value=True), + patch("pipeline.ensure_pr", return_value=None), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + ): + import contextlib + + from pipeline import run_task + + with contextlib.suppress(RuntimeError): + run_task( + repo_url="owner/repo", + task_description="crash case", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-crash", + trace=True, + user_id="u-1", + ) # pipeline re-raises after writing FAILED + + # Upload was invoked on the crash path. + assert mock_upload.called + call_kwargs = mock_upload.call_args.kwargs + assert call_kwargs["task_id"] == "t-crash" + assert call_kwargs["user_id"] == "u-1" + + # Terminal was written as FAILED WITH trace_s3_uri threaded in. + mock_task_state.write_terminal.assert_called() + args, _ = mock_task_state.write_terminal.call_args + assert args[1] == "FAILED" + crash_result = args[2] + assert crash_result["trace_s3_uri"] == "s3://b/traces/u-1/t-crash.jsonl.gz" + + @patch("pipeline.upload_trace_to_s3") + @patch("pipeline.run_agent") + @patch("pipeline.build_system_prompt") + @patch("pipeline.discover_project_config") + @patch("repo.setup_repo") + @patch("pipeline.task_span") + @patch("pipeline.task_state") + def test_crash_path_upload_exception_does_not_mask_original_error( + self, + mock_task_state, + mock_task_span, + mock_setup_repo, + _mock_discover, + _mock_build_prompt, + mock_run_agent, + mock_upload, + monkeypatch, + ): + """If the crash-path upload itself raises, the original + pipeline exception must still be the one that propagates.""" + monkeypatch.setenv("GITHUB_TOKEN", "ghp_test") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_setup_repo.return_value = RepoSetup( + repo_dir="/workspace/repo", + branch="bgagent/test/branch", + build_before=True, + ) + + async def fake_run_agent(_prompt, _system_prompt, _config, cwd=None, trajectory=None): + if trajectory is not None: + trajectory._put_event({"event": "TURN", "turn": 1}) + return AgentResult(status="success", turns=1, cost_usd=0.01, num_turns=1) + + mock_run_agent.side_effect = fake_run_agent + mock_upload.side_effect = RuntimeError("upload explode") + + mock_span = MagicMock() + mock_span.__enter__ = MagicMock(return_value=mock_span) + mock_span.__exit__ = MagicMock(return_value=False) + mock_task_span.return_value = mock_span + + with ( + patch("pipeline.ensure_committed", return_value=False), + patch("pipeline.verify_build", side_effect=ValueError("original pipeline error")), + patch("pipeline.verify_lint", return_value=True), + patch("pipeline.ensure_pr", return_value=None), + patch("pipeline.get_disk_usage", return_value=0), + patch("pipeline.print_metrics"), + ): + import pytest + + from pipeline import run_task + + with pytest.raises(ValueError, match="original pipeline error"): + run_task( + repo_url="owner/repo", + task_description="mask test", + github_token="ghp_test", + aws_region="us-east-1", + task_id="t-mask", + trace=True, + user_id="u-1", + ) + + # Terminal still written despite the upload failure. + mock_task_state.write_terminal.assert_called() diff --git a/agent/tests/test_pipeline_outcomes.py b/agent/tests/test_pipeline_outcomes.py index 4b22258..0c00cc7 100644 --- a/agent/tests/test_pipeline_outcomes.py +++ b/agent/tests/test_pipeline_outcomes.py @@ -1,7 +1,11 @@ """Unit tests for pipeline task outcome resolution and error chaining.""" from models import AgentResult -from pipeline import _chain_prior_agent_error, _resolve_overall_task_status +from pipeline import ( + _chain_prior_agent_error, + _compute_turns_completed, + _resolve_overall_task_status, +) class TestResolveOverallTaskStatus: @@ -57,3 +61,44 @@ def test_chains_status_error_without_message(self): msg = _chain_prior_agent_error(ar, OSError("network")) assert "Agent reported status=error" in msg assert "network" in msg + + +class TestComputeTurnsCompleted: + """Rev-5 DATA-1: ``turns_completed`` must clamp to ``max_turns`` only on + ``error_max_turns``; all other statuses pass ``turns_attempted`` through.""" + + def test_success_passes_turns_through_unchanged(self): + # Agent finished cleanly in 5 SDK turns; max allowed 10. No clamp. + assert _compute_turns_completed("success", 5, max_turns=10) == 5 + + def test_end_turn_passes_turns_through_unchanged(self): + assert _compute_turns_completed("end_turn", 3, max_turns=10) == 3 + + def test_error_status_without_max_turns_does_not_clamp(self): + # Generic error — e.g. tool failure — should NOT clamp. + assert _compute_turns_completed("error", 7, max_turns=10) == 7 + + def test_error_max_turns_clamps_when_sdk_overreports(self): + # SDK reports max_turns + 1 on error_max_turns; clamp to the declared + # ceiling so ``turns_completed`` reflects what actually executed. + assert _compute_turns_completed("error_max_turns", 11, max_turns=10) == 10 + + def test_error_max_turns_does_not_increase_below_the_clamp(self): + # Defensive: if the SDK reported fewer turns than max (shouldn't + # happen, but we don't want to invent turns). + assert _compute_turns_completed("error_max_turns", 6, max_turns=10) == 6 + + def test_error_max_turns_with_exact_max_passes_through(self): + # Edge case: attempted == max_turns; min() is a no-op. + assert _compute_turns_completed("error_max_turns", 10, max_turns=10) == 10 + + def test_none_turns_attempted_round_trips(self): + # Missing SDK count must round-trip as None so writers can detect + # "no turn counter available" vs "zero turns". + assert _compute_turns_completed("success", None, max_turns=10) is None + assert _compute_turns_completed("error_max_turns", None, max_turns=10) is None + + def test_zero_turns_attempted_round_trips(self): + # Zero is treated the same as None (falsy) so we don't clamp it to a + # negative / nonsensical value. + assert _compute_turns_completed("error_max_turns", 0, max_turns=10) == 0 diff --git a/agent/tests/test_progress_writer.py b/agent/tests/test_progress_writer.py new file mode 100644 index 0000000..1fbb1ec --- /dev/null +++ b/agent/tests/test_progress_writer.py @@ -0,0 +1,747 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +"""Unit tests for progress_writer._ProgressWriter.""" + +from __future__ import annotations + +from decimal import Decimal +from unittest.mock import MagicMock, patch + +import pytest + +from progress_writer import ( + _classify_ddb_error, + _generate_ulid, + _ProgressWriter, + _reset_circuit_breakers, + _truncate_preview, +) + + +# Reset the shared circuit-breaker state between every test so a tripped +# breaker in one test does not silently disable the writer-under-test in +# the next. Forgetting this was the single largest hazard flagged when +# review finding #8 consolidated per-writer state into a shared map. +@pytest.fixture(autouse=True) +def _reset_shared_circuit_breaker_state(): + _reset_circuit_breakers() + yield + _reset_circuit_breakers() + + +# --------------------------------------------------------------------------- +# _generate_ulid +# --------------------------------------------------------------------------- + + +class TestGenerateUlid: + def test_length_is_26(self): + assert len(_generate_ulid()) == 26 + + def test_monotonic_ordering_across_milliseconds(self): + """ULIDs generated across different milliseconds are lexicographically ordered.""" + import time + + ids = [] + for _ in range(5): + ids.append(_generate_ulid()) + time.sleep(0.002) # 2ms gap to ensure different timestamp + assert ids == sorted(ids) + + def test_uniqueness(self): + ids = {_generate_ulid() for _ in range(100)} + assert len(ids) == 100 + + +# --------------------------------------------------------------------------- +# _truncate_preview +# --------------------------------------------------------------------------- + + +class TestTruncatePreview: + def test_short_string_unchanged(self): + assert _truncate_preview("hello") == "hello" + + def test_none_returns_empty(self): + assert _truncate_preview(None) == "" + + def test_empty_returns_empty(self): + assert _truncate_preview("") == "" + + def test_long_string_truncated(self): + long = "x" * 300 + result = _truncate_preview(long) + assert len(result) <= 203 # 200 + "..." + assert result.endswith("...") + + def test_custom_max_len(self): + result = _truncate_preview("abcdef", max_len=3) + assert result == "abc..." + + def test_exact_length_not_truncated(self): + s = "a" * 200 + assert _truncate_preview(s) == s + + +# --------------------------------------------------------------------------- +# _ProgressWriter — init and disable +# --------------------------------------------------------------------------- + + +class TestProgressWriterInit: + def test_noop_when_env_var_unset(self, monkeypatch): + monkeypatch.delenv("TASK_EVENTS_TABLE_NAME", raising=False) + pw = _ProgressWriter("task-1") + pw.write_agent_milestone("test", "detail") + # Should not raise — silently no-ops (table_name is None so _put_event returns early) + assert pw._table_name is None + assert pw._table is None + + def test_enabled_when_env_var_set(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "my-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + pw = _ProgressWriter("task-1") + assert pw._table_name == "my-table" + assert pw._disabled is False + + +# --------------------------------------------------------------------------- +# _ProgressWriter — DDB writes +# --------------------------------------------------------------------------- + + +class TestProgressWriterPutEvent: + @pytest.fixture() + def writer(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "events-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + return _ProgressWriter("task-42") + + @pytest.fixture() + def mock_table(self, writer): + table = MagicMock() + writer._table = table + return table + + def test_write_agent_turn(self, writer, mock_table): + writer.write_agent_turn( + turn=1, + model="claude-4", + thinking="deep thoughts", + text="hello world", + tool_calls_count=3, + ) + mock_table.put_item.assert_called_once() + item = mock_table.put_item.call_args[1]["Item"] + assert item["task_id"] == "task-42" + assert item["event_type"] == "agent_turn" + assert item["metadata"]["turn"] == 1 + assert item["metadata"]["model"] == "claude-4" + assert item["metadata"]["thinking_preview"] == "deep thoughts" + assert item["metadata"]["text_preview"] == "hello world" + assert item["metadata"]["tool_calls_count"] == 3 + assert "event_id" in item + assert "timestamp" in item + assert "ttl" in item + + def test_write_agent_tool_call(self, writer, mock_table): + writer.write_agent_tool_call(tool_name="Bash", tool_input="ls -la", turn=2) + item = mock_table.put_item.call_args[1]["Item"] + assert item["event_type"] == "agent_tool_call" + assert item["metadata"]["tool_name"] == "Bash" + assert item["metadata"]["tool_input_preview"] == "ls -la" + assert item["metadata"]["turn"] == 2 + + def test_write_agent_tool_result(self, writer, mock_table): + writer.write_agent_tool_result( + tool_name="Bash", + is_error=True, + content="command not found", + turn=2, + ) + item = mock_table.put_item.call_args[1]["Item"] + assert item["event_type"] == "agent_tool_result" + assert item["metadata"]["is_error"] is True + assert item["metadata"]["content_preview"] == "command not found" + + def test_write_agent_milestone(self, writer, mock_table): + writer.write_agent_milestone("repo_setup_complete", "branch=main") + item = mock_table.put_item.call_args[1]["Item"] + assert item["event_type"] == "agent_milestone" + assert item["metadata"]["milestone"] == "repo_setup_complete" + assert item["metadata"]["details"] == "branch=main" + + def test_write_agent_cost_update(self, writer, mock_table): + writer.write_agent_cost_update( + cost_usd=0.0512, + input_tokens=1000, + output_tokens=500, + turn=5, + ) + item = mock_table.put_item.call_args[1]["Item"] + assert item["event_type"] == "agent_cost_update" + assert item["metadata"]["cost_usd"] == Decimal("0.0512") + assert item["metadata"]["input_tokens"] == 1000 + assert item["metadata"]["output_tokens"] == 500 + + def test_write_agent_error(self, writer, mock_table): + writer.write_agent_error(error_type="RuntimeError", message="something broke") + item = mock_table.put_item.call_args[1]["Item"] + assert item["event_type"] == "agent_error" + assert item["metadata"]["error_type"] == "RuntimeError" + assert item["metadata"]["message_preview"] == "something broke" + + def test_preview_fields_truncated(self, writer, mock_table): + long_text = "x" * 500 + writer.write_agent_turn( + turn=1, + model="claude-4", + thinking=long_text, + text=long_text, + tool_calls_count=0, + ) + item = mock_table.put_item.call_args[1]["Item"] + assert len(item["metadata"]["thinking_preview"]) <= 203 + assert len(item["metadata"]["text_preview"]) <= 203 + + def test_ttl_is_90_days_from_now(self, writer, mock_table): + import time + + before = int(time.time()) + writer.write_agent_milestone("test", "") + item = mock_table.put_item.call_args[1]["Item"] + after = int(time.time()) + + ttl_90_days = 90 * 24 * 60 * 60 + assert before + ttl_90_days <= item["ttl"] <= after + ttl_90_days + 1 + + +# --------------------------------------------------------------------------- +# _ProgressWriter — --trace preview cap (design §10.1) +# --------------------------------------------------------------------------- + + +class TestProgressWriterTrace: + """Trace-enabled writers use a 4 KB preview cap instead of 200 chars. + + The cap is per-instance, not a mutable global: two writers in the + same process (unit tests, local batch mode) can coexist with + different caps without cross-contamination. + """ + + @pytest.fixture() + def trace_writer(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "events-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + return _ProgressWriter("task-trace", trace=True) + + @pytest.fixture() + def normal_writer(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "events-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + return _ProgressWriter("task-normal") + + def test_trace_raises_preview_cap_to_4kb(self, trace_writer): + table = MagicMock() + trace_writer._table = table + long_text = "x" * 3000 + trace_writer.write_agent_turn( + turn=1, model="c4", thinking=long_text, text="", tool_calls_count=0 + ) + item = table.put_item.call_args[1]["Item"] + # 3000 chars fits inside the 4 KB trace cap → returned verbatim, + # no "..." suffix appended. + assert item["metadata"]["thinking_preview"] == long_text + + def test_trace_still_caps_at_4096_plus_ellipsis(self, trace_writer): + table = MagicMock() + trace_writer._table = table + long_text = "y" * 5000 + trace_writer.write_agent_turn( + turn=1, model="c4", thinking=long_text, text="", tool_calls_count=0 + ) + item = table.put_item.call_args[1]["Item"] + preview = item["metadata"]["thinking_preview"] + # 4096 content chars + "..." = 4099. Assert the prefix content so + # a regression that kept the last 4096 (instead of the first) + # surfaces here instead of passing silently. + assert len(preview) == 4099 + assert preview[:4096] == "y" * 4096 + assert preview.endswith("...") + + @pytest.mark.parametrize( + "length,expected_len,has_ellipsis", + [ + (4095, 4095, False), # below cap: passes through verbatim + (4096, 4096, False), # exactly at cap (``<= max_len`` branch) + (4097, 4099, True), # one over: truncated with ellipsis + ], + ) + def test_trace_cap_boundary_conditions(self, trace_writer, length, expected_len, has_ellipsis): + # Lock the ``<=`` vs ``<`` off-by-one at the exact cap boundary. + table = MagicMock() + trace_writer._table = table + trace_writer.write_agent_milestone("m", "x" * length) + preview = table.put_item.call_args[1]["Item"]["metadata"]["details"] + assert len(preview) == expected_len + assert preview.endswith("...") is has_ellipsis + + @pytest.mark.parametrize( + "length,expected_len,has_ellipsis", + [ + (199, 199, False), + (200, 200, False), + (201, 203, True), + ], + ) + def test_normal_cap_boundary_conditions( + self, + normal_writer, + length, + expected_len, + has_ellipsis, + ): + # Same off-by-one guard on the default 200-char path. + table = MagicMock() + normal_writer._table = table + normal_writer.write_agent_milestone("m", "x" * length) + preview = table.put_item.call_args[1]["Item"]["metadata"]["details"] + assert len(preview) == expected_len + assert preview.endswith("...") is has_ellipsis + + def test_normal_writer_default_200_char_cap_preserved(self, normal_writer): + # Regression guard: trace=False must keep the 200-char cap. + table = MagicMock() + normal_writer._table = table + long_text = "z" * 500 + normal_writer.write_agent_turn( + turn=1, model="c4", thinking=long_text, text="", tool_calls_count=0 + ) + item = table.put_item.call_args[1]["Item"] + preview = item["metadata"]["thinking_preview"] + assert len(preview) == 203 # 200 + "..." + + def test_trace_flag_applies_to_all_preview_fields(self, trace_writer): + # Cover every preview site so a future ``write_agent_X`` that + # forgets ``self._preview(...)`` gets caught. + table = MagicMock() + trace_writer._table = table + long = "L" * 1000 + + trace_writer.write_agent_tool_call(tool_name="Bash", tool_input=long, turn=1) + assert table.put_item.call_args[1]["Item"]["metadata"]["tool_input_preview"] == long + + trace_writer.write_agent_tool_result(tool_name="Bash", is_error=False, content=long, turn=1) + assert table.put_item.call_args[1]["Item"]["metadata"]["content_preview"] == long + + trace_writer.write_agent_milestone("ms", long) + assert table.put_item.call_args[1]["Item"]["metadata"]["details"] == long + + trace_writer.write_agent_error("E", long) + assert table.put_item.call_args[1]["Item"]["metadata"]["message_preview"] == long + + def test_two_writers_in_same_process_have_independent_caps(self, normal_writer, trace_writer): + # Per-instance cap — not a mutable module global. + normal_table = MagicMock() + trace_table = MagicMock() + normal_writer._table = normal_table + trace_writer._table = trace_table + + long = "x" * 1000 + normal_writer.write_agent_milestone("n", long) + trace_writer.write_agent_milestone("t", long) + + n_details = normal_table.put_item.call_args[1]["Item"]["metadata"]["details"] + t_details = trace_table.put_item.call_args[1]["Item"]["metadata"]["details"] + assert len(n_details) == 203 # 200 + "..." + assert t_details == long # under 4096, full pass-through + + +# --------------------------------------------------------------------------- +# _ProgressWriter — fail-open behavior +# --------------------------------------------------------------------------- + + +class TestProgressWriterFailOpen: + @pytest.fixture() + def writer(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "events-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + return _ProgressWriter("task-fail") + + @pytest.fixture() + def failing_table(self, writer): + table = MagicMock() + table.put_item.side_effect = Exception("DDB unavailable") + writer._table = table + return table + + def test_single_failure_does_not_raise(self, writer, failing_table): + writer.write_agent_milestone("test", "") + # No exception raised + assert writer._failure_count == 1 + assert writer._disabled is False + + def test_circuit_breaker_disables_after_max_failures(self, writer, failing_table): + for _ in range(3): + writer.write_agent_milestone("test", "") + assert writer._disabled is True + assert writer._failure_count == 3 + + def test_no_writes_after_circuit_breaker(self, writer, failing_table): + for _ in range(3): + writer.write_agent_milestone("test", "") + assert writer._disabled is True + + # Reset mock to track new calls + failing_table.put_item.reset_mock() + writer.write_agent_milestone("test", "") + failing_table.put_item.assert_not_called() + + def test_success_resets_failure_count(self, writer): + table = MagicMock() + # Fail once, then succeed + table.put_item.side_effect = [Exception("fail"), None] + writer._table = table + + writer.write_agent_milestone("test1", "") + assert writer._failure_count == 1 + + writer.write_agent_milestone("test2", "") + assert writer._failure_count == 0 + + +# --------------------------------------------------------------------------- +# _ProgressWriter — lazy boto3 init +# --------------------------------------------------------------------------- + + +class TestProgressWriterLazyInit: + def test_boto3_imported_lazily(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "events-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + pw = _ProgressWriter("task-lazy") + # Table should not be initialized until first write + assert pw._table is None + + def test_boto3_import_error_disables(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "events-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + pw = _ProgressWriter("task-no-boto") + + with patch.dict("sys.modules", {"boto3": None}): + pw.write_agent_milestone("test", "") + + assert pw._disabled is True + + +# --------------------------------------------------------------------------- +# krokoko PR #52 review finding #6 — error classification +# --------------------------------------------------------------------------- + + +def _make_client_error(code: str, message: str = "boom") -> Exception: + """Build a duck-typed ``ClientError``-like exception. + + We avoid ``from botocore.exceptions import ClientError`` to keep the + test module importable in environments where ``botocore`` is missing + — matching the classifier's own structural duck-typing in + :func:`progress_writer._classify_ddb_error`. + """ + err = Exception(message) + setattr( # noqa: B010 — intentional dynamic attr to duck-type ClientError + err, + "response", + { + "Error": {"Code": code, "Message": message}, + "ResponseMetadata": {"HTTPStatusCode": 400}, + }, + ) + return err + + +class TestClassifyDdbError: + """Unit-level coverage of the classifier so higher-level tests can + focus on the writer's flow rather than re-testing the taxonomy.""" + + @pytest.mark.parametrize( + "code", + [ + "ValidationException", + "ItemCollectionSizeLimitExceededException", + "ResourceNotFoundException", + "AccessDeniedException", + "UnauthorizedOperation", + ], + ) + def test_permanent_aws_codes(self, code): + assert _classify_ddb_error(_make_client_error(code)) == "permanent" + + @pytest.mark.parametrize( + "code", + [ + "ProvisionedThroughputExceededException", + "RequestLimitExceeded", + "ThrottlingException", + "ServiceUnavailable", + "InternalServerError", + ], + ) + def test_transient_aws_codes(self, code): + assert _classify_ddb_error(_make_client_error(code)) == "transient" + + def test_unknown_aws_code_falls_through_to_unknown(self): + assert _classify_ddb_error(_make_client_error("SomeNewException")) == "unknown" + + def test_network_class_name_treated_as_transient(self): + class EndpointConnectionError(Exception): + pass + + assert _classify_ddb_error(EndpointConnectionError("no route to host")) == "transient" + + def test_arbitrary_exception_is_unknown(self): + assert _classify_ddb_error(RuntimeError("wat")) == "unknown" + + +class TestProgressWriterFailOpenClassified: + """Finding #6: bare ``except Exception`` folded permanent and + transient errors into the same breaker. These tests lock the new + contract so a regression (e.g. re-introducing a bare handler) fails + immediately.""" + + @pytest.fixture() + def writer(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "events-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + return _ProgressWriter("task-finding6") + + def test_permanent_error_does_not_trip_breaker(self, writer): + # ValidationException is the canonical case: a trace-heavy event + # pushes the item over the 400 KB DDB limit. Subsequent events + # are smaller and would succeed — so we must NOT trip the + # counter on this class of error. + table = MagicMock() + table.put_item.side_effect = _make_client_error("ValidationException") + writer._table = table + + # Fire WELL past the transient threshold. Any bare-except + # regression would trip the breaker here. + for _ in range(10): + writer.write_agent_milestone("test", "") + + assert writer._failure_count == 0, "Permanent errors must not increment the shared counter" + assert writer._disabled is False, ( + "ValidationException must keep the stream alive for smaller events" + ) + + def test_transient_error_trips_breaker_as_before(self, writer): + # Regression guard: the original circuit-breaker contract is + # preserved for transient errors. + table = MagicMock() + table.put_item.side_effect = _make_client_error("ProvisionedThroughputExceededException") + writer._table = table + + for _ in range(_ProgressWriter._MAX_FAILURES): + writer.write_agent_milestone("test", "") + + assert writer._disabled is True + assert writer._failure_count == _ProgressWriter._MAX_FAILURES + + def test_access_denied_disables_writer_immediately_with_loud_log(self, writer, capsys): + # AccessDeniedException is permanent AND catastrophic: IAM + # misconfig means every single future write will fail the same + # way. Flip the breaker on the FIRST occurrence so we don't + # waste three rounds of CloudWatch noise discovering it. + table = MagicMock() + table.put_item.side_effect = _make_client_error("AccessDeniedException") + writer._table = table + + writer.write_agent_milestone("test", "") + + assert writer._disabled is True, ( + "AccessDeniedException must flip the breaker on first occurrence" + ) + # Loud log line: operators need to spot this during rollouts. + captured = capsys.readouterr() + assert "permanent error" in captured.out.lower() + assert "AccessDeniedException" in captured.out + assert "disabling" in captured.out.lower() + + def test_resource_not_found_disables_writer_immediately(self, writer): + # Same fast-path as AccessDeniedException: a missing table will + # never un-miss itself, so retry is pointless. + table = MagicMock() + table.put_item.side_effect = _make_client_error("ResourceNotFoundException") + writer._table = table + + writer.write_agent_milestone("test", "") + + assert writer._disabled is True + + def test_unknown_exception_treated_as_transient_with_error_log(self, writer, capsys): + # Unknown exceptions default to transient-style counting (so a + # new botocore release adding a transient code does not instantly + # silence the stream) but log at ERROR level so operators notice + # and add the code to the classifier. + table = MagicMock() + table.put_item.side_effect = RuntimeError("mystery") + writer._table = table + + writer.write_agent_milestone("test", "") + + assert writer._failure_count == 1 + assert writer._disabled is False # below threshold + captured = capsys.readouterr() + # Loud ERROR marker so it stands out in CloudWatch Logs. + assert "ERROR" in captured.out + assert "UNKNOWN" in captured.out + + +# --------------------------------------------------------------------------- +# krokoko PR #52 review finding #8 — shared circuit-breaker state +# --------------------------------------------------------------------------- + + +class TestSharedCircuitBreaker: + """Before this change the runner and pipeline writers kept + independent state, so a throttling burst would trip one while the + other kept emitting milestones — producing a visibly half-alive + stream. These tests lock the shared-state contract.""" + + @pytest.fixture() + def env(self, monkeypatch): + monkeypatch.setenv("TASK_EVENTS_TABLE_NAME", "events-table") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + def test_shared_circuit_breaker_across_writers_same_task_id(self, env): + # Two writers for the same task: tripping the first must disable + # the second. This is the core half-alive-stream regression + # guard. + w1 = _ProgressWriter("task-shared") + w2 = _ProgressWriter("task-shared") + + t1 = MagicMock() + t1.put_item.side_effect = _make_client_error("ProvisionedThroughputExceededException") + w1._table = t1 + + for _ in range(_ProgressWriter._MAX_FAILURES): + w1.write_agent_milestone("turn", "") + + assert w1._disabled is True + assert w2._disabled is True, "Shared breaker must also disable the sibling writer" + + # Second writer must not hit DDB once the shared breaker is open + # — early-return on ``_disabled`` check at the top of + # ``_put_event``. + t2 = MagicMock() + w2._table = t2 + w2.write_agent_milestone("milestone", "") + t2.put_item.assert_not_called() + + def test_separate_tasks_have_independent_breakers(self, env): + # State is keyed by ``task_id``, not shared globally — tripping + # task A must not disable task B (critical when two tasks run in + # the same process, e.g. local batch mode or future shared-runtime + # deploys). + w_a = _ProgressWriter("task-a") + w_b = _ProgressWriter("task-b") + + t_a = MagicMock() + t_a.put_item.side_effect = _make_client_error("ProvisionedThroughputExceededException") + w_a._table = t_a + + for _ in range(_ProgressWriter._MAX_FAILURES): + w_a.write_agent_milestone("x", "") + assert w_a._disabled is True + + # Task B is untouched. + assert w_b._disabled is False + assert w_b._failure_count == 0 + + # And still writes. + t_b = MagicMock() + w_b._table = t_b + w_b.write_agent_milestone("y", "") + t_b.put_item.assert_called_once() + + def test_unknown_sentinel_task_id_is_isolated(self, env): + # ``runner.py`` falls back to ``config.task_id or "unknown"`` — + # lock that the sentinel does not bleed state across two real + # tasks that both end up using it. (Two ``"unknown"`` writers + # legitimately share state; this test pins that a real task_id + # and the sentinel remain distinct.) + w_real = _ProgressWriter("task-real") + w_unknown = _ProgressWriter("unknown") + + t = MagicMock() + t.put_item.side_effect = _make_client_error("ProvisionedThroughputExceededException") + w_unknown._table = t + + for _ in range(_ProgressWriter._MAX_FAILURES): + w_unknown.write_agent_milestone("x", "") + assert w_unknown._disabled is True + + # Real task is unaffected. + assert w_real._disabled is False + + def test_reset_helper_clears_shared_state_between_tests(self, env): + # Pinned because forgetting to reset is the single largest + # hazard of shared state — every test in this module relies on + # the autouse fixture clearing the map. + w = _ProgressWriter("task-reset") + t = MagicMock() + t.put_item.side_effect = _make_client_error("ProvisionedThroughputExceededException") + w._table = t + for _ in range(_ProgressWriter._MAX_FAILURES): + w.write_agent_milestone("x", "") + assert w._disabled is True + + # Reset and confirm a fresh writer for the same task starts + # clean. + _reset_circuit_breakers() + w2 = _ProgressWriter("task-reset") + assert w2._disabled is False + assert w2._failure_count == 0 + + def test_success_on_one_writer_resets_shared_counter(self, env): + # A successful write on any writer for the task must reset the + # shared failure counter — otherwise transient errors on two + # writers interleaved with successes would still trip the + # breaker counter as if they were consecutive. + w1 = _ProgressWriter("task-share-success") + w2 = _ProgressWriter("task-share-success") + + t1 = MagicMock() + t1.put_item.side_effect = _make_client_error("ProvisionedThroughputExceededException") + w1._table = t1 + w1.write_agent_milestone("turn", "") + assert w1._failure_count == 1 + + # w2 writes successfully — shared counter must reset, so the + # sibling writer sees it as fresh too. + t2 = MagicMock() + w2._table = t2 + w2.write_agent_milestone("milestone", "") + assert w1._failure_count == 0 + assert w2._failure_count == 0 + + def test_permanent_error_on_one_writer_does_not_affect_sibling_breaker(self, env): + # Cross-check of the #6 + #8 interaction: a permanent error on + # one writer must NOT trip the shared breaker, so the sibling + # writer continues to function. + w1 = _ProgressWriter("task-perm-cross") + w2 = _ProgressWriter("task-perm-cross") + + t1 = MagicMock() + t1.put_item.side_effect = _make_client_error("ValidationException") + w1._table = t1 + for _ in range(10): + w1.write_agent_milestone("oversized", "x" * 10) + assert w1._disabled is False + + # Sibling is still writing. + t2 = MagicMock() + w2._table = t2 + w2.write_agent_milestone("normal", "y") + t2.put_item.assert_called_once() diff --git a/agent/tests/test_server.py b/agent/tests/test_server.py index fdc951f..ed4c0d8 100644 --- a/agent/tests/test_server.py +++ b/agent/tests/test_server.py @@ -1,6 +1,10 @@ """Tests for AgentCore FastAPI server behavior.""" +from __future__ import annotations + +import threading import time +from typing import Any from unittest.mock import MagicMock import pytest @@ -63,6 +67,13 @@ def boom(**_kwargs): assert body["status"] == "unhealthy" assert body["reason"] == "background_pipeline_failed" + # Race: /ping flips to 503 as soon as ``_background_pipeline_failed = True`` + # is set in the except block, but ``task_state.write_terminal(...)`` happens + # a few lines later (after ``print()`` + ``traceback.print_exc()``). Wait + # for the mock to actually be invoked before asserting. + deadline2 = time.time() + 5.0 + while time.time() < deadline2 and not mock_write.called: + time.sleep(0.05) mock_write.assert_called() call_kw = mock_write.call_args assert call_kw[0][0] == "task-crash-1" @@ -71,3 +82,422 @@ def boom(**_kwargs): assert "error" in dumped assert "Background pipeline thread" in dumped["error"] assert "RuntimeError" in dumped["error"] + + +def _invocation_payload(task_id: str = "task-sync-1") -> dict: + return { + "input": { + "task_id": task_id, + "repo_url": "o/r", + "prompt": "do a thing", + "github_token": "ghp_x", + "aws_region": "us-east-1", + } + } + + +def test_sync_path_regression_when_accept_is_missing(client, monkeypatch): + """No Accept header → JSON acceptance shape preserved.""" + started = threading.Event() + + def fake_run_task(**kwargs): + started.set() + + monkeypatch.setattr(server, "run_task", fake_run_task) + monkeypatch.setattr(server.task_state, "write_terminal", MagicMock()) + + r = client.post("/invocations", json=_invocation_payload("t-sync")) + assert r.status_code == 200 + body = r.json() + assert body["output"]["result"] == {"status": "accepted", "task_id": "t-sync"} + assert "message" in body["output"] + # Background thread ran + assert started.wait(timeout=3) + + +def test_sync_path_preserved_for_application_json_accept(client, monkeypatch): + """Accept: application/json → sync JSON path.""" + monkeypatch.setattr(server, "run_task", lambda **_: None) + monkeypatch.setattr(server.task_state, "write_terminal", MagicMock()) + + r = client.post( + "/invocations", + json=_invocation_payload("t-json"), + headers={"Accept": "application/json"}, + ) + assert r.status_code == 200 + assert r.headers["content-type"].startswith("application/json") + assert r.json()["output"]["result"]["status"] == "accepted" + + +def test_event_stream_accept_header_ignored_returns_sync_json(client, monkeypatch): + """Accept: text/event-stream is ignored; sync JSON is always returned.""" + monkeypatch.setattr(server, "run_task", lambda **_: None) + monkeypatch.setattr(server.task_state, "write_terminal", MagicMock()) + + r = client.post( + "/invocations", + json=_invocation_payload("t-accept-sse"), + headers={"Accept": "text/event-stream"}, + ) + assert r.status_code == 200 + assert r.headers["content-type"].startswith("application/json") + assert r.json()["output"]["result"] == {"status": "accepted", "task_id": "t-accept-sse"} + + +def test_ping_reports_healthy_when_idle(client, monkeypatch): + """/ping returns {"status": "healthy"} with no active pipeline threads.""" + monkeypatch.setattr(server, "_background_pipeline_failed", False) + with server._threads_lock: + server._active_threads.clear() + r = client.get("/ping") + assert r.status_code == 200 + assert r.json() == {"status": "healthy"} + + +def test_ping_reports_healthybusy_when_pipeline_alive(client, monkeypatch): + """/ping returns HealthyBusy while a pipeline thread is alive (idle-evict guard).""" + monkeypatch.setattr(server, "_background_pipeline_failed", False) + + stop = threading.Event() + + def worker(): + stop.wait(timeout=5) + + t = threading.Thread(target=worker, name="test-live-pipeline") + t.start() + try: + with server._threads_lock: + server._active_threads.clear() + server._active_threads.append(t) + r = client.get("/ping") + assert r.status_code == 200 + assert r.json() == {"status": "HealthyBusy"} + finally: + stop.set() + t.join(timeout=2) + with server._threads_lock: + server._active_threads.clear() + + +def test_invocations_rejects_missing_required_params_with_400(client, monkeypatch): + """A task record missing required fields is rejected up front with 400. + + Regression guard for wiring `_validate_required_params` into the handler + — without it, bad payloads would spawn a background thread that crashes + deep inside `setup_repo` or hydration, producing a cryptic terminal + failure instead of a structured `TASK_RECORD_INCOMPLETE` 400. + """ + # Patch _spawn_background so if validation ever fails to trigger we'd + # see the test spawn a real pipeline thread. + spawn_calls: list[dict] = [] + monkeypatch.setattr(server, "_spawn_background", lambda params: spawn_calls.append(params)) + + response = client.post( + "/invocations", + json={"input": {"task_id": "t-missing", "task_type": "pr_review"}}, + ) + + assert response.status_code == 400 + body = response.json() + assert body["code"] == "TASK_RECORD_INCOMPLETE" + assert "repo_url" in body["missing"] + assert "pr_number" in body["missing"] + # Background pipeline must NOT be spawned on validation failure. + assert spawn_calls == [] + + +def test_spawn_background_resets_pipeline_failed_flag(monkeypatch): + """A new spawn clears ``_background_pipeline_failed`` when no prior threads are alive. + + AgentCore reconciliation keys off ``/ping`` status; a stale + ``_background_pipeline_failed = True`` after a crashed pipeline would + route new traffic around a healthy container forever. + """ + server._background_pipeline_failed = True + with server._threads_lock: + server._active_threads.clear() + + # Stub the actual pipeline so we don't try to run a real task. + monkeypatch.setattr(server, "_run_task_background", lambda **_kwargs: None) + + thread = server._spawn_background( + {"task_id": "t-reset", "repo_url": "o/r", "task_description": "x"} + ) + thread.join(timeout=2) + + assert server._background_pipeline_failed is False + + with server._threads_lock: + server._active_threads.clear() + + +def test_run_task_background_starts_and_stops_heartbeat(monkeypatch): + """The heartbeat worker thread runs while the pipeline runs and stops after. + + Regression guard: if someone accidentally drops the heartbeat thread + start/stop, the stranded-task reconciler would start flagging healthy + long-running tasks as stuck. + """ + heartbeat_calls: list[str] = [] + + def fake_write_heartbeat(task_id: str) -> None: + heartbeat_calls.append(task_id) + + monkeypatch.setattr(server.task_state, "write_heartbeat", fake_write_heartbeat) + monkeypatch.setattr(server, "_HEARTBEAT_INTERVAL_SECONDS", 0.05) + + # Stub run_task to sleep briefly so the heartbeat has time to fire. + def fake_run_task(**_kwargs): + time.sleep(0.15) + + monkeypatch.setattr(server, "run_task", fake_run_task) + # Stub terminal write so the fake pipeline doesn't try to hit DDB. + monkeypatch.setattr(server.task_state, "write_terminal", lambda *a, **kw: None) + + server._run_task_background( + task_id="t-heartbeat", + repo_url="o/r", + task_description="x", + issue_number="", + github_token="", + anthropic_model="", + max_turns=10, + max_budget_usd=None, + aws_region="us-east-1", + ) + + # Heartbeat should have fired at least once during the 0.15s pipeline + # with a 0.05s cadence. + assert len(heartbeat_calls) >= 1 + assert heartbeat_calls[0] == "t-heartbeat" + + +def test_validate_required_params_pr_types_require_pr_number(): + """PR-iteration and PR-review task_types need a pr_number regardless.""" + missing = server._validate_required_params( + { + "repo_url": "o/r", + "task_type": "pr_iteration", + "pr_number": "", + } + ) + assert missing == ["pr_number"] + + missing = server._validate_required_params( + { + "repo_url": "o/r", + "task_type": "pr_review", + "pr_number": "42", + } + ) + assert missing == [] + + # new_task needs issue OR description. + missing = server._validate_required_params( + { + "repo_url": "o/r", + "task_type": "new_task", + } + ) + assert missing == ["issue_number_or_task_description"] + + missing = server._validate_required_params( + { + "repo_url": "o/r", + "task_type": "new_task", + "task_description": "do the thing", + } + ) + assert missing == [] + + +def test_drain_threads_joins_active_threads(): + """_drain_threads joins live background threads on shutdown.""" + stop = threading.Event() + + def worker(): + stop.wait(timeout=1) + + t = threading.Thread(target=worker, name="drain-test") + t.start() + with server._threads_lock: + server._active_threads.clear() + server._active_threads.append(t) + + # Signal thread to exit, then drain. + stop.set() + server._drain_threads(timeout=5) + # Thread must have finished by now. + assert not t.is_alive() + + with server._threads_lock: + server._active_threads.clear() + + +def test_debug_cw_write_blocking_no_log_group_is_noop(monkeypatch): + """_debug_cw is a no-op when LOG_GROUP_NAME is unset.""" + monkeypatch.delenv("LOG_GROUP_NAME", raising=False) + # Should not raise, even if boto3 would fail — we never reach it. + server._debug_cw("hello", task_id="t") + + +def test_debug_cw_write_blocking_bumps_failure_counter_on_boto_error(monkeypatch): + """On boto errors the failure counter increments so operators can alarm. + + AgentCore doesn't forward container stdout to APPLICATION_LOGS, so a + broken ``_debug_cw`` is invisible except for this counter. If the + counter ever stops bumping on error the blind-debug alarm breaks + silently. + """ + # Seed the counter to a known value so we can assert the delta without + # being sensitive to other tests. + with server._debug_cw_failures_lock: + server._debug_cw_failures = 0 + + # Stub ``boto3.client`` to raise so the except branch (which bumps + # the counter) runs. + class _BrokenBoto3: + @staticmethod + def client(*args, **kwargs): + raise RuntimeError("simulated boto failure") + + monkeypatch.setitem(__import__("sys").modules, "boto3", _BrokenBoto3) + + server._debug_cw_write_blocking( + log_group="/some/log-group", + task_id="t-1", + stamped="2026-01-01T00:00:00Z hello", + ) + + with server._debug_cw_failures_lock: + assert server._debug_cw_failures == 1 + + +# --------------------------------------------------------------------------- +# Chunk K: trace flag extraction (design §10.1) +# --------------------------------------------------------------------------- + + +class _FakeRequest: + """Minimal stand-in for starlette.Request — only ``.headers.get`` is used.""" + + def __init__(self, headers=None): + self.headers = headers or {} + + +class TestExtractTrace: + """_extract_invocation_params is the boundary where the orchestrator's + ``trace`` payload becomes the agent's ``trace`` kwarg. The flag is + strictly opt-in — only a real boolean ``True`` counts.""" + + def _base_payload(self, **extra): + return { + "repo_url": "org/repo", + "task_description": "Fix it", + "task_id": "t-1", + **extra, + } + + def _fake_req(self) -> Any: + # ``_extract_invocation_params`` only calls ``request.headers.get``, + # so a duck-typed stub suffices. Return ``Any`` to silence the + # ty type checker without importing starlette at runtime. + return _FakeRequest() + + def test_trace_true_in_payload_extracts_to_True(self): + params = server._extract_invocation_params( + self._base_payload(trace=True), + self._fake_req(), + ) + assert params["trace"] is True + + def test_trace_absent_defaults_to_False(self): + params = server._extract_invocation_params( + self._base_payload(), + self._fake_req(), + ) + assert params["trace"] is False + + def test_trace_string_true_does_NOT_enable_trace(self): + # Guard against a misbehaving client sending "true" (truthy + # string) — the extractor uses ``is True`` so only real + # booleans flip the flag. + params = server._extract_invocation_params( + self._base_payload(trace="true"), + self._fake_req(), + ) + assert params["trace"] is False + + def test_trace_1_does_NOT_enable_trace(self): + params = server._extract_invocation_params( + self._base_payload(trace=1), + self._fake_req(), + ) + assert params["trace"] is False + + +class TestExtractUserId: + """K2 Stage 3: ``user_id`` is the platform Cognito ``sub`` threaded + from the orchestrator. The agent uses it to construct the trace S3 + key ``traces//.jsonl.gz``. A non-string value + must be coerced to empty so a surprise ``None`` / int doesn't flow + into an S3 PutObject call later.""" + + def _base_payload(self, **extra): + return { + "repo_url": "org/repo", + "task_description": "Fix it", + "task_id": "t-1", + **extra, + } + + def _fake_req(self) -> Any: + return _FakeRequest() + + def test_user_id_string_extracts_verbatim(self): + params = server._extract_invocation_params( + self._base_payload(user_id="sub-abc-123"), + self._fake_req(), + ) + assert params["user_id"] == "sub-abc-123" + + def test_user_id_absent_defaults_to_empty_string(self): + params = server._extract_invocation_params( + self._base_payload(), + self._fake_req(), + ) + assert params["user_id"] == "" + + def test_user_id_none_coerced_to_empty(self): + params = server._extract_invocation_params( + self._base_payload(user_id=None), + self._fake_req(), + ) + assert params["user_id"] == "" + + def test_user_id_non_string_coerced_to_empty(self): + # Defend against a misbehaving caller sending an int or dict — + # the agent writes ``user_id`` into an S3 object key, so a + # non-string would blow up at upload time (or worse, silently + # stringify to something like ``"None"`` or ``"123"``). + params = server._extract_invocation_params( + self._base_payload(user_id=12345), + self._fake_req(), + ) + assert params["user_id"] == "" + + def test_user_id_non_string_logs_warn(self, capsys): + # Silent coercion is a documented anti-pattern in project + # guidelines — if Stage 4 later skips the S3 upload because + # ``user_id`` is empty, a user investigating "my trace never + # appeared" needs a signal in CloudWatch to correlate. + server._extract_invocation_params( + self._base_payload(user_id=12345, task_id="t-warn"), + self._fake_req(), + ) + captured = capsys.readouterr() + assert "[server/warn]" in captured.out + assert "user_id payload field is not a string" in captured.out + assert "type=int" in captured.out + assert "'t-warn'" in captured.out diff --git a/agent/tests/test_task_state.py b/agent/tests/test_task_state.py index 5db6ea9..0cab27d 100644 --- a/agent/tests/test_task_state.py +++ b/agent/tests/test_task_state.py @@ -1,6 +1,9 @@ """Unit tests for pure functions in task_state.py.""" -from task_state import _build_logs_url, _now_iso +import pytest + +import task_state +from task_state import TaskFetchError, _build_logs_url, _now_iso class TestNowIso: @@ -49,3 +52,471 @@ def test_uses_default_region(self, monkeypatch): url = _build_logs_url("t1") assert url is not None assert "eu-west-1" in url + + +class TestGetTask: + """Verify the NotFound vs FetchFailed distinction. + + Callers must be able to tell "record doesn't exist" (``None``) from + "couldn't read it" (``TaskFetchError``). Collapsing the two to ``None`` + would let a transient DDB blip look like a legitimate absence. + """ + + def test_returns_none_when_no_table(self, monkeypatch): + monkeypatch.setattr(task_state, "_get_table", lambda: None) + assert task_state.get_task("t-any") is None + + def test_returns_item_when_found(self, monkeypatch): + class _FakeTable: + def get_item(self, Key): + assert Key == {"task_id": "t-present"} + return {"Item": {"task_id": "t-present", "status": "RUNNING"}} + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + item = task_state.get_task("t-present") + assert item == {"task_id": "t-present", "status": "RUNNING"} + + def test_returns_none_when_item_absent(self, monkeypatch): + class _FakeTable: + def get_item(self, Key): + return {} # DDB returns no "Item" key when not found. + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + assert task_state.get_task("t-missing") is None + + def test_raises_TaskFetchError_on_ddb_failure(self, monkeypatch): + class _FakeTable: + def get_item(self, Key): + raise RuntimeError("ProvisionedThroughputExceededException") + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + with pytest.raises(TaskFetchError) as exc_info: + task_state.get_task("t-throttled") + assert "ProvisionedThroughputExceededException" in str(exc_info.value) + + +class TestWriteSessionInfo: + """Rev-5 OBS-4: interactive path writes session_id + agent_runtime_arn.""" + + def test_writes_session_id_and_arn(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + + task_state.write_session_info( + "t-interactive", + "sess-abc123", + "arn:aws:bedrock-agentcore:us-east-1:123:runtime/jwt-xyz", + ) + + assert len(calls) == 1 + call = calls[0] + assert call["Key"] == {"task_id": "t-interactive"} + assert "session_id = :sid" in call["UpdateExpression"] + assert "agent_runtime_arn = :arn" in call["UpdateExpression"] + assert "compute_type = :ct" in call["UpdateExpression"] + assert "compute_metadata = :cm" in call["UpdateExpression"] + values = call["ExpressionAttributeValues"] + assert values[":sid"] == "sess-abc123" + assert values[":arn"] == "arn:aws:bedrock-agentcore:us-east-1:123:runtime/jwt-xyz" + assert values[":ct"] == "agentcore" + assert values[":cm"] == { + "runtimeArn": "arn:aws:bedrock-agentcore:us-east-1:123:runtime/jwt-xyz" + } + + def test_noop_when_both_empty(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + + task_state.write_session_info("t-empty", "", "") + assert calls == [] + + def test_skips_silently_when_task_already_advanced(self, monkeypatch): + from botocore.exceptions import ClientError + + class _FakeTable: + def update_item(self, **kwargs): + raise ClientError( + {"Error": {"Code": "ConditionalCheckFailedException"}}, + "UpdateItem", + ) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + + # Must NOT raise — the conditional failure is expected when the + # task has already transitioned past SUBMITTED/HYDRATING. + task_state.write_session_info("t-raced", "sess-x", "arn:x") + + def test_writes_only_session_when_arn_missing(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + + task_state.write_session_info("t-partial", "sess-only", "") + assert len(calls) == 1 + assert "session_id = :sid" in calls[0]["UpdateExpression"] + assert "agent_runtime_arn" not in calls[0]["UpdateExpression"] + + +class TestWriteRunningMaintainsStatusCreatedAt: + """Regression guard: ``write_running`` must rewrite ``status_created_at`` + so the ``UserStatusIndex`` GSI sort key reflects the current status. + Without this, ``bga list`` sorts by the stale SUBMITTED prefix and newly + running / completed / cancelled tasks appear after stale SUBMITTED rows. + """ + + def test_writes_status_created_at_with_running_prefix(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + task_state.write_running("t-run") + + assert len(calls) == 1 + call = calls[0] + assert "status_created_at = :sca" in call["UpdateExpression"] + sca = call["ExpressionAttributeValues"][":sca"] + assert sca.startswith("RUNNING#") + # The timestamp after the '#' matches _now_iso()'s ISO-Z format. + ts = sca.split("#", 1)[1] + assert ts.endswith("Z") + assert len(ts) == 20 + + +class TestWriteTerminalMaintainsStatusCreatedAt: + def test_completed_rewrites_sca_with_completed_prefix(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + task_state.write_terminal("t-done", "COMPLETED") + + assert len(calls) == 1 + call = calls[0] + assert "status_created_at = :sca" in call["UpdateExpression"] + sca = call["ExpressionAttributeValues"][":sca"] + assert sca.startswith("COMPLETED#") + + def test_failed_rewrites_sca_with_failed_prefix(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + task_state.write_terminal("t-fail", "FAILED", {"error": "boom"}) + + assert len(calls) == 1 + sca = calls[0]["ExpressionAttributeValues"][":sca"] + assert sca.startswith("FAILED#") + + def test_sca_and_completed_at_share_timestamp(self, monkeypatch): + """The SCA timestamp and completed_at should match so operators can + cross-reference the GSI row against the base table without wondering + which write happened first.""" + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + task_state.write_terminal("t-sync", "COMPLETED") + + values = calls[0]["ExpressionAttributeValues"] + sca_ts = values[":sca"].split("#", 1)[1] + completed_at = values[":t"] + assert sca_ts == completed_at + + +class TestWriteTerminalTraceS3Uri: + """K2 Stage 4 — ``write_terminal`` persists ``trace_s3_uri`` from + the result dict so the ``get-trace-url`` handler (which reads the + field off the TaskRecord) sees a consistent view the moment the + task reaches terminal.""" + + def test_trace_s3_uri_written_when_present_in_result(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + task_state.write_terminal( + "t-trace", + "COMPLETED", + {"trace_s3_uri": "s3://bucket/traces/u-1/t-trace.jsonl.gz"}, + ) + assert len(calls) == 1 + update_expr = calls[0]["UpdateExpression"] + assert "trace_s3_uri = :ts3" in update_expr + values = calls[0]["ExpressionAttributeValues"] + assert values[":ts3"] == "s3://bucket/traces/u-1/t-trace.jsonl.gz" + + def test_trace_s3_uri_omitted_when_result_has_no_uri(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + task_state.write_terminal( + "t-plain", + "COMPLETED", + {"pr_url": "https://github.com/o/r/pull/1"}, + ) + assert len(calls) == 1 + update_expr = calls[0]["UpdateExpression"] + assert "trace_s3_uri" not in update_expr + values = calls[0]["ExpressionAttributeValues"] + assert ":ts3" not in values + + def test_trace_s3_uri_none_omitted(self, monkeypatch): + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + task_state.write_terminal( + "t-null", + "COMPLETED", + {"trace_s3_uri": None}, + ) + update_expr = calls[0]["UpdateExpression"] + assert "trace_s3_uri" not in update_expr + + def test_conditional_check_failed_with_trace_uri_logs_orphan_diagnostic( + self, + monkeypatch, + capsys, + ): + """K2 final review SIG-1: when ``write_terminal``'s precondition + fails (typically: concurrent cancel) and a ``trace_s3_uri`` was + already uploaded, the orphaned S3 object needs a dedicated log + line — otherwise the generic ``skipped: precondition not met`` + message hides silently-lost trace URIs. + + L4 extension: after the orphan log prints, the self-heal + ``write_trace_uri_conditional`` fires; when the second + UpdateItem succeeds, the self-heal log also prints.""" + from botocore.exceptions import ClientError + + class _FakeTable: + def __init__(self): + self.calls = 0 + + def update_item(self, **_kwargs): + self.calls += 1 + # First call (write_terminal) raises CCF. + # Second call (self-heal) succeeds. + if self.calls == 1: + raise ClientError( + {"Error": {"Code": "ConditionalCheckFailedException", "Message": "!"}}, + "UpdateItem", + ) + return {} + + fake = _FakeTable() + monkeypatch.setattr(task_state, "_get_table", lambda: fake) + task_state.write_terminal( + "t-orphan", + "COMPLETED", + {"trace_s3_uri": "s3://bucket/traces/u-1/t-orphan.jsonl.gz"}, + ) + out = capsys.readouterr().out + # Generic skip message still prints (benign-case compatibility). + assert "write_terminal skipped" in out + # And the specific orphan log calls out the URI + actionable + # detail (7-day lifecycle) so operators can reason about cost. + assert "orphaned by ConditionalCheckFailed" in out + assert "s3://bucket/traces/u-1/t-orphan.jsonl.gz" in out + assert "7-day lifecycle" in out + # L4: self-heal fired (second update_item call) and logged success. + assert fake.calls == 2 + assert "self-healed" in out + + def test_conditional_check_failed_without_trace_uri_skips_orphan_log( + self, + monkeypatch, + capsys, + ): + """The orphan diagnostic must NOT fire on the common + benign-cancel case (where no S3 write happened) — otherwise + operators get log noise that blunts the signal of a real + orphan.""" + from botocore.exceptions import ClientError + + class _FakeTable: + def update_item(self, **_kwargs): + raise ClientError( + {"Error": {"Code": "ConditionalCheckFailedException", "Message": "!"}}, + "UpdateItem", + ) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + task_state.write_terminal("t-benign", "COMPLETED", {"pr_url": "https://pr"}) + out = capsys.readouterr().out + assert "write_terminal skipped" in out + assert "orphaned" not in out + + +class TestWriteTraceUriConditional: + """L4 item 1a — ``write_trace_uri_conditional`` persists + ``trace_s3_uri`` on an already-terminal record as a self-heal + after ``write_terminal`` loses a race with cancel / reconciler. + + The helper is scoped to ``attribute_not_exists(trace_s3_uri) AND + status IN (CANCELLED, COMPLETED, FAILED, TIMED_OUT)`` so it cannot + clobber an existing URI or write on a non-terminal record.""" + + def test_happy_path_writes_uri_and_returns_true(self, monkeypatch): + """Status=COMPLETED, no existing trace_s3_uri → write succeeds.""" + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + return {} + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + healed = task_state.write_trace_uri_conditional( + "t-heal", "s3://bucket/traces/u-1/t-heal.jsonl.gz" + ) + assert healed is True + assert len(calls) == 1 + kwargs = calls[0] + assert kwargs["Key"] == {"task_id": "t-heal"} + assert kwargs["UpdateExpression"] == "SET trace_s3_uri = :ts3" + # ConditionExpression must be scoped to both "URI not set" and + # "status terminal" — either one alone would be unsafe. + cond = kwargs["ConditionExpression"] + assert "attribute_not_exists(trace_s3_uri)" in cond + assert "#s IN" in cond + assert kwargs["ExpressionAttributeNames"] == {"#s": "status"} + values = kwargs["ExpressionAttributeValues"] + assert values[":ts3"] == "s3://bucket/traces/u-1/t-heal.jsonl.gz" + # All four terminal-status literals must appear in the IN-list + # (the helper's contract is terminal-agnostic). + assert values[":cancelled"] == "CANCELLED" + assert values[":completed"] == "COMPLETED" + assert values[":failed"] == "FAILED" + assert values[":timed_out"] == "TIMED_OUT" + + def test_uri_already_present_returns_false_and_logs_info(self, monkeypatch, capsys): + """``ConditionalCheckFailedException`` → returns False, INFO log (benign).""" + from botocore.exceptions import ClientError + + class _FakeTable: + def update_item(self, **_kwargs): + raise ClientError( + {"Error": {"Code": "ConditionalCheckFailedException", "Message": "!"}}, + "UpdateItem", + ) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + healed = task_state.write_trace_uri_conditional( + "t-already", "s3://bucket/traces/u/t-already.jsonl.gz" + ) + assert healed is False + out = capsys.readouterr().out + assert "write_trace_uri_conditional skipped" in out + assert "t-already" in out + + def test_non_terminal_status_returns_false(self, monkeypatch): + """Non-terminal status raises CCF (status IN clause rejects) → False.""" + from botocore.exceptions import ClientError + + class _FakeTable: + def update_item(self, **_kwargs): + raise ClientError( + {"Error": {"Code": "ConditionalCheckFailedException", "Message": "!"}}, + "UpdateItem", + ) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + healed = task_state.write_trace_uri_conditional( + "t-running", "s3://b/traces/u/t-running.jsonl.gz" + ) + assert healed is False + + def test_transient_ddb_error_returns_false_and_logs_warn(self, monkeypatch, capsys): + """A non-CCF ClientError (e.g., throttling) → returns False, WARN log.""" + from botocore.exceptions import ClientError + + class _FakeTable: + def update_item(self, **_kwargs): + raise ClientError( + { + "Error": { + "Code": "ProvisionedThroughputExceededException", + "Message": "!", + } + }, + "UpdateItem", + ) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + healed = task_state.write_trace_uri_conditional( + "t-throttle", "s3://b/traces/u/t-throttle.jsonl.gz" + ) + assert healed is False + out = capsys.readouterr().out + assert "write_trace_uri_conditional failed" in out + # Log surfaces the exception type name to aid triage. + assert "ClientError" in out + + def test_empty_uri_is_a_noop(self, monkeypatch): + """Guard: empty URI → no DDB call, returns False.""" + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + healed = task_state.write_trace_uri_conditional("t-x", "") + assert healed is False + assert calls == [] + + def test_empty_task_id_is_a_noop(self, monkeypatch): + """Guard: empty task_id → no DDB call, returns False.""" + calls: list[dict] = [] + + class _FakeTable: + def update_item(self, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(task_state, "_get_table", lambda: _FakeTable()) + healed = task_state.write_trace_uri_conditional("", "s3://b/x.gz") + assert healed is False + assert calls == [] + + def test_no_table_returns_false(self, monkeypatch): + """When ``_get_table`` returns None (TASK_TABLE_NAME unset) → False.""" + monkeypatch.setattr(task_state, "_get_table", lambda: None) + healed = task_state.write_trace_uri_conditional("t-x", "s3://b/x.gz") + assert healed is False diff --git a/agent/tests/test_trace_key_contract.py b/agent/tests/test_trace_key_contract.py new file mode 100644 index 0000000..72a26ef --- /dev/null +++ b/agent/tests/test_trace_key_contract.py @@ -0,0 +1,122 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +"""Cross-language contract test: trace S3 key layout (design §10.1). + +The trace artifact key ``traces//.jsonl.gz`` is +asserted by three independent codepaths that MUST agree: + +1. Agent ``upload_trace_to_s3`` in ``agent/src/telemetry.py`` — constructs + the key it writes to S3. +2. CDK handler ``expectedKeyPrefix`` in + ``cdk/src/handlers/get-trace-url.ts`` — refuses to presign keys + outside the caller's user prefix. +3. CDK construct ``TRACE_OBJECT_KEY_PREFIX`` in + ``cdk/src/constructs/trace-artifacts-bucket.ts`` — exports the + canonical prefix that the handler imports. + +This test is a drift detector: if anyone renames ``traces/`` to +``trajectories/`` on one side, it fails immediately. The agent side is +exercised end-to-end (mocked boto3); the TypeScript sides are parsed as +source text so no TS runtime is needed. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from telemetry import upload_trace_to_s3 + +# Repository root relative to this test file: +# agent/tests/test_trace_key_contract.py -> parents[2] +REPO_ROOT = Path(__file__).resolve().parents[2] +CONSTRUCT_TS = REPO_ROOT / "cdk" / "src" / "constructs" / "trace-artifacts-bucket.ts" +HANDLER_TS = REPO_ROOT / "cdk" / "src" / "handlers" / "get-trace-url.ts" + + +def _read_trace_object_key_prefix_from_construct() -> str: + """Parse ``export const TRACE_OBJECT_KEY_PREFIX = 'traces/';`` from + the CDK construct file without importing TypeScript.""" + text = CONSTRUCT_TS.read_text(encoding="utf-8") + match = re.search( + r"""export\s+const\s+TRACE_OBJECT_KEY_PREFIX\s*=\s*['"]([^'"]+)['"]""", + text, + ) + assert match is not None, f"Could not find TRACE_OBJECT_KEY_PREFIX in {CONSTRUCT_TS}" + return match.group(1) + + +def _read_expected_key_prefix_expr_from_handler() -> str: + """Parse the ``expectedKeyPrefix = ...`` assignment from the handler. + + Returns the raw right-hand-side expression as a string. We only + care that it interpolates ``TRACE_OBJECT_KEY_PREFIX`` and builds a + ``${userId}/`` shape — that's what binds the three sides + together. + """ + text = HANDLER_TS.read_text(encoding="utf-8") + match = re.search( + r"""const\s+expectedKeyPrefix\s*=\s*(`[^`]+`)""", + text, + ) + assert match is not None, f"Could not find 'const expectedKeyPrefix = `...`' in {HANDLER_TS}" + return match.group(1) + + +class TestTraceKeyLayoutContract: + """All three codepaths must agree on ``traces//.jsonl.gz``.""" + + def test_construct_prefix_is_traces_slash(self): + prefix = _read_trace_object_key_prefix_from_construct() + assert prefix == "traces/", ( + f"TRACE_OBJECT_KEY_PREFIX drifted to {prefix!r}; agent uploader still " + "writes under 'traces/' — update agent/src/telemetry.py in lock-step." + ) + + def test_handler_expected_prefix_uses_construct_constant(self): + """The handler must compose its expected prefix from the shared + constant (not a hardcoded string), and append the caller's user + id with a trailing slash.""" + expr = _read_expected_key_prefix_expr_from_handler() + # Template literal must reference TRACE_OBJECT_KEY_PREFIX — if + # someone inlines the string, the drift detector in + # test_construct_prefix_is_traces_slash stops catching renames. + assert "TRACE_OBJECT_KEY_PREFIX" in expr, ( + f"expectedKeyPrefix expression {expr!r} must reference " + "TRACE_OBJECT_KEY_PREFIX so the construct is the single source of truth." + ) + # Must interpolate the user id and end with a trailing slash. + assert "${userId}" in expr, expr + assert expr.rstrip("`").endswith("/"), expr + + def test_agent_uploader_writes_key_under_traces_prefix(self, monkeypatch): + """Round-trip: the agent's actual put_object call uses the same + prefix the construct exports.""" + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "b") + mock_client = MagicMock() + with patch("boto3.client", return_value=mock_client): + uri = upload_trace_to_s3(task_id="TASK-1", user_id="user-abc", body=b"x") + + prefix = _read_trace_object_key_prefix_from_construct() + _, kwargs = mock_client.put_object.call_args + key = kwargs["Key"] + assert key.startswith(prefix), ( + f"Agent wrote key {key!r} but construct declares prefix {prefix!r}" + ) + assert key == f"{prefix}user-abc/TASK-1.jsonl.gz" + assert uri == f"s3://b/{prefix}user-abc/TASK-1.jsonl.gz" + + def test_full_key_shape_matches_design_10_1(self, monkeypatch): + """Pin the full shape ``traces//.jsonl.gz``.""" + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "b") + mock_client = MagicMock() + with patch("boto3.client", return_value=mock_client): + upload_trace_to_s3(task_id="t-42", user_id="sub-123", body=b"x") + + _, kwargs = mock_client.put_object.call_args + assert kwargs["Key"] == "traces/sub-123/t-42.jsonl.gz" diff --git a/agent/tests/test_trace_upload.py b/agent/tests/test_trace_upload.py new file mode 100644 index 0000000..f9f54a4 --- /dev/null +++ b/agent/tests/test_trace_upload.py @@ -0,0 +1,500 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +"""Tests for the K2 Stage 4 --trace upload path (design §10.1). + +Covers: + * ``_TrajectoryWriter`` accumulator behavior (enabled/disabled, + bounded, JSONL header shape) + * ``upload_trace_to_s3`` fail-open semantics and contract enforcement + from the K2 Stage 3 review (empty user_id -> skip + warn, never + write ``traces//`` keys) +""" + +from __future__ import annotations + +import gzip +import io +import json +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from telemetry import _TrajectoryWriter, upload_trace_to_s3 + + +class TestTrajectoryAccumulator: + """``_TrajectoryWriter`` with ``accumulate=True`` retains events + in memory for the terminal-state S3 dump. With ``accumulate=False`` + (default) nothing is kept — the writer stays CW-only and zero-cost + for non-trace tasks.""" + + def test_accumulator_disabled_by_default(self): + w = _TrajectoryWriter("t-1") + w._put_event({"event": "X", "n": 1}) + assert w._events == [] + assert w._accumulated_bytes == 0 + assert w.dump_gzipped_jsonl() is None + + def test_accumulator_retains_events_when_enabled(self): + w = _TrajectoryWriter("t-1", accumulate=True) + w._put_event({"event": "A", "n": 1}) + w._put_event({"event": "B", "n": 2}) + assert len(w._events) == 2 + assert w._accumulated_bytes > 0 + + def test_dump_gzipped_jsonl_produces_well_formed_output(self): + w = _TrajectoryWriter("task-42", accumulate=True) + w._put_event({"event": "TURN", "turn": 1}) + w._put_event({"event": "TURN", "turn": 2}) + + body = w.dump_gzipped_jsonl() + assert body is not None + # Decompress and parse every line as JSON + lines = gzip.decompress(body).decode("utf-8").splitlines() + assert len(lines) == 3 # header + 2 events + header = json.loads(lines[0]) + assert header["event"] == "TRAJECTORY_ARTIFACT_HEADER" + assert header["task_id"] == "task-42" + assert header["accumulated_events"] == 2 + assert header["dropped"] == 0 + assert header["max_bytes"] == _TrajectoryWriter._ACCUMULATOR_MAX_BYTES + # Events are in insertion order + events = [json.loads(line) for line in lines[1:]] + assert events[0] == {"event": "TURN", "turn": 1} + assert events[1] == {"event": "TURN", "turn": 2} + + def test_accumulator_is_bounded(self): + # Force a low cap so the test runs fast without allocating 50 MB. + # Instance-level attribute override — does NOT leak across tests + # (each test constructs a fresh writer). A future refactor that + # converts this to a class-level assignment would silently leak; + # keep the assignment on ``w`` (the instance), never on the class. + w = _TrajectoryWriter("t-1", accumulate=True) + w._ACCUMULATOR_MAX_BYTES = 100 # tiny budget + # Each event serializes to ~30+ bytes; a few will fit, the + # rest should bounce off the cap. + for i in range(10): + w._put_event({"event": "X", "i": i}) + # At least one event must have been captured before the cap + # tripped — otherwise this test would pass trivially against a + # buggy ``_put_event`` that rejects everything (pr-test-analyzer + # Finding S1). + assert len(w._events) >= 1 + assert w._accumulator_dropped > 0 + assert w._accumulated_bytes <= 100 + # Header must report the drop so a consumer can tell a + # truncated trace from a complete one. + body = w.dump_gzipped_jsonl() + assert body is not None + header = json.loads(gzip.decompress(body).decode("utf-8").splitlines()[0]) + assert header["dropped"] > 0 + assert header["accumulated_events"] < 10 + + def test_dump_returns_none_when_no_events(self): + w = _TrajectoryWriter("t-1", accumulate=True) + # No events appended + assert w.dump_gzipped_jsonl() is None + + def test_accumulator_cap_uses_inclusive_upper_bound(self): + """Pin ``<=`` boundary on ``_put_event``. + + An event whose serialized size is EXACTLY the cap must be + accepted (``<=``). A subsequent 1-byte-added event must be + rejected. This test guards against a future ``<`` off-by-one + refactor that would silently drop events sitting right at the + cap. + + The exact serialized JSON length is measured below so the test + is deterministic under default ``json.dumps`` spacing; the + padding length (75) is chosen to land the event on 100 bytes + on the nose. + """ + import json as _json + + w = _TrajectoryWriter("t", accumulate=True) + w._ACCUMULATOR_MAX_BYTES = 100 + + # Craft an event whose JSON byte length is exactly 100. + exact_event = {"event": "X", "pad": "A" * 75} + assert len(_json.dumps(exact_event).encode("utf-8")) == 100, ( + "Padding recipe drifted; recompute the 'pad' length so the " + "serialized event is 100 bytes." + ) + + w._put_event(exact_event) + # Accepted at the boundary (<=). + assert len(w._events) == 1 + assert w._accumulated_bytes == 100 + assert w._accumulator_dropped == 0 + + # Any further event (even 1 byte over the remaining budget) is + # dropped — remaining budget is 0. + w._put_event({"event": "X"}) + assert len(w._events) == 1 # unchanged + assert w._accumulator_dropped == 1 + + def test_accumulator_handles_non_serializable_gracefully(self, capsys): + w = _TrajectoryWriter("t-1", accumulate=True) + + class Unserializable: + def __repr__(self): + # Make repr clean so the default=str fallback produces + # deterministic output rather than a random address. + return "Unserializable()" + + # json.dumps with default=str will stringify most objects, but + # we still pin the fail-open branch in case a future refactor + # removes the fallback. + w._put_event({"event": "OK"}) + # Force a TypeError by sneaking in something json actually can't + # serialize even with default=str — use a circular dict. + bad: dict = {} + bad["self"] = bad + w._put_event(bad) + # First event still captured. + assert len(w._events) >= 1 + + +class TestUploadTraceToS3: + """``upload_trace_to_s3`` is the agent's S3 write path. It is + fail-open and enforces the Stage 3 review contract (empty user_id + -> skip, never write ``traces//...``).""" + + def test_skip_when_user_id_empty(self, capsys, monkeypatch): + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "bucket") + result = upload_trace_to_s3(task_id="t-1", user_id="", body=b"payload") + assert result is None + captured = capsys.readouterr() + assert "skip" in captured.out + assert "unreachable key" in captured.out + assert "t-1" in captured.out + + def test_skip_when_task_id_empty(self, capsys, monkeypatch): + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "bucket") + result = upload_trace_to_s3(task_id="", user_id="u-1", body=b"x") + assert result is None + assert "empty task_id" in capsys.readouterr().out + + def test_skip_when_bucket_env_unset(self, capsys, monkeypatch): + monkeypatch.delenv("TRACE_ARTIFACTS_BUCKET_NAME", raising=False) + result = upload_trace_to_s3(task_id="t-1", user_id="u-1", body=b"x") + assert result is None + assert "TRACE_ARTIFACTS_BUCKET_NAME unset" in capsys.readouterr().out + + def test_happy_path_returns_s3_uri(self, monkeypatch): + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "my-bucket") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_client = MagicMock() + with patch("boto3.client", return_value=mock_client): + result = upload_trace_to_s3(task_id="t-9", user_id="u-1", body=b"gz-payload") + + assert result == "s3://my-bucket/traces/u-1/t-9.jsonl.gz" + # ContentEncoding=gzip intentionally omitted — it triggers Node's + # fetch (undici) auto-decompression, which breaks the CLI's trace + # download paths. See telemetry.upload_trace_to_s3 comment. + mock_client.put_object.assert_called_once_with( + Bucket="my-bucket", + Key="traces/u-1/t-9.jsonl.gz", + Body=b"gz-payload", + ContentType="application/gzip", + ) + _, kwargs = mock_client.put_object.call_args + assert "ContentEncoding" not in kwargs + + def test_fail_open_on_s3_error(self, capsys, monkeypatch): + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "my-bucket") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + mock_client = MagicMock() + mock_client.put_object.side_effect = RuntimeError("boom") + with patch("boto3.client", return_value=mock_client): + result = upload_trace_to_s3(task_id="t-9", user_id="u-1", body=b"x") + + # Fail-open: returns None but does NOT raise. + assert result is None + captured = capsys.readouterr() + assert "S3 put_object failed" in captured.out + assert "boom" in captured.out + + def test_flags_iam_misconfiguration_in_error_path(self, capsys, monkeypatch): + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "my-bucket") + + mock_client = MagicMock() + mock_client.put_object.side_effect = PermissionError( + "AccessDenied: agent role lacks s3:PutObject", + ) + with patch("boto3.client", return_value=mock_client): + upload_trace_to_s3(task_id="t-9", user_id="u-1", body=b"x") + + captured = capsys.readouterr() + assert "IAM misconfiguration likely" in captured.out + + def test_object_key_uses_design_layout(self, monkeypatch): + # Pin the key layout from design §10.1: + # ``traces//.jsonl.gz`` + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "b") + mock_client = MagicMock() + with patch("boto3.client", return_value=mock_client): + upload_trace_to_s3(task_id="TASK-XYZ", user_id="sub-123", body=b"x") + + _, kwargs = mock_client.put_object.call_args + assert kwargs["Key"] == "traces/sub-123/TASK-XYZ.jsonl.gz" + + def test_region_env_unset_passes_none_to_boto_client(self, monkeypatch): + """Both ``AWS_REGION`` and ``AWS_DEFAULT_REGION`` unset — the + uploader must still proceed and delegate region resolution to + boto3's default credential/region provider chain by passing + ``region_name=None``.""" + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "bucket") + monkeypatch.delenv("AWS_REGION", raising=False) + monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) + + mock_client = MagicMock() + with patch("boto3.client", return_value=mock_client) as mock_factory: + result = upload_trace_to_s3(task_id="t-1", user_id="u-1", body=b"x") + + # Upload proceeded (did not short-circuit on missing region). + assert result == "s3://bucket/traces/u-1/t-1.jsonl.gz" + # boto3.client was invoked with region_name=None so boto's default + # chain (IMDS, config file, env var precedence) resolves it. + args, kwargs = mock_factory.call_args + assert args[0] == "s3" + assert kwargs.get("region_name") is None + + def test_empty_body_still_calls_put_object(self, monkeypatch): + """Empty ``body=b""`` must be passed to ``put_object`` without + short-circuiting. Boto3 accepts empty Body; we pin that behavior + so a future refactor can't silently skip zero-byte uploads.""" + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "my-bucket") + mock_client = MagicMock() + with patch("boto3.client", return_value=mock_client): + result = upload_trace_to_s3(task_id="t-empty", user_id="u-1", body=b"") + + assert result == "s3://my-bucket/traces/u-1/t-empty.jsonl.gz" + mock_client.put_object.assert_called_once() + _, kwargs = mock_client.put_object.call_args + assert kwargs["Body"] == b"" + assert kwargs["Key"] == "traces/u-1/t-empty.jsonl.gz" + + def test_none_body_raises_when_put_object_rejects(self, monkeypatch): + """``body=None`` is a contract violation from the caller's side. + + The current implementation passes ``None`` straight through to + ``put_object`` and relies on boto3's ``ParamValidationError`` + (or TypeError in the mocked case) to fail visibly. Pin whatever + the current behavior is — if L3 wants to harden this with an + early ``if body is None: skip``, it can. + + We use ``side_effect=TypeError`` to simulate boto3's rejection + of ``Body=None`` in a way that's deterministic without requiring + the real SDK. + """ + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "my-bucket") + mock_client = MagicMock() + mock_client.put_object.side_effect = TypeError( + "put_object() Body expected bytes-like, got NoneType" + ) + from typing import cast + + # ``cast`` launders ``None`` through the ``bytes``-typed parameter + # so the static type checker accepts the contract violation this + # test is deliberately exercising. + bad_body: bytes = cast("bytes", None) + with patch("boto3.client", return_value=mock_client): + # Fail-open on the generic except path — returns None, does + # not raise. This pins the CURRENT behavior: None body is + # swallowed as an upload failure rather than rejected up + # front. Flagged for L3 in the report. + result = upload_trace_to_s3(task_id="t-none", user_id="u-1", body=bad_body) + assert result is None + mock_client.put_object.assert_called_once() + _, kwargs = mock_client.put_object.call_args + assert kwargs["Body"] is None + + +class TestEndToEndAccumulatorRoundTrip: + """A small end-to-end test: accumulate events, dump, upload — + covers the shape the pipeline will actually produce.""" + + def test_accumulated_payload_is_uploadable(self, monkeypatch): + monkeypatch.setenv("TRACE_ARTIFACTS_BUCKET_NAME", "b") + + w = _TrajectoryWriter("task-rt", accumulate=True) + w._put_event({"event": "TRAJECTORY_TURN", "turn": 1, "text": "hi"}) + w._put_event({"event": "TRAJECTORY_RESULT", "num_turns": 1}) + + body = w.dump_gzipped_jsonl() + assert body is not None + # gzip roundtrip produces parseable JSONL + lines = gzip.decompress(body).decode("utf-8").splitlines() + assert len(lines) == 3 # header + 2 + for line in lines: + json.loads(line) # raises if invalid + + # Upload path accepts the dumped payload. + mock_client = MagicMock() + with patch("boto3.client", return_value=mock_client): + uri = upload_trace_to_s3(task_id="task-rt", user_id="u-1", body=body) + + assert uri == "s3://b/traces/u-1/task-rt.jsonl.gz" + put_kwargs = mock_client.put_object.call_args.kwargs + # Uploaded bytes exactly match what dump_gzipped_jsonl produced. + assert put_kwargs["Body"] == body + + +def _decompress_jsonl(body: bytes) -> list[dict]: + return [json.loads(line) for line in gzip.decompress(body).decode("utf-8").splitlines()] + + +class TestDumpEmptyVsNonEmpty: + """Regression pin: the accumulator must emit the header only when + there are events to describe. An empty accumulator -> ``None`` so + callers do not upload a zero-event artifact.""" + + def test_empty_accumulator_returns_none(self): + w = _TrajectoryWriter("t", accumulate=True) + assert w.dump_gzipped_jsonl() is None + + def test_single_event_produces_header_plus_event(self): + w = _TrajectoryWriter("t", accumulate=True) + w._put_event({"event": "ONLY"}) + body = w.dump_gzipped_jsonl() + assert body is not None + assert len(_decompress_jsonl(body)) == 2 + + +class TestWriterByteTracking: + """``_accumulated_bytes`` must track the uncompressed JSON size — + not the Python object size — because that's what the bound guards.""" + + def test_bytes_counter_advances_on_each_event(self): + w = _TrajectoryWriter("t", accumulate=True) + size0 = w._accumulated_bytes + w._put_event({"event": "X"}) + size1 = w._accumulated_bytes + w._put_event({"event": "Y"}) + size2 = w._accumulated_bytes + assert size0 < size1 < size2 + + def test_bytes_counter_matches_serialized_length(self): + w = _TrajectoryWriter("t", accumulate=True) + payload = {"event": "X", "content": "hello"} + w._put_event(payload) + expected = len(io.BytesIO(json.dumps(payload).encode("utf-8")).getvalue()) + assert w._accumulated_bytes == expected + + +class TestAccumulatorFlagsAreIndependent: + """The ``accumulate`` flag must not affect the existing CW-write + path for non-trace tasks. Explicitly assert that disabling the + accumulator leaves the writer at zero memory cost.""" + + def test_non_accumulating_writer_retains_no_state(self): + w = _TrajectoryWriter("t", accumulate=False) + # Even after many events, accumulator state stays at zero. + for i in range(100): + w._put_event({"event": "X", "i": i}) + assert w._events == [] + assert w._accumulated_bytes == 0 + assert w._accumulator_dropped == 0 + + +class TestAccumulatorWhenCloudWatchDisabled: + """K2 review Finding #9: accumulator must capture events even when + the CloudWatch path is disabled (no log group env, or circuit + breaker open). The S3 artifact is independent of CW health by + design.""" + + def test_captures_when_log_group_unset(self, monkeypatch): + monkeypatch.delenv("LOG_GROUP_NAME", raising=False) + w = _TrajectoryWriter("t", accumulate=True) + w._put_event({"event": "X"}) + assert len(w._events) == 1 + + def test_captures_when_circuit_breaker_open(self): + w = _TrajectoryWriter("t", accumulate=True) + w._disabled = True # simulate circuit breaker open + w._put_event({"event": "X"}) + assert len(w._events) == 1 + + +class TestTruncationCallback: + """K2 review Finding #3: accumulator cap trips fire a one-shot + callback so the pipeline can surface ``trace_truncated`` in + ``bgagent watch``.""" + + def test_callback_fires_on_first_drop_only(self): + w = _TrajectoryWriter("t", accumulate=True) + w._ACCUMULATOR_MAX_BYTES = 50 + calls: list[tuple[int, int]] = [] + w.set_truncation_callback(lambda maxb, dropped: calls.append((maxb, dropped))) + + # First N events fit; later ones trip the cap repeatedly. + for i in range(20): + w._put_event({"event": "X", "i": i}) + + # Fire-once: callback called exactly one time even though + # many events dropped. + assert len(calls) == 1 + assert calls[0][0] == 50 # max_bytes arg + assert calls[0][1] >= 1 # at least one drop at the moment of first announcement + + def test_callback_not_fired_when_cap_never_trips(self): + w = _TrajectoryWriter("t", accumulate=True) + calls: list[tuple[int, int]] = [] + w.set_truncation_callback(lambda maxb, dropped: calls.append((maxb, dropped))) + + for i in range(5): + w._put_event({"event": "X", "i": i}) + assert calls == [] + + def test_callback_errors_are_swallowed(self, capsys): + w = _TrajectoryWriter("t", accumulate=True) + w._ACCUMULATOR_MAX_BYTES = 50 + + def broken_cb(_maxb, _dropped): + raise RuntimeError("cb boom") + + w.set_truncation_callback(broken_cb) + # Should not raise even though the callback raises. + for i in range(20): + w._put_event({"event": "X", "i": i}) + assert w._accumulator_dropped > 0 + assert "truncation callback raised" in capsys.readouterr().out + + def test_accumulator_dropped_continues_past_announcement(self): + """Debounce semantics: the callback fires once, but + ``_accumulator_dropped`` must keep incrementing for every + subsequent rejected event so the header reports the true final + drop count (not just the count at the moment of announcement).""" + w = _TrajectoryWriter("t", accumulate=True) + w._ACCUMULATOR_MAX_BYTES = 50 + calls: list[tuple[int, int]] = [] + w.set_truncation_callback(lambda maxb, dropped: calls.append((maxb, dropped))) + + # Many events past the cap — force multiple drops. + for i in range(50): + w._put_event({"event": "X", "i": i}) + + # Fire-once: the callback was called exactly one time. + assert len(calls) == 1 + announced_drops = calls[0][1] + # Counter kept climbing after the one-shot announcement. + assert w._accumulator_dropped > announced_drops, ( + f"dropped counter stuck at announcement value " + f"{announced_drops}; final={w._accumulator_dropped}" + ) + + def test_callback_not_fired_when_accumulator_disabled(self): + w = _TrajectoryWriter("t", accumulate=False) + calls: list[tuple[int, int]] = [] + w.set_truncation_callback(lambda maxb, dropped: calls.append((maxb, dropped))) + # Even with huge event volume, non-accumulating writer skips + # the bookkeeping branch entirely. + for i in range(1000): + w._put_event({"event": "X", "i": i}) + assert calls == [] diff --git a/cdk/package.json b/cdk/package.json index 0feaf5b..d82018a 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -21,8 +21,10 @@ "@aws-sdk/client-ecs": "^3.1021.0", "@aws-sdk/client-dynamodb": "^3.1021.0", "@aws-sdk/client-lambda": "^3.1021.0", + "@aws-sdk/client-s3": "^3.1021.0", "@aws-sdk/client-secrets-manager": "^3.1021.0", "@aws-sdk/lib-dynamodb": "^3.1021.0", + "@aws-sdk/s3-request-presigner": "^3.1021.0", "@aws/durable-execution-sdk-js": "^1.1.0", "aws-cdk-lib": "^2.238.0", "cdk-nag": "^2.37.55", diff --git a/cdk/src/constructs/fanout-consumer.ts b/cdk/src/constructs/fanout-consumer.ts new file mode 100644 index 0000000..a81ede0 --- /dev/null +++ b/cdk/src/constructs/fanout-consumer.ts @@ -0,0 +1,168 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import * as path from 'path'; +import { Duration } from 'aws-cdk-lib'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import { StartingPosition, Architecture, Runtime } from 'aws-cdk-lib/aws-lambda'; +import { DynamoEventSource, SqsDlq } from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as lambda from 'aws-cdk-lib/aws-lambda-nodejs'; +import * as sm from 'aws-cdk-lib/aws-secretsmanager'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; +import { NagSuppressions } from 'cdk-nag'; +import { Construct } from 'constructs'; + +/** + * Properties for `FanOutConsumer` — the Phase 1b §8.9 fan-out plane + * consumer that reads `TaskEventsTable` via DynamoDB Streams and + * dispatches interesting events to non-interactive channels (Slack, + * GitHub PR comments, email). + */ +export interface FanOutConsumerProps { + /** The TaskEventsTable whose stream this consumer reads from. Must + * have `stream: NEW_IMAGE` enabled (see `TaskEventsTable`). */ + readonly taskEventsTable: dynamodb.ITable; + + /** + * TaskTable — the GitHub dispatcher needs read access to resolve + * repo + pr_number + existing github_comment_id for a task, and + * write access to persist the comment_id + etag after an upsert. + * Optional: if omitted, the GitHub dispatcher skips (log-only) and + * Slack / Email continue to run as stubs. + */ + readonly taskTable?: dynamodb.ITable; + + /** + * RepoTable — GitHub dispatcher reads per-repo + * `github_token_secret_arn` overrides. Optional: if omitted, falls + * back to the platform default secret. + */ + readonly repoTable?: dynamodb.ITable; + + /** + * Platform default GitHub token secret. Used by the GitHub + * dispatcher when the per-repo config has no override. Optional: if + * omitted and the repo has no override, the dispatcher skips. + */ + readonly githubTokenSecret?: sm.ISecret; + + /** + * Maximum batch size delivered to the Lambda per invocation. + * + * @default 100 (DynamoDB Stream default) + */ + readonly batchSize?: number; + + /** + * Max age of records in the batch before Lambda is invoked even if + * batch isn't full. Keeps fan-out latency bounded for low-volume + * periods. + * + * @default Duration.seconds(5) + */ + readonly maxBatchingWindow?: Duration; +} + +/** + * DynamoDB Stream → Lambda consumer that fans out task events to + * non-interactive channels. Ships as a skeleton per design §8.9 — + * per-channel dispatcher integrations land incrementally without any + * change to the agent or CLI. + * + * Errors in individual records do NOT fail the batch. Persistent + * failures land in the DLQ attached to the event source mapping so + * operators can replay. + */ +export class FanOutConsumer extends Construct { + public readonly fn: lambda.NodejsFunction; + public readonly dlq: sqs.Queue; + + constructor(scope: Construct, id: string, props: FanOutConsumerProps) { + super(scope, id); + + const handlersDir = path.join(__dirname, '..', 'handlers'); + + this.dlq = new sqs.Queue(this, 'FanOutDlq', { + // Persistent failures (e.g., dispatcher throws non-caught error + // five times in a row) land here for operator inspection. + retentionPeriod: Duration.days(14), + enforceSSL: true, + }); + + this.fn = new lambda.NodejsFunction(this, 'FanOutFn', { + entry: path.join(handlersDir, 'fanout-task-events.ts'), + handler: 'handler', + runtime: Runtime.NODEJS_24_X, + architecture: Architecture.ARM_64, + timeout: Duration.minutes(1), + memorySize: 256, + bundling: { + externalModules: ['@aws-sdk/*'], + }, + }); + + // GitHub dispatcher plumbing. Each grant/env var is guarded so the + // fan-out plane still deploys cleanly in a dev environment that + // hasn't onboarded the RepoTable or a platform GitHub token yet — + // the dispatcher will log-and-skip rather than crash. + if (props.taskTable) { + props.taskTable.grantReadWriteData(this.fn); + this.fn.addEnvironment('TASK_TABLE_NAME', props.taskTable.tableName); + } + if (props.repoTable) { + props.repoTable.grantReadData(this.fn); + this.fn.addEnvironment('REPO_TABLE_NAME', props.repoTable.tableName); + } + if (props.githubTokenSecret) { + props.githubTokenSecret.grantRead(this.fn); + this.fn.addEnvironment('GITHUB_TOKEN_SECRET_ARN', props.githubTokenSecret.secretArn); + } + + this.fn.addEventSource(new DynamoEventSource(props.taskEventsTable, { + startingPosition: StartingPosition.LATEST, + batchSize: props.batchSize ?? 100, + maxBatchingWindow: props.maxBatchingWindow ?? Duration.seconds(5), + // Fan-out delivery is best-effort; don't block the stream if one + // poisonous record blows up the Lambda. After 3 retries, send the + // record batch to the DLQ and advance the iterator. + retryAttempts: 3, + onFailure: new SqsDlq(this.dlq), + reportBatchItemFailures: true, + })); + + NagSuppressions.addResourceSuppressions(this.fn, [ + { + id: 'AwsSolutions-IAM4', + reason: 'AWSLambdaBasicExecutionRole is required for CloudWatch Logs access', + }, + { + id: 'AwsSolutions-IAM5', + reason: + 'DynamoDB stream/index wildcards generated by CDK for event-source-mapping read access', + }, + ], true); + NagSuppressions.addResourceSuppressions(this.dlq, [ + { + id: 'AwsSolutions-SQS3', + reason: + 'This queue IS the DLQ for the fan-out Lambda — having its own DLQ would be infinite recursion', + }, + ]); + } +} diff --git a/cdk/src/constructs/stranded-task-reconciler.ts b/cdk/src/constructs/stranded-task-reconciler.ts new file mode 100644 index 0000000..7c094ab --- /dev/null +++ b/cdk/src/constructs/stranded-task-reconciler.ts @@ -0,0 +1,134 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import * as path from 'path'; +import { Duration } from 'aws-cdk-lib'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as targets from 'aws-cdk-lib/aws-events-targets'; +import { Architecture, Runtime } from 'aws-cdk-lib/aws-lambda'; +import * as lambda from 'aws-cdk-lib/aws-lambda-nodejs'; +import { NagSuppressions } from 'cdk-nag'; +import { Construct } from 'constructs'; + +/** + * Properties for StrandedTaskReconciler construct. + */ +export interface StrandedTaskReconcilerProps { + /** TaskTable (has StatusIndex GSI used by the query). */ + readonly taskTable: dynamodb.ITable; + + /** TaskEventsTable (handler writes task_stranded + task_failed events). */ + readonly taskEventsTable: dynamodb.ITable; + + /** UserConcurrencyTable (handler decrements active_count on fail). */ + readonly userConcurrencyTable: dynamodb.ITable; + + /** + * How often to run the reconciler. Short enough to clear stranded + * tasks in a reasonable user-facing time, long enough to amortise the + * Lambda + DDB cost. Defaults to 5 minutes. + * + * @default Duration.minutes(5) + */ + readonly schedule?: Duration; + + /** + * Stranded-timeout (seconds). Tasks in SUBMITTED or HYDRATING older + * than this are transitioned to FAILED. Set via + * `STRANDED_TIMEOUT_SECONDS`. + * + * @default 1200 (20 minutes) + */ + readonly strandedTimeoutSeconds?: number; + + /** Forwarded to the handler for event TTL. @default 90 */ + readonly taskRetentionDays?: number; +} + +/** + * Scheduled Lambda that fails stranded tasks. + * + * A stranded task is one admitted into TaskTable (SUBMITTED or HYDRATING) + * whose pipeline never started — typically because the orchestrator + * Lambda crashed between admission and InvokeAgentRuntime, or because the + * agent container crashed during startup. Left alone these permanently + * consume a user's concurrency slot. + * + * RUNNING / FINALIZING tasks are handled separately by `pollTaskStatus` + * in `orchestrator.ts` via the `agent_heartbeat_at` timeout. + */ +export class StrandedTaskReconciler extends Construct { + public readonly fn: lambda.NodejsFunction; + + constructor(scope: Construct, id: string, props: StrandedTaskReconcilerProps) { + super(scope, id); + + const handlersDir = path.join(__dirname, '..', 'handlers'); + + const strandedTimeout = props.strandedTimeoutSeconds ?? 1200; + const retentionDays = props.taskRetentionDays ?? 90; + + this.fn = new lambda.NodejsFunction(this, 'ReconcilerFn', { + entry: path.join(handlersDir, 'reconcile-stranded-tasks.ts'), + handler: 'handler', + runtime: Runtime.NODEJS_24_X, + architecture: Architecture.ARM_64, + timeout: Duration.minutes(5), + memorySize: 256, + environment: { + TASK_TABLE_NAME: props.taskTable.tableName, + TASK_EVENTS_TABLE_NAME: props.taskEventsTable.tableName, + USER_CONCURRENCY_TABLE_NAME: props.userConcurrencyTable.tableName, + STRANDED_TIMEOUT_SECONDS: String(strandedTimeout), + TASK_RETENTION_DAYS: String(retentionDays), + }, + bundling: { + externalModules: ['@aws-sdk/*'], + }, + }); + + // TaskTable: read (query by StatusIndex) + conditional UpdateItem to + // transition stranded rows to FAILED. + props.taskTable.grantReadWriteData(this.fn); + // TaskEvents: write task_stranded + task_failed events. + props.taskEventsTable.grantWriteData(this.fn); + // Concurrency: decrement active_count on fail. + props.userConcurrencyTable.grantReadWriteData(this.fn); + + const schedule = props.schedule ?? Duration.minutes(5); + const rule = new events.Rule(this, 'ReconcilerSchedule', { + schedule: events.Schedule.rate(schedule), + }); + rule.addTarget(new targets.LambdaFunction(this.fn)); + + NagSuppressions.addResourceSuppressions(this.fn, [ + { + id: 'AwsSolutions-IAM4', + reason: 'AWSLambdaBasicExecutionRole is required for CloudWatch Logs access', + }, + { + id: 'AwsSolutions-IAM5', + reason: + 'DynamoDB index/* wildcards generated by CDK grantReadWriteData for ' + + 'StatusIndex query access + Item update path', + }, + ], true); + } +} diff --git a/cdk/src/constructs/task-api.ts b/cdk/src/constructs/task-api.ts index a69b02a..2309fe0 100644 --- a/cdk/src/constructs/task-api.ts +++ b/cdk/src/constructs/task-api.ts @@ -26,6 +26,7 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import { Runtime, Architecture } from 'aws-cdk-lib/aws-lambda'; import * as lambda from 'aws-cdk-lib/aws-lambda-nodejs'; import * as logs from 'aws-cdk-lib/aws-logs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; import * as wafv2 from 'aws-cdk-lib/aws-wafv2'; import { NagSuppressions } from 'cdk-nag'; import { Construct } from 'constructs'; @@ -44,6 +45,18 @@ export interface TaskApiProps { */ readonly taskEventsTable: dynamodb.ITable; + /** + * The DynamoDB task nudges table (Phase 2). When provided, the + * `POST /tasks/{task_id}/nudge` endpoint is created. + */ + readonly taskNudgesTable?: dynamodb.ITable; + + /** + * Per-task per-minute nudge rate limit. + * @default 10 + */ + readonly nudgeRateLimitPerMinute?: number; + /** * The DynamoDB repo config table. When provided, task creation checks * that the target repository is onboarded before accepting the task. @@ -96,10 +109,18 @@ export interface TaskApiProps { readonly webhookRetentionDays?: number; /** - * AgentCore runtime ARNs for which cancel-task may call `StopRuntimeSession`. - * First ARN is also passed as `RUNTIME_ARN` when the task record has no `agent_runtime_arn`. + * AgentCore runtime ARN for which cancel-task may call `StopRuntimeSession`. + * Also passed as `RUNTIME_ARN` to cancel-task so it can resolve the target + * runtime when a task record lacks `agent_runtime_arn`. */ - readonly agentCoreStopSessionRuntimeArns?: string[]; + readonly agentCoreStopSessionRuntimeArn?: string; + + /** + * S3 bucket storing ``--trace`` trajectory artifacts. When provided, + * a ``GET /v1/tasks/{task_id}/trace`` route is created that issues + * short-lived presigned download URLs (design §10.1). + */ + readonly traceArtifactsBucket?: s3.IBucket; /** * ECS cluster ARN for cancel-task to stop ECS-backed tasks. @@ -134,6 +155,11 @@ export class TaskApi extends Construct { */ public readonly userPool: cognito.UserPool; + /** + * The Cognito User Pool App Client. + */ + public readonly appClient: cognito.UserPoolClient; + /** * The Cognito User Pool App Client ID. */ @@ -159,14 +185,14 @@ export class TaskApi extends Construct { removalPolicy, }); - const appClient = this.userPool.addClient('AppClient', { + this.appClient = this.userPool.addClient('AppClient', { authFlows: { userPassword: true, userSrp: true, }, generateSecret: false, }); - this.appClientId = appClient.userPoolClientId; + this.appClientId = this.appClient.userPoolClientId; // Suppress Cognito rules not applicable for dev environment NagSuppressions.addResourceSuppressions(this.userPool, [ @@ -286,8 +312,24 @@ export class TaskApi extends Construct { TASK_EVENTS_TABLE_NAME: props.taskEventsTable.tableName, TASK_RETENTION_DAYS: String(props.taskRetentionDays ?? 90), }; + // The Node.js Lambda runtime ships an AWS SDK, but its pinned version + // lags current. `@aws-sdk/client-bedrock-agentcore` in particular has + // shipped new commands (e.g. StopRuntimeSessionCommand) that are not in + // the runtime's bundled SDK, so externalizing it causes Lambdas to throw + // ` is not a constructor` at runtime — a silent failure mode + // because catch blocks swallow the error and log a best-effort warning. + // Bundle bedrock-agentcore explicitly; keep stable clients external to + // keep Lambda sizes small. const commonBundling: lambda.BundlingOptions = { - externalModules: ['@aws-sdk/*'], + externalModules: [ + '@aws-sdk/client-dynamodb', + '@aws-sdk/client-ecs', + '@aws-sdk/client-lambda', + '@aws-sdk/client-bedrock-runtime', + '@aws-sdk/client-secrets-manager', + '@aws-sdk/lib-dynamodb', + '@aws-sdk/util-dynamodb', + ], }; // --- Lambda handlers --- @@ -331,9 +373,9 @@ export class TaskApi extends Construct { }); const cancelTaskEnv: Record = { ...commonEnv }; - const stopSessionArns = props.agentCoreStopSessionRuntimeArns ?? []; - if (stopSessionArns.length > 0) { - cancelTaskEnv.RUNTIME_ARN = stopSessionArns[0]!; + const stopSessionArn = props.agentCoreStopSessionRuntimeArn; + if (stopSessionArn) { + cancelTaskEnv.RUNTIME_ARN = stopSessionArn; } if (props.ecsClusterArn) { cancelTaskEnv.ECS_CLUSTER_ARN = props.ecsClusterArn; @@ -346,6 +388,12 @@ export class TaskApi extends Construct { architecture: Architecture.ARM_64, environment: cancelTaskEnv, bundling: commonBundling, + // Cancel performs: DDB GetItem + DDB UpdateItem + ECS StopTask or + // AgentCore StopRuntimeSession + DDB PutItem. The default 3s timeout + // is not enough once cold-start TLS handshakes for bedrock-agentcore + // are added. 15s gives comfortable headroom. + timeout: Duration.seconds(15), + memorySize: 256, }); const getTaskEventsFn = new lambda.NodejsFunction(this, 'GetTaskEventsFn', { @@ -364,11 +412,10 @@ export class TaskApi extends Construct { props.taskTable.grantReadWriteData(cancelTaskFn); props.taskEventsTable.grantReadWriteData(cancelTaskFn); - if (stopSessionArns.length > 0) { - const runtimeResources = stopSessionArns.flatMap(arn => [arn, `${arn}/*`]); + if (stopSessionArn) { cancelTaskFn.addToRolePolicy(new iam.PolicyStatement({ actions: ['bedrock-agentcore:StopRuntimeSession'], - resources: runtimeResources, + resources: [stopSessionArn, `${stopSessionArn}/*`], })); } @@ -432,6 +479,101 @@ export class TaskApi extends Construct { const events = taskById.addResource('events'); events.addMethod('GET', new apigw.LambdaIntegration(getTaskEventsFn), cognitoAuthOptions); + // --- Trace URL endpoint (design §10.1): GET /tasks/{task_id}/trace --- + if (props.traceArtifactsBucket) { + const traceBucket = props.traceArtifactsBucket; + const getTraceUrlFn = new lambda.NodejsFunction(this, 'GetTraceUrlFn', { + entry: path.join(handlersDir, 'get-trace-url.ts'), + handler: 'handler', + runtime: Runtime.NODEJS_24_X, + architecture: Architecture.ARM_64, + environment: { + ...commonEnv, + TRACE_ARTIFACTS_BUCKET_NAME: traceBucket.bucketName, + }, + bundling: { + ...commonBundling, + // Defensive future-proofing: if ``@aws-sdk/client-s3`` or + // ``@aws-sdk/s3-request-presigner`` are ever added to + // ``commonBundling.externalModules`` (e.g. because a future + // Node runtime ships them), this filter ensures they stay + // bundled for *this* function — the Node 24 Lambda runtime + // does not ship either, and ``getSignedUrl`` will throw + // ``Cannot find module`` at cold start if it's externalized. + // Today this is a no-op (neither module is in the common + // external list); the filter exists to guard against drift. + externalModules: commonBundling.externalModules?.filter( + m => m !== '@aws-sdk/client-s3' && m !== '@aws-sdk/s3-request-presigner', + ), + }, + // Cold-start SDK load (s3-client + s3-request-presigner + lib-dynamodb) + // exceeds Lambda's 3s default, causing INIT timeout → 502 Bad Gateway. + timeout: Duration.seconds(15), + memorySize: 512, + }); + + props.taskTable.grantReadData(getTraceUrlFn); + // Minimal grant — the handler only needs ``s3:GetObject`` (which + // implicitly covers ``s3:HeadObject``) on trace objects to sign + // presigned URLs and HEAD-check for existence before presigning. + // ``grantRead`` would expand to ``s3:GetObject*`` + ``s3:GetBucket*`` + // + ``s3:List*``; ``ListBucket`` / ``GetBucketLocation`` / etc. are + // unnecessary scope. Tightening to an explicit statement (L3 item 2). + getTraceUrlFn.addToRolePolicy(new iam.PolicyStatement({ + actions: ['s3:GetObject'], + resources: [`${traceBucket.bucketArn}/*`], + })); + + const trace = taskById.addResource('trace'); + trace.addMethod('GET', new apigw.LambdaIntegration(getTraceUrlFn), cognitoAuthOptions); + + allFunctions.push(getTraceUrlFn); + } + + // --- Nudge endpoint (Phase 2): POST /tasks/{task_id}/nudge --- + if (props.taskNudgesTable) { + const nudgeTaskEnv: Record = { + ...commonEnv, + NUDGES_TABLE_NAME: props.taskNudgesTable.tableName, + NUDGE_RATE_LIMIT_PER_MINUTE: String(props.nudgeRateLimitPerMinute ?? 10), + }; + if (props.guardrailId && props.guardrailVersion) { + nudgeTaskEnv.GUARDRAIL_ID = props.guardrailId; + nudgeTaskEnv.GUARDRAIL_VERSION = props.guardrailVersion; + } + + const nudgeTaskFn = new lambda.NodejsFunction(this, 'NudgeTaskFn', { + entry: path.join(handlersDir, 'nudge-task.ts'), + handler: 'handler', + runtime: Runtime.NODEJS_24_X, + architecture: Architecture.ARM_64, + environment: nudgeTaskEnv, + bundling: commonBundling, + }); + + // Read tasks (ownership + state), read/write nudges (persist + rate-limit counter). + props.taskTable.grantReadData(nudgeTaskFn); + props.taskNudgesTable.grantReadWriteData(nudgeTaskFn); + + if (props.guardrailId) { + nudgeTaskFn.addToRolePolicy(new iam.PolicyStatement({ + actions: ['bedrock:ApplyGuardrail'], + resources: [ + Stack.of(this).formatArn({ + service: 'bedrock', + resource: 'guardrail', + resourceName: props.guardrailId, + }), + ], + })); + } + + const nudge = taskById.addResource('nudge'); + nudge.addMethod('POST', new apigw.LambdaIntegration(nudgeTaskFn), cognitoAuthOptions); + + allFunctions.push(nudgeTaskFn); + } + // --- Webhook endpoints (only when webhookTable is provided) --- if (props.webhookTable) { const webhookEnv: Record = { diff --git a/cdk/src/constructs/task-events-table.ts b/cdk/src/constructs/task-events-table.ts index 61cb209..cd41574 100644 --- a/cdk/src/constructs/task-events-table.ts +++ b/cdk/src/constructs/task-events-table.ts @@ -76,6 +76,11 @@ export class TaskEventsTable extends Construct { pointInTimeRecoverySpecification: { pointInTimeRecoveryEnabled: props.pointInTimeRecovery ?? true, }, + // DynamoDB Streams feed the Phase 1b fan-out plane (Slack, GitHub comments, + // email) via a Lambda subscriber. NEW_IMAGE is sufficient — consumers only + // need the event payload, not old/new diffs. Enabling Streams on an existing + // table is an in-place CloudFormation update (no table replacement). + stream: dynamodb.StreamViewType.NEW_IMAGE, removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, }); } diff --git a/cdk/src/constructs/task-nudges-table.ts b/cdk/src/constructs/task-nudges-table.ts new file mode 100644 index 0000000..c0f8490 --- /dev/null +++ b/cdk/src/constructs/task-nudges-table.ts @@ -0,0 +1,93 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { RemovalPolicy } from 'aws-cdk-lib'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import { Construct } from 'constructs'; + +/** + * Properties for TaskNudgesTable construct. + */ +export interface TaskNudgesTableProps { + /** + * Optional table name override. + * @default - auto-generated by CloudFormation + */ + readonly tableName?: string; + + /** + * Removal policy for the table. + * @default RemovalPolicy.DESTROY + */ + readonly removalPolicy?: RemovalPolicy; + + /** + * Whether to enable point-in-time recovery. + * @default true + */ + readonly pointInTimeRecovery?: boolean; +} + +/** + * DynamoDB table for Phase 2 nudge messages. + * + * Schema: task_id (PK) + nudge_id (SK, ULID). + * + * Nudges are short steering messages from a user to a running agent, + * submitted via `POST /v1/tasks/{task_id}/nudge`. They are persisted + * here after guardrail screening and per-task rate limiting, then + * consumed by the agent between turns (polling Query by task_id with + * `consumed = false`). Unlike TaskEventsTable, this table has NO + * DynamoDB stream — nudges are poll-consumed by the agent, not + * fanned out to external channels. + * + * Rate-limit rows share the table with synthetic keys + * (`task_id = 'RATE#'`, `nudge_id = 'MINUTE#'`) + * and carry a short TTL (~120s) so they are cleaned up automatically. + */ +export class TaskNudgesTable extends Construct { + /** + * The underlying DynamoDB table. Use this to grant access or read the table name. + */ + public readonly table: dynamodb.Table; + + constructor(scope: Construct, id: string, props: TaskNudgesTableProps = {}) { + super(scope, id); + + this.table = new dynamodb.Table(this, 'Table', { + tableName: props.tableName, + partitionKey: { + name: 'task_id', + type: dynamodb.AttributeType.STRING, + }, + sortKey: { + name: 'nudge_id', + type: dynamodb.AttributeType.STRING, + }, + billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, + timeToLiveAttribute: 'ttl', + pointInTimeRecoverySpecification: { + pointInTimeRecoveryEnabled: props.pointInTimeRecovery ?? true, + }, + // No DynamoDB stream — nudges are poll-consumed by the agent runtime, + // not fanned out to external channels like TaskEventsTable. + removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, + }); + } +} diff --git a/cdk/src/constructs/task-orchestrator.ts b/cdk/src/constructs/task-orchestrator.ts index 7ebf432..ceb8273 100644 --- a/cdk/src/constructs/task-orchestrator.ts +++ b/cdk/src/constructs/task-orchestrator.ts @@ -60,7 +60,15 @@ export interface TaskOrchestratorProps { /** * Maximum concurrent tasks per user. - * @default 3 + * + * Raised from 3 to 10 in rev 5 to accommodate power-user CLI flows + * (developer running `bgagent run` a few times while iterating on a + * feature, reviewing queued PRs, etc.). 3 was a conservative starter + * that led to surprise admission rejections in practice. The + * stranded-task reconciler (scheduled handler) prevents abandoned + * tasks from permanently consuming slots. + * + * @default 10 */ readonly maxConcurrentTasksPerUser?: number; @@ -160,7 +168,7 @@ export class TaskOrchestrator extends Construct { } const handlersDir = path.join(__dirname, '..', 'handlers'); - const maxConcurrent = props.maxConcurrentTasksPerUser ?? 3; + const maxConcurrent = props.maxConcurrentTasksPerUser ?? 10; this.fn = new lambda.NodejsFunction(this, 'OrchestratorFn', { entry: path.join(handlersDir, 'orchestrate-task.ts'), @@ -197,7 +205,19 @@ export class TaskOrchestrator extends Construct { }), }, bundling: { - externalModules: ['@aws-sdk/*'], + // Bundle `@aws-sdk/client-bedrock-agentcore` — newer commands (e.g. + // StopRuntimeSessionCommand) are not in the Lambda runtime's pinned + // SDK and throw ` is not a constructor` if externalized. + // See cancel-task silent-failure mode (task-api.ts commonBundling). + externalModules: [ + '@aws-sdk/client-dynamodb', + '@aws-sdk/client-ecs', + '@aws-sdk/client-lambda', + '@aws-sdk/client-bedrock-runtime', + '@aws-sdk/client-secrets-manager', + '@aws-sdk/lib-dynamodb', + '@aws-sdk/util-dynamodb', + ], }, }); diff --git a/cdk/src/constructs/trace-artifacts-bucket.ts b/cdk/src/constructs/trace-artifacts-bucket.ts new file mode 100644 index 0000000..cb61c65 --- /dev/null +++ b/cdk/src/constructs/trace-artifacts-bucket.ts @@ -0,0 +1,106 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { Duration, RemovalPolicy } from 'aws-cdk-lib'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import { Construct } from 'constructs'; + +/** Lifecycle expiry for trace artifacts (design §10.1). */ +export const TRACE_ARTIFACT_TTL_DAYS = 7; + +/** + * Object-key prefix used for all trace artifacts. Key layout: + * ``traces//.jsonl.gz`` (design §10.1). The per-user + * prefix is load-bearing — the ``get-trace-url`` handler relies on the + * caller's Cognito ``sub`` matching the TaskRecord's ``user_id`` to + * authorize a presigned read, so the agent MUST write under its own + * user prefix and never another user's. + */ +export const TRACE_OBJECT_KEY_PREFIX = 'traces/'; + +/** + * Properties for TraceArtifactsBucket construct. + */ +export interface TraceArtifactsBucketProps { + /** + * Removal policy for the bucket. + * @default RemovalPolicy.DESTROY + */ + readonly removalPolicy?: RemovalPolicy; + + /** + * Whether to auto-delete objects when the bucket is removed. Mirrors + * the developer-ergonomic goal of ``TaskTable`` / ``TaskEventsTable`` + * (no hand-cleanup required before ``cdk destroy``). Unlike DynamoDB + * tables — which auto-empty on table delete — S3 requires a custom + * resource to clear out the bucket, so enabling this flag deploys + * CDK's ``Custom::S3AutoDeleteObjects`` Lambda with delete permissions + * on this bucket (wider IAM surface than ``grantPut(runtime)`` alone + * suggests; the auto-delete Lambda role is active, not just during + * destroy). + * @default true + */ + readonly autoDeleteObjects?: boolean; +} + +/** + * S3 bucket for ``--trace`` trajectory artifacts (design §10.1). + * + * On terminal state, agents submitted with ``--trace`` upload a gzipped + * JSONL dump of the full trajectory (SDK message log + tool I/O + hook + * callbacks) to ``s3:///traces//.jsonl.gz``. + * The CLI retrieves it via a presigned URL issued by the + * ``get-trace-url`` handler. + * + * Security / hygiene: + * - ``blockPublicAccess: BLOCK_ALL`` + ``enforceSSL: true`` — + * no public read, TLS-only transport. + * - ``encryption: S3_MANAGED`` — server-side encryption at rest. + * - 7-day lifecycle expiry per §10.1 (debug captures are not an + * archival concern; tight TTL keeps storage cost bounded and caps + * the blast radius of an accidental permission leak). + */ +export class TraceArtifactsBucket extends Construct { + /** The underlying S3 bucket. */ + public readonly bucket: s3.Bucket; + + constructor(scope: Construct, id: string, props: TraceArtifactsBucketProps = {}) { + super(scope, id); + + this.bucket = new s3.Bucket(this, 'Bucket', { + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + encryption: s3.BucketEncryption.S3_MANAGED, + enforceSSL: true, + lifecycleRules: [ + { + id: 'trace-artifacts-ttl', + enabled: true, + expiration: Duration.days(TRACE_ARTIFACT_TTL_DAYS), + // Reap incomplete multipart uploads after 1 day. Object + // expiration does not apply to in-flight MPUs (they are not + // objects yet), so a separate reaper is needed to keep stale + // upload parts from lingering and accruing storage cost. + abortIncompleteMultipartUploadAfter: Duration.days(1), + }, + ], + removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, + autoDeleteObjects: props.autoDeleteObjects ?? true, + }); + } +} diff --git a/cdk/src/handlers/cancel-task.ts b/cdk/src/handlers/cancel-task.ts index 7da0812..72b3864 100644 --- a/cdk/src/handlers/cancel-task.ts +++ b/cdk/src/handlers/cancel-task.ts @@ -81,6 +81,10 @@ export async function handler(event: APIGatewayProxyEvent): Promise> = { + // Slack is the "on-call" channel per §6.2 — all terminal outcomes + // (including cancellations and strands) plus agent_error and the + // Phase 2/3 interactive signals. + slack: new Set([ + ...TERMINAL_EVENT_TYPES, + 'pr_created', + 'agent_error', + 'approval_required', // Phase 3 (not yet emitted) + 'status_response', // Phase 2 (not yet emitted) + ]), + // Email is deliberately minimal per §6.2: only task_completed, + // task_failed, and approval_required. Cancellations and strands are + // intentionally NOT delivered — the user already knows they cancelled + // the task, and strands are an operator signal. Keep these in sync + // with the design doc's per-channel defaults table. + email: new Set([ + 'task_completed', + 'task_failed', + 'approval_required', // Phase 3 (not yet emitted) + ]), + // GitHub edits a single issue comment in place (§6.4) covering + // pr_created + terminal — including cancellations and strands so + // the comment reflects the task's final outcome. + github: new Set([ + ...TERMINAL_EVENT_TYPES, + 'pr_created', + ]), +}; + +/** + * Resolve the effective event-type filter for a channel. + * + * For v1 this is always the channel's default set — per-task + * overrides (design §6.5 `TaskRecord.notifications`) are forward- + * compatible plumbing: when Chunk K adds a DDB read, a caller can + * pass `overrides` and enable/disable the channel or override its + * event list. Today the value is always `undefined`, so every task + * inherits the defaults. + * + * Resolution rules: + * - ``{ enabled: false }`` → empty set (channel opted out). + * - ``events`` absent → channel default. + * - ``events: []`` → empty set (treated as opt-out with + * a WARN, since an empty explicit list + * is almost always a submission mistake — + * we surface it rather than silently mute). + * - ``events: ["default", …]`` → ``"default"`` expands to the + * channel default, other entries are + * added on top. + * - ``events: [only literals]`` → the explicit list REPLACES the + * default entirely. + */ +export function resolveChannelFilter( + channel: NotificationChannel, + overrides?: TaskNotificationsConfig, +): ReadonlySet { + const channelOverride = overrides?.[channel]; + if (channelOverride?.enabled === false) return new Set(); + if (!channelOverride?.events) return CHANNEL_DEFAULTS[channel]; + if (channelOverride.events.length === 0) { + // An empty explicit list silently muting a channel would be a + // footgun once Chunk K exposes this at the submit-time API. Log + // a WARN so operators see the mute; downstream validation should + // catch this at submission, but defense-in-depth matters here + // because the DDB path is cheap to bypass. + logger.warn('[fanout] channel override has empty events list — muting channel', { + event: 'fanout.resolve.empty_events_override', + channel, + }); + return new Set(); + } + const expanded = new Set(); + for (const e of channelOverride.events) { + if (e === 'default') { + for (const d of CHANNEL_DEFAULTS[channel]) expanded.add(d); + } else { + expanded.add(e); + } + } + return expanded; +} + +/** Stable channel iteration order, derived from ``CHANNEL_DEFAULTS``'s + * insertion order so adding a fourth channel (append to + * ``NotificationChannel`` + ``CHANNEL_DEFAULTS`` + ``DISPATCHERS``) + * does not require a matching edit here. */ +const CHANNELS = Object.keys(CHANNEL_DEFAULTS) as readonly NotificationChannel[]; + +/** Union of every channel's currently-subscribed events. Used as the + * outer guard: events no channel cares about short-circuit before we + * spin up dispatchers, keeping the stream-processor narrow. */ +function unionSubscribedTypes(overrides?: TaskNotificationsConfig): ReadonlySet { + const u = new Set(); + for (const ch of CHANNELS) { + for (const t of resolveChannelFilter(ch, overrides)) u.add(t); + } + return u; +} + +/** Tight-loop suppression to bound spam per task for chatty agents. The + * hard cap is per Lambda invocation (not global) so a pathological + * agent can at worst emit `MAX_EVENTS_PER_TASK_PER_INVOCATION` events + * to each channel per stream poll (~1 s). A future follow-up can + * promote this to a DDB-backed rate limiter if needed. */ +const MAX_EVENTS_PER_TASK_PER_INVOCATION = 20; + +export interface FanOutEvent { + readonly task_id: string; + readonly event_id: string; + readonly event_type: string; + readonly timestamp: string; + readonly metadata?: Record; +} + +/** + * Flatten a DynamoDB Stream NEW_IMAGE record to a plain `FanOutEvent`. + * Returns `null` for records we can't parse (deletes, garbage, test + * harness events) — let them fall out rather than crash the batch. + */ +export function parseStreamRecord(record: DynamoDBRecord): FanOutEvent | null { + if (record.eventName !== 'INSERT' && record.eventName !== 'MODIFY') return null; + const img = record.dynamodb?.NewImage; + if (!img) return null; + + const task_id = img.task_id?.S; + const event_id = img.event_id?.S; + const event_type = img.event_type?.S; + const timestamp = img.timestamp?.S; + if (!task_id || !event_id || !event_type || !timestamp) return null; + + let metadata: Record | undefined; + const metaImg = img.metadata; + if (metaImg?.M) { + metadata = {}; + for (const [k, v] of Object.entries(metaImg.M)) { + if (v.S !== undefined) metadata[k] = v.S; + else if (v.N !== undefined) metadata[k] = Number(v.N); + else if (v.BOOL !== undefined) metadata[k] = v.BOOL; + else if (v.NULL !== undefined) metadata[k] = null; + } + } + + return { task_id, event_id, event_type, timestamp, metadata }; +} + +/** + * Allowlist of ``agent_milestone`` names that are eligible to be + * unwrapped into their effective routing type. Keeping this narrow is + * a **structural** defense against naming drift: a future refactor + * that accidentally renames an unrelated milestone (e.g. + * ``task_cancelled_acknowledged`` → ``task_cancelled``) must not + * silently start fanning out as a terminal. If a new milestone should + * reach channels, add it here AND to the relevant channel default. + * + * The milestones the agent emits today (see + * ``agent/src/progress_writer.py``, ``agent/src/pipeline.py``, and + * ``agent/src/hooks.py``) are: ``pr_created``, ``nudge_acknowledged``, + * ``repo_setup_complete``, ``agent_execution_complete``, + * ``task_cancelled_acknowledged``, ``cancel_detected``, + * ``trajectory_uploaded``, ``trace_truncated``. Only ``pr_created`` + * is currently in any channel's default filter (§6.2 Slack + GitHub). + */ +const ROUTABLE_MILESTONES: ReadonlySet = new Set(['pr_created']); + +/** + * Unwrap ``agent_milestone`` events to their milestone name for + * routing and rendering purposes. + * + * The agent writes named checkpoints (``pr_created``, + * ``nudge_acknowledged``, ``repo_setup_complete``, …) as a single + * ``agent_milestone`` event with ``metadata.milestone`` carrying the + * name — see ``agent/src/progress_writer.py::write_agent_milestone`` + * and the design doc §4.2 event-types table. The watch CLI already + * reads ``metadata.milestone`` when rendering those events. + * + * The fan-out filters are expressed against **effective** event types + * (e.g. ``pr_created``, design §6.2 GitHub default set), so the + * router must unwrap before matching — otherwise every milestone + * routes as the string ``agent_milestone`` and gets dropped. + * + * Unwrap is restricted to ``ROUTABLE_MILESTONES`` so a future + * milestone whose name happens to collide with a terminal / error + * event type cannot silently fan out. Non-milestone events, bare + * ``agent_milestone`` events without a well-formed milestone name, + * and milestones outside the allowlist all keep their original + * routing (i.e. match on the wrapper ``agent_milestone``). + */ +export function effectiveEventType(event: FanOutEvent): string { + if (event.event_type !== 'agent_milestone') return event.event_type; + const milestone = event.metadata?.milestone; + if (typeof milestone !== 'string' || milestone.length === 0) return event.event_type; + if (!ROUTABLE_MILESTONES.has(milestone)) return event.event_type; + return milestone; +} + +/** True if any subscribed channel wants this event. Used as the outer + * guard so events nobody cares about short-circuit before we spin + * dispatchers. Matches on the unwrapped effective event type so + * ``agent_milestone`` carriers route by their milestone name. */ +export function shouldFanOut(event: FanOutEvent, overrides?: TaskNotificationsConfig): boolean { + return unionSubscribedTypes(overrides).has(effectiveEventType(event)); +} + +/** + * Per-channel dispatcher stubs. Each currently just logs what it + * WOULD have sent. Replace the body when a real integration lands — + * the interface stays the same. + * + * Dispatchers do NOT catch their own errors. Error isolation lives in + * ``routeEvent`` where ``Promise.allSettled`` records per-channel + * outcomes and a single ``fanout.dispatcher.rejected`` warn fires on + * rejection. Keeping one error sink ensures batch telemetry + * (`dispatched` count) reflects reality: a channel whose dispatcher + * threw is NOT counted as dispatched. + */ +async function dispatchToSlack(event: FanOutEvent): Promise { + logger.info('[fanout/slack] would dispatch', { + event: 'fanout.slack.dispatch_stub', + task_id: event.task_id, + event_id: event.event_id, + event_type: event.event_type, + effective_event_type: effectiveEventType(event), + }); +} + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); + +/** + * Load the TaskRecord fields the GitHub dispatcher needs. Returns + * ``null`` if the task vanished (race with TTL cleanup) or if the + * TaskTable env var is missing in a broken deployment — the dispatcher + * logs and skips instead of failing the batch. + */ +async function loadTaskForComment(taskId: string): Promise { + const tableName = process.env.TASK_TABLE_NAME; + if (!tableName) { + logger.warn('[fanout/github] TASK_TABLE_NAME not set — cannot dispatch', { + event: 'fanout.github.missing_env', + }); + return null; + } + const result = await ddb.send(new GetCommand({ + TableName: tableName, + Key: { task_id: taskId }, + })); + return (result.Item as TaskRecord | undefined) ?? null; +} + +/** + * Persist the ``github_comment_id`` on the TaskRecord after a + * successful POST (either the first-ever dispatch or a 404 re-POST + * fallback). Subsequent PATCHes are no-ops on the TaskRecord because + * there is no additional state to carry — per-comment concurrency + * relies on DDB Stream ordering, not on a stored ETag. + * + * The ConditionExpression guards two races: + * 1. ``attribute_exists(task_id)`` — a concurrent TTL eviction would + * otherwise create a zombie record with only this field. + * 2. Comment-id overwrite guard — the write is only allowed if (a) + * no comment has ever been persisted for this task, or (b) the + * stored id matches the one the caller thought was there. Without + * this clause, a 404 → POST fallback racing a concurrent fanout + * invocation could overwrite a sibling's freshly-posted comment id + * with our own new id, silently orphaning the sibling's comment. + * Under the normal single-writer flow the guard is a no-op. + * + * The caller (``dispatchToGitHubComment``) decides how to react to + * each failure mode: ConditionalCheckFailedException (task evicted or + * sibling-writer won the race) is benign; any other error is a real + * persistence bug that risks a duplicate comment on the next event + * (logged at ERROR with a dedicated ``FANOUT_GITHUB_PERSIST_FAILED`` + * error_id so operators can alarm). + */ +async function saveCommentState( + taskId: string, + commentId: number, + previousCommentId: number | undefined, +): Promise { + const tableName = process.env.TASK_TABLE_NAME; + if (!tableName) return; + const base = { + TableName: tableName, + Key: { task_id: taskId }, + UpdateExpression: 'SET github_comment_id = :cid', + }; + if (previousCommentId === undefined) { + // First-ever POST: require the field to be absent so a sibling + // invocation that beat us cannot be silently overwritten. + await ddb.send(new UpdateCommand({ + ...base, + ExpressionAttributeValues: { ':cid': commentId }, + ConditionExpression: 'attribute_exists(task_id) AND attribute_not_exists(github_comment_id)', + })); + } else { + // 404 re-POST fallback: require the stored id to match the one we + // thought was there before racing to overwrite it. + await ddb.send(new UpdateCommand({ + ...base, + ExpressionAttributeValues: { + ':cid': commentId, + ':prev': previousCommentId, + }, + ConditionExpression: 'attribute_exists(task_id) AND github_comment_id = :prev', + })); + } +} + +/** Name of the AWS SDK v3 conditional-failure error. Checking ``name`` + * rather than ``instanceof`` keeps the check decoupled from the + * specific SDK client class the DocumentClient wraps. */ +const CONDITIONAL_CHECK_FAILED = 'ConditionalCheckFailedException'; + +/** + * Resolve the GitHub comment target for this task. Prefers ``pr_number`` + * (the design-intent surface for pr_iteration / pr_review tasks) and + * falls back to ``issue_number``. Returns ``null`` if the task has + * neither — new_task tasks submitted via the API (no webhook) have no + * upstream surface to comment on. + */ +function resolveCommentTarget(task: TaskRecord): number | null { + return task.pr_number ?? task.issue_number ?? null; +} + +/** + * Resolve the GitHub token ARN for a task. Per-repo config wins; fall + * back to the Lambda's platform default env var so freshly-onboarded + * repos without an override still work. + * + * Error classification: + * - ``ResourceNotFoundException`` (RepoTable absent in dev) → fall + * back to the platform default silently. + * - ``AccessDeniedException`` → hard fail. An IAM misconfig means + * the dispatcher would use the wrong token for every repo, and + * silently falling back would mask the deployment bug. + * - Anything else (throttling, transient DDB errors, schema + * violations) → log at error and fall back so one flaky DDB + * invocation doesn't black-hole GitHub comments platform-wide. + */ +async function resolveTokenSecretArn(repo: string): Promise { + let repoConfig: Awaited> = null; + try { + repoConfig = await loadRepoConfig(repo); + } catch (err) { + const name = err instanceof Error ? err.name : ''; + if (name === 'AccessDeniedException') { + // Hard fail — IAM deny means every task in this deploy would + // silently fall back to the platform default, hiding the bug. + throw err; + } + if (name === 'ResourceNotFoundException') { + logger.info('[fanout/github] RepoTable not present — using platform default token', { + event: 'fanout.github.repo_table_absent', + repo, + }); + } else { + logger.error('[fanout/github] loadRepoConfig transient error — falling back to platform token', { + event: 'fanout.github.repo_config_failed', + error_id: 'FANOUT_REPO_CONFIG_FAILED', + repo, + error_name: name, + error: err instanceof Error ? err.message : String(err), + }); + } + } + return repoConfig?.github_token_secret_arn + ?? process.env.GITHUB_TOKEN_SECRET_ARN + ?? null; +} + +async function dispatchToGitHubComment(event: FanOutEvent): Promise { + const task = await loadTaskForComment(event.task_id); + if (!task) { + logger.warn('[fanout/github] task not found — skipping comment', { + event: 'fanout.github.task_missing', + task_id: event.task_id, + }); + return; + } + + const targetNumber = resolveCommentTarget(task); + if (targetNumber === null) { + // No issue / PR to comment on (API-submitted new_task with only a + // task_description). Skip silently at debug level. + logger.info('[fanout/github] no issue/pr target for task — skipping', { + event: 'fanout.github.no_target', + task_id: event.task_id, + }); + return; + } + + const tokenArn = await resolveTokenSecretArn(task.repo); + if (!tokenArn) { + logger.warn('[fanout/github] no GitHub token ARN configured — skipping', { + event: 'fanout.github.no_token_arn', + task_id: event.task_id, + repo: task.repo, + }); + return; + } + + let token: string; + try { + token = await resolveGitHubToken(tokenArn); + } catch (err) { + logger.warn('[fanout/github] token resolution failed — skipping', { + event: 'fanout.github.token_resolve_failed', + task_id: event.task_id, + error: err instanceof Error ? err.message : String(err), + }); + return; + } + + // Render the effective event type so comment bodies read + // ``pr_created`` / ``nudge_acknowledged`` rather than the wrapper + // ``agent_milestone``. Matches the watch CLI's rendering of these + // milestones (``cli/src/commands/watch.ts``). + const renderedEventType = effectiveEventType(event); + const body = renderCommentBody({ + taskId: task.task_id, + status: task.status, + repo: task.repo, + latestEventType: renderedEventType, + latestEventAt: event.timestamp, + prUrl: task.pr_url ?? null, + // DDB returns numeric attributes as strings at the Document-client + // boundary (see ``shared/numeric.ts``). Without coercion + // ``costUsd.toFixed(4)`` throws ``TypeError`` and the dispatcher + // is rejected for every terminal event. + durationS: coerceNumericOrNull( + task.duration_s, + { field: 'duration_s', task_id: task.task_id, event_id: event.event_id }, + logger, + ), + costUsd: coerceNumericOrNull( + task.cost_usd, + { field: 'cost_usd', task_id: task.task_id, event_id: event.event_id }, + logger, + ), + }); + + const upsertParams = { + repo: task.repo, + issueOrPrNumber: targetNumber, + body, + token, + existingCommentId: task.github_comment_id, + }; + + let result; + try { + result = await upsertTaskComment(upsertParams); + } catch (err) { + // On 401 we treat the cached token as stale (rotation / expiry), + // evict the cache, and retry exactly once. A cold token fetch is + // cheap (one Secrets Manager call) and this self-heals the common + // rotation case without operator intervention. Identify by duck- + // typing on ``name`` + ``httpStatus`` rather than ``instanceof`` so + // downstream callers (and tests that mock the module) can throw + // a compatible shape without being the exact same class instance. + const isGhErr = err instanceof Error && err.name === 'GitHubCommentError'; + const httpStatus = (err as { httpStatus?: unknown }).httpStatus; + if (isGhErr && httpStatus === 401) { + logger.warn('[fanout/github] 401 from GitHub — evicting token cache and retrying once', { + event: 'fanout.github.token_stale_retry', + task_id: event.task_id, + token_arn: tokenArn, + }); + clearTokenCache(); + const freshToken = await resolveGitHubToken(tokenArn); + result = await upsertTaskComment({ ...upsertParams, token: freshToken }); + } else { + throw err; + } + } + + // Only the upserts that POSTed (either first-ever or 404 re-POST + // fallback) have new state to persist. Steady-state PATCHes reuse + // the same ``github_comment_id``, and we no longer track an ETag + // since GitHub's PATCH endpoint doesn't honor ``If-Match`` + // (concurrency is handled upstream by DDB Stream ordering; see + // ``shared/github-comment.ts`` file header). + if (result.created) { + try { + await saveCommentState(task.task_id, result.commentId, task.github_comment_id); + } catch (err) { + const errName = err instanceof Error ? err.name : ''; + if (errName === CONDITIONAL_CHECK_FAILED) { + // Benign: either the task was TTL-evicted between our GetItem + // and this UpdateItem (subsequent events for this task will + // also GetItem-miss and skip), or a sibling fanout invocation + // that raced us already wrote a comment id (our comment + // survives as an orphan with the bgagent marker, safe to + // reconcile offline). Either way no duplicate-comment-runaway + // risk to chase here. + logger.info('[fanout/github] saveCommentState condition failed — benign (eviction or sibling race)', { + event: 'fanout.github.persist_benign_evicted', + task_id: task.task_id, + }); + } else { + // Non-conditional failure (DDB throttling, IAM deny, etc.) is a + // real persistence bug: the comment WAS posted but its id is + // not on the TaskRecord. The next event will POST a second + // comment instead of PATCHing. Log at ERROR with an error_id so + // operators can alarm on persistent GitHub dispatch failures + // distinctly from the generic dispatcher-rejected stream. + logger.error('[fanout/github] saveCommentState failed — next event may duplicate comment', { + event: 'fanout.github.persist_failed', + error_id: 'FANOUT_GITHUB_PERSIST_FAILED', + task_id: task.task_id, + comment_id: result.commentId, + created: result.created, + error_name: errName, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } + + logger.info('[fanout/github] comment dispatched', { + event: 'fanout.github.dispatched', + task_id: task.task_id, + comment_id: result.commentId, + created: result.created, + event_type: event.event_type, + effective_event_type: renderedEventType, + }); +} + +async function dispatchToEmail(event: FanOutEvent): Promise { + logger.info('[fanout/email] would send', { + event: 'fanout.email.dispatch_stub', + task_id: event.task_id, + event_type: event.event_type, + effective_event_type: effectiveEventType(event), + }); +} + +/** Exposed for testing: the per-channel dispatcher callable by the + * handler. Each key's absence from the routing map disables its + * dispatcher; the signature is uniform so adding a channel is one + * entry. */ +const DISPATCHERS: Record Promise> = { + slack: dispatchToSlack, + github: dispatchToGitHubComment, + email: dispatchToEmail, +}; + +/** + * Route an event to every subscribed channel. A dispatcher that + * rejects must NOT fail the whole batch: we swallow per-channel + * rejections so one Slack outage can't block GitHub comment delivery + * or drop an email notification. + * + * Returns the list of channels that **successfully** dispatched — a + * channel whose dispatcher rejected is omitted so batch telemetry + * (`dispatched` count in the handler) reflects reality. A rejected + * dispatcher is still logged with a ``fanout.dispatcher.rejected`` + * warn line that operators can alert on. + */ +export async function routeEvent( + ev: FanOutEvent, + overrides?: TaskNotificationsConfig, +): Promise { + const attempted: NotificationChannel[] = []; + const tasks: Promise[] = []; + // Match against the effective type so ``agent_milestone`` carriers + // (``pr_created``, ``nudge_acknowledged``, …) reach the channels + // subscribed to those milestone names. + const effective = effectiveEventType(ev); + for (const ch of CHANNELS) { + const filter = resolveChannelFilter(ch, overrides); + if (!filter.has(effective)) continue; + attempted.push(ch); + tasks.push(DISPATCHERS[ch](ev)); + } + // Parallelism is bounded by the dispatcher list (at most 3 channels), + // not by program input, so the unbounded-parallelism lint does not apply. + // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism + const results = await Promise.allSettled(tasks); + + const dispatched: NotificationChannel[] = []; + results.forEach((r, i) => { + const ch = attempted[i]; + if (r.status === 'fulfilled') { + dispatched.push(ch); + return; + } + // Belt-and-braces — the dispatcher stubs catch inside their own + // try/catch so this branch only fires if a future refactor drops + // the inner catch or if the dispatcher throws synchronously before + // entering its try. Record at warn so the signal isn't lost. + const reason = r.reason instanceof Error ? r.reason.message : String(r.reason); + logger.warn('[fanout] dispatcher rejected — continuing batch', { + event: 'fanout.dispatcher.rejected', + channel: ch, + task_id: ev.task_id, + event_id: ev.event_id, + event_type: ev.event_type, + effective_event_type: effectiveEventType(ev), + error: reason, + }); + }); + return dispatched; +} + +/** + * Lambda entry point. Invoked by the DynamoDB Streams event source + * mapping with batches of NEW_IMAGE records from `TaskEventsTable`. + * + * Returns a ``DynamoDBBatchResponse`` so the event-source-mapping's + * ``reportBatchItemFailures: true`` setting (see + * ``constructs/fanout-consumer.ts``) can honor partial-batch semantics. + * Without a structured return, a single poisonous record would cause + * Lambda to retry the **entire batch** from the stream checkpoint, + * replaying every sibling event and defeating the per-task ordering + * guarantee promised by ``ParallelizationFactor: 1`` upstream. + * + * Partial-failure surface (per-record try/catch below): + * - ``routeEvent`` wraps each dispatcher in ``Promise.allSettled``, so + * dispatcher rejections are already caught at the channel granularity + * and do not reach here. What DOES reach here is a throw BEFORE the + * ``allSettled`` — e.g. ``resolveTokenSecretArn`` throwing + * ``AccessDeniedException`` on an IAM misconfig (deliberate hard fail + * inside ``dispatchToGitHubComment``), a synchronous throw in + * ``loadTaskForComment`` on a broken DDB env, or any future writer + * that opens a non-``allSettled`` code path. + * - Parse / filter / rate-limit errors are defensive — today they + * cannot throw, but catching them keeps one stray ``throw`` in a + * future refactor (e.g. a stricter ``parseStreamRecord``) from + * crashing the whole batch. + * + * On any caught throw we push ``{ itemIdentifier: record.eventID }`` so + * Lambda retries ONLY that record, isolating the poison pill per + * design §6 + §8.9 expectations. Successful records are NOT in + * ``batchItemFailures`` and advance the stream checkpoint normally. + * + * Refs: PR #52 krokoko code review findings #1 and #5 (the fanout + * handler returned ``void`` despite ``reportBatchItemFailures: true``, + * and a ``routeEvent`` throw from ``resolveTokenSecretArn`` could crash + * the whole batch). + */ +// ``DynamoDBStreamHandler`` constrains the return to ``void | Promise``, +// which blocks the ``DynamoDBBatchResponse`` we must return for +// ``reportBatchItemFailures: true`` to work (finding #1). Typing the +// handler as a plain 1-arg async function lets us return a structured +// response; Lambda's nodejs24.x runtime detects any 3-arg shape as +// callback-style and rejects it at init with +// ``Runtime.CallbackHandlerDeprecated`` (observed 2026-05-05 post- +// redeploy). Tests still invoke with trailing args — JS silently +// ignores extra params, so ``handler(event, ctx, cb)`` keeps working. +export const handler = async ( + event: DynamoDBStreamEvent, +): Promise => { + const perTaskCounts = new Map(); + const batchItemFailures: DynamoDBBatchItemFailure[] = []; + let processed = 0; + let dispatched = 0; + let dropped = 0; + + // v1: no per-task override; every event uses the channel defaults. + // Chunk K wires a DDB read here to load ``TaskRecord.notifications``. + const overrides: TaskNotificationsConfig | undefined = undefined; + + for (const record of event.Records) { + processed++; + try { + const ev = parseStreamRecord(record); + if (!ev) { + dropped++; + continue; + } + if (!shouldFanOut(ev, overrides)) { + dropped++; + continue; + } + + const seen = perTaskCounts.get(ev.task_id) ?? 0; + if (seen >= MAX_EVENTS_PER_TASK_PER_INVOCATION) { + logger.warn('[fanout] per-task cap hit — dropping event', { + event: 'fanout.rate_limit.hit', + task_id: ev.task_id, + event_id: ev.event_id, + event_type: ev.event_type, + effective_event_type: effectiveEventType(ev), + cap: MAX_EVENTS_PER_TASK_PER_INVOCATION, + }); + dropped++; + continue; + } + perTaskCounts.set(ev.task_id, seen + 1); + + const channels = await routeEvent(ev, overrides); + if (channels.length > 0) dispatched++; + } catch (err) { + // Poison-pill isolation: one record's unhandled throw must not + // crash the batch. See the handler doc block for the full list of + // paths that can reach here (notably AccessDeniedException from + // ``resolveTokenSecretArn``, finding #5). + // + // ``eventID`` is the stream-record identifier Lambda uses for the + // retry cursor; on Kinesis-style event-source-mappings with + // ``reportBatchItemFailures: true`` the service retries all + // records at-or-after the lowest-sequence failure. Returning even + // one failed itemIdentifier is enough to preserve ordering across + // the whole batch for that task. + const eventID = record.eventID; + logger.warn('[fanout] record threw — flagging for partial-batch retry', { + event: 'fanout.record.failed', + event_id: eventID, + error: err instanceof Error ? err.message : String(err), + error_name: err instanceof Error ? err.name : undefined, + }); + if (eventID !== undefined) { + batchItemFailures.push({ itemIdentifier: eventID }); + } + } + } + + logger.info('[fanout] batch complete', { + event: 'fanout.batch.complete', + records: event.Records.length, + processed, + dispatched, + dropped, + failed: batchItemFailures.length, + unique_tasks: perTaskCounts.size, + }); + + return { batchItemFailures }; +}; diff --git a/cdk/src/handlers/get-task-events.ts b/cdk/src/handlers/get-task-events.ts index b82041d..f631a18 100644 --- a/cdk/src/handlers/get-task-events.ts +++ b/cdk/src/handlers/get-task-events.ts @@ -25,14 +25,47 @@ import { extractUserId } from './shared/gateway'; import { logger } from './shared/logger'; import { ErrorCode, errorResponse, paginatedResponse } from './shared/response'; import type { EventRecord, TaskRecord } from './shared/types'; -import { decodePaginationToken, encodePaginationToken, parseLimit } from './shared/validation'; +import { + decodePaginationToken, + encodePaginationToken, + isValidUlid, + parseLimit, +} from './shared/validation'; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); const TABLE_NAME = process.env.TASK_TABLE_NAME!; const EVENTS_TABLE_NAME = process.env.TASK_EVENTS_TABLE_NAME!; +const LOG_LEVEL = (process.env.LOG_LEVEL ?? 'INFO').toUpperCase(); +const DEBUG_ENABLED = LOG_LEVEL === 'DEBUG'; + +/** Query mode resolved from query parameters for structured logging. */ +type QueryMode = 'from_beginning' | 'next_token' | 'after' | 'desc'; /** * GET /v1/tasks/{task_id}/events — Get task event audit trail. + * + * Supports three alternative query modes (plus the default "from the beginning"): + * + * - ``?after=`` — ULID cursor. Query returns events with + * ``event_id > after``. Used by CLI polling and webhook replay to + * resume from a known event id. ULIDs are lexicographically sortable + * by timestamp, so string ``>`` compare is correct. + * - ``?next_token=`` — opaque DynamoDB ``LastEvaluatedKey``, + * used for normal forward pagination. + * - ``?desc=1`` — return the newest events first. Used by ``bgagent + * status`` to render a recency-biased snapshot in O(limit) rather + * than walking the full event stream. Mutually exclusive with + * ``after`` (a forward cursor has no meaning against a descending + * scan); the combination is rejected as 400. + * + * If both ``after`` and ``next_token`` are provided, ``after`` wins (a WARN + * is logged — likely a client bug). If none are provided, the query starts + * from the oldest event. In all modes, when the result is truncated at + * ``limit`` a ``next_token`` is emitted so the caller can continue + * paginating. + * + * @param event - API Gateway proxy event. + * @returns API Gateway proxy result. */ export async function handler(event: APIGatewayProxyEvent): Promise { const requestId = ulid(); @@ -50,7 +83,77 @@ export async function handler(event: APIGatewayProxyEvent): Promise :after`` — safe because ULIDs are lexicographic. + // ``desc`` flips ``ScanIndexForward`` so the newest events return first, + // which is what ``bgagent status`` needs to render a recency-biased + // snapshot cheaply. const queryInput: Record = { TableName: EVENTS_TABLE_NAME, - KeyConditionExpression: 'task_id = :tid', - ExpressionAttributeValues: { ':tid': taskId }, - ScanIndexForward: true, + KeyConditionExpression: afterValid + ? 'task_id = :tid AND event_id > :after' + : 'task_id = :tid', + ExpressionAttributeValues: afterValid + ? { ':tid': taskId, ':after': afterValid } + : { ':tid': taskId }, + ScanIndexForward: !desc, Limit: limit, }; - if (startKey) { + if (!afterValid && startKey) { queryInput.ExclusiveStartKey = startKey; } + if (DEBUG_ENABLED) { + logger.info('DDB query prepared (debug)', { + level_override: 'DEBUG', + request_id: requestId, + task_id: taskId, + key_condition: queryInput.KeyConditionExpression, + has_exclusive_start_key: Boolean(queryInput.ExclusiveStartKey), + }); + } + const result = await ddb.send(new QueryCommand(queryInput as any)); const events = (result.Items ?? []) as EventRecord[]; + if (DEBUG_ENABLED) { + logger.info('DDB query returned (debug)', { + level_override: 'DEBUG', + request_id: requestId, + task_id: taskId, + count: events.length, + has_last_evaluated_key: Boolean(result.LastEvaluatedKey), + }); + } + // 6. Strip task_id from event records (redundant in response context) const eventData = events.map(e => ({ event_id: e.event_id, @@ -94,11 +220,41 @@ export async function handler(event: APIGatewayProxyEvent): Promise | undefined); + // For descending scans we intentionally suppress ``next_token``. DDB's + // ``LastEvaluatedKey`` carries no direction — a follow-up request that + // passes ``?next_token=...`` without also passing ``desc=1`` would + // silently scan ascending from mid-stream and interleave results. + // ``bgagent status`` only ever requests one page anyway; surfacing a + // token here would invite future callers into that footgun. + const nextToken = desc + ? null + : encodePaginationToken(result.LastEvaluatedKey as Record | undefined); + + // 7. Warn on unexpectedly empty catch-up — helps debug CLI reconnect logic. + // We only warn for ``after`` mode because "no events yet" is normal on cold start. + if (afterValid && events.length === 0) { + logger.warn('after cursor returned empty page (caller may be at tail)', { + request_id: requestId, + task_id: taskId, + after: afterValid, + }); + } + + logger.info('get-task-events complete', { + request_id: requestId, + task_id: taskId, + event_count: events.length, + has_more: nextToken !== null, + query_mode: queryMode, + }); return paginatedResponse(eventData, nextToken, requestId); } catch (err) { - logger.error('Failed to get task events', { error: String(err), request_id: requestId }); + logger.error('Failed to get task events', { + error: String(err), + error_type: err instanceof Error ? err.constructor.name : typeof err, + request_id: requestId, + }); return errorResponse(500, ErrorCode.INTERNAL_ERROR, 'Internal server error.', requestId); } } diff --git a/cdk/src/handlers/get-trace-url.ts b/cdk/src/handlers/get-trace-url.ts new file mode 100644 index 0000000..bdf1a55 --- /dev/null +++ b/cdk/src/handlers/get-trace-url.ts @@ -0,0 +1,233 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { GetObjectCommand, HeadObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { DynamoDBDocumentClient, GetCommand } from '@aws-sdk/lib-dynamodb'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import type { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda'; +import { ulid } from 'ulid'; +import { TRACE_OBJECT_KEY_PREFIX } from '../constructs/trace-artifacts-bucket'; +import { extractUserId } from './shared/gateway'; +import { logger } from './shared/logger'; +import { ErrorCode, errorResponse, successResponse } from './shared/response'; +import type { TaskRecord } from './shared/types'; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const s3 = new S3Client({}); +const TABLE_NAME = process.env.TASK_TABLE_NAME!; +const TRACE_BUCKET_NAME = process.env.TRACE_ARTIFACTS_BUCKET_NAME!; + +/** + * Presigned URL TTL. 15 minutes is long enough for a multi-MB trajectory + * download on a slow link, short enough to bound the window for a leaked + * URL to be useful. Also short enough that the caller's Cognito session + * is still valid — if a user wants a fresh URL, they re-issue via + * ``bgagent trace download``. + */ +export const TRACE_URL_TTL_SECONDS = 900; + +/** + * ``GET /v1/tasks/{task_id}/trace`` — return a presigned S3 URL for the + * ``--trace`` trajectory dump. + * + * Response shape (200): + * ``` + * { data: { url: string, expires_at: string } } + * ``` + * + * Errors: + * - 401 UNAUTHORIZED — Cognito auth missing + * - 400 VALIDATION_ERROR — missing ``task_id`` path parameter + * - 403 FORBIDDEN — caller does not own this task + * - 404 TASK_NOT_FOUND — task_id not in the table + * - 404 TRACE_NOT_AVAILABLE — task exists but was not submitted with ``--trace``, + * or the upload has not yet completed + * - 500 INTERNAL_ERROR — DDB or S3 presign failure + */ +export async function handler(event: APIGatewayProxyEvent): Promise { + const requestId = ulid(); + + try { + const userId = extractUserId(event); + if (!userId) { + return errorResponse(401, ErrorCode.UNAUTHORIZED, 'Missing or invalid authentication.', requestId); + } + + const taskId = event.pathParameters?.task_id; + if (!taskId) { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, 'Missing task_id path parameter.', requestId); + } + + const result = await ddb.send(new GetCommand({ + TableName: TABLE_NAME, + Key: { task_id: taskId }, + })); + + if (!result.Item) { + return errorResponse(404, ErrorCode.TASK_NOT_FOUND, `Task ${taskId} not found.`, requestId); + } + + const record = result.Item as TaskRecord; + if (record.user_id !== userId) { + return errorResponse(403, ErrorCode.FORBIDDEN, 'You do not have access to this task.', requestId); + } + + const s3Uri = record.trace_s3_uri; + if (!s3Uri) { + // Covers two cases with one status code — the CLI can't disambiguate + // "never enabled" from "not yet uploaded" without racing the agent, + // and the user-facing remedy is the same: re-submit with --trace (or + // wait for the task to reach terminal). + return errorResponse( + 404, + ErrorCode.TRACE_NOT_AVAILABLE, + 'Task did not run with --trace, or the trace has not been uploaded yet.', + requestId, + ); + } + + const parsed = parseS3Uri(s3Uri); + if (!parsed) { + logger.error('TaskRecord.trace_s3_uri is not a valid s3:// URI', { + task_id: taskId, + trace_s3_uri: s3Uri, + request_id: requestId, + }); + return errorResponse(500, ErrorCode.INTERNAL_ERROR, 'Trace URI is malformed.', requestId); + } + + // Defense in depth: refuse to presign URLs for objects in a bucket + // we don't own. Prevents a DDB-injection attack that spoofs a + // ``trace_s3_uri`` pointing at an attacker-controlled bucket from + // turning this handler into an open URL signer. + if (parsed.bucket !== TRACE_BUCKET_NAME) { + logger.error('TaskRecord.trace_s3_uri bucket does not match TRACE_ARTIFACTS_BUCKET_NAME', { + task_id: taskId, + record_bucket: parsed.bucket, + expected_bucket: TRACE_BUCKET_NAME, + request_id: requestId, + }); + return errorResponse(500, ErrorCode.INTERNAL_ERROR, 'Trace URI references an unexpected bucket.', requestId); + } + + // Second defense-in-depth guard: the object key must live under the + // caller's own user prefix. The agent writes with + // ``traces//.jsonl.gz`` by construction; an + // ownership-check mismatch here signals either a stale record or a + // cross-user write that escaped the runtime's per-prefix policy. + // + // Note: the comparator is the CALLER's ``userId`` (from Cognito), + // NOT ``record.user_id``. That is the stronger invariant: it defends + // against a compromised agent pointing one user's record at another + // user's artifact. ``record.user_id !== userId`` already short- + // circuited cross-user RECORD access above; this guard additionally + // prevents cross-user ARTIFACT access when the record is legitimately + // owned by the caller but ``trace_s3_uri`` was tampered with to + // point elsewhere. Do NOT "simplify" by using ``record.user_id``. + const expectedKeyPrefix = `${TRACE_OBJECT_KEY_PREFIX}${userId}/`; + if (!parsed.key.startsWith(expectedKeyPrefix)) { + logger.error('TaskRecord.trace_s3_uri key is not under the caller\'s user prefix', { + task_id: taskId, + user_id: userId, + record_key: parsed.key, + expected_prefix: expectedKeyPrefix, + request_id: requestId, + }); + return errorResponse(403, ErrorCode.FORBIDDEN, 'Trace artifact is not owned by the caller.', requestId); + } + + // HEAD-check the object before presigning. The agent may have + // written ``trace_s3_uri`` to DDB before the S3 PUT propagated, or + // a lifecycle policy / operator action may have deleted the + // artifact after the record was stamped. Issuing a URL that 404s + // XML from S3 would leave the user debugging a broken download with + // no obvious remedy; returning the same ``TRACE_NOT_AVAILABLE`` 404 + // the CLI already knows how to message (re-submit with --trace) is + // strictly more user-friendly. ``s3:GetObject`` implicitly grants + // HeadObject per AWS IAM docs, so no extra permission is required. + try { + await s3.send(new HeadObjectCommand({ Bucket: parsed.bucket, Key: parsed.key })); + } catch (err) { + // S3 SDK v3 returns either ``NotFound`` (object-level 404) or + // ``NoSuchKey`` (key-level 404) depending on operation; both map + // to the same user-facing outcome. HTTP 403 can also mean the + // object is missing in a bucket the principal can't probe, but + // since this handler signs for its own bucket and the CLI already + // received ``trace_s3_uri``, 404 is the only case we hide behind + // TRACE_NOT_AVAILABLE. + const name = (err as { name?: string })?.name; + const httpStatus = (err as { $metadata?: { httpStatusCode?: number } })?.$metadata?.httpStatusCode; + if (name === 'NotFound' || name === 'NoSuchKey' || httpStatus === 404) { + logger.warn('Trace artifact S3 object not found at HEAD time', { + task_id: taskId, + bucket: parsed.bucket, + key: parsed.key, + request_id: requestId, + }); + return errorResponse( + 404, + ErrorCode.TRACE_NOT_AVAILABLE, + 'Task did not run with --trace, or the trace has not been uploaded yet.', + requestId, + ); + } + logger.error('HeadObject failed for trace artifact', { + task_id: taskId, + bucket: parsed.bucket, + key: parsed.key, + error: err instanceof Error ? err.message : String(err), + error_name: name, + request_id: requestId, + }); + return errorResponse(500, ErrorCode.INTERNAL_ERROR, 'Internal server error.', requestId); + } + + const url = await getSignedUrl( + s3, + new GetObjectCommand({ Bucket: parsed.bucket, Key: parsed.key }), + { expiresIn: TRACE_URL_TTL_SECONDS }, + ); + + const expiresAt = new Date(Date.now() + TRACE_URL_TTL_SECONDS * 1000).toISOString(); + + return successResponse(200, { url, expires_at: expiresAt }, requestId); + } catch (err) { + logger.error('Failed to issue trace download URL', { + error: err instanceof Error ? err.message : String(err), + request_id: requestId, + }); + return errorResponse(500, ErrorCode.INTERNAL_ERROR, 'Internal server error.', requestId); + } +} + +/** + * Parse an ``s3://bucket/key`` URI into its components. Returns ``null`` + * if the string is malformed. + */ +export function parseS3Uri(uri: string): { bucket: string; key: string } | null { + if (!uri.startsWith('s3://')) return null; + const rest = uri.slice('s3://'.length); + const slash = rest.indexOf('/'); + if (slash <= 0 || slash === rest.length - 1) return null; + const bucket = rest.slice(0, slash); + const key = rest.slice(slash + 1); + if (!bucket || !key) return null; + return { bucket, key }; +} diff --git a/cdk/src/handlers/nudge-task.ts b/cdk/src/handlers/nudge-task.ts new file mode 100644 index 0000000..c519aab --- /dev/null +++ b/cdk/src/handlers/nudge-task.ts @@ -0,0 +1,246 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient, GetCommand, PutCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; +import type { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda'; +import { ulid } from 'ulid'; +import { TERMINAL_STATUSES } from '../constructs/task-status'; +import { GuardrailScreeningError, screenWithGuardrail } from './shared/context-hydration'; +import { extractUserId } from './shared/gateway'; +import { logger } from './shared/logger'; +import { ErrorCode, errorResponse, successResponse } from './shared/response'; +import { NUDGE_MAX_MESSAGE_LENGTH, type NudgeRecord, type NudgeRequest, type TaskRecord } from './shared/types'; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const TASK_TABLE_NAME = process.env.TASK_TABLE_NAME; +const NUDGES_TABLE_NAME = process.env.NUDGES_TABLE_NAME; +if (!TASK_TABLE_NAME || !NUDGES_TABLE_NAME) { + throw new Error( + 'nudge-task handler requires TASK_TABLE_NAME and NUDGES_TABLE_NAME env vars to be set', + ); +} +const RATE_LIMIT_PER_MINUTE = Number(process.env.NUDGE_RATE_LIMIT_PER_MINUTE ?? '10'); +/** TTL for stored nudge rows (~30 days). */ +const NUDGE_RETENTION_SECONDS = 30 * 24 * 60 * 60; +/** TTL for rate-limit counter rows (~2 minutes — only need the current minute bucket). */ +const RATE_LIMIT_ROW_TTL_SECONDS = 120; + +/** + * POST /v1/tasks/{task_id}/nudge — submit a steering message to a running agent. + * + * Flow: auth → validate → ownership → state → guardrail → rate-limit → persist. + * + * Note ordering: guardrail screening runs BEFORE the rate-limit counter + * increment so that a guardrail-blocked message does NOT consume a slot in + * the user's per-minute budget. ApplyGuardrail is cheap compared to DDB, + * and the user-facing UX of "accidentally blocked, lost a slot" is worse + * than the alternative. Authenticated users are already rate-limited + * upstream by API Gateway / Cognito. + * + * Returns 202 Accepted with the nudge_id. The nudge will be picked up by + * the agent runtime at the next between-turns seam. + * @param event - the API Gateway proxy event. + * @returns the API Gateway proxy result. + */ +export async function handler(event: APIGatewayProxyEvent): Promise { + const requestId = ulid(); + + try { + // 1. Auth + const userId = extractUserId(event); + if (!userId) { + return errorResponse(401, ErrorCode.UNAUTHORIZED, 'Missing or invalid authentication.', requestId); + } + + // 2. Path param + const taskId = event.pathParameters?.task_id; + if (!taskId) { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, 'Missing task_id path parameter.', requestId); + } + + // 3. Body validation + let parsed: NudgeRequest | null = null; + try { + parsed = event.body ? JSON.parse(event.body) as NudgeRequest : null; + } catch { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, 'Request body must be valid JSON.', requestId); + } + if (!parsed || typeof parsed.message !== 'string') { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, 'Missing required field: message (string).', requestId); + } + const message = parsed.message.trim(); + if (message.length === 0) { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, 'Field "message" must be a non-empty string.', requestId); + } + if (message.length > NUDGE_MAX_MESSAGE_LENGTH) { + return errorResponse( + 400, + ErrorCode.VALIDATION_ERROR, + `Field "message" exceeds maximum length of ${NUDGE_MAX_MESSAGE_LENGTH} characters.`, + requestId, + ); + } + + // 4. Ownership + state check + const getResult = await ddb.send(new GetCommand({ + TableName: TASK_TABLE_NAME, + Key: { task_id: taskId }, + })); + if (!getResult.Item) { + return errorResponse(404, ErrorCode.TASK_NOT_FOUND, `Task ${taskId} not found.`, requestId); + } + const record = getResult.Item as TaskRecord; + if (record.user_id !== userId) { + return errorResponse(403, ErrorCode.FORBIDDEN, 'You do not have access to this task.', requestId); + } + if (TERMINAL_STATUSES.includes(record.status)) { + return errorResponse( + 409, + ErrorCode.TASK_ALREADY_TERMINAL, + `Task ${taskId} is in terminal state ${record.status}; cannot accept nudges.`, + requestId, + ); + } + + const now = new Date(); + const minuteBucket = formatMinuteBucket(now); + const nowIso = now.toISOString(); + const nowEpoch = Math.floor(now.getTime() / 1000); + + // 5. Guardrail screening (fail-closed). + // + // Runs BEFORE the rate-limit counter so a guardrail-blocked message + // does not consume a rate-limit slot (see handler docstring). + try { + const screenResult = await screenWithGuardrail(message, taskId); + if (screenResult?.action === 'GUARDRAIL_INTERVENED') { + const details = screenResult.assessments + ?.map(a => `${a.filter_type}/${a.filter_name}${a.confidence ? ` (${a.confidence})` : ''}`) + .join(', '); + const reason = `Nudge message blocked by content policy${details ? ': ' + details : ''}`; + return errorResponse(400, ErrorCode.VALIDATION_ERROR, reason, requestId); + } + } catch (err) { + if (err instanceof GuardrailScreeningError) { + logger.error('Guardrail screening failed for nudge (fail-closed)', { + task_id: taskId, + request_id: requestId, + error: err.message, + }); + return errorResponse( + 503, + ErrorCode.SERVICE_UNAVAILABLE, + 'Content screening is temporarily unavailable.', + requestId, + ); + } + throw err; + } + + // 6. Per-task per-minute rate limit. + // + // Uses a synthetic row in the nudges table to take advantage of the + // existing grantReadWriteData wiring and atomic UpdateItem semantics. + // PK = `RATE#` (distinct from real PKs which are task ULIDs), + // SK = `MINUTE#` (one bucket per wall-clock minute). + // Short TTL (~2m) cleans up stale counters automatically. + try { + await ddb.send(new UpdateCommand({ + TableName: NUDGES_TABLE_NAME, + Key: { + task_id: `RATE#${taskId}`, + nudge_id: `MINUTE#${minuteBucket}`, + }, + UpdateExpression: 'ADD #count :one SET #ttl = :ttl', + ConditionExpression: 'attribute_not_exists(#count) OR #count < :max', + ExpressionAttributeNames: { + '#count': 'count', + '#ttl': 'ttl', + }, + ExpressionAttributeValues: { + ':one': 1, + ':max': RATE_LIMIT_PER_MINUTE, + ':ttl': nowEpoch + RATE_LIMIT_ROW_TTL_SECONDS, + }, + })); + } catch (err: unknown) { + const name = (err as { name?: string })?.name; + if (name === 'ConditionalCheckFailedException') { + return errorResponse( + 429, + ErrorCode.RATE_LIMIT_EXCEEDED, + `Rate limit exceeded: at most ${RATE_LIMIT_PER_MINUTE} nudges per minute per task.`, + requestId, + ); + } + throw err; + } + + // 7. Persist nudge record. + const nudgeId = ulid(); + const nudgeRecord: NudgeRecord = { + task_id: taskId, + nudge_id: nudgeId, + user_id: userId, + message, + created_at: nowIso, + consumed: false, + ttl: nowEpoch + NUDGE_RETENTION_SECONDS, + }; + await ddb.send(new PutCommand({ + TableName: NUDGES_TABLE_NAME, + Item: nudgeRecord, + })); + + logger.info('Nudge submitted', { + task_id: taskId, + nudge_id: nudgeId, + user_id: userId, + request_id: requestId, + message_length: message.length, + }); + + return successResponse(202, { + task_id: taskId, + nudge_id: nudgeId, + submitted_at: nowIso, + }, requestId); + } catch (err) { + logger.error('Failed to submit nudge', { + error: err instanceof Error ? err.message : String(err), + request_id: requestId, + }); + return errorResponse(500, ErrorCode.INTERNAL_ERROR, 'Internal server error.', requestId); + } +} + +/** + * Format the current minute as a `yyyymmddhhmm` UTC bucket identifier. + * @param date - the timestamp to format. + * @returns 12-character bucket string. + */ +function formatMinuteBucket(date: Date): string { + const y = date.getUTCFullYear().toString().padStart(4, '0'); + const m = (date.getUTCMonth() + 1).toString().padStart(2, '0'); + const d = date.getUTCDate().toString().padStart(2, '0'); + const h = date.getUTCHours().toString().padStart(2, '0'); + const mi = date.getUTCMinutes().toString().padStart(2, '0'); + return `${y}${m}${d}${h}${mi}`; +} diff --git a/cdk/src/handlers/reconcile-stranded-tasks.ts b/cdk/src/handlers/reconcile-stranded-tasks.ts new file mode 100644 index 0000000..760049c --- /dev/null +++ b/cdk/src/handlers/reconcile-stranded-tasks.ts @@ -0,0 +1,329 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Scheduled handler: find and fail stranded tasks. + * + * A stranded task is one whose admission write landed in TaskTable but + * whose pipeline never started — typically because the orchestrator + * Lambda crashed between the TaskTable write and the InvokeAgentRuntime + * call, or because the agent container crashed during startup before + * writing its first heartbeat. + * + * RUNNING / FINALIZING tasks are handled separately by `pollTaskStatus` + * in `orchestrator.ts` via the `agent_heartbeat_at` timeout path — this + * reconciler only targets `SUBMITTED` and `HYDRATING`. + */ + +import { + DynamoDBClient, + QueryCommand, + UpdateItemCommand, + PutItemCommand, +} from '@aws-sdk/client-dynamodb'; +import { ulid } from 'ulid'; +import { logger } from './shared/logger'; + +const ddb = new DynamoDBClient({}); +const TASK_TABLE = process.env.TASK_TABLE_NAME!; +const EVENTS_TABLE = process.env.TASK_EVENTS_TABLE_NAME!; +const CONCURRENCY_TABLE = process.env.USER_CONCURRENCY_TABLE_NAME!; + +/** Stranded-task timeout. The orchestrator Lambda is async-invoked and + * the agent runtime has a cold-start path; 1200 s covers Lambda retries + * + AgentCore container warm-up without false positives. */ +const STRANDED_TIMEOUT_SECONDS = Number( + process.env.STRANDED_TIMEOUT_SECONDS ?? '1200', +); + +const TASK_RETENTION_DAYS = Number(process.env.TASK_RETENTION_DAYS ?? '90'); + +interface StrandedCandidate { + readonly task_id: string; + readonly user_id: string; + readonly status: string; + readonly created_at: string; + readonly age_seconds: number; +} + +/** + * Query TaskTable by (status, created_at) via the StatusIndex GSI and + * return rows older than the stranded timeout. + * + * One query per status (SUBMITTED, HYDRATING) using a sort-key condition + * `created_at < :cutoff`. + */ +async function findStrandedCandidates( + status: 'SUBMITTED' | 'HYDRATING', + now: Date, +): Promise { + const cutoff = new Date(now.getTime() - STRANDED_TIMEOUT_SECONDS * 1000); + + const matches: StrandedCandidate[] = []; + let lastKey: Record | undefined; + + do { + const resp = await ddb.send(new QueryCommand({ + TableName: TASK_TABLE, + IndexName: 'StatusIndex', + KeyConditionExpression: '#s = :status AND created_at < :cutoff', + ExpressionAttributeNames: { '#s': 'status' }, + ExpressionAttributeValues: { + ':status': { S: status }, + ':cutoff': { S: cutoff.toISOString() }, + }, + ExclusiveStartKey: lastKey as Record | undefined, + })); + + for (const item of resp.Items ?? []) { + const taskId = item.task_id?.S; + const userId = item.user_id?.S; + const createdAt = item.created_at?.S; + if (!taskId || !userId || !createdAt) continue; + + const createdMs = Date.parse(createdAt); + const ageSec = Math.floor((now.getTime() - createdMs) / 1000); + + matches.push({ + task_id: taskId, + user_id: userId, + status, + created_at: createdAt, + age_seconds: ageSec, + }); + } + + lastKey = resp.LastEvaluatedKey; + } while (lastKey); + + return matches; +} + +/** + * Transition a stranded task to FAILED, emit a task_stranded event, and + * release its concurrency slot. Best-effort and idempotent — a concurrent + * legitimate status transition wins (conditional check fails cleanly). + */ +async function failStrandedTask(task: StrandedCandidate): Promise { + const now = new Date().toISOString(); + const errorMessage = `Stranded: ${task.status} for ${task.age_seconds}s — ` + + 'no pipeline attached before the stranded-task timeout. ' + + 'This usually means the orchestrator Lambda crashed before invoking ' + + 'the runtime, or the agent container crashed during startup.'; + + // 1. Conditional status transition — only if still in the stranded state. + try { + await ddb.send(new UpdateItemCommand({ + TableName: TASK_TABLE, + Key: { task_id: { S: task.task_id } }, + UpdateExpression: + 'SET #s = :failed, updated_at = :now, completed_at = :now, ' + + 'error_message = :err, status_created_at = :sca', + ConditionExpression: '#s = :expected', + ExpressionAttributeNames: { '#s': 'status' }, + ExpressionAttributeValues: { + ':failed': { S: 'FAILED' }, + ':expected': { S: task.status }, + ':now': { S: now }, + ':err': { S: errorMessage }, + ':sca': { S: `FAILED#${now}` }, + }, + })); + } catch (err: unknown) { + if (err && typeof err === 'object' && 'name' in err && err.name === 'ConditionalCheckFailedException') { + // The task advanced out of SUBMITTED/HYDRATING while we were looking + // at it — legit, no action needed. + logger.info('Task advanced before transition — skipping', { + task_id: task.task_id, + reason: 'advanced_during_reconcile', + }); + return false; + } + throw err; + } + + // 2. Emit task_stranded + task_failed events. Best-effort — loss of an + // event is acceptable; the task record is the source of truth. + const ttl = Math.floor(Date.now() / 1000) + TASK_RETENTION_DAYS * 24 * 3600; + try { + await ddb.send(new PutItemCommand({ + TableName: EVENTS_TABLE, + Item: { + task_id: { S: task.task_id }, + event_id: { S: ulid() }, + event_type: { S: 'task_stranded' }, + timestamp: { S: now }, + ttl: { N: String(ttl) }, + metadata: { + M: { + code: { S: 'STRANDED_NO_HEARTBEAT' }, + prior_status: { S: task.status }, + age_seconds: { N: String(task.age_seconds) }, + }, + }, + }, + })); + } catch (eventErr) { + logger.warn('Failed to write task_stranded event (best-effort)', { + task_id: task.task_id, + error: eventErr instanceof Error ? eventErr.message : String(eventErr), + }); + } + + try { + await ddb.send(new PutItemCommand({ + TableName: EVENTS_TABLE, + Item: { + task_id: { S: task.task_id }, + event_id: { S: ulid() }, + event_type: { S: 'task_failed' }, + timestamp: { S: now }, + ttl: { N: String(ttl) }, + metadata: { M: { error_message: { S: errorMessage } } }, + }, + })); + } catch (eventErr) { + logger.warn('Failed to write task_failed event (best-effort)', { + task_id: task.task_id, + error: eventErr instanceof Error ? eventErr.message : String(eventErr), + }); + } + + // 3. Release the concurrency slot. Best-effort; drift is later corrected + // by the concurrency reconciler. + try { + await ddb.send(new UpdateItemCommand({ + TableName: CONCURRENCY_TABLE, + Key: { user_id: { S: task.user_id } }, + UpdateExpression: 'SET active_count = active_count - :one, updated_at = :now', + ConditionExpression: 'active_count > :zero', + ExpressionAttributeValues: { + ':one': { N: '1' }, + ':zero': { N: '0' }, + ':now': { S: now }, + }, + })); + } catch (decrErr: unknown) { + if (decrErr && typeof decrErr === 'object' && 'name' in decrErr + && decrErr.name !== 'ConditionalCheckFailedException') { + logger.warn('Failed to decrement concurrency for stranded task', { + task_id: task.task_id, + user_id: task.user_id, + error: decrErr instanceof Error ? decrErr.message : String(decrErr), + }); + } + // ConditionalCheckFailedException means the counter is already 0 — + // drift the concurrency reconciler will eventually catch. + } + + return true; +} + +export async function handler(): Promise { + logger.info('Stranded-task reconciler started', { + stranded_timeout_s: STRANDED_TIMEOUT_SECONDS, + }); + + const now = new Date(); + const statuses: ('SUBMITTED' | 'HYDRATING')[] = ['SUBMITTED', 'HYDRATING']; + let totalStranded = 0; + let totalFailed = 0; + let totalSkipped = 0; + let totalErrors = 0; + + for (const status of statuses) { + let candidates: StrandedCandidate[]; + try { + candidates = await findStrandedCandidates(status, now); + } catch (queryErr) { + logger.error('Query for stranded candidates failed — skipping status', { + status, + error: queryErr instanceof Error ? queryErr.message : String(queryErr), + }); + totalErrors++; + continue; + } + + totalStranded += candidates.length; + for (const task of candidates) { + logger.info('Detected stranded task', { + task_id: task.task_id, + status: task.status, + age_seconds: task.age_seconds, + }); + try { + const applied = await failStrandedTask(task); + if (applied) { + totalFailed++; + } else { + totalSkipped++; + } + } catch (err) { + totalErrors++; + logger.warn('Per-task failStrandedTask failed, continuing', { + task_id: task.task_id, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } + + // Severity escalation for the final log line. + // + // Per-task failures upstream are caught and swallowed (logged at WARN) + // so one flaky DDB call doesn't abort the entire reconcile window. But + // a systemic failure — IAM outage, table-level throttling, schema + // corruption — can silently strand 100% of candidates while each + // individual WARN line looks ignorable. We classify the terminal log + // three ways so CloudWatch Log Insights / metric filters can alarm on + // the dedicated `error_id` strings: + // + // 1. totalStranded > 0 AND totalFailed == 0 AND totalErrors > 0 + // → SYSTEMIC failure. Every candidate hit an exception. Log ERROR + // with error_id='RECONCILER_TOTAL_FAILURE' (alarm-worthy). + // 2. totalErrors > 0 AND totalFailed > 0 + // → PARTIAL failure. Some tasks transitioned, some didn't. Log + // WARN with error_id='RECONCILER_PARTIAL_FAILURE' (dashboard + // signal, not an alarm — expected under occasional DDB flakes). + // 3. Otherwise (no stranded, or all-success with zero errors) + // → SUCCESS. Log INFO as before. + // + // We do NOT throw — the EventBridge schedule invocation should still + // complete "normally" (no retry storm against an already-degraded + // DDB). The log-level escalation IS the alarm signal. + const finalPayload = { + stranded: totalStranded, + failed: totalFailed, + skipped: totalSkipped, + errors: totalErrors, + }; + if (totalStranded > 0 && totalFailed === 0 && totalErrors > 0) { + logger.error('Stranded-task reconciler finished — every candidate failed to transition', { + ...finalPayload, + error_id: 'RECONCILER_TOTAL_FAILURE', + }); + } else if (totalErrors > 0 && totalFailed > 0) { + logger.warn('Stranded-task reconciler finished with partial failures', { + ...finalPayload, + error_id: 'RECONCILER_PARTIAL_FAILURE', + }); + } else { + logger.info('Stranded-task reconciler finished', finalPayload); + } +} diff --git a/cdk/src/handlers/shared/create-task-core.ts b/cdk/src/handlers/shared/create-task-core.ts index 5f003c4..b65f898 100644 --- a/cdk/src/handlers/shared/create-task-core.ts +++ b/cdk/src/handlers/shared/create-task-core.ts @@ -31,7 +31,7 @@ import { logger } from './logger'; import { checkRepoOnboarded } from './repo-config'; import { ErrorCode, errorResponse, successResponse } from './response'; import { type CreateTaskRequest, isPrTaskType, type TaskRecord, type TaskType, toTaskDetail } from './types'; -import { computeTtlEpoch, DEFAULT_MAX_TURNS, hasTaskSpec, isValidIdempotencyKey, isValidRepo, isValidTaskDescriptionLength, isValidTaskType, MAX_TASK_DESCRIPTION_LENGTH, validateMaxBudgetUsd, validateMaxTurns, validatePrNumber } from './validation'; +import { computeTtlEpoch, hasTaskSpec, isValidIdempotencyKey, isValidRepo, isValidTaskDescriptionLength, isValidTaskType, MAX_TASK_DESCRIPTION_LENGTH, validateMaxBudgetUsd, validateMaxTurns, validatePrNumber } from './validation'; import { TaskStatus } from '../../constructs/task-status'; /** @@ -123,6 +123,14 @@ export async function createTaskCore( } const userMaxBudgetUsd = maxBudgetResult; + // --trace is a strict boolean — reject strings / numbers so a + // misbehaving client can't accidentally enable it with ``"trace": + // "false"`` (which would be truthy). + if (body.trace !== undefined && typeof body.trace !== 'boolean') { + return errorResponse(400, ErrorCode.VALIDATION_ERROR, 'Invalid trace. Must be a boolean.', requestId); + } + const userTrace = body.trace === true; + // 2. Screen task description with Bedrock Guardrail (fail-closed: unscreened content // must not reach the agent — a Bedrock outage blocks task submissions) if (bedrockClient && body.task_description) { @@ -196,6 +204,7 @@ export async function createTaskCore( branch_name: branchName, ...(userMaxTurns !== undefined && { max_turns: userMaxTurns }), ...(userMaxBudgetUsd !== undefined && { max_budget_usd: userMaxBudgetUsd }), + ...(userTrace && { trace: true }), ...(context.idempotencyKey && { idempotency_key: context.idempotencyKey }), channel_source: context.channelSource, channel_metadata: context.channelMetadata, @@ -253,9 +262,17 @@ export async function createTaskCore( InvocationType: 'Event', Payload: new TextEncoder().encode(JSON.stringify({ task_id: taskId })), })); - logger.info('Orchestrator invoked', { task_id: taskId }); + logger.info('Orchestrator invoked', { + event: 'task.admitted.orchestrator_invoked', + task_id: taskId, + request_id: requestId, + }); } catch (orchErr) { - logger.error('Failed to invoke orchestrator', { error: String(orchErr), task_id: taskId }); + logger.error('Failed to invoke orchestrator', { + event: 'task.admitted.orchestrator_invoke_failed', + error: String(orchErr), + task_id: taskId, + }); } } diff --git a/cdk/src/handlers/shared/error-classifier.ts b/cdk/src/handlers/shared/error-classifier.ts index 838290e..16908b7 100644 --- a/cdk/src/handlers/shared/error-classifier.ts +++ b/cdk/src/handlers/shared/error-classifier.ts @@ -201,6 +201,43 @@ const PATTERNS: readonly ErrorPattern[] = [ retryable: true, }, }, + // Specific agent_status classifiers — ordered BEFORE the generic + // ``Task did not succeed.*agent_status=`` catch-all so the concrete + // cap / runtime-error signals surface to users rather than the + // opaque "Agent task did not succeed" title. Each matches the + // ``agent_status`` literals emitted by ``agent/src/pipeline.py`` + // (see ``_resolve_overall_task_status``) and + // ``agent/src/runner.py``. + { + pattern: /agent_status=['"]?error_max_turns['"]?/i, + classification: { + category: ErrorCategory.TIMEOUT, + title: 'Exceeded max turns', + description: 'The agent reached the configured ``max_turns`` limit before completing.', + remedy: 'Raise ``--max-turns`` on the submit call, simplify the task, or break it into smaller sub-tasks.', + retryable: true, + }, + }, + { + pattern: /agent_status=['"]?error_max_budget_usd['"]?/i, + classification: { + category: ErrorCategory.TIMEOUT, + title: 'Exceeded max budget', + description: 'The agent reached the configured ``max_budget_usd`` limit before completing.', + remedy: 'Raise ``--max-budget`` on the submit call, simplify the task, or break it into smaller sub-tasks.', + retryable: true, + }, + }, + { + pattern: /agent_status=['"]?error_during_execution['"]?/i, + classification: { + category: ErrorCategory.AGENT, + title: 'Agent errored during execution', + description: 'The agent raised an uncaught error mid-turn. The Claude Agent SDK reported the task as failed before a clean terminal.', + remedy: 'Retry the task. If persistent, check the agent container logs and the PR branch for partial state.', + retryable: true, + }, + }, { pattern: /Task did not succeed.*agent_status=/i, classification: { diff --git a/cdk/src/handlers/shared/github-comment.ts b/cdk/src/handlers/shared/github-comment.ts new file mode 100644 index 0000000..7b42a3b --- /dev/null +++ b/cdk/src/handlers/shared/github-comment.ts @@ -0,0 +1,428 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * GitHub issue-comment edit-in-place helper (design §6.4). + * + * The fan-out plane maintains a single GitHub comment per task, edited + * in place as the agent progresses through terminal states + pr_created + * by default. Concurrency is handled entirely upstream: DDB Streams on + * ``TaskEventsTable`` with ``ParallelizationFactor: 1`` guarantee + * per-task ordering, and the fanout Lambda is the only writer on its + * own comment. A second writer cannot race us, so last-writer-wins is + * safe — there is no concurrent edit to lose. + * + * (An earlier revision used GitHub's ``If-Match`` / ETag for optimistic + * concurrency. That approach was abandoned after in-account validation + * proved GitHub's REST API does not support ``If-Match`` on + * ``PATCH /issues/comments/{id}``: every conditional PATCH returns + * HTTP 400 with + * ``"Conditional request headers are not allowed in unsafe requests + * unless supported by the endpoint"``. The ETag returned on GET is a + * cache validator only. See PR #52 Scenario 7-extended deploy + * validation.) + * + * The 404 fallback path is preserved: if the target comment was + * deleted upstream (e.g. a user cleaned up the PR thread), we POST a + * fresh one rather than losing the task's status surface. + * + * Raw ``fetch`` is used rather than octokit to match the existing + * codebase pattern (``preflight.ts``, ``context-hydration.ts``). + */ + +import { logger } from './logger'; +import { coerceNumericOrNull } from './numeric'; + +/** GitHub REST v3 media type — required on writes for stable behavior. */ +const GITHUB_ACCEPT = 'application/vnd.github.v3+json'; + +/** Per-request timeout. GitHub's API is usually sub-second; 5 s is a + * generous ceiling for edge cases like region failover. */ +const GITHUB_TIMEOUT_MS = 5_000; + +/** User-Agent required by the GitHub API on all writes. */ +const USER_AGENT = 'abca-fanout/1.0'; + +/** Rate-limit WARN threshold. GitHub's authenticated limit is 5000 req/h. + * Below 500 remaining we're within 10% of the 403 window — ops should + * see a signal well before the next-poll-storm exhausts the budget. */ +const RATE_LIMIT_WARN_THRESHOLD = 500; + +/** + * Inspect ``X-RateLimit-Remaining`` on every GitHub response and emit a + * WARN when the budget falls below ``RATE_LIMIT_WARN_THRESHOLD`` (L3 + * item 4). Does NOT block the request — the goal is an ops-visible + * trail leading up to the 403 that would otherwise arrive unannounced. + * + * Called from both the POST and PATCH paths so partial-burst scenarios + * (e.g. a reconciliation wave patching every comment) surface early. + */ +function logRateLimitIfLow(response: Response, repo: string): void { + const remainingHeader = response.headers.get('x-ratelimit-remaining'); + if (remainingHeader === null) return; + const remaining = Number(remainingHeader); + if (!Number.isFinite(remaining) || remaining >= RATE_LIMIT_WARN_THRESHOLD) { + return; + } + logger.warn('GitHub rate limit low', { + event: 'github.rate_limit_low', + remaining, + reset_at: response.headers.get('x-ratelimit-reset') ?? undefined, + repo, + }); +} + +/** Result of a comment upsert. ``created`` distinguishes the initial + * POST from subsequent PATCHes so the caller can gate the TaskRecord + * UpdateItem (first call persists the comment_id; later calls are + * no-ops on the TaskRecord since we no longer track an ETag). */ +export interface UpsertCommentResult { + readonly commentId: number; + readonly created: boolean; +} + +/** Minimal shape of the GitHub issue-comment API response. */ +interface GitHubCommentResponse { + readonly id: number; + readonly body: string; +} + +/** Error that escapes this module. All HTTP errors funnel through here + * so the caller can log once and continue without introducing a new + * exception taxonomy. */ +export class GitHubCommentError extends Error { + readonly httpStatus: number | undefined; + constructor(message: string, httpStatus?: number) { + super(message); + this.name = 'GitHubCommentError'; + this.httpStatus = httpStatus; + } +} + +/** + * Create or update the single in-place comment for a task. + * + * Flow: + * - If ``existingCommentId`` is undefined, POST a new comment and + * return its id. + * - Otherwise PATCH the existing comment directly (one GitHub call + * per event, no GET round-trip). + * - On 404, treat the comment as deleted upstream and POST a fresh + * one. + * + * All errors are thrown as ``GitHubCommentError`` — the caller is + * expected to ``try/catch`` and log rather than propagating. + */ +export async function upsertTaskComment(params: { + repo: string; + issueOrPrNumber: number; + body: string; + token: string; + existingCommentId: number | undefined; +}): Promise { + const { repo, issueOrPrNumber, body, token, existingCommentId } = params; + + if (existingCommentId === undefined) { + return createComment({ repo, issueOrPrNumber, body, token }); + } + + try { + return await patchExistingComment({ + repo, + commentId: existingCommentId, + body, + token, + }); + } catch (err) { + if (err instanceof GitHubCommentError && err.httpStatus === 404) { + // Upstream deletion — fall back to POSTing a fresh comment. + logger.warn('[github-comment] previous comment deleted upstream, re-creating', { + event: 'github.comment.recreated', + repo, + comment_id: existingCommentId, + }); + return createComment({ repo, issueOrPrNumber, body, token }); + } + throw err; + } +} + +/** + * PATCH an existing comment with the given body. One GitHub call per + * event — no GET round-trip, no conditional headers (see file-level + * rationale above). 404 propagates so the caller can POST-fallback. + */ +async function patchExistingComment(params: { + repo: string; + commentId: number; + body: string; + token: string; +}): Promise { + const { repo, commentId, body, token } = params; + const url = `https://api.github.com/repos/${repo}/issues/comments/${commentId}`; + + let res: Response; + try { + res = await fetch(url, { + method: 'PATCH', + headers: { + 'Authorization': `token ${token}`, + 'Accept': GITHUB_ACCEPT, + 'User-Agent': USER_AGENT, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ body }), + signal: AbortSignal.timeout(GITHUB_TIMEOUT_MS), + }); + } catch (err) { + throw new GitHubCommentError( + `PATCH /repos/${repo}/issues/comments/${commentId} network error: ${String(err)}`, + ); + } + + logRateLimitIfLow(res, repo); + if (!res.ok) { + throw new GitHubCommentError( + `PATCH /repos/${repo}/issues/comments/${commentId} failed: HTTP ${res.status}`, + res.status, + ); + } + + return { commentId, created: false }; +} + +async function createComment(params: { + repo: string; + issueOrPrNumber: number; + body: string; + token: string; +}): Promise { + const { repo, issueOrPrNumber, body, token } = params; + const url = `https://api.github.com/repos/${repo}/issues/${issueOrPrNumber}/comments`; + + let res: Response; + try { + res = await fetch(url, { + method: 'POST', + headers: { + 'Authorization': `token ${token}`, + 'Accept': GITHUB_ACCEPT, + 'User-Agent': USER_AGENT, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ body }), + signal: AbortSignal.timeout(GITHUB_TIMEOUT_MS), + }); + } catch (err) { + throw new GitHubCommentError( + `POST /repos/${repo}/issues/${issueOrPrNumber}/comments network error: ${String(err)}`, + ); + } + + logRateLimitIfLow(res, repo); + if (!res.ok) { + throw new GitHubCommentError( + `POST /repos/${repo}/issues/${issueOrPrNumber}/comments failed: HTTP ${res.status}`, + res.status, + ); + } + + let payload: GitHubCommentResponse; + try { + payload = (await res.json()) as GitHubCommentResponse; + } catch { + throw new GitHubCommentError( + `POST comment response was not JSON (repo=${repo}, issue=${issueOrPrNumber})`, + ); + } + if (typeof payload.id !== 'number') { + throw new GitHubCommentError( + `POST comment response missing numeric id (repo=${repo}, issue=${issueOrPrNumber})`, + ); + } + return { commentId: payload.id, created: true }; +} + +// --------------------------------------------------------------------------- +// Body rendering +// --------------------------------------------------------------------------- + +/** Hidden HTML marker prefix that tags every bgagent-owned comment so + * a future reconciliation tool, user grep, or rehydration path can + * identify the in-place comment in a long PR thread. Exported so + * downstream callers (Chunk K forensics, Phase 3 audit trail, etc.) + * don't have to re-invent the regex. */ +export const BGAGENT_COMMENT_MARKER_PREFIX = 'bgagent:task-id='; + +/** GitHub issue-comment body hard cap is 65 536 UTF-16 code units. We + * leave 5 KB of headroom for the truncation marker and for rough + * utf-8-vs-utf-16 skew. Any body exceeding this is truncated at + * ``renderCommentBody`` time rather than failing the PATCH with 422. */ +const MAX_COMMENT_BODY_CHARS = 60_000; + +/** Sanitize a server-sourced event type for inclusion in a Markdown + * table cell. Strips backticks, pipes, and newlines that would break + * the table layout. Event types today are enum-like (snake_case), so + * this is defensive against future writers emitting freer-form + * values, not a live vulnerability. */ +function sanitizeEventType(eventType: string): string { + return eventType.replace(/[`|\r\n]/g, ''); +} + +/** + * Sanitize a ``prUrl`` before interpolating it into a Markdown link + * target (``[link]()``). Without this guard a crafted URL could + * break the table layout or inject trailing Markdown: ``)`` closes the + * link prematurely, ``|`` starts a new table column, and CR / LF / ``]`` + * / ``"`` each let an attacker extend the comment body past the link. + * + * Strategy: strip the six characters that are meaningful inside + * ``[text](target)`` Markdown AND reject anything that does not parse + * as an ``http://`` or ``https://`` URL after the strip. The strip is + * deliberately conservative — a legitimate GitHub PR URL + * (``https://github.com/owner/repo/pull/42``) never contains any of the + * listed characters, so false positives are effectively zero. Any URL + * that fails validation returns ``null`` and the caller must omit the + * Pull-request row entirely rather than risk a broken-layout comment. + * + * Refs: PR #52 krokoko code review finding #12 (Markdown injection + * possible via ``prUrl`` in GitHub comment body). + */ +export function sanitizeMarkdownLinkTarget(url: string | null | undefined): string | null { + if (url === null || url === undefined) return null; + // Reject anything carrying a Markdown-significant character. We do + // NOT attempt to URL-encode these — encoded ``)`` (``%29``) would + // render correctly, but encoding opens a harder surface to reason + // about (e.g. an attacker who gets ``%0A`` past this function could + // still break the table on some Markdown renderers). A flat reject + // keeps the contract simple and the comment row trustworthy. + if (/[\r\n\t\s)|\]"<>`]/.test(url)) return null; + // Validate the parsed URL is http(s). A ``javascript:`` link target + // is also attacker-controlled content and has no place in a status + // comment. ``new URL`` throws on malformed input. + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; + } + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') return null; + return url; +} + +function bgagentMarker(taskId: string): string { + return ``; +} + +/** + * A compact terminal-friendly summary the GitHub comment displays as + * the task progresses. Kept small on purpose — GitHub truncates long + * comments in mobile / email notifications and the PR activity log + * accumulates the full history anyway. + * + * The numeric fields accept ``string`` alongside ``number | null`` to + * honestly model the DynamoDB Document-client boundary: ``Number`` + * attributes deserialize as JS ``string`` in some code paths (see + * ``shared/numeric.ts``). ``renderCommentBody`` coerces defensively so + * a future caller that forgets to run the shared coercion helper at + * its own boundary does not crash the way the fanout dispatcher did in + * Scenario 7-ext (``costUsd.toFixed is not a function``, commit + * ``9fe704e``). + */ +export interface CommentBodyInput { + readonly taskId: string; + readonly status: string; + readonly repo: string; + readonly latestEventType: string; + readonly latestEventAt: string; + readonly prUrl: string | null; + readonly durationS: number | string | null; + readonly costUsd: number | string | null; +} + +/** + * Render the Markdown body for the in-place comment. Pure: no timing, + * no network — callers can snapshot-test the exact bytes. The only + * side effect is a ``logger.warn`` via ``coerceNumericOrNull`` if a + * numeric field arrives with a non-finite value (e.g. ``'NaN'``), which + * surfaces upstream writer bugs instead of silently dropping the row. + * + * Defense-in-depth vs. the caller's coercion (finding #9): + * - Callers SHOULD coerce DDB numerics at their own boundary using + * ``coerceNumericOrNull`` so the warn log carries their context + * (task_id, event_id). The fanout dispatcher does this today. + * - ``renderCommentBody`` coerces again internally so a future caller + * that forgets the boundary step (e.g. a Chunk K reconciler + * reading raw DDB items) still degrades to a null-omitted row + * instead of throwing ``TypeError: .toFixed is not a function``. + * Non-finite values (``NaN``, ``Infinity``) collapse to null and + * omit the row; finite values (including parseable strings) render + * normally. + * + * Markdown-link target sanitization (finding #12): + * - ``prUrl`` is interpolated into a Markdown link (``[link]()``). + * Without sanitization a crafted URL containing ``)`` / ``|`` / CR + * / LF could break the table layout or inject trailing content. + * ``sanitizeMarkdownLinkTarget`` strips the injection surface and + * validates the URL is http(s); a rejected URL omits the row + * rather than rendering a broken or misleading link. + */ +export function renderCommentBody(input: CommentBodyInput): string { + // Coerce DDB-string numerics defensively — see doc block above. The + // context object gives the ``numeric.coercion_failed`` warn enough + // breadcrumbs (field + task_id) to trace back to the upstream writer. + const durationS = coerceNumericOrNull( + input.durationS, + { field: 'duration_s', task_id: input.taskId }, + logger, + ); + const costUsd = coerceNumericOrNull( + input.costUsd, + { field: 'cost_usd', task_id: input.taskId }, + logger, + ); + // Sanitize the PR link target before interpolation. A rejected URL + // returns null and the row is omitted. + const safePrUrl = sanitizeMarkdownLinkTarget(input.prUrl); + + const lines: string[] = []; + lines.push(bgagentMarker(input.taskId)); + lines.push(`### Background agent — ${input.status}`); + lines.push(''); + lines.push('| Field | Value |'); + lines.push('|-------|-------|'); + lines.push(`| Task | \`${input.taskId}\` |`); + lines.push(`| Repo | \`${input.repo}\` |`); + lines.push(`| Status | **${input.status}** |`); + lines.push(`| Last event | \`${sanitizeEventType(input.latestEventType)}\` @ ${input.latestEventAt} |`); + if (safePrUrl) { + lines.push(`| Pull request | [link](${safePrUrl}) |`); + } + if (durationS !== null) { + lines.push(`| Duration | ${durationS}s |`); + } + if (costUsd !== null) { + lines.push(`| Cost | $${costUsd.toFixed(4)} |`); + } + const rendered = lines.join('\n'); + if (rendered.length <= MAX_COMMENT_BODY_CHARS) return rendered; + // Truncate mid-body with a visible marker so the GitHub API accepts + // the edit and a human inspecting the PR sees that data was lost. + return rendered.slice(0, MAX_COMMENT_BODY_CHARS) + '\n\n…(truncated — body exceeded 60 000 chars)'; +} diff --git a/cdk/src/handlers/shared/numeric.ts b/cdk/src/handlers/shared/numeric.ts new file mode 100644 index 0000000..c7b242e --- /dev/null +++ b/cdk/src/handlers/shared/numeric.ts @@ -0,0 +1,88 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Numeric coercion helpers for the DynamoDB / Document-client boundary. + * + * Rationale: the AWS SDK v3 Document client deserializes DynamoDB + * ``Number`` attributes as JavaScript ``string``s in some code paths + * (notably ``TaskRecord.{duration_s,cost_usd}`` from ``TaskTable``), + * even though our TypeScript types declare them as ``number | null``. + * Callers that go on to call ``.toFixed()`` / do arithmetic on those + * fields silently fail with ``TypeError: input.costUsd.toFixed is not + * a function`` (Scenario 7-extended deploy validation uncovered this + * in the GitHub fan-out dispatcher). + * + * Use these helpers at any boundary that passes those fields on to + * numeric consumers. Adding a third call site is a signal that the + * underlying type declaration should be widened (``number | string | + * null``) at the DDB load boundary; for now, coerce locally. + */ + +export interface CoerceContext { + readonly field: string; + readonly task_id?: string; + readonly event_id?: string; +} + +/** Minimal logger shape used by the coercion helper. Structural so the + * shared ``logger`` export and test mocks both satisfy it without an + * explicit interface import. */ +export interface CoerceLogger { + warn(message: string, meta?: Record): void; +} + +/** + * Coerce a value that should be a number but may arrive as a string + * (DynamoDB Document-client deserializes ``Number`` attributes as + * strings) into a finite ``number`` or ``null``. + * + * Rules: + * - ``null`` / ``undefined`` / empty-string → ``null`` (treated as + * "absent"; no warn). + * - Finite number (either a real ``number`` or a parseable string) + * → that number. + * - Non-finite coercion (``NaN``, ``Infinity``) → ``null`` AND emits + * a warn via the provided logger so writer bugs surface in + * CloudWatch rather than silently dropping the consumer's render. + * + * The logger argument keeps this helper free of a direct import of + * ``./logger`` so the same shape is usable from tests without a full + * mock. + */ +export function coerceNumericOrNull( + value: number | string | null | undefined, + context: CoerceContext, + logger: CoerceLogger, +): number | null { + if (value === null || value === undefined) return null; + if (typeof value === 'string' && value.length === 0) return null; + const n = typeof value === 'number' ? value : Number(value); + if (!Number.isFinite(n)) { + logger.warn('[numeric] non-finite coercion — dropping field', { + event: 'numeric.coercion_failed', + field: context.field, + raw: String(value), + task_id: context.task_id, + event_id: context.event_id, + }); + return null; + } + return n; +} diff --git a/cdk/src/handlers/shared/orchestrator.ts b/cdk/src/handlers/shared/orchestrator.ts index 98befe9..c6a0bf8 100644 --- a/cdk/src/handlers/shared/orchestrator.ts +++ b/cdk/src/handlers/shared/orchestrator.ts @@ -23,6 +23,7 @@ import { ulid } from 'ulid'; import { hydrateContext } from './context-hydration'; import { logger } from './logger'; import { writeMinimalEpisode } from './memory'; +import { coerceNumericOrNull } from './numeric'; import { computePromptVersion } from './prompt-version'; import { loadRepoConfig, type BlueprintConfig, type ComputeType } from './repo-config'; import type { TaskRecord } from './types'; @@ -324,6 +325,14 @@ export async function hydrateAndTransition(task: TaskRecord, blueprintConfig?: B const payload: Record = { repo_url: task.repo, task_id: task.task_id, + // user_id is required by the agent ONLY when ``trace`` is true — + // the agent writes the trajectory to + // ``traces//.jsonl.gz`` (design §10.1) and the + // handler's per-caller-prefix guard relies on the agent landing + // under the submitter's prefix. Threaded unconditionally so + // scripts that inspect the payload can always see it; costs one + // Cognito-sub-sized string in the JSON. + user_id: task.user_id, branch_name: hydratedContext.resolved_branch_name ?? task.branch_name, ...(task.issue_number !== undefined && { issue_number: String(task.issue_number) }), task_type: task.task_type ?? 'new_task', @@ -332,6 +341,10 @@ export async function hydrateAndTransition(task: TaskRecord, blueprintConfig?: B ...(task.task_description && { prompt: task.task_description }), max_turns: task.max_turns ?? blueprintConfig?.max_turns ?? DEFAULT_MAX_TURNS, ...(effectiveBudget !== undefined && { max_budget_usd: effectiveBudget }), + // Only include when true so the agent's ``inp.get("trace", False)`` + // default semantics remain the no-op path. Keeps the wire payload + // slim for the common non-trace case. + ...(task.trace === true && { trace: true }), ...(blueprintConfig?.model_id && { model_id: blueprintConfig.model_id }), ...(blueprintConfig?.system_prompt_overrides && { system_prompt_overrides: blueprintConfig.system_prompt_overrides }), ...(blueprintConfig?.cedar_policies && blueprintConfig.cedar_policies.length > 0 && { cedar_policies: blueprintConfig.cedar_policies }), @@ -507,13 +520,29 @@ export async function finalizeTask( if (MEMORY_ID && !task.memory_written) { logger.info('Agent did not write memory — writing fallback episode', { task_id: taskId }); try { + // Coerce at the shared helper rather than ``Number(...)`` so a + // corrupt string ``cost_usd`` from the DDB Document client + // collapses to ``undefined`` (and logs a warn) rather than + // rendering ``Cost: $NaN.`` into the episode text — see + // ``memory.ts::writeMinimalEpisode`` line 325 which calls + // ``.toFixed(4)`` on the value. + const durationS = coerceNumericOrNull( + task.duration_s, + { field: 'duration_s', task_id: taskId }, + logger, + ); + const costUsd = coerceNumericOrNull( + task.cost_usd, + { field: 'cost_usd', task_id: taskId }, + logger, + ); const written = await writeMinimalEpisode( MEMORY_ID, task.repo, taskId, currentStatus, - task.duration_s !== undefined ? Number(task.duration_s) : undefined, - task.cost_usd !== undefined ? Number(task.cost_usd) : undefined, + durationS ?? undefined, + costUsd ?? undefined, ); if (!written) { logger.warn('Fallback episode write returned false', { task_id: taskId }); diff --git a/cdk/src/handlers/shared/response.ts b/cdk/src/handlers/shared/response.ts index 0e91f01..411ae2b 100644 --- a/cdk/src/handlers/shared/response.ts +++ b/cdk/src/handlers/shared/response.ts @@ -27,12 +27,14 @@ export const ErrorCode = { UNAUTHORIZED: 'UNAUTHORIZED', FORBIDDEN: 'FORBIDDEN', TASK_NOT_FOUND: 'TASK_NOT_FOUND', + TRACE_NOT_AVAILABLE: 'TRACE_NOT_AVAILABLE', DUPLICATE_TASK: 'DUPLICATE_TASK', TASK_ALREADY_TERMINAL: 'TASK_ALREADY_TERMINAL', RATE_LIMIT_EXCEEDED: 'RATE_LIMIT_EXCEEDED', WEBHOOK_NOT_FOUND: 'WEBHOOK_NOT_FOUND', WEBHOOK_ALREADY_REVOKED: 'WEBHOOK_ALREADY_REVOKED', REPO_NOT_ONBOARDED: 'REPO_NOT_ONBOARDED', + SERVICE_UNAVAILABLE: 'SERVICE_UNAVAILABLE', INTERNAL_ERROR: 'INTERNAL_ERROR', } as const; diff --git a/cdk/src/handlers/shared/types.ts b/cdk/src/handlers/shared/types.ts index 231e5cd..094b8ba 100644 --- a/cdk/src/handlers/shared/types.ts +++ b/cdk/src/handlers/shared/types.ts @@ -18,12 +18,25 @@ */ import { classifyError, type ErrorClassification } from './error-classifier'; +import { logger } from './logger'; +import { coerceNumericOrNull } from './numeric'; import type { ComputeType } from './repo-config'; import type { TaskStatusType } from '../../constructs/task-status'; /** Valid task types for task creation. */ export type TaskType = 'new_task' | 'pr_iteration' | 'pr_review'; +/** + * Provenance of a task's submission. ``api`` covers CLI / Cognito-authenticated + * submissions; ``webhook`` covers HMAC-signed inbound webhook submissions. + * + * Narrowed from ``string`` so switches and predicates that read + * ``channel_source`` get exhaustiveness checking at compile time; matches the + * internal ``CreateTaskContext.channelSource`` literal in ``create-task-core.ts``. + * Keep in sync with ``cli/src/types.ts::ChannelSource``. + */ +export type ChannelSource = 'api' | 'webhook'; + /** Task types that operate on an existing pull request. */ export function isPrTaskType(taskType: TaskType): boolean { return taskType === 'pr_iteration' || taskType === 'pr_review'; @@ -51,7 +64,7 @@ export interface TaskRecord { readonly pr_url?: string; readonly error_message?: string; readonly idempotency_key?: string; - readonly channel_source: string; + readonly channel_source: ChannelSource; readonly channel_metadata?: Record; readonly status_created_at: string; readonly created_at: string; @@ -63,11 +76,74 @@ export interface TaskRecord { readonly build_passed?: boolean; readonly max_turns?: number; readonly max_budget_usd?: number; + /** + * Whether the task was submitted with ``--trace`` (design §10.1). + * When true the orchestrator threads a ``trace: true`` flag into the + * agent payload; the agent's ``_ProgressWriter`` raises its preview + * cap from 200 chars to 4 KB so debug captures aren't silently + * clipped. Opt-in per task — not routine observability. + */ + readonly trace?: boolean; + /** + * S3 URI of the gzipped JSONL trajectory dump written by the agent on + * terminal state when ``trace`` is true (design §10.1). Shape: + * ``s3:///traces//.jsonl.gz``. Absent + * until the agent finishes the upload; also absent for tasks that ran + * without ``--trace`` or whose upload failed. The + * ``get-trace-url`` handler reads this to issue presigned download URLs. + */ + readonly trace_s3_uri?: string; + /** Rev-5 DATA-1: authoritative SDK counter including the attempt that + * tripped any cap. Equals the legacy `turns` value. */ + readonly turns_attempted?: number; + /** Rev-5 DATA-1: turns that actually completed (clamped to + * `max_turns` when `agent_status='error_max_turns'`). */ + readonly turns_completed?: number; readonly prompt_version?: string; readonly memory_written?: boolean; readonly compute_type?: ComputeType; readonly compute_metadata?: Record; readonly ttl?: number; + /** + * Optional per-task override for the FanOutConsumer's channel filters + * (design §6.5). When present, the router uses these settings instead + * of the per-channel defaults. Chunk I introduced the type and the + * resolution path; Chunk K adds a submit-time API parameter and the + * DDB read that populates this field — until then it is always + * ``undefined`` at runtime and every task inherits the defaults. + */ + readonly notifications?: TaskNotificationsConfig; + /** + * ID of the single GitHub issue comment the fan-out plane maintains + * for this task (design §6.4 — edit-in-place). Written by the + * GitHub dispatcher on the first delivery; read on subsequent + * deliveries to PATCH instead of POST. Absent until the first + * dispatch fires successfully. + */ + readonly github_comment_id?: number; +} + +/** Per-channel override for one notification channel. See + * ``handlers/fanout-task-events.ts::resolveChannelFilter`` for the + * resolution semantics — explicit ``events`` REPLACE the channel + * default; a ``"default"`` token inside ``events`` expands to the + * default set. */ +export interface ChannelConfig { + /** If false, the channel is opted-out and no events dispatch. */ + readonly enabled?: boolean; + /** Override the subscribed event types. ``["default"]`` resolves to + * the channel default; an explicit list replaces defaults entirely. */ + readonly events?: readonly string[]; +} + +/** Per-task notification overrides (design §6.5). Single source of truth; + * imported by both ``TaskRecord`` (producer side) and + * ``fanout-task-events.ts`` (consumer side) so a Chunk K schema change + * lands in one place and both sides pick it up at compile time. */ +export interface TaskNotificationsConfig { + readonly slack?: ChannelConfig; + readonly email?: ChannelConfig; + readonly github?: ChannelConfig; } /** @@ -87,6 +163,12 @@ export interface TaskDetail { readonly pr_url: string | null; readonly error_message: string | null; readonly error_classification: ErrorClassification | null; + /** Provenance of the task's submission — ``api`` for CLI/Cognito + * submissions, ``webhook`` for HMAC-signed inbound webhooks. Present + * on every task record at creation time (``create-task-core.ts``) + * and surfaced here so CLI / dashboard / audit consumers do not have + * to spelunk CloudWatch to learn which channel created a task. */ + readonly channel_source: ChannelSource; readonly created_at: string; readonly updated_at: string; readonly started_at: string | null; @@ -96,7 +178,23 @@ export interface TaskDetail { readonly build_passed: boolean | null; readonly max_turns: number | null; readonly max_budget_usd: number | null; + /** Rev-5 DATA-1: SDK-attempted turn count (may exceed `max_turns` by 1 + * under `agent_status='error_max_turns'`). */ + readonly turns_attempted: number | null; + /** Rev-5 DATA-1: actually-completed turns, clamped to `max_turns` + * when the cap tripped. */ + readonly turns_completed: number | null; readonly prompt_version: string | null; + /** True when the task was submitted with ``--trace`` — surfaces the + * opt-in state to scripts / CLI consumers without making them + * guess from secondary signals. */ + readonly trace: boolean; + /** S3 URI of the uploaded ``--trace`` trajectory dump, or ``null`` + * until the agent finishes the terminal upload (or for tasks that + * ran without ``--trace``). Non-optional so scripts can rely on + * the field being present; CLI download resolves this via the + * ``get-trace-url`` handler rather than hitting S3 directly. */ + readonly trace_s3_uri: string | null; } /** @@ -128,8 +226,40 @@ export interface EventRecord { readonly ttl?: number; } +/** + * Query parameters accepted by ``GET /v1/tasks/{task_id}/events``. + * + * Pagination is mutually exclusive: prefer ``after`` (a ULID event_id cursor + * used by CLI polling and webhook replay to resume from a known event id) + * over ``next_token`` (an opaque DynamoDB pagination token). If both are + * provided, the handler uses ``after`` and logs a WARN. Neither is required + * — callers may start from the beginning of the task's event stream. + * + * When a page is truncated at ``limit``, the response includes a + * ``next_token`` so the caller can continue paginating forward regardless + * of which mode they started with. + * + * Keep in sync with ``cli/src/types.ts``. + */ +export interface GetTaskEventsQuery { + readonly limit?: number; + readonly next_token?: string; + /** ULID event_id cursor. Returns events with ``event_id > after``. */ + readonly after?: string; + /** + * When truthy (``"1"`` or ``"true"``), return events in descending + * ``event_id`` order (newest first). Used by ``bgagent status`` to + * render a recency-biased snapshot without walking the full event + * stream. Mutually exclusive with ``after`` — the handler rejects + * the combination with 400. + */ + readonly desc?: string; +} + /** * Create task request body. + * + * Keep in sync with ``cli/src/types.ts``. */ export interface CreateTaskRequest { readonly repo: string; @@ -140,6 +270,8 @@ export interface CreateTaskRequest { readonly task_type?: TaskType; readonly pr_number?: number; readonly attachments?: Attachment[]; + /** Enable 4 KB debug previews (design §10.1, opt-in per task). */ + readonly trace?: boolean; } /** @@ -155,10 +287,21 @@ export interface Attachment { /** * Map a DynamoDB task record to the API detail response shape. + * + * All numeric fields sourced from the DDB record are routed through + * ``coerceNumericOrNull`` — the Document-client deserializes DynamoDB + * ``Number`` attributes as JavaScript ``string``s in some code paths + * (see ``shared/numeric.ts`` for rationale), and any downstream caller + * doing arithmetic (``.toFixed``, comparison, math) on a string-typed + * "number" crashes at runtime. Coercing uniformly here means no caller + * has to guess which TaskDetail numeric fields are safe — do not bypass + * the helper when adding new numeric fields. + * * @param record - the DynamoDB task record. * @returns the API-facing task detail. */ export function toTaskDetail(record: TaskRecord): TaskDetail { + const ctx = { task_id: record.task_id }; return { task_id: record.task_id, status: record.status, @@ -172,19 +315,84 @@ export function toTaskDetail(record: TaskRecord): TaskDetail { pr_url: record.pr_url ?? null, error_message: record.error_message ?? null, error_classification: classifyError(record.error_message), + channel_source: record.channel_source, created_at: record.created_at, updated_at: record.updated_at, started_at: record.started_at ?? null, completed_at: record.completed_at ?? null, - duration_s: record.duration_s ?? null, - cost_usd: record.cost_usd ?? null, + duration_s: coerceNumericOrNull(record.duration_s, { ...ctx, field: 'duration_s' }, logger), + cost_usd: coerceNumericOrNull(record.cost_usd, { ...ctx, field: 'cost_usd' }, logger), build_passed: record.build_passed ?? null, - max_turns: record.max_turns ?? null, - max_budget_usd: record.max_budget_usd ?? null, + max_turns: coerceNumericOrNull(record.max_turns, { ...ctx, field: 'max_turns' }, logger), + max_budget_usd: coerceNumericOrNull(record.max_budget_usd, { ...ctx, field: 'max_budget_usd' }, logger), + turns_attempted: coerceNumericOrNull(record.turns_attempted, { ...ctx, field: 'turns_attempted' }, logger), + turns_completed: coerceNumericOrNull(record.turns_completed, { ...ctx, field: 'turns_completed' }, logger), prompt_version: record.prompt_version ?? null, + trace: record.trace === true, + trace_s3_uri: record.trace_s3_uri ?? null, }; } +/** + * Maximum length (in characters, after trim) of a nudge ``message``. + * + * Mirrored in ``cli/src/types.ts`` as ``NUDGE_MAX_MESSAGE_LENGTH`` and + * consumed both client-side (for fail-fast rejection without a round-trip) + * and server-side (in ``cdk/src/handlers/nudge-task.ts``). + */ +export const NUDGE_MAX_MESSAGE_LENGTH = 2000; + +/** + * Nudge request body for POST /v1/tasks/{task_id}/nudge (Phase 2). + * + * A nudge is a short, between-turns steering message from the user to a + * running agent. It is written to `TaskNudgesTable` after guardrail + * screening + rate limiting, then picked up by the agent's nudge_reader + * at the next between-turns seam and injected as an authoritative + * `` XML block. + * + * Keep in sync with `cli/src/types.ts`. + */ +export interface NudgeRequest { + /** Free-text steering message. Max 2000 chars after trim; guardrail-screened. */ + readonly message: string; +} + +/** + * Nudge response body. Returned with HTTP 202 Accepted — the nudge has + * been persisted but has not yet reached the agent; it will be injected + * at the next between-turns seam. Callers wanting confirmation that the + * agent saw the nudge should watch task events for `nudge_consumed`. + */ +export interface NudgeResponse { + readonly task_id: string; + readonly nudge_id: string; + readonly submitted_at: string; +} + +/** + * Full nudge record as stored in `TaskNudgesTable`. + * + * - PK = `task_id` (groups all nudges for a task together) + * - SK = `nudge_id` (ULID — lexicographic sort == chronological sort) + * + * The agent-side reader queries by `task_id` with `consumed = false` + * filter, orders by `nudge_id` (implicit sort-key order), and marks + * each consumed nudge with an atomic conditional UpdateItem + * (ConditionExpression: `consumed = :false`) for idempotency across + * restarts mid-consume. + */ +export interface NudgeRecord { + readonly task_id: string; + readonly nudge_id: string; + readonly user_id: string; + readonly message: string; + readonly created_at: string; + readonly consumed: boolean; + readonly consumed_at?: string; + readonly ttl?: number; +} + /** * Full webhook record as stored in DynamoDB. */ diff --git a/cdk/src/handlers/shared/validation.ts b/cdk/src/handlers/shared/validation.ts index 356c399..11398c5 100644 --- a/cdk/src/handlers/shared/validation.ts +++ b/cdk/src/handlers/shared/validation.ts @@ -17,7 +17,7 @@ * SOFTWARE. */ -import { isPrTaskType, type CreateTaskRequest, type TaskType } from './types'; +import { type CreateTaskRequest, type TaskType } from './types'; import { TaskStatus } from '../../constructs/task-status'; /** Default maximum agent turns per task. */ @@ -36,6 +36,9 @@ export const MAX_MAX_BUDGET_USD = 100; const REPO_PATTERN = /^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/; const IDEMPOTENCY_KEY_PATTERN = /^[a-zA-Z0-9_-]{1,128}$/; const WEBHOOK_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9 _-]{0,62}[a-zA-Z0-9]$/; +// ULID format: 26 chars, Crockford Base32 alphabet (0-9, A-Z excluding I, L, O, U). +// Matches the ``_generate_ulid`` output in ``agent/src/progress_writer.py``. +const ULID_PATTERN = /^[0-9A-HJKMNP-TV-Z]{26}$/; const ALL_STATUSES = new Set(Object.values(TaskStatus)); /** @@ -139,6 +142,20 @@ export function encodePaginationToken(lastKey: Record | undefin return Buffer.from(JSON.stringify(lastKey)).toString('base64'); } +/** + * Validate a ULID string (26-char Crockford Base32, case-insensitive). + * ULIDs are lexicographically sortable by timestamp prefix, so string comparison + * on valid ULIDs behaves correctly for "events after this id" queries. The + * canonical alphabet excludes the letters I, L, O, and U to avoid visual + * ambiguity — we accept upper- or lower-case callers by uppercasing first. + * @param value - the candidate ULID string. + * @returns true if the value matches the ULID shape. + */ +export function isValidUlid(value: string): boolean { + if (typeof value !== 'string' || value.length !== 26) return false; + return ULID_PATTERN.test(value.toUpperCase()); +} + /** * Validate a webhook name (2-64 characters, alphanumeric, spaces, hyphens, underscores). * Must start and end with an alphanumeric character. diff --git a/cdk/src/stacks/agent.ts b/cdk/src/stacks/agent.ts index da99401..98a3be1 100644 --- a/cdk/src/stacks/agent.ts +++ b/cdk/src/stacks/agent.ts @@ -21,7 +21,7 @@ import * as path from 'path'; import * as agentcore from '@aws-cdk/aws-bedrock-agentcore-alpha'; import * as bedrock from '@aws-cdk/aws-bedrock-alpha'; import * as agentcoremixins from '@aws-cdk/mixins-preview/aws-bedrockagentcore'; -import { Stack, StackProps, RemovalPolicy, CfnOutput, CfnResource } from 'aws-cdk-lib'; +import { Stack, StackProps, RemovalPolicy, CfnOutput, CfnResource, Duration, Lazy } from 'aws-cdk-lib'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; // ecr_assets import is only needed when the ECS block below is uncommented // import * as ecr_assets from 'aws-cdk-lib/aws-ecr-assets'; @@ -36,13 +36,17 @@ import { AgentVpc } from '../constructs/agent-vpc'; import { Blueprint } from '../constructs/blueprint'; import { ConcurrencyReconciler } from '../constructs/concurrency-reconciler'; import { DnsFirewall } from '../constructs/dns-firewall'; -// import { EcsAgentCluster } from '../constructs/ecs-agent-cluster'; +import { FanOutConsumer } from '../constructs/fanout-consumer'; import { RepoTable } from '../constructs/repo-table'; +import { StrandedTaskReconciler } from '../constructs/stranded-task-reconciler'; +// import { EcsAgentCluster } from '../constructs/ecs-agent-cluster'; import { TaskApi } from '../constructs/task-api'; import { TaskDashboard } from '../constructs/task-dashboard'; import { TaskEventsTable } from '../constructs/task-events-table'; +import { TaskNudgesTable } from '../constructs/task-nudges-table'; import { TaskOrchestrator } from '../constructs/task-orchestrator'; import { TaskTable } from '../constructs/task-table'; +import { TraceArtifactsBucket } from '../constructs/trace-artifacts-bucket'; import { UserConcurrencyTable } from '../constructs/user-concurrency-table'; import { WebhookTable } from '../constructs/webhook-table'; @@ -57,10 +61,38 @@ export class AgentStack extends Stack { // Task state persistence const taskTable = new TaskTable(this, 'TaskTable'); const taskEventsTable = new TaskEventsTable(this, 'TaskEventsTable'); + const taskNudgesTable = new TaskNudgesTable(this, 'TaskNudgesTable'); const userConcurrencyTable = new UserConcurrencyTable(this, 'UserConcurrencyTable'); const webhookTable = new WebhookTable(this, 'WebhookTable'); const repoTable = new RepoTable(this, 'RepoTable'); + // --trace trajectory storage (design §10.1). Opt-in per task; only + // written when the submit payload sets ``trace: true``. + const traceArtifactsBucket = new TraceArtifactsBucket(this, 'TraceArtifactsBucket'); + + // Server access logging intentionally disabled. Rationale: + // - writes: only the agent runtime IAM role (``grantPut`` below). + // - reads: only via short-lived presigned URL issued by + // ``get-trace-url`` after a Cognito auth check + ownership + // check against the TaskRecord. + // - 7-day object TTL bounds blast radius. + // - adding a log bucket would double S3 footprint for a debug-only + // feature users explicitly opt into with ``--trace``. + // Note: default CloudTrail does NOT capture S3 object-level + // events (PutObject / GetObject via presigned URL), so there is + // intentionally no object-level audit trail for this bucket. That + // is an accepted trade-off for a sample-project debug feature — + // the cost/complexity of CloudTrail data events or a log bucket + // is not justified for opt-in ``--trace`` usage. If a future + // requirement needs audit, the right fix is a CloudTrail data + // event selector on this bucket, not server access logs. + NagSuppressions.addResourceSuppressions(traceArtifactsBucket.bucket, [ + { + id: 'AwsSolutions-S1', + reason: 'Debug-only artifacts (design §10.1) with 7-day TTL; writes confined to runtime IAM role by grantPut; reads only via short-lived presigned URLs from an authn\'d handler. Object-level audit intentionally omitted — cost/complexity of CloudTrail data events or a log bucket is not justified for opt-in --trace usage.', + }, + ]); + // --- Repository onboarding --- const agentPluginsBlueprint = new Blueprint(this, 'AgentPluginsBlueprint', { repo: 'krokoko/agent-plugins', @@ -126,46 +158,142 @@ export class AgentStack extends Stack { // --- AgentCore Memory (cross-task learning) --- const agentMemory = new AgentMemory(this, 'AgentMemory'); + // --- Bedrock Guardrail for prompt injection detection --- + // (Declared early so TaskApi — constructed before the runtimes — can reference it.) + const inputGuardrail = new bedrock.Guardrail(this, 'InputGuardrail', { + guardrailName: 'task-input-guardrail', + description: 'Screens task submissions for prompt injection attacks', + contentFilters: [ + { + type: bedrock.ContentFilterType.PROMPT_ATTACK, + // MEDIUM blocks on MEDIUM+HIGH confidence; LOW-confidence + // detections are ignored. Observed during PR #52 Scenario + // 7-extended deploy validation: at HIGH (blocks LOW too) the + // PROMPT_ATTACK classifier is stochastic at the LOW tier and + // flags ordinary imperative-mood task descriptions and + // ordinary PR bodies (pr_iteration hydration). MEDIUM matches + // the Bedrock documentation's default for non-adversarial + // user input. The previous threshold blocked legitimate + // natural-language submissions (e.g. "Make no changes, just + // inspect README.md and finish.", "enumerate every plugin in + // extreme detail") and legitimate pr_iteration hydrations + // against PRs containing normal imperative documentation. + inputStrength: bedrock.ContentFilterStrength.MEDIUM, + outputStrength: bedrock.ContentFilterStrength.NONE, + }, + ], + }); + + inputGuardrail.createVersion('Initial version'); + + // --- TaskApi is constructed before the orchestrator (which it needs the + // ARN of) and before the Runtime (which it needs the ARN of, for the + // cancel-task Lambda's stop-session permission). We break both cycles + // with Lazy strings that resolve to CloudFormation tokens at synth time. + let orchestratorArnHolder: string | undefined; + const lazyOrchestratorArn = Lazy.string({ + produce: () => { + if (!orchestratorArnHolder) { + throw new Error('Orchestrator ARN was accessed before the TaskOrchestrator was created'); + } + return orchestratorArnHolder; + }, + }); + + // Runtime ARN placeholder — the runtime is created AFTER TaskApi so the + // Lambda handlers can get their env var via a Lazy.string reference. + let runtimeArnHolder: string | undefined; + const lazyRuntimeArn = Lazy.string({ + produce: () => { + if (!runtimeArnHolder) { + throw new Error('Runtime ARN was accessed before Runtime was created'); + } + return runtimeArnHolder; + }, + }); + + // --- Task API (REST API + Cognito + Lambda handlers) --- + const taskApi = new TaskApi(this, 'TaskApi', { + taskTable: taskTable.table, + taskEventsTable: taskEventsTable.table, + taskNudgesTable: taskNudgesTable.table, + repoTable: repoTable.table, + webhookTable: webhookTable.table, + orchestratorFunctionArn: lazyOrchestratorArn, + guardrailId: inputGuardrail.guardrailId, + guardrailVersion: inputGuardrail.guardrailVersion, + agentCoreStopSessionRuntimeArn: lazyRuntimeArn, + traceArtifactsBucket: traceArtifactsBucket.bucket, + }); + + // --- AgentCore Runtime (IAM-authed orchestrator path) --- + // + // One runtime, invoked by OrchestratorFn via SigV4. See + // `docs/design/INTERACTIVE_AGENTS.md` §3.1 and AD-1. + const runtimeEnvironmentVariables = { + GITHUB_TOKEN_SECRET_ARN: githubTokenSecret.secretArn, + AWS_REGION: process.env.AWS_REGION ?? 'us-east-1', + CLAUDE_CODE_USE_BEDROCK: '1', + ANTHROPIC_LOG: 'debug', + ANTHROPIC_DEFAULT_HAIKU_MODEL: 'anthropic.claude-haiku-4-5-20251001-v1:0', + TASK_TABLE_NAME: taskTable.table.tableName, + TASK_EVENTS_TABLE_NAME: taskEventsTable.table.tableName, + NUDGES_TABLE_NAME: taskNudgesTable.table.tableName, + USER_CONCURRENCY_TABLE_NAME: userConcurrencyTable.table.tableName, + // --trace artifact store (§10.1). The agent writes the JSONL + // trajectory to ``traces//.jsonl.gz`` on + // terminal state when the submit payload enabled ``trace``. + TRACE_ARTIFACTS_BUCKET_NAME: traceArtifactsBucket.bucket.bucketName, + LOG_GROUP_NAME: applicationLogGroup.logGroupName, + MEMORY_ID: agentMemory.memory.memoryId, + MAX_TURNS: '100', + // Session storage: the S3-backed FUSE mount at /mnt/workspace does NOT + // support flock(). Only caches whose tools never call flock() go there. + // Everything else stays on local ephemeral disk. + // + // Local disk (tools use flock): + // AGENT_WORKSPACE — omitted, defaults to /workspace + // MISE_DATA_DIR — mise's pipx backend sets UV_TOOL_DIR inside installs/, + // and uv flocks that directory → must be local. + MISE_DATA_DIR: '/tmp/mise-data', + UV_CACHE_DIR: '/tmp/uv-cache', + // Persistent mount (no flock): + CLAUDE_CONFIG_DIR: '/mnt/workspace/.claude-config', + npm_config_cache: '/mnt/workspace/.npm-cache', + // ENABLE_CLI_TELEMETRY: '1', + }; + + const runtimeNetworkConfig = agentcore.RuntimeNetworkConfiguration.usingVpc(this, { + vpc: agentVpc.vpc, + vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, + securityGroups: [agentVpc.runtimeSecurityGroup], + }); + + // LifecycleConfiguration — both timers set to the AgentCore 8h maximum so + // long-running tasks (approval waits, heavy builds) are not evicted. + const lifecycleConfiguration: agentcore.LifecycleConfiguration = { + idleRuntimeSessionTimeout: Duration.hours(8), + maxLifetime: Duration.hours(8), + }; + + // Construct id 'Runtime' is load-bearing — renaming it forces CFN to + // CREATE the new resource before DELETING the old one, violating + // AgentCore's account-level runtimeName uniqueness and triggering an + // UPDATE_ROLLBACK. const runtime = new agentcore.Runtime(this, 'Runtime', { runtimeName, agentRuntimeArtifact: artifact, - networkConfiguration: agentcore.RuntimeNetworkConfiguration.usingVpc(this, { - vpc: agentVpc.vpc, - vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }, - securityGroups: [agentVpc.runtimeSecurityGroup], - }), - environmentVariables: { - GITHUB_TOKEN_SECRET_ARN: githubTokenSecret.secretArn, - AWS_REGION: process.env.AWS_REGION ?? 'us-east-1', - CLAUDE_CODE_USE_BEDROCK: '1', - ANTHROPIC_LOG: 'debug', - ANTHROPIC_DEFAULT_HAIKU_MODEL: 'anthropic.claude-haiku-4-5-20251001-v1:0', - TASK_TABLE_NAME: taskTable.table.tableName, - TASK_EVENTS_TABLE_NAME: taskEventsTable.table.tableName, - USER_CONCURRENCY_TABLE_NAME: userConcurrencyTable.table.tableName, - LOG_GROUP_NAME: applicationLogGroup.logGroupName, - MEMORY_ID: agentMemory.memory.memoryId, - MAX_TURNS: '100', - // Session storage: the S3-backed FUSE mount at /mnt/workspace does NOT - // support flock(). Only caches whose tools never call flock() go there. - // Everything else stays on local ephemeral disk. - // - // Local disk (tools use flock): - // AGENT_WORKSPACE — omitted, defaults to /workspace - // MISE_DATA_DIR — mise's pipx backend sets UV_TOOL_DIR inside installs/, - // and uv flocks that directory → must be local. - MISE_DATA_DIR: '/tmp/mise-data', - UV_CACHE_DIR: '/tmp/uv-cache', - // Persistent mount (no flock): - CLAUDE_CONFIG_DIR: '/mnt/workspace/.claude-config', - npm_config_cache: '/mnt/workspace/.npm-cache', - // ENABLE_CLI_TELEMETRY: '1', - }, + networkConfiguration: runtimeNetworkConfig, + environmentVariables: runtimeEnvironmentVariables, + lifecycleConfiguration: lifecycleConfiguration, }); + runtimeArnHolder = runtime.agentRuntimeArn; + // --- Session storage (preview) --- - // The L2 construct does not yet expose filesystemConfigurations. - // Use a CFN escape hatch until the L2 adds native support. + // The L2 construct does not yet expose filesystemConfigurations; use the + // CFN escape hatch. /mnt/workspace mount backs the persistent cache + // shared across tasks in the same repo. const cfnRuntime = runtime.node.defaultChild as CfnResource; cfnRuntime.addPropertyOverride('FilesystemConfigurations', [ { @@ -175,60 +303,69 @@ export class AgentStack extends Stack { }, ]); + // --- IAM grants --- taskTable.table.grantReadWriteData(runtime); taskEventsTable.table.grantReadWriteData(runtime); + taskNudgesTable.table.grantReadWriteData(runtime); userConcurrencyTable.table.grantReadWriteData(runtime); githubTokenSecret.grantRead(runtime); applicationLogGroup.grantWrite(runtime); agentMemory.grantReadWrite(runtime); + // Runtime only ever writes trace artifacts (read happens via presigned + // URL from the ``get-trace-url`` handler, not the runtime). + // + // TODO(K2 Stage 2+): tighten to a per-prefix condition so the runtime + // cannot write outside its own task's ``traces//`` prefix. + // The current grant expands to ``Resource: /*`` with no + // ``s3:prefix`` / ``aws:PrincipalTag`` condition — per-user isolation + // is enforced in *agent code* (object-key construction), which is a + // trust boundary, not an enforcement boundary. Options: propagate + // ``user_id`` as an IAM session tag on the runtime invocation and + // condition the policy on ``aws:PrincipalTag/UserId``; or run the + // upload from a short-lived Lambda with a scoped policy instead of + // the runtime itself. Deferred because the session-tag plumbing is + // orthogonal to landing the feature behavior. + traceArtifactsBucket.bucket.grantPut(runtime); const model = new bedrock.BedrockFoundationModel('anthropic.claude-sonnet-4-6', { supportsAgents: true, supportsCrossRegion: true, }); - model.grantInvoke(runtime); - // Create a cross-region inference profile for Claude Sonnet 4.6 const inferenceProfile = bedrock.CrossRegionInferenceProfile.fromConfig({ geoRegion: bedrock.CrossRegionInferenceProfileRegion.US, model: model, }); - // Grant the runtime permissions to invoke the inference profile - inferenceProfile.grantInvoke(runtime); - const model3 = new bedrock.BedrockFoundationModel('anthropic.claude-opus-4-20250514-v1:0', { supportsAgents: true, supportsCrossRegion: true, }); - model3.grantInvoke(runtime); - const inferenceProfile3 = bedrock.CrossRegionInferenceProfile.fromConfig({ geoRegion: bedrock.CrossRegionInferenceProfileRegion.US, model: model3, }); - inferenceProfile3.grantInvoke(runtime); - const model2 = new bedrock.BedrockFoundationModel('anthropic.claude-haiku-4-5-20251001-v1:0', { supportsAgents: true, supportsCrossRegion: true, }); - model2.grantInvoke(runtime); - // Create a cross-region inference profile for Claude Haiku 4.5 const inferenceProfile2 = bedrock.CrossRegionInferenceProfile.fromConfig({ geoRegion: bedrock.CrossRegionInferenceProfileRegion.US, model: model2, }); - // Grant the runtime permissions to invoke the inference profile + model.grantInvoke(runtime); + inferenceProfile.grantInvoke(runtime); + model3.grantInvoke(runtime); + inferenceProfile3.grantInvoke(runtime); + model2.grantInvoke(runtime); inferenceProfile2.grantInvoke(runtime); - // Runtime logs and traces runtime.with(agentcoremixins.mixins.CfnRuntimeLogsMixin.APPLICATION_LOGS.toLogGroup(applicationLogGroup)); runtime.with(agentcoremixins.mixins.CfnRuntimeLogsMixin.TRACES.toXRay()); runtime.with(agentcoremixins.mixins.CfnRuntimeLogsMixin.USAGE_LOGS.toLogGroup(usageLogGroup)); @@ -255,6 +392,11 @@ export class AgentStack extends Stack { description: 'Name of the DynamoDB task events audit table', }); + new CfnOutput(this, 'TaskNudgesTableName', { + value: taskNudgesTable.table.tableName, + description: 'Name of the DynamoDB task nudges table (Phase 2)', + }); + new CfnOutput(this, 'UserConcurrencyTableName', { value: userConcurrencyTable.table.tableName, description: 'Name of the DynamoDB user concurrency table', @@ -275,21 +417,11 @@ export class AgentStack extends Stack { description: 'ARN of the Secrets Manager secret for the GitHub token', }); - // --- Bedrock Guardrail for prompt injection detection --- - const inputGuardrail = new bedrock.Guardrail(this, 'InputGuardrail', { - guardrailName: 'task-input-guardrail', - description: 'Screens task submissions for prompt injection attacks', - contentFilters: [ - { - type: bedrock.ContentFilterType.PROMPT_ATTACK, - inputStrength: bedrock.ContentFilterStrength.HIGH, - outputStrength: bedrock.ContentFilterStrength.NONE, - }, - ], + new CfnOutput(this, 'TraceArtifactsBucketName', { + value: traceArtifactsBucket.bucket.bucketName, + description: 'Name of the S3 bucket storing --trace trajectory artifacts (design §10.1)', }); - inputGuardrail.createVersion('Initial version'); - // --- ECS Fargate compute backend (optional) --- // To enable ECS as an alternative compute backend, uncomment the block below // and the EcsAgentCluster import at the top of this file. Repos can then use @@ -333,6 +465,9 @@ export class AgentStack extends Stack { // }, }); + // Now that the orchestrator exists, resolve the Lazy used by TaskApi at synth. + orchestratorArnHolder = orchestrator.alias.functionArn; + // Grant the orchestrator Lambda read+write access to memory // (reads during context hydration, writes for fallback episodes) agentMemory.grantReadWrite(orchestrator.fn); @@ -343,18 +478,27 @@ export class AgentStack extends Stack { userConcurrencyTable: userConcurrencyTable.table, }); - // --- Task API (REST API + Cognito + Lambda handlers) --- - const taskApi = new TaskApi(this, 'TaskApi', { + // --- Stranded-task reconciler --- + // Catches SUBMITTED / HYDRATING tasks whose pipeline never started + // (orchestrator Lambda crash between TaskTable write and InvokeAgentRuntime, + // container crash during startup, etc.). Transitions to FAILED with a + // `task_stranded` event. + new StrandedTaskReconciler(this, 'StrandedTaskReconciler', { taskTable: taskTable.table, taskEventsTable: taskEventsTable.table, + userConcurrencyTable: userConcurrencyTable.table, + }); + + // --- Fan-out plane consumer --- + // Consumes TaskEventsTable DynamoDB Streams and dispatches events to + // Slack / GitHub / email per per-channel default filters. GitHub + // dispatcher (Chunk J) edits a single issue comment in place with + // If-Match ETag; Slack / Email remain log-only until Phase 2. + new FanOutConsumer(this, 'FanOutConsumer', { + taskEventsTable: taskEventsTable.table, + taskTable: taskTable.table, repoTable: repoTable.table, - webhookTable: webhookTable.table, - orchestratorFunctionArn: orchestrator.alias.functionArn, - guardrailId: inputGuardrail.guardrailId, - guardrailVersion: inputGuardrail.guardrailVersion, - agentCoreStopSessionRuntimeArns: [runtime.agentRuntimeArn], - // To allow cancel-task to stop ECS-backed tasks, uncomment: - // ecsClusterArn: ecsCluster.cluster.clusterArn, + githubTokenSecret, }); // --- Operator dashboard --- diff --git a/cdk/test/constructs/task-api.test.ts b/cdk/test/constructs/task-api.test.ts index ee415a8..da23a60 100644 --- a/cdk/test/constructs/task-api.test.ts +++ b/cdk/test/constructs/task-api.test.ts @@ -20,6 +20,7 @@ import { App, Stack } from 'aws-cdk-lib'; import { Template, Match } from 'aws-cdk-lib/assertions'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as s3 from 'aws-cdk-lib/aws-s3'; import { TaskApi, type TaskApiProps } from '../../src/constructs/task-api'; function createStack(overrides?: Partial): { stack: Stack; template: Template } { @@ -432,3 +433,293 @@ describe('TaskApi construct with webhooks', () => { }); }); }); + +describe('TaskApi construct — nudge endpoint (Phase 2)', () => { + function createStackWithNudges(overrides?: Partial): Template { + const app = new App(); + const stack = new Stack(app, 'NudgeStack'); + const taskTable = new dynamodb.Table(stack, 'TaskTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + }); + const taskEventsTable = new dynamodb.Table(stack, 'TaskEventsTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + sortKey: { name: 'event_id', type: dynamodb.AttributeType.STRING }, + }); + const taskNudgesTable = new dynamodb.Table(stack, 'TaskNudgesTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + sortKey: { name: 'nudge_id', type: dynamodb.AttributeType.STRING }, + }); + new TaskApi(stack, 'TaskApi', { + taskTable, + taskEventsTable, + taskNudgesTable, + guardrailId: 'gr-abc', + guardrailVersion: '1', + ...overrides, + }); + return Template.fromStack(stack); + } + + test('does NOT create a nudge resource when taskNudgesTable is absent', () => { + const app = new App(); + const stack = new Stack(app, 'NoNudgeStack'); + const taskTable = new dynamodb.Table(stack, 'TaskTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + }); + const taskEventsTable = new dynamodb.Table(stack, 'TaskEventsTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + sortKey: { name: 'event_id', type: dynamodb.AttributeType.STRING }, + }); + new TaskApi(stack, 'TaskApi', { taskTable, taskEventsTable }); + const template = Template.fromStack(stack); + + const resources = template.findResources('AWS::ApiGateway::Resource'); + const nudgeRes = Object.values(resources).filter( + r => (r as { Properties?: { PathPart?: string } }).Properties?.PathPart === 'nudge', + ); + expect(nudgeRes).toHaveLength(0); + }); + + test('creates a /nudge resource when taskNudgesTable is provided', () => { + const template = createStackWithNudges(); + template.hasResourceProperties('AWS::ApiGateway::Resource', { + PathPart: 'nudge', + }); + }); + + test('nudge route uses Cognito authorization on POST', () => { + const template = createStackWithNudges(); + const methods = template.findResources('AWS::ApiGateway::Method'); + const nudgePost = Object.values(methods).filter(m => { + const p = (m as { Properties?: { HttpMethod?: string } }).Properties ?? {}; + return p.HttpMethod === 'POST'; + }); + // At least one POST is for nudge — assert at least one POST uses COGNITO. + const cognitoPosts = nudgePost.filter(m => + (m as { Properties?: { AuthorizationType?: string } }).Properties?.AuthorizationType === 'COGNITO_USER_POOLS', + ); + expect(cognitoPosts.length).toBeGreaterThanOrEqual(1); + }); + + test('nudge Lambda has NUDGES_TABLE_NAME and NUDGE_RATE_LIMIT_PER_MINUTE env vars', () => { + const template = createStackWithNudges(); + template.hasResourceProperties('AWS::Lambda::Function', { + Environment: { + Variables: Match.objectLike({ + NUDGES_TABLE_NAME: Match.anyValue(), + NUDGE_RATE_LIMIT_PER_MINUTE: '10', + }), + }, + }); + }); + + test('nudge Lambda has guardrail env vars when provided', () => { + const template = createStackWithNudges(); + template.hasResourceProperties('AWS::Lambda::Function', { + Environment: { + Variables: Match.objectLike({ + NUDGES_TABLE_NAME: Match.anyValue(), + GUARDRAIL_ID: 'gr-abc', + GUARDRAIL_VERSION: '1', + }), + }, + }); + }); + + test('nudge Lambda has bedrock:ApplyGuardrail permission when guardrail configured', () => { + const template = createStackWithNudges(); + template.hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: Match.arrayWith([ + Match.objectLike({ + Action: 'bedrock:ApplyGuardrail', + Effect: 'Allow', + }), + ]), + }, + }); + }); + + test('respects custom nudgeRateLimitPerMinute', () => { + const template = createStackWithNudges({ nudgeRateLimitPerMinute: 25 }); + template.hasResourceProperties('AWS::Lambda::Function', { + Environment: { + Variables: Match.objectLike({ + NUDGE_RATE_LIMIT_PER_MINUTE: '25', + }), + }, + }); + }); +}); + +describe('TaskApi construct — trace endpoint (design §10.1)', () => { + function createStackWithTrace(overrides?: Partial): Template { + const app = new App(); + const stack = new Stack(app, 'TraceStack'); + const taskTable = new dynamodb.Table(stack, 'TaskTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + }); + const taskEventsTable = new dynamodb.Table(stack, 'TaskEventsTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + sortKey: { name: 'event_id', type: dynamodb.AttributeType.STRING }, + }); + const traceBucket = new s3.Bucket(stack, 'TraceBucket'); + new TaskApi(stack, 'TaskApi', { + taskTable, + taskEventsTable, + traceArtifactsBucket: traceBucket, + ...overrides, + }); + return Template.fromStack(stack); + } + + test('does NOT create a trace resource when traceArtifactsBucket is absent', () => { + const app = new App(); + const stack = new Stack(app, 'NoTraceStack'); + const taskTable = new dynamodb.Table(stack, 'TaskTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + }); + const taskEventsTable = new dynamodb.Table(stack, 'TaskEventsTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + sortKey: { name: 'event_id', type: dynamodb.AttributeType.STRING }, + }); + new TaskApi(stack, 'TaskApi', { taskTable, taskEventsTable }); + const template = Template.fromStack(stack); + + const resources = template.findResources('AWS::ApiGateway::Resource'); + const pathParts = Object.values(resources).map(r => r.Properties.PathPart); + expect(pathParts).not.toContain('trace'); + }); + + test('creates a GET /tasks/{task_id}/trace resource when traceArtifactsBucket is provided', () => { + const template = createStackWithTrace(); + + // There should be an API Gateway Resource with PathPart='trace' + const resources = template.findResources('AWS::ApiGateway::Resource'); + const tracePath = Object.values(resources).find(r => r.Properties.PathPart === 'trace'); + expect(tracePath).toBeDefined(); + + // And a GET method on it + template.hasResourceProperties('AWS::ApiGateway::Method', { + HttpMethod: 'GET', + AuthorizationType: 'COGNITO_USER_POOLS', + ResourceId: Match.anyValue(), + }); + }); + + test('creates the GetTraceUrlFn Lambda with TRACE_ARTIFACTS_BUCKET_NAME env var', () => { + const template = createStackWithTrace(); + + const functions = template.findResources('AWS::Lambda::Function'); + const traceFns = Object.entries(functions).filter(([id]) => + id.startsWith('TaskApiGetTraceUrlFn'), + ); + expect(traceFns).toHaveLength(1); + const [, resource] = traceFns[0]; + const envVars = resource.Properties.Environment?.Variables; + expect(envVars).toBeDefined(); + expect(envVars.TRACE_ARTIFACTS_BUCKET_NAME).toBeDefined(); + // TASK_TABLE_NAME must be present too (for the ownership check) + expect(envVars.TASK_TABLE_NAME).toBeDefined(); + }); + + test('grants the handler read-only access to the trace bucket (GetObject, not PutObject)', () => { + const template = createStackWithTrace(); + + // Find the IAM policy attached to the GetTraceUrlFn role + const policies = template.findResources('AWS::IAM::Policy'); + const handlerPolicies = Object.entries(policies).filter(([id]) => + id.includes('GetTraceUrlFn'), + ); + expect(handlerPolicies.length).toBeGreaterThan(0); + + // Walk every policy attached to the handler and check S3 actions. + const allS3Actions: string[] = []; + for (const [, resource] of handlerPolicies) { + const statements = resource.Properties.PolicyDocument?.Statement ?? []; + for (const stmt of statements) { + const actionList = Array.isArray(stmt.Action) ? stmt.Action : [stmt.Action]; + for (const a of actionList) { + if (typeof a === 'string' && a.startsWith('s3:')) { + allS3Actions.push(a); + } + } + } + } + + // Must be able to GetObject (to presign + HeadObject). L3 item 2 + // tightens this from CDK's ``grantRead`` (which expands to + // ``s3:GetObject*`` / ``s3:GetBucket*`` / ``s3:List*``) down to an + // explicit ``s3:GetObject`` — AWS grants HeadObject implicitly on + // the same permission, so the handler's HEAD-before-presign check + // is still authorized. + expect(allS3Actions).toContain('s3:GetObject'); + // The wildcarded ``s3:GetObject*`` form must be absent — L3 pinned + // the handler to the exact action, not the wildcard. + expect(allS3Actions).not.toContain('s3:GetObject*'); + // ``ListBucket`` is unnecessary scope (the handler never lists). A + // regression here would reintroduce the ``grantRead`` expansion. + expect(allS3Actions).not.toContain('s3:ListBucket'); + expect(allS3Actions.some(a => a.startsWith('s3:List'))).toBe(false); + expect(allS3Actions.some(a => a.startsWith('s3:GetBucket'))).toBe(false); + // Must NOT have write permissions (including wildcarded forms). + expect(allS3Actions.some(a => a.startsWith('s3:PutObject'))).toBe(false); + expect(allS3Actions.some(a => a.startsWith('s3:DeleteObject'))).toBe(false); + expect(allS3Actions).not.toContain('s3:*'); + }); + + test('grants the handler read access to the task table for ownership checks', () => { + const template = createStackWithTrace(); + + const policies = template.findResources('AWS::IAM::Policy'); + const handlerPolicies = Object.entries(policies).filter(([id]) => + id.includes('GetTraceUrlFn'), + ); + + const allDdbActions: string[] = []; + for (const [, resource] of handlerPolicies) { + const statements = resource.Properties.PolicyDocument?.Statement ?? []; + for (const stmt of statements) { + const actionList = Array.isArray(stmt.Action) ? stmt.Action : [stmt.Action]; + for (const a of actionList) { + if (typeof a === 'string' && a.startsWith('dynamodb:')) { + allDdbActions.push(a); + } + } + } + } + expect(allDdbActions).toContain('dynamodb:GetItem'); + // Must NOT have write permissions + expect(allDdbActions).not.toContain('dynamodb:PutItem'); + expect(allDdbActions).not.toContain('dynamodb:UpdateItem'); + }); + + test('trace endpoint uses Cognito authorization (same as other task endpoints)', () => { + const template = createStackWithTrace(); + + // The trace resource's method must require Cognito auth. + const methods = template.findResources('AWS::ApiGateway::Method'); + const traceMethods = Object.values(methods).filter(m => + m.Properties.HttpMethod === 'GET', + ); + // Gather all Cognito-authorized GET methods; the trace one must be among them. + const cognitoGetMethods = traceMethods.filter(m => m.Properties.AuthorizationType === 'COGNITO_USER_POOLS'); + // There should be at least 3 Cognito GET methods: get-task, list-tasks, get-events, get-trace. + // But we only test that `get-trace` auth is Cognito (which is implied by the creation test above). + expect(cognitoGetMethods.length).toBeGreaterThanOrEqual(4); + }); + + test('GetTraceUrlFn has adequate timeout and memory for SDK cold-start', () => { + const template = createStackWithTrace(); + // Find the GetTraceUrlFn by looking for the function whose env has TRACE_ARTIFACTS_BUCKET_NAME. + const functions = template.findResources('AWS::Lambda::Function'); + const traceFn = Object.values(functions).find( + f => f.Properties.Environment?.Variables?.TRACE_ARTIFACTS_BUCKET_NAME !== undefined, + ); + expect(traceFn).toBeDefined(); + // 15s matches CancelTaskFn precedent for cold-start SDK loads; + // 512 MB is headroom above the observed 126 MB cold-start peak. + expect(traceFn!.Properties.Timeout).toBe(15); + expect(traceFn!.Properties.MemorySize).toBe(512); + }); +}); diff --git a/cdk/test/constructs/task-events-table.test.ts b/cdk/test/constructs/task-events-table.test.ts index 93a226c..fa49001 100644 --- a/cdk/test/constructs/task-events-table.test.ts +++ b/cdk/test/constructs/task-events-table.test.ts @@ -84,6 +84,14 @@ describe('TaskEventsTable', () => { ], }); }); + + test('enables DynamoDB Streams with NEW_IMAGE view type (Phase 1b fan-out plane)', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + StreamSpecification: { + StreamViewType: 'NEW_IMAGE', + }, + }); + }); }); describe('TaskEventsTable with custom props', () => { diff --git a/cdk/test/constructs/task-nudges-table.test.ts b/cdk/test/constructs/task-nudges-table.test.ts new file mode 100644 index 0000000..e20285d --- /dev/null +++ b/cdk/test/constructs/task-nudges-table.test.ts @@ -0,0 +1,131 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { App, RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { TaskNudgesTable } from '../../src/constructs/task-nudges-table'; + +describe('TaskNudgesTable', () => { + let template: Template; + + beforeEach(() => { + const app = new App(); + const stack = new Stack(app, 'TestStack'); + new TaskNudgesTable(stack, 'TaskNudgesTable'); + template = Template.fromStack(stack); + }); + + test('creates a DynamoDB table with task_id PK and nudge_id SK', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + KeySchema: [ + { AttributeName: 'task_id', KeyType: 'HASH' }, + { AttributeName: 'nudge_id', KeyType: 'RANGE' }, + ], + }); + }); + + test('uses PAY_PER_REQUEST billing mode', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + BillingMode: 'PAY_PER_REQUEST', + }); + }); + + test('enables point-in-time recovery by default', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + PointInTimeRecoverySpecification: { + PointInTimeRecoveryEnabled: true, + }, + }); + }); + + test('sets DESTROY removal policy by default', () => { + template.hasResource('AWS::DynamoDB::Table', { + DeletionPolicy: 'Delete', + UpdateReplacePolicy: 'Delete', + }); + }); + + test('does not create any GSIs', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + GlobalSecondaryIndexes: Match.absent(), + }); + }); + + test('enables TTL on ttl attribute', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + TimeToLiveSpecification: { + AttributeName: 'ttl', + Enabled: true, + }, + }); + }); + + test('declares exactly two attribute definitions', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + AttributeDefinitions: [ + { AttributeName: 'task_id', AttributeType: 'S' }, + { AttributeName: 'nudge_id', AttributeType: 'S' }, + ], + }); + }); + + test('does not enable DynamoDB Streams (nudges are poll-consumed)', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + StreamSpecification: Match.absent(), + }); + }); +}); + +describe('TaskNudgesTable with custom props', () => { + test('accepts custom table name', () => { + const app = new App(); + const stack = new Stack(app, 'TestStack'); + new TaskNudgesTable(stack, 'TaskNudgesTable', { tableName: 'my-nudges' }); + const template = Template.fromStack(stack); + + template.hasResourceProperties('AWS::DynamoDB::Table', { + TableName: 'my-nudges', + }); + }); + + test('accepts custom removal policy', () => { + const app = new App(); + const stack = new Stack(app, 'TestStack'); + new TaskNudgesTable(stack, 'TaskNudgesTable', { removalPolicy: RemovalPolicy.RETAIN }); + const template = Template.fromStack(stack); + + template.hasResource('AWS::DynamoDB::Table', { + DeletionPolicy: 'Retain', + UpdateReplacePolicy: 'Retain', + }); + }); + + test('accepts point-in-time recovery disabled', () => { + const app = new App(); + const stack = new Stack(app, 'TestStack'); + new TaskNudgesTable(stack, 'TaskNudgesTable', { pointInTimeRecovery: false }); + const template = Template.fromStack(stack); + + template.hasResourceProperties('AWS::DynamoDB::Table', { + PointInTimeRecoverySpecification: { + PointInTimeRecoveryEnabled: false, + }, + }); + }); +}); diff --git a/cdk/test/constructs/task-orchestrator.test.ts b/cdk/test/constructs/task-orchestrator.test.ts index 0612020..6cf903a 100644 --- a/cdk/test/constructs/task-orchestrator.test.ts +++ b/cdk/test/constructs/task-orchestrator.test.ts @@ -114,7 +114,7 @@ describe('TaskOrchestrator construct', () => { TASK_EVENTS_TABLE_NAME: Match.anyValue(), USER_CONCURRENCY_TABLE_NAME: Match.anyValue(), RUNTIME_ARN: 'arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/test-runtime', - MAX_CONCURRENT_TASKS_PER_USER: '3', + MAX_CONCURRENT_TASKS_PER_USER: '10', TASK_RETENTION_DAYS: '90', }), }, diff --git a/cdk/test/constructs/trace-artifacts-bucket.test.ts b/cdk/test/constructs/trace-artifacts-bucket.test.ts new file mode 100644 index 0000000..3135904 --- /dev/null +++ b/cdk/test/constructs/trace-artifacts-bucket.test.ts @@ -0,0 +1,161 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { App, RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { TRACE_ARTIFACT_TTL_DAYS, TRACE_OBJECT_KEY_PREFIX, TraceArtifactsBucket } from '../../src/constructs/trace-artifacts-bucket'; + +describe('TraceArtifactsBucket', () => { + let template: Template; + + beforeEach(() => { + const app = new App(); + const stack = new Stack(app, 'TestStack'); + new TraceArtifactsBucket(stack, 'TraceArtifactsBucket'); + template = Template.fromStack(stack); + }); + + test('creates an S3 bucket with all public access blocked', () => { + template.hasResourceProperties('AWS::S3::Bucket', { + PublicAccessBlockConfiguration: { + BlockPublicAcls: true, + BlockPublicPolicy: true, + IgnorePublicAcls: true, + RestrictPublicBuckets: true, + }, + }); + }); + + test('enables S3-managed server-side encryption', () => { + template.hasResourceProperties('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + }, + }, + ], + }, + }); + }); + + test('attaches a bucket policy enforcing TLS-only access', () => { + template.hasResourceProperties('AWS::S3::BucketPolicy', { + PolicyDocument: { + Statement: Match.arrayWith([ + Match.objectLike({ + Effect: 'Deny', + Action: 's3:*', + Condition: { + Bool: { + 'aws:SecureTransport': 'false', + }, + }, + }), + ]), + }, + }); + }); + + test('configures a 7-day expiration lifecycle rule', () => { + template.hasResourceProperties('AWS::S3::Bucket', { + LifecycleConfiguration: { + Rules: Match.arrayWith([ + Match.objectLike({ + Id: 'trace-artifacts-ttl', + Status: 'Enabled', + ExpirationInDays: TRACE_ARTIFACT_TTL_DAYS, + }), + ]), + }, + }); + }); + + test('aborts incomplete multipart uploads within the TTL window', () => { + template.hasResourceProperties('AWS::S3::Bucket', { + LifecycleConfiguration: { + Rules: Match.arrayWith([ + Match.objectLike({ + AbortIncompleteMultipartUpload: { + DaysAfterInitiation: 1, + }, + }), + ]), + }, + }); + }); + + test('sets DESTROY removal policy by default', () => { + template.hasResource('AWS::S3::Bucket', { + DeletionPolicy: 'Delete', + UpdateReplacePolicy: 'Delete', + }); + }); + + test('enables autoDeleteObjects by default (matches TaskTable pattern)', () => { + // autoDeleteObjects is implemented via a CDK custom resource that + // empties the bucket before deletion. Its presence is the signal + // that autoDeleteObjects is on. + template.hasResourceProperties('Custom::S3AutoDeleteObjects', { + BucketName: Match.anyValue(), + }); + }); + + test('exposes a bucket handle via the `bucket` property', () => { + const app = new App(); + const stack = new Stack(app, 'TestStack'); + const trace = new TraceArtifactsBucket(stack, 'TraceArtifactsBucket'); + expect(trace.bucket).toBeDefined(); + // Sanity: the construct's public handle and the synthesized resource + // are the same bucket. + expect(trace.bucket.bucketName).toBeDefined(); + }); +}); + +describe('TraceArtifactsBucket with custom props', () => { + test('accepts custom removal policy', () => { + const app = new App(); + const stack = new Stack(app, 'TestStack'); + new TraceArtifactsBucket(stack, 'TraceArtifactsBucket', { + removalPolicy: RemovalPolicy.RETAIN, + autoDeleteObjects: false, + }); + const template = Template.fromStack(stack); + + template.hasResource('AWS::S3::Bucket', { + DeletionPolicy: 'Retain', + UpdateReplacePolicy: 'Retain', + }); + + // With autoDeleteObjects disabled, the custom resource is not synthesized. + const customResources = template.findResources('Custom::S3AutoDeleteObjects'); + expect(Object.keys(customResources)).toHaveLength(0); + }); +}); + +describe('TraceArtifactsBucket exported constants', () => { + test('TTL matches design §10.1', () => { + expect(TRACE_ARTIFACT_TTL_DAYS).toBe(7); + }); + + test('object key prefix matches design §10.1', () => { + expect(TRACE_OBJECT_KEY_PREFIX).toBe('traces/'); + }); +}); diff --git a/cdk/test/handlers/fanout-task-events.test.ts b/cdk/test/handlers/fanout-task-events.test.ts new file mode 100644 index 0000000..14b1511 --- /dev/null +++ b/cdk/test/handlers/fanout-task-events.test.ts @@ -0,0 +1,1301 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import type { DynamoDBRecord, DynamoDBStreamEvent } from 'aws-lambda'; + +// -- DDB + downstream-module mocks (hoisted before handler import) -- +// Default resolves to an empty-item Get so routing tests that don't +// care about DDB see the dispatcher short-circuit on "task not found" +// rather than throwing a TypeError. Per-test code can override with +// ``mockDdbSend.mockReset()`` + ``.mockResolvedValueOnce(...)`` as +// needed. +const mockDdbSend = jest.fn().mockResolvedValue({ Item: undefined }); +// Stub the DDB client + command constructors. Using ``jest.fn`` for +// each command class gives us ``new GetCommand(input)`` producing a +// plain object we can inspect; the DocumentClient's ``send`` is routed +// to the mock above. ``requireActual`` on ``lib-dynamodb`` would pull +// in the real command implementations which internally instantiate +// ``client-dynamodb`` classes we've stubbed — that's the import cycle +// that surfaces as ``GetItemCommand is not a constructor``. +jest.mock('@aws-sdk/client-dynamodb', () => ({ DynamoDBClient: jest.fn(() => ({})) })); +jest.mock('@aws-sdk/lib-dynamodb', () => ({ + DynamoDBDocumentClient: { from: jest.fn(() => ({ send: mockDdbSend })) }, + GetCommand: jest.fn((input: unknown) => ({ _type: 'Get', input })), + UpdateCommand: jest.fn((input: unknown) => ({ _type: 'Update', input })), +})); + +const mockUpsertTaskComment: jest.Mock = jest.fn(); +const mockRenderCommentBody: jest.Mock = jest.fn().mockReturnValue('rendered body'); +jest.mock('../../src/handlers/shared/github-comment', () => ({ + upsertTaskComment: (args: unknown) => mockUpsertTaskComment(args), + renderCommentBody: (args: unknown) => mockRenderCommentBody(args), + // Stub class mirrors the production shape so the handler's + // ``instanceof GitHubCommentError && err.httpStatus === 401`` check + // fires correctly in the token-rotation test. + GitHubCommentError: class GitHubCommentError extends Error { + readonly httpStatus: number | undefined; + constructor(message: string, httpStatus?: number) { + super(message); + this.name = 'GitHubCommentError'; + this.httpStatus = httpStatus; + } + }, +})); + +const mockLoadRepoConfig: jest.Mock = jest.fn(); +jest.mock('../../src/handlers/shared/repo-config', () => ({ + loadRepoConfig: (repo: string) => mockLoadRepoConfig(repo), +})); + +const mockResolveGitHubToken: jest.Mock = jest.fn(); +const mockClearTokenCache: jest.Mock = jest.fn(); +jest.mock('../../src/handlers/shared/context-hydration', () => ({ + resolveGitHubToken: (arn: string) => mockResolveGitHubToken(arn), + clearTokenCache: () => mockClearTokenCache(), +})); + +process.env.TASK_TABLE_NAME = 'Tasks'; +process.env.GITHUB_TOKEN_SECRET_ARN = 'arn:aws:secretsmanager:us-east-1:0:secret:platform'; + +import { + CHANNEL_DEFAULTS, + parseStreamRecord, + resolveChannelFilter, + routeEvent, + shouldFanOut, + handler, + type FanOutEvent, + type TaskNotificationsConfig, +} from '../../src/handlers/fanout-task-events'; + +function mkRecord( + eventName: 'INSERT' | 'MODIFY' | 'REMOVE', + newImage: Record }> | undefined, +): DynamoDBRecord { + return { + eventID: `evt-${Math.random().toString(36).slice(2)}`, + eventName, + eventSource: 'aws:dynamodb', + dynamodb: newImage ? { NewImage: newImage as never } : {}, + } as unknown as DynamoDBRecord; +} + +function mkEvent(type: string, taskId = 't-1'): DynamoDBRecord { + return mkRecord('INSERT', { + task_id: { S: taskId }, + event_id: { S: `01ABC${type}` }, + event_type: { S: type }, + timestamp: { S: '2026-04-22T04:00:00Z' }, + metadata: { M: { code: { S: 'OK' } } }, + }); +} + +describe('fanout-task-events: parseStreamRecord', () => { + test('parses a well-formed INSERT into FanOutEvent', () => { + const rec = mkEvent('task_completed', 't-parse-1'); + const parsed = parseStreamRecord(rec); + expect(parsed).not.toBeNull(); + expect(parsed!.task_id).toBe('t-parse-1'); + expect(parsed!.event_type).toBe('task_completed'); + expect(parsed!.metadata).toEqual({ code: 'OK' }); + }); + + test('returns null on REMOVE (tombstones are ignored)', () => { + const rec = mkRecord('REMOVE', undefined); + expect(parseStreamRecord(rec)).toBeNull(); + }); + + test('returns null when NewImage is missing required fields', () => { + const rec = mkRecord('INSERT', { + task_id: { S: 't-bad' }, + // missing event_id, event_type, timestamp + }); + expect(parseStreamRecord(rec)).toBeNull(); + }); +}); + +describe('fanout-task-events: shouldFanOut filter (union of per-channel defaults)', () => { + const make = (event_type: string): FanOutEvent => ({ + task_id: 't-1', + event_id: 'e-1', + event_type, + timestamp: '2026-04-22T04:00:00Z', + }); + + // Rev-6 design §6.2: chattier event types (task_created, agent_milestone) + // are intentionally dropped from defaults so users don't mute integrations + // on day one. The ``--verbose`` opt-in (Chunk K follow-up) will re-enable + // milestone delivery. + test.each([ + 'task_failed', + 'task_completed', + 'task_cancelled', + 'task_stranded', + 'agent_error', + 'pr_created', + 'approval_required', // Phase 3 forward-compat + 'status_response', // Phase 2 forward-compat + ])('%s is fanned out (matches at least one channel default)', (t) => { + expect(shouldFanOut(make(t))).toBe(true); + }); + + test.each([ + 'task_created', // intentionally dropped in rev-6 defaults + // Bare ``agent_milestone`` (no ``metadata.milestone``) stays + // dropped; wrapped milestones on the ``ROUTABLE_MILESTONES`` + // allowlist route by name — see the agent_milestone routing + // suite below. + 'agent_milestone', + 'agent_turn', + 'agent_tool_call', + 'agent_tool_result', + 'agent_cost_update', + 'session_started', + 'hydration_started', + 'hydration_complete', + 'admission_rejected', + 'something_else', + ])('%s is NOT fanned out (verbose / internal)', (t) => { + expect(shouldFanOut(make(t))).toBe(false); + }); +}); + +describe('fanout-task-events: per-channel filter contract (design §6.2)', () => { + // Lock in the exact sets from the design doc so a drift in + // CHANNEL_DEFAULTS surfaces here instead of in production telemetry. + test('Slack subscribes to terminal + PR + error + approval + status_response', () => { + const f = CHANNEL_DEFAULTS.slack; + expect([...f].sort()).toEqual([ + 'agent_error', + 'approval_required', + 'pr_created', + 'status_response', + 'task_cancelled', + 'task_completed', + 'task_failed', + 'task_stranded', + ]); + }); + + test('Email subscribes to task_completed + task_failed + approval_required only (minimal per §6.2)', () => { + // Design §6.2 explicitly limits Email to these three types. + // task_cancelled and task_stranded are NOT delivered via email — + // the user already knows they cancelled; strands are an operator + // signal handled via Slack / dashboards. + const f = CHANNEL_DEFAULTS.email; + expect([...f].sort()).toEqual([ + 'approval_required', + 'task_completed', + 'task_failed', + ]); + expect(f.has('task_cancelled')).toBe(false); + expect(f.has('task_stranded')).toBe(false); + }); + + test('GitHub subscribes to pr_created + terminal (edit-in-place surface)', () => { + const f = CHANNEL_DEFAULTS.github; + expect([...f].sort()).toEqual([ + 'pr_created', + 'task_cancelled', + 'task_completed', + 'task_failed', + 'task_stranded', + ]); + }); + + test('agent_error routes only to Slack, not Email or GitHub', () => { + // Operator-focused event. Email fires once per outcome; GitHub + // edits in place on PR activity; only Slack surfaces errors + // directly so on-call can jump in. + expect(CHANNEL_DEFAULTS.slack.has('agent_error')).toBe(true); + expect(CHANNEL_DEFAULTS.email.has('agent_error')).toBe(false); + expect(CHANNEL_DEFAULTS.github.has('agent_error')).toBe(false); + }); +}); + +describe('fanout-task-events: resolveChannelFilter overrides', () => { + test('no overrides → channel default', () => { + expect(resolveChannelFilter('slack')).toBe(CHANNEL_DEFAULTS.slack); + }); + + test('enabled=false returns empty set so no events dispatch', () => { + const overrides: TaskNotificationsConfig = { email: { enabled: false } }; + expect(resolveChannelFilter('email', overrides).size).toBe(0); + }); + + test('explicit events replace defaults entirely', () => { + const overrides: TaskNotificationsConfig = { + slack: { events: ['task_completed'] }, + }; + const f = resolveChannelFilter('slack', overrides); + expect([...f]).toEqual(['task_completed']); + // Must NOT include the default agent_error — explicit overrides + // replace, not augment. + expect(f.has('agent_error')).toBe(false); + }); + + test('"default" token in an explicit list expands to the channel defaults', () => { + const overrides: TaskNotificationsConfig = { + slack: { events: ['default', 'agent_milestone'] }, + }; + const f = resolveChannelFilter('slack', overrides); + // Inherits every default + the extra opt-in. + for (const t of CHANNEL_DEFAULTS.slack) expect(f.has(t)).toBe(true); + expect(f.has('agent_milestone')).toBe(true); + }); + + test('empty events list mutes the channel AND emits a footgun warn', async () => { + // An empty explicit list is almost always a submission mistake + // (e.g. ``jq '.events=[]'`` accident). Silent mute would be + // a silent-failure trap; surface the WARN so operators see it. + const loggerModule = await import('../../src/handlers/shared/logger'); + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation(() => undefined); + try { + const overrides: TaskNotificationsConfig = { slack: { events: [] } }; + expect(resolveChannelFilter('slack', overrides).size).toBe(0); + const warnMeta = warnSpy.mock.calls.map(c => c[1] as Record | undefined); + const emptyWarn = warnMeta.find(m => m?.event === 'fanout.resolve.empty_events_override'); + expect(emptyWarn).toBeDefined(); + expect(emptyWarn?.channel).toBe('slack'); + } finally { + warnSpy.mockRestore(); + } + }); + + test('other channels are unaffected when one is overridden', () => { + const overrides: TaskNotificationsConfig = { + slack: { enabled: false }, + }; + // Slack silenced — but email still sees terminal events. + expect(resolveChannelFilter('slack', overrides).size).toBe(0); + expect(resolveChannelFilter('email', overrides)).toBe(CHANNEL_DEFAULTS.email); + }); +}); + +describe('fanout-task-events: routeEvent (per-channel dispatch)', () => { + const mk = (event_type: string): FanOutEvent => ({ + task_id: 't-route', + event_id: 'e-route', + event_type, + timestamp: '2026-04-22T04:00:00Z', + }); + + test('task_completed routes to all three channels', async () => { + const channels = await routeEvent(mk('task_completed')); + expect(channels.sort()).toEqual(['email', 'github', 'slack']); + }); + + test('task_cancelled skips Email per §6.2 (only Slack + GitHub)', async () => { + // Regression guard against accidentally folding cancelled+stranded + // into Email via a shared TERMINAL spread — design says Email is + // minimal (task_completed, task_failed, approval_required only). + const channels = await routeEvent(mk('task_cancelled')); + expect(channels.sort()).toEqual(['github', 'slack']); + }); + + test('task_stranded skips Email per §6.2', async () => { + const channels = await routeEvent(mk('task_stranded')); + expect(channels.sort()).toEqual(['github', 'slack']); + }); + + test('agent_error routes only to Slack', async () => { + const channels = await routeEvent(mk('agent_error')); + expect(channels).toEqual(['slack']); + }); + + test('pr_created routes to Slack + GitHub but not Email', async () => { + const channels = await routeEvent(mk('pr_created')); + expect(channels.sort()).toEqual(['github', 'slack']); + }); + + test('event with no subscribers returns an empty channel list', async () => { + // ``agent_milestone`` is not in any channel's default — routing + // must produce an empty list so the handler records dispatched=0. + const channels = await routeEvent(mk('agent_milestone')); + expect(channels).toEqual([]); + }); + + test('per-task override silences one channel without affecting others', async () => { + const overrides: TaskNotificationsConfig = { slack: { enabled: false } }; + const channels = await routeEvent(mk('task_completed'), overrides); + expect(channels.sort()).toEqual(['email', 'github']); + expect(channels).not.toContain('slack'); + }); +}); + +describe('fanout-task-events: channel isolation', () => { + test('one channel rejecting does NOT prevent the others from dispatching', async () => { + // Simulate a Slack-side failure by making the Slack dispatcher's + // inner ``logger.info`` throw, which escapes its own try-block via + // the caught-and-rethrown path in the stub. The router's + // ``Promise.allSettled`` must record Slack as rejected while + // Email + GitHub complete normally. The assertions verify two + // independent signals: + // (1) the other two dispatchers' stub log calls actually ran + // (proving the work was done, not just that the router + // reported success) + // (2) Slack is omitted from the ``dispatched`` return so batch + // telemetry reflects reality + const loggerModule = await import('../../src/handlers/shared/logger'); + const originalInfo = loggerModule.logger.info.bind(loggerModule.logger); + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation(() => undefined); + const observedEvents: string[] = []; + const infoSpy = jest.spyOn(loggerModule.logger, 'info').mockImplementation( + (msg: string, meta?: Record) => { + const ev = meta?.event as string | undefined; + if (ev) observedEvents.push(ev); + if (ev === 'fanout.slack.dispatch_stub') { + throw new Error('slack is down'); + } + return originalInfo(msg, meta); + }, + ); + try { + const channels = await routeEvent({ + task_id: 't-isol', + event_id: 'e-isol', + event_type: 'task_completed', + timestamp: '2026-04-22T04:00:00Z', + }); + + // (1) Email actually ran its dispatch path (GitHub short-circuits + // on "task not found" because the shared DDB mock returns no + // Item — that's fine; the key invariant is that one channel's + // failure doesn't block the others). + expect(observedEvents).toContain('fanout.email.dispatch_stub'); + // Slack also ran (it threw), so its log line was emitted before the throw. + expect(observedEvents).toContain('fanout.slack.dispatch_stub'); + + // (2) Telemetry truthfulness: Slack must NOT be in ``dispatched`` + // because its dispatcher rejected. Email + GitHub are. + expect(channels.sort()).toEqual(['email', 'github']); + expect(channels).not.toContain('slack'); + + // The rejection surfaces in a warn log so operators can alert on it. + const warnCalls = warnSpy.mock.calls.map(c => c[1] as Record | undefined); + const rejectedWarn = warnCalls.find(meta => meta?.event === 'fanout.dispatcher.rejected'); + expect(rejectedWarn).toBeDefined(); + expect(rejectedWarn?.channel).toBe('slack'); + } finally { + infoSpy.mockRestore(); + warnSpy.mockRestore(); + } + }); +}); + +describe('fanout-task-events: handler', () => { + test('dispatches only filtered events', async () => { + const event: DynamoDBStreamEvent = { + Records: [ + mkEvent('agent_turn'), // dropped (verbose) + mkEvent('task_completed'), // dispatched + mkEvent('agent_cost_update'), // dropped + mkEvent('pr_created'), // dispatched + ], + }; + // Must not throw; the log-only dispatchers just call logger.info. + // Handler returns a ``DynamoDBBatchResponse`` so ``reportBatchItemFailures`` + // semantics are honored end-to-end (finding #1). Empty ``batchItemFailures`` + // means every record succeeded from the event-source-mapping's perspective. + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + }); + + test('per-task cap drops events beyond 20 per invocation', async () => { + const records: DynamoDBRecord[] = []; + // 25 milestones for the same task. + for (let i = 0; i < 25; i++) { + records.push(mkEvent('agent_milestone', 't-chatty')); + } + const event: DynamoDBStreamEvent = { Records: records }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + // No strong assertion possible without mocking logger — but the + // call must not throw, and the cap path is exercised. + }); + + test('malformed records are dropped, not thrown', async () => { + const event: DynamoDBStreamEvent = { + Records: [ + mkRecord('INSERT', undefined), + mkRecord('INSERT', { task_id: { S: 'x' } }), // missing fields + mkEvent('task_completed'), + ], + }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + }); + + test('REMOVE events are skipped', async () => { + const event: DynamoDBStreamEvent = { + Records: [mkRecord('REMOVE', undefined)], + }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + }); +}); + +// --------------------------------------------------------------------------- +// Chunk J — GitHub dispatcher integration +// --------------------------------------------------------------------------- + +describe('fanout-task-events: GitHub dispatcher (Chunk J)', () => { + const TASK_RECORD_BASE = { + task_id: 't-gh', + user_id: 'u-1', + status: 'COMPLETED', + repo: 'owner/repo', + pr_number: 42, + branch_name: 'bgagent/t-gh/fix', + channel_source: 'api', + status_created_at: 'COMPLETED#2026-04-30T12:00:00Z', + created_at: '2026-04-30T11:50:00Z', + updated_at: '2026-04-30T12:00:00Z', + }; + + beforeEach(() => { + // Per-test-suite reset. After ``mockReset`` we re-establish a + // permissive default so a test that forgets to script GetCommand + // doesn't crash with a TypeError. + mockDdbSend.mockReset().mockResolvedValue({ Item: undefined }); + mockUpsertTaskComment.mockReset(); + mockRenderCommentBody.mockReset().mockReturnValue('rendered body'); + mockLoadRepoConfig.mockReset().mockResolvedValue(null); + mockResolveGitHubToken.mockReset().mockResolvedValue('ghp_fake'); + mockClearTokenCache.mockReset(); + }); + + test('first terminal event POSTs a new comment and persists the comment_id to TaskTable', async () => { + // Get task record → upsert creates → UpdateItem persists. + mockDdbSend + .mockResolvedValueOnce({ Item: TASK_RECORD_BASE }) // GetCommand + .mockResolvedValueOnce({}); // UpdateCommand + mockUpsertTaskComment.mockResolvedValueOnce({ + commentId: 555, + created: true, + }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + expect(mockUpsertTaskComment).toHaveBeenCalledTimes(1); + const upsertArg = mockUpsertTaskComment.mock.calls[0][0]; + expect(upsertArg).toMatchObject({ + repo: 'owner/repo', + issueOrPrNumber: 42, + token: 'ghp_fake', + existingCommentId: undefined, + }); + // Scenario 7-ext (redeploy) BLOCKER regression: the dispatcher + // used to carry ``existingEtag`` for an ``If-Match`` PATCH header + // that GitHub rejects with HTTP 400. The field must no longer be + // passed on. + expect(upsertArg).not.toHaveProperty('existingEtag'); + // UpdateCommand fired with the new id (no etag persistence). + const update = mockDdbSend.mock.calls[1][0] as { + input: { + ExpressionAttributeValues: Record; + UpdateExpression: string; + ConditionExpression: string; + }; + }; + expect(update.input.ExpressionAttributeValues[':cid']).toBe(555); + expect(update.input.UpdateExpression).toBe('SET github_comment_id = :cid'); + expect(update.input.UpdateExpression).not.toMatch(/etag/); + // First-ever POST guard: refuse to overwrite a sibling's comment id + // that might have landed between our GetItem and this UpdateItem. + expect(update.input.ConditionExpression).toContain('attribute_not_exists(github_comment_id)'); + }); + + test('subsequent event passes the persisted comment_id so the helper PATCHes', async () => { + mockDdbSend + .mockResolvedValueOnce({ Item: { ...TASK_RECORD_BASE, github_comment_id: 555 } }); + // No UpdateCommand on a PATCH — nothing new to persist. + mockUpsertTaskComment.mockResolvedValueOnce({ + commentId: 555, + created: false, + }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + const upsertArg = mockUpsertTaskComment.mock.calls[0][0]; + expect(upsertArg.existingCommentId).toBe(555); + // No second DDB call (no UpdateCommand) — the PATCH path skips + // ``saveCommentState`` since there's no new state. + expect(mockDdbSend).toHaveBeenCalledTimes(1); + }); + + test('task with no issue_number and no pr_number skips the GitHub dispatcher', async () => { + mockDdbSend.mockResolvedValueOnce({ + Item: { ...TASK_RECORD_BASE, pr_number: undefined, issue_number: undefined }, + }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + expect(mockUpsertTaskComment).not.toHaveBeenCalled(); + // No UpdateItem either — nothing to persist. + expect(mockDdbSend).toHaveBeenCalledTimes(1); + }); + + test('missing task record (TTL race) → skip without throwing', async () => { + mockDdbSend.mockResolvedValueOnce({ Item: undefined }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-missing')] }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + + expect(mockUpsertTaskComment).not.toHaveBeenCalled(); + }); + + test('upsertTaskComment rejection does NOT break the batch (routeEvent catches)', async () => { + mockDdbSend.mockResolvedValueOnce({ Item: TASK_RECORD_BASE }); + mockUpsertTaskComment.mockRejectedValueOnce(new Error('github 500')); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + // No UpdateCommand fires (no id to persist from a failed upsert). + const updateCalls = mockDdbSend.mock.calls.filter( + c => (c[0] as { _type?: string })._type === 'Update', + ); + expect(updateCalls).toHaveLength(0); + }); + + test('dispatcher does NOT forward an If-Match-style ETag to upsertTaskComment (BLOCKER regression)', async () => { + // Scenario 7-ext (redeploy) found that GitHub rejects any PATCH + // on an issue comment carrying a conditional header with HTTP 400 + // ("Conditional request headers are not allowed in unsafe requests + // unless supported by the endpoint"). The fanout dispatcher must + // not carry an etag through to the helper, even when stray + // ``github_comment_etag`` data exists on legacy TaskRecords from + // before this fix landed. + mockDdbSend + .mockResolvedValueOnce({ + Item: { + ...TASK_RECORD_BASE, + github_comment_id: 555, + // Legacy field — must be ignored by the new code path. + github_comment_etag: '"legacy-etag-from-before-fix"', + }, + }); + mockUpsertTaskComment.mockResolvedValueOnce({ + commentId: 555, + created: false, + }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + const upsertArg = mockUpsertTaskComment.mock.calls[0][0]; + expect(upsertArg.existingCommentId).toBe(555); + expect(upsertArg).not.toHaveProperty('existingEtag'); + }); + + test('404 → POST fallback persists new comment id with a prev-id condition guard', async () => { + // Race guard (silent-failure review SIG-3): when the cached + // comment was deleted upstream and the helper POSTed a new one, + // the UpdateItem must require ``github_comment_id = :prev`` so + // we cannot silently overwrite a sibling fanout invocation that + // already re-posted (or that beat us to writing a fresh id). + mockDdbSend + .mockResolvedValueOnce({ + Item: { ...TASK_RECORD_BASE, github_comment_id: 555 }, + }) + .mockResolvedValueOnce({}); // UpdateCommand for the re-POST + mockUpsertTaskComment.mockResolvedValueOnce({ + commentId: 999, // new id from the fallback POST + created: true, + }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + const update = mockDdbSend.mock.calls[1][0] as { + input: { + ExpressionAttributeValues: Record; + UpdateExpression: string; + ConditionExpression: string; + }; + }; + expect(update.input.ExpressionAttributeValues[':cid']).toBe(999); + expect(update.input.ExpressionAttributeValues[':prev']).toBe(555); + expect(update.input.ConditionExpression).toContain('github_comment_id = :prev'); + expect(update.input.ConditionExpression).not.toContain('attribute_not_exists(github_comment_id)'); + }); + + test('400 from PATCH surfaces as fanout.dispatcher.rejected without duplicate POST (If-Match regression guard)', async () => { + // End-to-end version of silent-failure review MINOR-1: if a + // future refactor accidentally reintroduces an If-Match (or any + // conditional header) header, GitHub returns HTTP 400 for the + // PATCH. The fanout handler must NOT retry via POST (only 404 + // triggers the fallback) and must NOT persist anything new. The + // 400 surfaces as a warn through the batch-level + // ``fanout.dispatcher.rejected`` log instead. + const loggerModule = await import('../../src/handlers/shared/logger'); + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation(() => undefined); + try { + mockDdbSend.mockResolvedValueOnce({ + Item: { ...TASK_RECORD_BASE, github_comment_id: 555 }, + }); + const { GitHubCommentError } = jest.requireMock( + '../../src/handlers/shared/github-comment', + ); + mockUpsertTaskComment.mockRejectedValueOnce( + new GitHubCommentError( + 'PATCH /repos/owner/repo/issues/comments/555 failed: HTTP 400', + 400, + ), + ); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + + // No UpdateCommand fires — the 400 path has nothing to persist. + const updateCalls = mockDdbSend.mock.calls.filter( + c => (c[0] as { _type?: string })._type === 'Update', + ); + expect(updateCalls).toHaveLength(0); + + // The 400 surfaced as a dispatcher-rejected warn, not as a + // silent swallow. + const rejectedWarn = warnSpy.mock.calls.find( + c => (c[1] as Record | undefined)?.event === 'fanout.dispatcher.rejected', + ); + expect(rejectedWarn).toBeDefined(); + expect((rejectedWarn?.[1] as Record).channel).toBe('github'); + expect(String((rejectedWarn?.[1] as Record).error)).toContain('HTTP 400'); + } finally { + warnSpy.mockRestore(); + } + }); + + test('falls back to issue_number when pr_number is absent', async () => { + // Webhook-submitted issue tasks are the common real-world surface. + mockDdbSend + .mockResolvedValueOnce({ + Item: { ...TASK_RECORD_BASE, pr_number: undefined, issue_number: 7 }, + }) + .mockResolvedValueOnce({}); + mockUpsertTaskComment.mockResolvedValueOnce({ commentId: 1, created: true }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + expect(mockUpsertTaskComment.mock.calls[0][0].issueOrPrNumber).toBe(7); + }); + + test('loadRepoConfig throwing a transient error falls back to the platform default token', async () => { + // SFH-S2: DDB throttling must not black-hole GitHub comments; + // the dispatcher falls back to the platform default ARN so + // one flaky invocation doesn't silence the whole fleet. + mockLoadRepoConfig.mockRejectedValueOnce( + Object.assign(new Error('rate exceeded'), { name: 'ProvisionedThroughputExceededException' }), + ); + mockDdbSend + .mockResolvedValueOnce({ Item: TASK_RECORD_BASE }) + .mockResolvedValueOnce({}); + mockUpsertTaskComment.mockResolvedValueOnce({ commentId: 1, created: true }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + // Fallback to the platform env-var ARN (set at the top of this file). + expect(mockResolveGitHubToken).toHaveBeenCalledWith('arn:aws:secretsmanager:us-east-1:0:secret:platform'); + }); + + test('resolveGitHubToken throwing causes the dispatcher to skip without calling upsertTaskComment', async () => { + // SFH-S1 adjacent: when Secrets Manager fails, we must NOT + // attempt to write a comment with an undefined token. + mockDdbSend.mockResolvedValueOnce({ Item: TASK_RECORD_BASE }); + mockResolveGitHubToken.mockRejectedValueOnce(new Error('secrets manager down')); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + + expect(mockUpsertTaskComment).not.toHaveBeenCalled(); + }); + + test('saveCommentState ConditionalCheckFailed (task evicted) logs at INFO not ERROR', async () => { + // Benign: the task was TTL-evicted between the Get and the + // Update. Subsequent events for this task will also skip, so + // no duplicate-comment risk. Must NOT alarm operators. + mockDdbSend + .mockResolvedValueOnce({ Item: TASK_RECORD_BASE }) + .mockRejectedValueOnce( + Object.assign(new Error('condition failed'), { name: 'ConditionalCheckFailedException' }), + ); + mockUpsertTaskComment.mockResolvedValueOnce({ commentId: 1, created: true }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + + // Upsert fired (comment posted); handler didn't throw. + expect(mockUpsertTaskComment).toHaveBeenCalledTimes(1); + }); + + test('saveCommentState non-conditional failure (DDB throttling) logs at ERROR with error_id', async () => { + // SFH-B2: non-ConditionalCheckFailed failures leave the task + // without a comment_id, so the next event will duplicate. This + // is a real persistence bug that must alarm distinctly. + const errorSpy = jest.fn(); + jest.spyOn( + (await import('../../src/handlers/shared/logger')).logger, + 'error', + ).mockImplementation(errorSpy); + + mockDdbSend + .mockResolvedValueOnce({ Item: TASK_RECORD_BASE }) + .mockRejectedValueOnce( + Object.assign(new Error('throttled'), { name: 'ProvisionedThroughputExceededException' }), + ); + mockUpsertTaskComment.mockResolvedValueOnce({ commentId: 1, created: true }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + // The dedicated error_id tag must fire so operators can alarm on it. + const errorCall = errorSpy.mock.calls.find( + c => (c[1] as Record | undefined)?.error_id === 'FANOUT_GITHUB_PERSIST_FAILED', + ); + expect(errorCall).toBeDefined(); + }); + + test('401 from GitHub clears the token cache and retries once with a fresh token', async () => { + // SFH-S1: token rotation recovery. The first upsert rejects with + // 401, the dispatcher evicts the cache, re-fetches, and retries. + // We import the (mocked) class fresh so ``instanceof`` in the + // handler matches the instance the test throws. + const { GitHubCommentError } = jest.requireMock( + '../../src/handlers/shared/github-comment', + ); + mockDdbSend + .mockResolvedValueOnce({ Item: TASK_RECORD_BASE }) + .mockResolvedValueOnce({}); + mockUpsertTaskComment + .mockRejectedValueOnce(new GitHubCommentError('unauthorized', 401)) + .mockResolvedValueOnce({ commentId: 1, created: true }); + // Two token fetches — stale then fresh. + mockResolveGitHubToken + .mockResolvedValueOnce('ghp_stale') + .mockResolvedValueOnce('ghp_fresh'); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + expect(mockClearTokenCache).toHaveBeenCalledTimes(1); + expect(mockUpsertTaskComment).toHaveBeenCalledTimes(2); + // Retry carried the fresh token. + expect(mockUpsertTaskComment.mock.calls[1][0].token).toBe('ghp_fresh'); + }); + + test('per-repo github_token_secret_arn override takes precedence over platform default', async () => { + mockLoadRepoConfig.mockResolvedValueOnce({ + repo: 'owner/repo', + status: 'active', + onboarded_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + github_token_secret_arn: 'arn:repo-specific', + }); + mockDdbSend + .mockResolvedValueOnce({ Item: TASK_RECORD_BASE }) + .mockResolvedValueOnce({}); + mockUpsertTaskComment.mockResolvedValueOnce({ commentId: 1, created: true }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + expect(mockResolveGitHubToken).toHaveBeenCalledWith('arn:repo-specific'); + }); + + // ---- Scenario 7-extended regression (post-K2 deploy validation) ---- + + test('TaskRecord with string-typed cost_usd/duration_s renders without throwing (DDB Number coercion)', async () => { + // Regression: the DynamoDB Document-client returns Number + // attributes as strings. ``renderCommentBody`` calls + // ``costUsd.toFixed(4)`` which throws TypeError on a string, + // causing every terminal event on a pr_iteration task to be + // rejected by the dispatcher (observed in Scenario 7-extended + // deploy validation, task ``01KQSPFXQMYQR0CNGCF56XB9ZM``). The + // fan-out boundary must coerce. + mockDdbSend + .mockResolvedValueOnce({ + Item: { + ...TASK_RECORD_BASE, + cost_usd: '0.20939010000000002', + duration_s: '96.0', + }, + }) + .mockResolvedValueOnce({}); + mockUpsertTaskComment.mockResolvedValueOnce({ commentId: 1, created: true }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await expect(handler(event)).resolves.toEqual({ batchItemFailures: [] }); + + expect(mockRenderCommentBody).toHaveBeenCalledTimes(1); + const renderArg = mockRenderCommentBody.mock.calls[0][0]; + // Coerced to finite numbers so ``.toFixed`` downstream works. + expect(typeof renderArg.costUsd).toBe('number'); + expect(renderArg.costUsd).toBeCloseTo(0.2094, 4); + expect(typeof renderArg.durationS).toBe('number'); + expect(renderArg.durationS).toBe(96); + // Upsert reached the HTTP layer — no TypeError short-circuit. + expect(mockUpsertTaskComment).toHaveBeenCalledTimes(1); + }); + + test('non-finite string cost collapses to null and emits a warn (surfaces writer bugs)', async () => { + // Defense-in-depth: a corrupt ``cost_usd`` that parses to ``NaN`` + // must not produce a ``$NaN`` row. The coercion returns ``null`` + // so the optional render branch stays off, but must also emit a + // ``fanout.numeric_coercion_failed`` warn so the writer bug is + // not silently absorbed. + const loggerModule = await import('../../src/handlers/shared/logger'); + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation(() => undefined); + try { + mockDdbSend + .mockResolvedValueOnce({ + Item: { ...TASK_RECORD_BASE, cost_usd: 'not-a-number', duration_s: null }, + }) + .mockResolvedValueOnce({}); + mockUpsertTaskComment.mockResolvedValueOnce({ commentId: 1, created: true }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + const renderArg = mockRenderCommentBody.mock.calls[0][0]; + expect(renderArg.costUsd).toBeNull(); + expect(renderArg.durationS).toBeNull(); + + const warnCall = warnSpy.mock.calls.find( + c => (c[1] as Record | undefined)?.event === 'numeric.coercion_failed', + ); + expect(warnCall).toBeDefined(); + expect((warnCall?.[1] as Record).field).toBe('cost_usd'); + expect((warnCall?.[1] as Record).raw).toBe('not-a-number'); + } finally { + warnSpy.mockRestore(); + } + }); + + test('absent cost_usd / duration_s fields (not just null) render as absent without warning', async () => { + // The DDB Item may simply omit the attributes (task still RUNNING + // at the time of the event). ``undefined`` inputs must not warn — + // they're not corrupt, they're just not set yet. + const loggerModule = await import('../../src/handlers/shared/logger'); + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation(() => undefined); + try { + const base = { ...TASK_RECORD_BASE } as Record; + delete base.cost_usd; + delete base.duration_s; + mockDdbSend.mockResolvedValueOnce({ Item: base }).mockResolvedValueOnce({}); + mockUpsertTaskComment.mockResolvedValueOnce({ commentId: 1, created: true }); + + const event: DynamoDBStreamEvent = { Records: [mkEvent('task_completed', 't-gh')] }; + await handler(event); + + const renderArg = mockRenderCommentBody.mock.calls[0][0]; + expect(renderArg.costUsd).toBeNull(); + expect(renderArg.durationS).toBeNull(); + + const coercionWarns = warnSpy.mock.calls.filter( + c => (c[1] as Record | undefined)?.event === 'numeric.coercion_failed', + ); + expect(coercionWarns).toHaveLength(0); + } finally { + warnSpy.mockRestore(); + } + }); +}); + +// --------------------------------------------------------------------------- +// Scenario 7-extended — agent_milestone routing regression +// --------------------------------------------------------------------------- + +/** Stream record for an ``agent_milestone`` event carrying a named + * milestone in ``metadata.milestone`` — the shape written by + * ``agent/src/progress_writer.py::write_agent_milestone``. */ +function mkMilestoneRecord(milestone: string, taskId = 't-1'): DynamoDBRecord { + return mkRecord('INSERT', { + task_id: { S: taskId }, + event_id: { S: `01MILE${milestone}` }, + event_type: { S: 'agent_milestone' }, + timestamp: { S: '2026-05-04T14:34:57Z' }, + metadata: { M: { milestone: { S: milestone } } }, + }); +} + +describe('fanout-task-events: agent_milestone routing (effective event type)', () => { + // The agent writes named checkpoints (``pr_created``, + // ``nudge_acknowledged``, …) with ``event_type = agent_milestone`` + // and ``metadata.milestone`` carrying the name (see + // ``agent/src/progress_writer.py::write_agent_milestone``). The + // channel-default filters are expressed against the milestone names + // directly (design §6.2), so routing unwraps the wrapper before + // matching. Without unwrap, ``pr_created`` would fan out to zero + // channels — observed in Scenario 7-extended. + + const makeMilestone = (milestone: string): FanOutEvent => ({ + task_id: 't-1', + event_id: 'e-1', + event_type: 'agent_milestone', + timestamp: '2026-05-04T14:34:57Z', + metadata: { milestone }, + }); + + test('shouldFanOut unwraps agent_milestone to its milestone name', () => { + // ``pr_created`` is in Slack + GitHub defaults → fan out. + expect(shouldFanOut(makeMilestone('pr_created'))).toBe(true); + }); + + test('shouldFanOut drops agent_milestone with a non-subscribed milestone', () => { + // ``repo_setup_complete`` is deliberately NOT in any channel's + // default — verbose opt-in only, per §6.2. + expect(shouldFanOut(makeMilestone('repo_setup_complete'))).toBe(false); + }); + + test('shouldFanOut keeps old behavior when metadata.milestone is missing or malformed', () => { + // Backwards-compat: a bare ``agent_milestone`` event (shouldn't + // happen in practice — the writer always sets ``milestone``) must + // not crash the router; it simply doesn't match any default. We + // cover: missing ``metadata`` entirely, empty ``metadata`` object, + // missing ``milestone`` key, empty-string milestone, and a + // non-string milestone value. + const bare: FanOutEvent = { + task_id: 't-1', + event_id: 'e-1', + event_type: 'agent_milestone', + timestamp: '2026-05-04T14:34:57Z', + }; + expect(shouldFanOut(bare)).toBe(false); + expect(shouldFanOut({ ...bare, metadata: {} })).toBe(false); + expect(shouldFanOut({ ...bare, metadata: { foo: 'bar' } })).toBe(false); + expect(shouldFanOut({ ...bare, metadata: { milestone: '' } })).toBe(false); + expect(shouldFanOut({ ...bare, metadata: { milestone: 42 as unknown as string } })).toBe(false); + }); + + test('shouldFanOut rejects milestones outside the routing allowlist even if they match a channel default', () => { + // Structural defense against naming drift: a future rename that + // accidentally makes ``metadata.milestone`` equal an existing + // channel-default entry (e.g. ``task_cancelled``) must NOT start + // silently fanning out. Only the allowlist (today: ``pr_created``) + // is eligible for unwrap. + const colliding: FanOutEvent = { + task_id: 't-collide', + event_id: 'e-collide', + event_type: 'agent_milestone', + timestamp: '2026-05-04T14:34:57Z', + metadata: { milestone: 'task_cancelled' }, + }; + // ``task_cancelled`` is in Slack + GitHub defaults as a terminal + // event type — but unwrap must still refuse because the milestone + // is outside ``ROUTABLE_MILESTONES``. + expect(shouldFanOut(colliding)).toBe(false); + }); + + test('routeEvent dispatches agent_milestone(pr_created) to Slack + GitHub, not Email', async () => { + const channels = await routeEvent(makeMilestone('pr_created')); + expect(channels.sort()).toEqual(['github', 'slack']); + }); + + test('routeEvent drops agent_milestone(agent_turn-like) that no channel subscribes to', async () => { + // ``nudge_acknowledged`` is in no channel default today. Must + // still route cleanly (empty list) rather than throw. + const channels = await routeEvent(makeMilestone('nudge_acknowledged')); + expect(channels).toEqual([]); + }); + + test('handler dispatches GitHub comment on agent_milestone(pr_created) stream record', async () => { + // End-to-end guard: the DynamoDB Stream shape for pr_created is + // an ``agent_milestone`` wrapper. The handler must read the + // milestone name from metadata, match the GitHub default filter, + // load the task, and reach ``upsertTaskComment``. + mockDdbSend + .mockResolvedValueOnce({ + Item: { + task_id: 't-milestone', + user_id: 'u-1', + status: 'RUNNING', + repo: 'owner/repo', + pr_number: 99, + branch_name: 'bgagent/t-milestone/fix', + channel_source: 'api', + status_created_at: 'RUNNING#2026-05-04T14:34:57Z', + created_at: '2026-05-04T14:30:00Z', + updated_at: '2026-05-04T14:34:57Z', + }, + }) + .mockResolvedValueOnce({}); + mockUpsertTaskComment.mockResolvedValueOnce({ + commentId: 777, + created: true, + }); + + const event: DynamoDBStreamEvent = { + Records: [mkMilestoneRecord('pr_created', 't-milestone')], + }; + await handler(event); + + expect(mockUpsertTaskComment).toHaveBeenCalledTimes(1); + // Comment body renders ``pr_created`` (the effective type), + // not the wrapper ``agent_milestone``. Cross-check: the watch + // CLI renders ``★ pr_created: ...`` on the same record, so the + // two surfaces stay consistent. + const renderArg = mockRenderCommentBody.mock.calls[0][0]; + expect(renderArg.latestEventType).toBe('pr_created'); + }); +}); + +// --------------------------------------------------------------------------- +// Krokoko code review findings #1 + #5 — partial-batch response contract +// --------------------------------------------------------------------------- + +/** + * Stream record with a caller-supplied ``eventID`` so the test can + * assert which record surfaces in ``batchItemFailures``. ``mkEvent`` + * uses ``Math.random()`` for the id which is fine for parse tests but + * useless when we need to cross-reference the failure identifier. + */ +function mkEventWithId(type: string, eventID: string, taskId = 't-fail'): DynamoDBRecord { + return { + eventID, + eventName: 'INSERT', + eventSource: 'aws:dynamodb', + dynamodb: { + NewImage: { + task_id: { S: taskId }, + event_id: { S: `01ABC${type}` }, + event_type: { S: type }, + timestamp: { S: '2026-05-05T00:00:00Z' }, + metadata: { M: { code: { S: 'OK' } } }, + } as never, + }, + } as unknown as DynamoDBRecord; +} + +describe('fanout-task-events: partial-batch response (findings #1 + #5)', () => { + // Finding #1: the construct sets ``reportBatchItemFailures: true`` on + // the event-source-mapping, but the handler used to return ``void``. + // That combination makes Lambda retry the WHOLE batch on any + // unhandled throw — replaying every sibling event and defeating the + // per-task ordering guarantee promised upstream by + // ``ParallelizationFactor: 1``. + // + // Finding #5: the architecturally reachable poison-pill path is a + // throw that bypasses ``routeEvent``'s ``Promise.allSettled``. The + // isolation works today for async rejections (``resolveTokenSecretArn`` + // → ``AccessDeniedException`` is caught), but a future refactor that + // drops ``allSettled`` or introduces a sync-throw path before the + // dispatcher list is built would surface that throw at the handler. + // The tests below exercise the handler's defensive try/catch by + // injecting a throw from a dependency the handler uses OUTSIDE + // ``routeEvent`` — the ``logger.warn`` call in the rate-limit path — + // which is the same failure shape the handler must tolerate for any + // future escape from ``allSettled`` containment. + + beforeEach(() => { + mockDdbSend.mockReset().mockResolvedValue({ Item: undefined }); + mockUpsertTaskComment.mockReset(); + mockRenderCommentBody.mockReset().mockReturnValue('rendered body'); + mockLoadRepoConfig.mockReset().mockResolvedValue(null); + mockResolveGitHubToken.mockReset().mockResolvedValue('ghp_fake'); + mockClearTokenCache.mockReset(); + }); + + test('AccessDeniedException from resolveTokenSecretArn stays isolated via allSettled; batch still succeeds (finding #5 today)', async () => { + // Baseline: today's ``routeEvent`` catches the AccessDenied throw + // via ``Promise.allSettled`` so it surfaces as a + // ``fanout.dispatcher.rejected`` warn, NOT as a handler-level + // throw. The structured response is therefore an empty + // ``batchItemFailures`` — the record advances past the cursor. + // This test pins the current containment so a future change that + // accidentally rethrows past ``allSettled`` will flip it from + // "empty failures" to "one failure" and fail loudly here. + const loggerModule = await import('../../src/handlers/shared/logger'); + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation(() => undefined); + try { + mockDdbSend.mockResolvedValueOnce({ + Item: { + task_id: 't-boom', + user_id: 'u-1', + status: 'COMPLETED', + repo: 'owner/repo', + pr_number: 42, + branch_name: 'bgagent/t-boom/fix', + channel_source: 'api', + status_created_at: 'COMPLETED#2026-05-05T00:00:00Z', + created_at: '2026-05-05T00:00:00Z', + updated_at: '2026-05-05T00:00:00Z', + }, + }); + mockLoadRepoConfig.mockRejectedValueOnce( + Object.assign(new Error('iam deny'), { name: 'AccessDeniedException' }), + ); + + const poisonId = 'evt-access-denied'; + const event: DynamoDBStreamEvent = { + Records: [mkEventWithId('task_completed', poisonId, 't-boom')], + }; + + const result = await handler(event); + + // Containment invariant: ``Promise.allSettled`` caught the + // rejection; the handler sees no throw. + expect(result).toEqual({ batchItemFailures: [] }); + // … but the rejection WAS observed by operators through the + // dispatcher-rejected warn (existing coverage path). + const rejectedWarn = warnSpy.mock.calls.find( + c => (c[1] as Record | undefined)?.event === 'fanout.dispatcher.rejected', + ); + expect(rejectedWarn).toBeDefined(); + expect((rejectedWarn?.[1] as Record).channel).toBe('github'); + } finally { + warnSpy.mockRestore(); + } + }); + + test('unhandled throw OUTSIDE routeEvent flags the record as a batch item failure (finding #1 defense)', async () => { + // Defense-in-depth proof: when SOMETHING in the record-processing + // loop throws past ``routeEvent``'s containment (simulated here by + // making ``logger.warn`` throw on the rate-limit path — the + // closest real non-``routeEvent`` code path), the handler's + // per-record try/catch must push the record's ``eventID`` into + // ``batchItemFailures`` so Lambda retries ONLY that record. Pre-fix + // the handler returned void and Lambda would retry the ENTIRE + // batch, replaying every sibling event and defeating per-task + // ordering. + const loggerModule = await import('../../src/handlers/shared/logger'); + // Rate-limit warn on the 21st event throws; earlier events succeed. + let warnCalls = 0; + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation( + (_msg: string, meta?: Record) => { + if (meta?.event === 'fanout.rate_limit.hit') { + warnCalls++; + throw new Error('simulated: logger broke during rate-limit warn'); + } + }, + ); + try { + // 21 events for the same task — the 21st triggers the rate-limit + // warn, which throws, escaping ``routeEvent`` entirely (the + // cap check happens BEFORE ``routeEvent`` is called). + const records: DynamoDBRecord[] = []; + for (let i = 0; i < 21; i++) { + records.push(mkEventWithId('agent_milestone', `evt-${i}`, 't-chatty')); + } + // Only the 21st record should be in batchItemFailures — events + // 0..19 succeed (within cap), event 20 trips the cap and throws. + // Note that ``agent_milestone`` with no metadata.milestone does + // not match any filter (so it's dropped), but the cap check is + // purely per-task per invocation and fires regardless; to make + // the record reach the cap check we use ``task_completed`` which + // routes to all three channels and survives ``shouldFanOut``. + records.length = 0; + for (let i = 0; i < 21; i++) { + records.push(mkEventWithId('task_completed', `evt-${i}`, 't-chatty')); + } + + const result = await handler({ Records: records }); + + expect(warnCalls).toBeGreaterThan(0); + // The 21st record (index 20) is the one that hit the cap and + // threw via the broken warn. Everything before it succeeded + // from the handler's perspective (``routeEvent`` short-circuits + // on "task not found" since the shared DDB mock returns no Item). + expect(result.batchItemFailures).toEqual([ + { itemIdentifier: 'evt-20' }, + ]); + } finally { + warnSpy.mockRestore(); + } + }); + + test('successful records do NOT appear in batchItemFailures (mixed batch)', async () => { + // Mixed batch: one record throws past routeEvent (via the same + // rate-limit-warn trick as above but in a simpler shape — we make + // the second record specifically trigger the throw), the other + // routes cleanly. The response must list ONLY the failing eventID. + const loggerModule = await import('../../src/handlers/shared/logger'); + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation( + (_msg: string, meta?: Record) => { + if (meta?.event === 'fanout.rate_limit.hit') { + throw new Error('simulated broken logger'); + } + }, + ); + try { + // Send 21 events for 't-chatty' (trips the cap on #21 → throws) + // preceded by ONE event for 't-ok' (dispatches cleanly). + const records: DynamoDBRecord[] = []; + records.push(mkEventWithId('task_completed', 'evt-ok', 't-ok')); + for (let i = 0; i < 21; i++) { + records.push(mkEventWithId('task_completed', `evt-chatty-${i}`, 't-chatty')); + } + const result = await handler({ Records: records }); + + expect(result.batchItemFailures).toHaveLength(1); + expect(result.batchItemFailures[0]).toEqual({ itemIdentifier: 'evt-chatty-20' }); + // Specifically NOT the successful record. + expect(result.batchItemFailures.map(f => f.itemIdentifier)).not.toContain('evt-ok'); + } finally { + warnSpy.mockRestore(); + } + }); + + test('poisonous record emits a fanout.record.failed warn so operators can alarm', async () => { + // The warn is the observability counterpart to the structured + // retry response — operators grep CloudWatch for the event name + // and alarm on its rate. + const loggerModule = await import('../../src/handlers/shared/logger'); + const allWarns: Array<{ msg: string; meta?: Record }> = []; + const warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation( + (msg: string, meta?: Record) => { + allWarns.push({ msg, meta }); + if (meta?.event === 'fanout.rate_limit.hit') { + throw new Error('simulated broken logger for rate-limit path'); + } + }, + ); + try { + const records: DynamoDBRecord[] = []; + for (let i = 0; i < 21; i++) { + records.push(mkEventWithId('task_completed', `evt-${i}`, 't-chatty')); + } + await handler({ Records: records }); + + const failedWarn = allWarns.find(w => w.meta?.event === 'fanout.record.failed'); + expect(failedWarn).toBeDefined(); + expect(failedWarn?.meta?.event_id).toBe('evt-20'); + // The underlying error message propagates into the warn so the + // alarm can point at the root cause rather than just the fact of + // a failure. + expect(String(failedWarn?.meta?.error)).toContain('simulated broken logger'); + } finally { + warnSpy.mockRestore(); + } + }); + + test('batch with zero throws returns an empty batchItemFailures array', async () => { + // Regression guard: the structured-response shape must hold even + // when nothing fails. Lambda's event-source-mapping treats an + // empty array as "all records succeeded" and advances the cursor. + const event: DynamoDBStreamEvent = { + Records: [ + mkEvent('agent_turn'), // dropped (verbose) + mkEvent('task_completed'), // dispatched (GitHub short-circuits on missing task) + ], + }; + const result = await handler(event); + expect(result).toEqual({ batchItemFailures: [] }); + }); +}); diff --git a/cdk/test/handlers/get-task-events.test.ts b/cdk/test/handlers/get-task-events.test.ts index 466ea0b..ad8c70f 100644 --- a/cdk/test/handlers/get-task-events.test.ts +++ b/cdk/test/handlers/get-task-events.test.ts @@ -230,4 +230,227 @@ describe('get-task-events handler', () => { expect(body.data[0].metadata).toEqual({}); }); + + // -------- ?after= support (Phase 1b Step 4) -------- + + const VALID_AFTER = '01ARZ3NDEKTSV4RRFFQ69G5FAV'; + const ANOTHER_ULID = '01ARZ3NDEKTSV4RRFFQ69G5FAW'; + + test('valid ?after routes to KeyConditionExpression with event_id > :after', async () => { + const event = makeEvent({ queryStringParameters: { after: VALID_AFTER } }); + await handler(event); + + const queryInput = MockQueryCommand.mock.calls[0][0]; + expect(queryInput.KeyConditionExpression).toBe('task_id = :tid AND event_id > :after'); + expect(queryInput.ExpressionAttributeValues).toEqual({ + ':tid': 'task-1', + ':after': VALID_AFTER, + }); + // ``after`` must NOT carry ExclusiveStartKey through + expect(queryInput.ExclusiveStartKey).toBeUndefined(); + }); + + test('both after and next_token → after wins + WARN logged', async () => { + const stdoutWrite = jest.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + const bogusToken = Buffer.from('{"task_id":"task-1","event_id":"z"}').toString('base64'); + const event = makeEvent({ + queryStringParameters: { after: VALID_AFTER, next_token: bogusToken }, + }); + const result = await handler(event); + + expect(result.statusCode).toBe(200); + + const queryInput = MockQueryCommand.mock.calls[0][0]; + // Confirms ``after`` was used, not ``next_token``. + expect(queryInput.KeyConditionExpression).toBe('task_id = :tid AND event_id > :after'); + expect(queryInput.ExclusiveStartKey).toBeUndefined(); + + // WARN about the conflict must be emitted. + const warnLines = stdoutWrite.mock.calls + .map(c => String(c[0])) + .filter(line => line.includes('"level":"WARN"')); + const conflictWarn = warnLines.find(line => + line.includes('Both after and next_token provided'), + ); + expect(conflictWarn).toBeDefined(); + } finally { + stdoutWrite.mockRestore(); + } + }); + + test('invalid ?after (wrong length) returns 400 with VALIDATION_ERROR', async () => { + mockSend.mockReset(); // Should short-circuit before any DDB call + const event = makeEvent({ queryStringParameters: { after: 'too-short' } }); + const result = await handler(event); + + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error.code).toBe('VALIDATION_ERROR'); + expect(body.error.message).toContain('after'); + expect(mockSend).not.toHaveBeenCalled(); + }); + + test('invalid ?after (illegal chars) returns 400 with VALIDATION_ERROR', async () => { + mockSend.mockReset(); + // Contains Crockford-excluded 'I' and 'L'. + const event = makeEvent({ queryStringParameters: { after: 'IIIIIIIIIIIIIIIIIIIIIIIIII' } }); + const result = await handler(event); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + expect(mockSend).not.toHaveBeenCalled(); + }); + + test('empty ?after falls through to from-beginning (regression)', async () => { + const event = makeEvent({ queryStringParameters: { after: '' } }); + const result = await handler(event); + + expect(result.statusCode).toBe(200); + const queryInput = MockQueryCommand.mock.calls[0][0]; + // No after filter applied. + expect(queryInput.KeyConditionExpression).toBe('task_id = :tid'); + expect(queryInput.ExpressionAttributeValues).toEqual({ ':tid': 'task-1' }); + }); + + test('neither after nor next_token preserves original from-beginning behavior', async () => { + await handler(makeEvent()); + + const queryInput = MockQueryCommand.mock.calls[0][0]; + expect(queryInput.KeyConditionExpression).toBe('task_id = :tid'); + expect(queryInput.ExpressionAttributeValues).toEqual({ ':tid': 'task-1' }); + expect(queryInput.ExclusiveStartKey).toBeUndefined(); + expect(queryInput.ScanIndexForward).toBe(true); + }); + + test('after mode with empty result returns events:[] and no next_token', async () => { + mockSend.mockReset(); + mockSend + .mockResolvedValueOnce({ Item: TASK_RECORD }) + .mockResolvedValueOnce({ Items: [] }); + + const result = await handler(makeEvent({ + queryStringParameters: { after: VALID_AFTER }, + })); + const body = JSON.parse(result.body); + + expect(result.statusCode).toBe(200); + expect(body.data).toEqual([]); + expect(body.pagination.has_more).toBe(false); + expect(body.pagination.next_token).toBeNull(); + }); + + test('after mode with truncated result emits next_token for continuation', async () => { + mockSend.mockReset(); + mockSend + .mockResolvedValueOnce({ Item: TASK_RECORD }) + .mockResolvedValueOnce({ + Items: [EVENT_ITEMS[0]], + LastEvaluatedKey: { task_id: 'task-1', event_id: ANOTHER_ULID }, + }); + + const result = await handler(makeEvent({ + queryStringParameters: { after: VALID_AFTER, limit: '1' }, + })); + const body = JSON.parse(result.body); + + expect(body.pagination.has_more).toBe(true); + expect(body.pagination.next_token).toBeTruthy(); + }); + + // -------- ?desc= support (status snapshot) -------- + + test('?desc=1 flips ScanIndexForward to false and logs query_mode=desc', async () => { + const stdoutWrite = jest.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + const event = makeEvent({ queryStringParameters: { desc: '1' } }); + const result = await handler(event); + + expect(result.statusCode).toBe(200); + + const queryInput = MockQueryCommand.mock.calls[0][0]; + expect(queryInput.ScanIndexForward).toBe(false); + // ``desc`` alone does not touch KeyConditionExpression or ExclusiveStartKey. + expect(queryInput.KeyConditionExpression).toBe('task_id = :tid'); + expect(queryInput.ExclusiveStartKey).toBeUndefined(); + + const infoLines = stdoutWrite.mock.calls + .map(c => String(c[0])) + .filter(line => line.includes('"level":"INFO"')); + const invoked = infoLines.find(line => line.includes('get-task-events invoked')); + expect(invoked).toContain('"query_mode":"desc"'); + } finally { + stdoutWrite.mockRestore(); + } + }); + + test('?desc=true is accepted as truthy', async () => { + await handler(makeEvent({ queryStringParameters: { desc: 'true' } })); + const queryInput = MockQueryCommand.mock.calls[0][0]; + expect(queryInput.ScanIndexForward).toBe(false); + }); + + test('unknown ?desc= value falls through to ascending (no 400)', async () => { + // Lenient parsing — anything other than "1"/"true" is treated as absent. + await handler(makeEvent({ queryStringParameters: { desc: 'yes' } })); + const queryInput = MockQueryCommand.mock.calls[0][0]; + expect(queryInput.ScanIndexForward).toBe(true); + }); + + test('?desc=1 suppresses next_token even when DDB returns a LastEvaluatedKey', async () => { + // DDB populates ``LastEvaluatedKey`` on truncated descending scans, but + // the token carries no direction. A follow-up caller that drops + // ``desc`` would silently interleave ascending results. The handler + // intentionally null-trims the token on descending pages; ``bgagent + // status`` only reads one page and does not need continuation. + mockSend.mockReset(); + mockSend + .mockResolvedValueOnce({ Item: TASK_RECORD }) + .mockResolvedValueOnce({ + Items: [EVENT_ITEMS[0]], + LastEvaluatedKey: { task_id: 'task-1', event_id: ANOTHER_ULID }, + }); + + const result = await handler(makeEvent({ + queryStringParameters: { desc: '1', limit: '1' }, + })); + const body = JSON.parse(result.body); + + expect(body.pagination.has_more).toBe(false); + expect(body.pagination.next_token).toBeNull(); + }); + + test('?desc=1 combined with ?after= returns 400 VALIDATION_ERROR before any DDB call', async () => { + mockSend.mockReset(); + const event = makeEvent({ queryStringParameters: { desc: '1', after: VALID_AFTER } }); + const result = await handler(event); + + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error.code).toBe('VALIDATION_ERROR'); + expect(body.error.message).toContain('mutually exclusive'); + expect(mockSend).not.toHaveBeenCalled(); + }); + + test('handler logs INFO on entry and exit with query_mode', async () => { + const stdoutWrite = jest.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + await handler(makeEvent({ queryStringParameters: { after: VALID_AFTER } })); + + const infoLines = stdoutWrite.mock.calls + .map(c => String(c[0])) + .filter(line => line.includes('"level":"INFO"')); + + const invoked = infoLines.find(line => line.includes('get-task-events invoked')); + const complete = infoLines.find(line => line.includes('get-task-events complete')); + + expect(invoked).toBeDefined(); + expect(invoked).toContain('"query_mode":"after"'); + expect(complete).toBeDefined(); + expect(complete).toContain('"event_count"'); + expect(complete).toContain('"has_more"'); + } finally { + stdoutWrite.mockRestore(); + } + }); }); diff --git a/cdk/test/handlers/get-task.test.ts b/cdk/test/handlers/get-task.test.ts index dfc3c63..3a59b6e 100644 --- a/cdk/test/handlers/get-task.test.ts +++ b/cdk/test/handlers/get-task.test.ts @@ -110,6 +110,27 @@ describe('get-task handler', () => { // Null fields should be present expect(body.data.pr_url).toBeNull(); expect(body.data.error_message).toBeNull(); + // Provenance — surfaced so CLI / dashboard consumers can distinguish + // webhook-submitted tasks from api-submitted tasks without spelunking + // CloudWatch. Pre-fix this field was present on the DDB record but + // dropped by ``toTaskDetail``. + expect(body.data.channel_source).toBe('api'); + }); + + test('surfaces channel_source=webhook for tasks created via the webhook path', async () => { + mockSend.mockReset(); + mockSend.mockResolvedValueOnce({ + Item: { + ...TASK_RECORD, + channel_source: 'webhook', + }, + }); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.data.channel_source).toBe('webhook'); }); test('returns 401 when user is not authenticated', async () => { diff --git a/cdk/test/handlers/get-trace-url.test.ts b/cdk/test/handlers/get-trace-url.test.ts new file mode 100644 index 0000000..8895ff8 --- /dev/null +++ b/cdk/test/handlers/get-trace-url.test.ts @@ -0,0 +1,373 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import type { APIGatewayProxyEvent } from 'aws-lambda'; + +// --- Mocks --- +const mockSend = jest.fn(); +jest.mock('@aws-sdk/client-dynamodb', () => ({ DynamoDBClient: jest.fn(() => ({})) })); +jest.mock('@aws-sdk/lib-dynamodb', () => ({ + DynamoDBDocumentClient: { from: jest.fn(() => ({ send: mockSend })) }, + GetCommand: jest.fn((input: unknown) => ({ _type: 'Get', input })), +})); + +const mockGetSignedUrl = jest.fn(); +const mockS3Send = jest.fn(); +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn(() => ({ send: (...args: unknown[]) => mockS3Send(...args) })), + GetObjectCommand: jest.fn((input: unknown) => ({ _type: 'S3Get', input })), + HeadObjectCommand: jest.fn((input: unknown) => ({ _type: 'S3Head', input })), +})); +jest.mock('@aws-sdk/s3-request-presigner', () => ({ + getSignedUrl: (...args: unknown[]) => mockGetSignedUrl(...args), +})); + +jest.mock('ulid', () => ({ ulid: jest.fn(() => 'REQ-ULID') })); + +process.env.TASK_TABLE_NAME = 'Tasks'; +process.env.TRACE_ARTIFACTS_BUCKET_NAME = 'trace-bucket'; + +import { handler, parseS3Uri, TRACE_URL_TTL_SECONDS } from '../../src/handlers/get-trace-url'; + +const TRACE_URI = 's3://trace-bucket/traces/user-123/task-1.jsonl.gz'; +const TASK_RECORD = { + task_id: 'task-1', + user_id: 'user-123', + status: 'COMPLETED', + repo: 'org/repo', + branch_name: 'bgagent/task-1/fix', + channel_source: 'api', + status_created_at: 'COMPLETED#2025-03-15T10:30:00Z', + created_at: '2025-03-15T10:30:00Z', + updated_at: '2025-03-15T10:31:00Z', + trace: true, + trace_s3_uri: TRACE_URI, +}; + +function makeEvent(overrides: Partial = {}): APIGatewayProxyEvent { + return { + body: null, + headers: {}, + multiValueHeaders: {}, + httpMethod: 'GET', + isBase64Encoded: false, + path: '/v1/tasks/task-1/trace', + pathParameters: { task_id: 'task-1' }, + queryStringParameters: null, + multiValueQueryStringParameters: null, + stageVariables: null, + resource: '/tasks/{task_id}/trace', + requestContext: { + accountId: '123456789012', + apiId: 'api-id', + authorizer: { claims: { sub: 'user-123' } }, + httpMethod: 'GET', + identity: { + sourceIp: '1.2.3.4', + userAgent: 'test/1.0', + accessKey: null, + accountId: null, + apiKey: null, + apiKeyId: null, + caller: null, + clientCert: null, + cognitoAuthenticationProvider: null, + cognitoAuthenticationType: null, + cognitoIdentityId: null, + cognitoIdentityPoolId: null, + principalOrgId: null, + user: null, + userArn: null, + }, + path: '/v1/tasks/task-1/trace', + protocol: 'HTTPS', + requestId: 'gw-req-1', + requestTimeEpoch: 0, + resourceId: 'res-id', + resourcePath: '/tasks/{task_id}/trace', + stage: 'v1', + }, + ...overrides, + }; +} + +beforeEach(() => { + jest.clearAllMocks(); + mockSend.mockResolvedValue({ Item: TASK_RECORD }); + // L3 item 3: HEAD the S3 object before presigning. Default: object exists. + mockS3Send.mockResolvedValue({ ContentLength: 1234 }); + mockGetSignedUrl.mockResolvedValue('https://example.com/presigned?sig=abc'); +}); + +describe('get-trace-url handler', () => { + test('returns presigned URL + expires_at for a trace-enabled task', async () => { + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.data.url).toBe('https://example.com/presigned?sig=abc'); + expect(typeof body.data.expires_at).toBe('string'); + // Should parse as a future ISO timestamp ~15 min out + const expiresAt = new Date(body.data.expires_at).getTime(); + const now = Date.now(); + const delta = expiresAt - now; + expect(delta).toBeGreaterThan((TRACE_URL_TTL_SECONDS - 5) * 1000); + expect(delta).toBeLessThanOrEqual((TRACE_URL_TTL_SECONDS + 5) * 1000); + }); + + test('calls getSignedUrl with the expected TTL (15 min)', async () => { + await handler(makeEvent()); + + expect(mockGetSignedUrl).toHaveBeenCalledTimes(1); + const [, command, options] = mockGetSignedUrl.mock.calls[0]; + expect(command.input.Bucket).toBe('trace-bucket'); + expect(command.input.Key).toBe('traces/user-123/task-1.jsonl.gz'); + expect(options).toEqual({ expiresIn: TRACE_URL_TTL_SECONDS }); + }); + + test('returns 401 when user is not authenticated', async () => { + const event = makeEvent(); + event.requestContext.authorizer = null; + const result = await handler(event); + + expect(result.statusCode).toBe(401); + expect(JSON.parse(result.body).error.code).toBe('UNAUTHORIZED'); + expect(mockSend).not.toHaveBeenCalled(); + }); + + test('returns 400 when task_id is missing', async () => { + const result = await handler(makeEvent({ pathParameters: null })); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + expect(mockSend).not.toHaveBeenCalled(); + }); + + test('returns 404 when task does not exist', async () => { + mockSend.mockResolvedValueOnce({ Item: undefined }); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(404); + expect(JSON.parse(result.body).error.code).toBe('TASK_NOT_FOUND'); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 403 when task belongs to another user', async () => { + mockSend.mockResolvedValueOnce({ + Item: { ...TASK_RECORD, user_id: 'other-user' }, + }); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(403); + expect(JSON.parse(result.body).error.code).toBe('FORBIDDEN'); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 404 TRACE_NOT_AVAILABLE when trace_s3_uri is absent', async () => { + const { trace_s3_uri: _unused, ...recordWithoutTrace } = TASK_RECORD; + mockSend.mockResolvedValueOnce({ Item: recordWithoutTrace }); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(404); + expect(JSON.parse(result.body).error.code).toBe('TRACE_NOT_AVAILABLE'); + expect(JSON.parse(result.body).error.message).toMatch(/--trace/); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 500 when trace_s3_uri is malformed', async () => { + mockSend.mockResolvedValueOnce({ + Item: { ...TASK_RECORD, trace_s3_uri: 'not-an-s3-uri' }, + }); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(500); + expect(JSON.parse(result.body).error.code).toBe('INTERNAL_ERROR'); + expect(JSON.parse(result.body).error.message).toMatch(/malformed/); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 500 when trace_s3_uri points at an unexpected bucket (defense in depth)', async () => { + mockSend.mockResolvedValueOnce({ + Item: { + ...TASK_RECORD, + trace_s3_uri: 's3://attacker-bucket/traces/user-123/task-1.jsonl.gz', + }, + }); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(500); + expect(JSON.parse(result.body).error.code).toBe('INTERNAL_ERROR'); + expect(JSON.parse(result.body).error.message).toMatch(/unexpected bucket/); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 403 when trace key is not under the caller\'s user prefix', async () => { + mockSend.mockResolvedValueOnce({ + Item: { + ...TASK_RECORD, + // Bucket matches, but the key is under a different user prefix + trace_s3_uri: 's3://trace-bucket/traces/other-user/task-1.jsonl.gz', + }, + }); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(403); + expect(JSON.parse(result.body).error.code).toBe('FORBIDDEN'); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 500 on DynamoDB error', async () => { + mockSend.mockRejectedValueOnce(new Error('DB failure')); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(500); + expect(JSON.parse(result.body).error.code).toBe('INTERNAL_ERROR'); + }); + + test('returns 500 on S3 presign error', async () => { + mockGetSignedUrl.mockRejectedValueOnce(new Error('presign boom')); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(500); + expect(JSON.parse(result.body).error.code).toBe('INTERNAL_ERROR'); + }); + + // -------- L3 item 3: HEAD-before-presign -------- + + test('HEADs the S3 object before signing (race-between-DDB-write-and-S3-propagation guard)', async () => { + await handler(makeEvent()); + // HeadObject must be called with the same bucket/key as the presign. + expect(mockS3Send).toHaveBeenCalledTimes(1); + const sentCommand = mockS3Send.mock.calls[0][0]; + expect(sentCommand._type).toBe('S3Head'); + expect(sentCommand.input).toEqual({ + Bucket: 'trace-bucket', + Key: 'traces/user-123/task-1.jsonl.gz', + }); + }); + + test('returns 404 TRACE_NOT_AVAILABLE when HEAD throws NotFound (SDK v3 error name)', async () => { + // SDK v3 surfaces a missing S3 object as an error with name='NotFound'. + const notFoundErr = new Error('Not Found'); + notFoundErr.name = 'NotFound'; + mockS3Send.mockRejectedValueOnce(notFoundErr); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(404); + expect(JSON.parse(result.body).error.code).toBe('TRACE_NOT_AVAILABLE'); + // Must NOT have attempted to sign a URL for a missing object. + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 404 TRACE_NOT_AVAILABLE when HEAD throws NoSuchKey', async () => { + // Some code paths (GET-style) surface missing as NoSuchKey; treat identically. + const err = new Error('The specified key does not exist'); + err.name = 'NoSuchKey'; + mockS3Send.mockRejectedValueOnce(err); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(404); + expect(JSON.parse(result.body).error.code).toBe('TRACE_NOT_AVAILABLE'); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 404 TRACE_NOT_AVAILABLE when HEAD returns HTTP 404 via $metadata', async () => { + // Belt-and-suspenders: catch a 404 that didn't tag error.name (older + // SDK versions or custom wrappers). + const err = Object.assign(new Error('NotFound'), { $metadata: { httpStatusCode: 404 } }); + mockS3Send.mockRejectedValueOnce(err); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(404); + expect(JSON.parse(result.body).error.code).toBe('TRACE_NOT_AVAILABLE'); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('returns 500 when HEAD throws a generic error (not a 404)', async () => { + // A non-404 HEAD error is a real AWS problem (throttle, 500, 503, + // etc.). Surface as INTERNAL_ERROR — retrying is fine, but hiding + // behind TRACE_NOT_AVAILABLE would mislead the user into re-submitting. + mockS3Send.mockRejectedValueOnce(new Error('AccessDenied')); + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(500); + expect(JSON.parse(result.body).error.code).toBe('INTERNAL_ERROR'); + expect(mockGetSignedUrl).not.toHaveBeenCalled(); + }); + + test('includes standard headers and X-Request-Id', async () => { + const result = await handler(makeEvent()); + + expect(result.headers?.['Content-Type']).toBe('application/json'); + expect(result.headers?.['X-Request-Id']).toBe('REQ-ULID'); + }); +}); + +describe('parseS3Uri', () => { + test('parses a valid s3:// URI', () => { + expect(parseS3Uri('s3://bucket/path/to/object.jsonl.gz')).toEqual({ + bucket: 'bucket', + key: 'path/to/object.jsonl.gz', + }); + }); + + test('rejects non-s3:// schemes', () => { + expect(parseS3Uri('https://bucket/key')).toBeNull(); + expect(parseS3Uri('s3:/bucket/key')).toBeNull(); + }); + + test('rejects missing bucket', () => { + expect(parseS3Uri('s3:///key')).toBeNull(); + }); + + test('rejects missing key', () => { + expect(parseS3Uri('s3://bucket/')).toBeNull(); + expect(parseS3Uri('s3://bucket')).toBeNull(); + }); + + test('preserves nested key paths', () => { + expect(parseS3Uri('s3://b/a/b/c/d.txt')).toEqual({ bucket: 'b', key: 'a/b/c/d.txt' }); + }); + + test('pins behavior on double-slash keys (leading / preserved in key)', () => { + // Current implementation produces ``key='/object'`` on ``s3://bucket//object``. + // S3 accepts this as a distinct key; the handler's prefix check then + // rejects it (key does not start with ``traces//``). Pinning + // the shape so a future parser refactor does not silently change + // it. + expect(parseS3Uri('s3://bucket//object')).toEqual({ bucket: 'bucket', key: '/object' }); + }); + + test('pins behavior on query-string-like suffixes (treated as literal key chars)', () => { + // S3 object keys can legally contain ``?`` and ``#``. The parser + // treats the entire post-bucket path as the key — it does NOT + // URL-decode or strip query fragments. The handler's bucket + + // prefix guards still gate these; pinning behavior here so a + // future ``new URL(...)`` rewrite has an explicit contract to + // maintain (``URL`` would split off ``?`` into ``search``). + expect(parseS3Uri('s3://bucket/traces/u/t.jsonl.gz?x=1')).toEqual({ + bucket: 'bucket', + key: 'traces/u/t.jsonl.gz?x=1', + }); + expect(parseS3Uri('s3://bucket/traces/u/t.jsonl.gz#frag')).toEqual({ + bucket: 'bucket', + key: 'traces/u/t.jsonl.gz#frag', + }); + }); +}); diff --git a/cdk/test/handlers/nudge-task.test.ts b/cdk/test/handlers/nudge-task.test.ts new file mode 100644 index 0000000..9a0f817 --- /dev/null +++ b/cdk/test/handlers/nudge-task.test.ts @@ -0,0 +1,411 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import type { APIGatewayProxyEvent } from 'aws-lambda'; + +// --- Mocks --- +const mockSend = jest.fn(); +jest.mock('@aws-sdk/client-dynamodb', () => ({ DynamoDBClient: jest.fn(() => ({})) })); +jest.mock('@aws-sdk/lib-dynamodb', () => ({ + DynamoDBDocumentClient: { from: jest.fn(() => ({ send: mockSend })) }, + GetCommand: jest.fn((input: unknown) => ({ _type: 'Get', input })), + PutCommand: jest.fn((input: unknown) => ({ _type: 'Put', input })), + UpdateCommand: jest.fn((input: unknown) => ({ _type: 'Update', input })), +})); + +const mockBedrockSend = jest.fn(); +jest.mock('@aws-sdk/client-bedrock-runtime', () => ({ + BedrockRuntimeClient: jest.fn(() => ({ send: mockBedrockSend })), + ApplyGuardrailCommand: jest.fn((input: unknown) => ({ _type: 'ApplyGuardrail', input })), +})); + +let ulidCounter = 0; +jest.mock('ulid', () => ({ ulid: jest.fn(() => `ULID${ulidCounter++}`) })); + +process.env.TASK_TABLE_NAME = 'Tasks'; +process.env.TASK_EVENTS_TABLE_NAME = 'TaskEvents'; +process.env.NUDGES_TABLE_NAME = 'Nudges'; +process.env.NUDGE_RATE_LIMIT_PER_MINUTE = '10'; +process.env.GUARDRAIL_ID = 'test-guardrail'; +process.env.GUARDRAIL_VERSION = '1'; + +import { handler } from '../../src/handlers/nudge-task'; + +const RUNNING_TASK = { + task_id: 'task-1', + user_id: 'user-123', + status: 'RUNNING', + repo: 'org/repo', + branch_name: 'bgagent/task-1/fix', + channel_source: 'api', + status_created_at: 'RUNNING#2025-03-15T10:30:00Z', + created_at: '2025-03-15T10:30:00Z', + updated_at: '2025-03-15T10:31:00Z', +}; + +function makeEvent(overrides: Partial = {}): APIGatewayProxyEvent { + return { + body: JSON.stringify({ message: 'please also add tests' }), + headers: {}, + multiValueHeaders: {}, + httpMethod: 'POST', + isBase64Encoded: false, + path: '/v1/tasks/task-1/nudge', + pathParameters: { task_id: 'task-1' }, + queryStringParameters: null, + multiValueQueryStringParameters: null, + stageVariables: null, + resource: '/tasks/{task_id}/nudge', + requestContext: { + accountId: '123456789012', + apiId: 'api-id', + authorizer: { claims: { sub: 'user-123' } }, + httpMethod: 'POST', + identity: { + sourceIp: '1.2.3.4', + userAgent: 'test/1.0', + accessKey: null, + accountId: null, + apiKey: null, + apiKeyId: null, + caller: null, + clientCert: null, + cognitoAuthenticationProvider: null, + cognitoAuthenticationType: null, + cognitoIdentityId: null, + cognitoIdentityPoolId: null, + principalOrgId: null, + user: null, + userArn: null, + }, + path: '/v1/tasks/task-1/nudge', + protocol: 'HTTPS', + requestId: 'gw-req-1', + requestTimeEpoch: 0, + resourceId: 'res-id', + resourcePath: '/tasks/{task_id}/nudge', + stage: 'v1', + }, + ...overrides, + }; +} + +/** + * Set up the happy-path mock sequence: Get (task) → Update (rate-limit) → Put (nudge). + */ +function primeHappyPath(task = RUNNING_TASK): void { + mockSend.mockReset(); + mockSend + .mockResolvedValueOnce({ Item: task }) // GetCommand (task) + .mockResolvedValueOnce({}) // UpdateCommand (rate-limit counter) + .mockResolvedValueOnce({}); // PutCommand (nudge record) +} + +beforeEach(() => { + jest.clearAllMocks(); + ulidCounter = 0; + mockBedrockSend.mockResolvedValue({ action: 'NONE' }); + primeHappyPath(); +}); + +describe('nudge-task handler — happy path', () => { + test('submits a nudge and returns 202 with the nudge_id', async () => { + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(202); + const body = JSON.parse(result.body); + expect(body.data.task_id).toBe('task-1'); + expect(body.data.nudge_id).toBeDefined(); + expect(typeof body.data.nudge_id).toBe('string'); + expect(body.data.submitted_at).toBeDefined(); + }); + + test('writes a NudgeRecord with consumed=false, user_id, message, ttl ~30 days out', async () => { + const before = Math.floor(Date.now() / 1000); + await handler(makeEvent()); + const after = Math.floor(Date.now() / 1000); + + // Third DDB call is the PutCommand for the nudge. + const putCall = mockSend.mock.calls[2][0]; + expect(putCall._type).toBe('Put'); + expect(putCall.input.TableName).toBe('Nudges'); + + const item = putCall.input.Item; + expect(item.task_id).toBe('task-1'); + expect(item.user_id).toBe('user-123'); + expect(item.message).toBe('please also add tests'); + expect(item.consumed).toBe(false); + expect(typeof item.nudge_id).toBe('string'); + expect(typeof item.created_at).toBe('string'); + expect(typeof item.ttl).toBe('number'); + + const thirtyDays = 30 * 24 * 60 * 60; + expect(item.ttl).toBeGreaterThanOrEqual(before + thirtyDays - 5); + expect(item.ttl).toBeLessThanOrEqual(after + thirtyDays + 5); + }); + + test('increments the per-task per-minute rate-limit counter', async () => { + await handler(makeEvent()); + + const updateCall = mockSend.mock.calls[1][0]; + expect(updateCall._type).toBe('Update'); + expect(updateCall.input.TableName).toBe('Nudges'); + // Synthetic PK/SK + expect(updateCall.input.Key.task_id).toBe('RATE#task-1'); + expect(updateCall.input.Key.nudge_id).toMatch(/^MINUTE#\d{12}$/); + // Counter + TTL + conditional max + expect(updateCall.input.UpdateExpression).toContain('ADD #count :one'); + expect(updateCall.input.UpdateExpression).toContain('SET #ttl = :ttl'); + expect(updateCall.input.ConditionExpression).toContain('#count < :max'); + expect(updateCall.input.ExpressionAttributeValues[':max']).toBe(10); + }); + + test('trims whitespace from message before storing', async () => { + await handler(makeEvent({ body: JSON.stringify({ message: ' hello world ' }) })); + + const putCall = mockSend.mock.calls[2][0]; + expect(putCall.input.Item.message).toBe('hello world'); + }); + + test('screens message via Bedrock guardrail when configured', async () => { + await handler(makeEvent()); + + expect(mockBedrockSend).toHaveBeenCalledTimes(1); + const cmd = mockBedrockSend.mock.calls[0][0]; + expect(cmd.input.guardrailIdentifier).toBe('test-guardrail'); + expect(cmd.input.guardrailVersion).toBe('1'); + expect(cmd.input.source).toBe('INPUT'); + }); +}); + +describe('nudge-task handler — auth and validation errors', () => { + test('returns 401 when user is not authenticated', async () => { + const event = makeEvent(); + event.requestContext.authorizer = null; + const result = await handler(event); + + expect(result.statusCode).toBe(401); + expect(JSON.parse(result.body).error.code).toBe('UNAUTHORIZED'); + }); + + test('returns 400 when task_id path parameter is missing', async () => { + const result = await handler(makeEvent({ pathParameters: null })); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + }); + + test('returns 400 when body is missing', async () => { + const result = await handler(makeEvent({ body: null })); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + }); + + test('returns 400 when body is not valid JSON', async () => { + const result = await handler(makeEvent({ body: '{not json' })); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + }); + + test('returns 400 when message is missing', async () => { + const result = await handler(makeEvent({ body: JSON.stringify({}) })); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + }); + + test('returns 400 when message is not a string', async () => { + const result = await handler(makeEvent({ body: JSON.stringify({ message: 42 }) })); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + }); + + test('returns 400 when message is empty after trim', async () => { + const result = await handler(makeEvent({ body: JSON.stringify({ message: ' ' }) })); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + }); + + test('returns 400 when message exceeds 2000 chars after trim', async () => { + const long = 'a'.repeat(2001); + const result = await handler(makeEvent({ body: JSON.stringify({ message: long }) })); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.code).toBe('VALIDATION_ERROR'); + expect(JSON.parse(result.body).error.message).toContain('2000'); + }); + + test('accepts message at exactly 2000 chars', async () => { + const boundary = 'a'.repeat(2000); + const result = await handler(makeEvent({ body: JSON.stringify({ message: boundary }) })); + + expect(result.statusCode).toBe(202); + }); +}); + +describe('nudge-task handler — task ownership and state', () => { + test('returns 404 when task does not exist', async () => { + mockSend.mockReset(); + mockSend.mockResolvedValueOnce({ Item: undefined }); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(404); + expect(JSON.parse(result.body).error.code).toBe('TASK_NOT_FOUND'); + }); + + test('returns 403 when task belongs to another user', async () => { + mockSend.mockReset(); + mockSend.mockResolvedValueOnce({ Item: { ...RUNNING_TASK, user_id: 'other-user' } }); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(403); + expect(JSON.parse(result.body).error.code).toBe('FORBIDDEN'); + }); + + test.each([ + ['COMPLETED'], + ['FAILED'], + ['CANCELLED'], + ['TIMED_OUT'], + ])('returns 409 when task is in terminal state %s', async (status) => { + mockSend.mockReset(); + mockSend.mockResolvedValueOnce({ Item: { ...RUNNING_TASK, status } }); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(409); + expect(JSON.parse(result.body).error.code).toBe('TASK_ALREADY_TERMINAL'); + }); + + test.each([ + ['SUBMITTED'], + ['HYDRATING'], + ['RUNNING'], + ['FINALIZING'], + ])('accepts a nudge for non-terminal state %s', async (status) => { + primeHappyPath({ ...RUNNING_TASK, status }); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(202); + }); +}); + +describe('nudge-task handler — rate limiting', () => { + test('returns 429 when rate-limit ConditionalCheckFailedException fires', async () => { + mockSend.mockReset(); + const condErr = new Error('Condition not met'); + condErr.name = 'ConditionalCheckFailedException'; + mockSend + .mockResolvedValueOnce({ Item: RUNNING_TASK }) // Get (task) + .mockRejectedValueOnce(condErr); // Update (rate-limit) — fails + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(429); + expect(JSON.parse(result.body).error.code).toBe('RATE_LIMIT_EXCEEDED'); + expect(JSON.parse(result.body).error.message).toContain('10'); + // Nudge must NOT have been persisted + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + test('returns 500 on unexpected rate-limit update error', async () => { + mockSend.mockReset(); + mockSend + .mockResolvedValueOnce({ Item: RUNNING_TASK }) + .mockRejectedValueOnce(new Error('DB failure')); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(500); + expect(JSON.parse(result.body).error.code).toBe('INTERNAL_ERROR'); + }); +}); + +describe('nudge-task handler — guardrail screening', () => { + test('returns 400 when guardrail blocks the message', async () => { + mockBedrockSend.mockReset(); + mockBedrockSend.mockResolvedValueOnce({ + action: 'GUARDRAIL_INTERVENED', + assessments: [ + { + contentPolicy: { + filters: [{ type: 'PROMPT_ATTACK', confidence: 'HIGH', action: 'BLOCKED' }], + }, + }, + ], + }); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error.code).toBe('VALIDATION_ERROR'); + expect(body.error.message.toLowerCase()).toContain('content policy'); + // Nudge must NOT have been persisted + const putCalls = mockSend.mock.calls.filter(c => c[0]._type === 'Put'); + expect(putCalls).toHaveLength(0); + // Guardrail-blocked messages must NOT consume a rate-limit slot + // (guardrail runs before rate-limit — see handler docstring). + const updateCalls = mockSend.mock.calls.filter(c => c[0]._type === 'Update'); + expect(updateCalls).toHaveLength(0); + }); + + test('returns 503 when guardrail API call fails (fail-closed)', async () => { + mockBedrockSend.mockReset(); + mockBedrockSend.mockRejectedValueOnce(new Error('Bedrock unavailable')); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(503); + expect(JSON.parse(result.body).error.code).toBe('SERVICE_UNAVAILABLE'); + expect(JSON.parse(result.body).error.message.toLowerCase()).toContain('screening'); + const putCalls = mockSend.mock.calls.filter(c => c[0]._type === 'Put'); + expect(putCalls).toHaveLength(0); + }); +}); + +describe('nudge-task handler — error paths', () => { + test('returns 500 on unexpected DynamoDB Get error', async () => { + mockSend.mockReset(); + mockSend.mockRejectedValueOnce(new Error('DB failure')); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(500); + }); + + test('returns 500 on unexpected DynamoDB Put error', async () => { + mockSend.mockReset(); + mockSend + .mockResolvedValueOnce({ Item: RUNNING_TASK }) + .mockResolvedValueOnce({}) // rate-limit ok + .mockRejectedValueOnce(new Error('Put failed')); + + const result = await handler(makeEvent()); + + expect(result.statusCode).toBe(500); + }); +}); diff --git a/cdk/test/handlers/orchestrate-task.test.ts b/cdk/test/handlers/orchestrate-task.test.ts index fcdf2b9..0151fe7 100644 --- a/cdk/test/handlers/orchestrate-task.test.ts +++ b/cdk/test/handlers/orchestrate-task.test.ts @@ -177,6 +177,56 @@ describe('hydrateAndTransition', () => { expect(payload.max_turns).toBe(100); }); + test('threads trace: true into the agent payload when set on the task record', async () => { + mockDdbSend.mockResolvedValue({}); + mockHydrateContext.mockResolvedValueOnce(mockHydratedContext); + const taskWithTrace = { ...baseTask, trace: true }; + const payload = await hydrateAndTransition(taskWithTrace as any); + expect(payload.trace).toBe(true); + }); + + test('omits trace from payload when task record has no trace flag (slim wire)', async () => { + mockDdbSend.mockResolvedValue({}); + mockHydrateContext.mockResolvedValueOnce(mockHydratedContext); + const payload = await hydrateAndTransition(baseTask as any); + expect(payload).not.toHaveProperty('trace'); + }); + + test('threads user_id into the agent payload (design §10.1 — required for trace S3 key)', async () => { + mockDdbSend.mockResolvedValue({}); + mockHydrateContext.mockResolvedValueOnce(mockHydratedContext); + const payload = await hydrateAndTransition(baseTask as any); + // user_id is threaded unconditionally (not just when trace=true) + // so that a future feature needing it does not have to re-plumb. + // The agent only USES it when trace=true. + expect(payload.user_id).toBe('user-123'); + }); + + test('threads user_id even when trace flag is absent', async () => { + mockDdbSend.mockResolvedValue({}); + mockHydrateContext.mockResolvedValueOnce(mockHydratedContext); + const taskNoTrace = { ...baseTask, trace: undefined }; + const payload = await hydrateAndTransition(taskNoTrace as any); + expect(payload.user_id).toBe('user-123'); + expect(payload).not.toHaveProperty('trace'); + }); + + test('payload.user_id is undefined when the task record lacks user_id (defensive)', async () => { + // Defends the Stage 3 ↔ Stage 4 contract: a legacy / corrupted + // record without ``user_id`` should surface as ``undefined`` in + // the payload (which JSON.stringify drops entirely), not get + // silently coerced to an empty string that would then combine + // with ``trace=true`` to produce an unreachable ``traces//...`` + // S3 key. Admission-control should catch this upstream; test + // pins the downstream behavior regardless. + mockDdbSend.mockResolvedValue({}); + mockHydrateContext.mockResolvedValueOnce(mockHydratedContext); + const taskNoUser = { ...baseTask }; + delete (taskNoUser as { user_id?: string }).user_id; + const payload = await hydrateAndTransition(taskNoUser as any); + expect(payload.user_id).toBeUndefined(); + }); + test('throws when guardrail_blocked is set on hydrated context', async () => { mockDdbSend.mockResolvedValue({}); mockHydrateContext.mockResolvedValueOnce({ diff --git a/cdk/test/handlers/reconcile-stranded-tasks.test.ts b/cdk/test/handlers/reconcile-stranded-tasks.test.ts new file mode 100644 index 0000000..a902a4f --- /dev/null +++ b/cdk/test/handlers/reconcile-stranded-tasks.test.ts @@ -0,0 +1,363 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// --- Mocks --- +const mockDdbSend = jest.fn(); +jest.mock('@aws-sdk/client-dynamodb', () => ({ + DynamoDBClient: jest.fn(() => ({ send: mockDdbSend })), + QueryCommand: jest.fn((input: unknown) => ({ _type: 'Query', input })), + UpdateItemCommand: jest.fn((input: unknown) => ({ _type: 'UpdateItem', input })), + PutItemCommand: jest.fn((input: unknown) => ({ _type: 'PutItem', input })), +})); + +process.env.TASK_TABLE_NAME = 'Tasks'; +process.env.TASK_EVENTS_TABLE_NAME = 'TaskEvents'; +process.env.USER_CONCURRENCY_TABLE_NAME = 'Concurrency'; +process.env.STRANDED_TIMEOUT_SECONDS = '1200'; +process.env.TASK_RETENTION_DAYS = '90'; + +import { handler } from '../../src/handlers/reconcile-stranded-tasks'; + +/** + * Build a dynamodb AttributeValue map mimicking a TaskTable StatusIndex hit. + */ +function mockTaskRow(opts: { + task_id: string; + user_id: string; + created_at: string; +}): Record { + return { + task_id: { S: opts.task_id }, + user_id: { S: opts.user_id }, + created_at: { S: opts.created_at }, + }; +} + +/** + * Run the handler after pre-seeding mockDdbSend with an array of responses. + * Commands are popped in order; throw test-visible error if we run out. + */ +function primeResponses(responses: unknown[]): void { + mockDdbSend.mockReset(); + let idx = 0; + mockDdbSend.mockImplementation(() => { + if (idx >= responses.length) { + throw new Error(`mockDdbSend ran out of responses after ${idx} calls`); + } + const r = responses[idx++]; + if (r instanceof Error) throw r; + return Promise.resolve(r); + }); +} + +describe('reconcile-stranded-tasks', () => { + beforeEach(() => { + mockDdbSend.mockReset(); + }); + + test('no candidates → handler is a no-op with no writes', async () => { + primeResponses([ + { Items: [] }, // Query SUBMITTED + { Items: [] }, // Query HYDRATING + ]); + + await handler(); + + // Exactly 2 queries, no updates. + expect(mockDdbSend).toHaveBeenCalledTimes(2); + }); + + test('task older than 1200s → fails + emits events + decrements concurrency', async () => { + const ancient = new Date(Date.now() - 25 * 60 * 1000).toISOString(); // 25 min ago + primeResponses([ + // Query SUBMITTED returns one stranded candidate. + { + Items: [mockTaskRow({ + task_id: 't-stranded', + user_id: 'u-1', + created_at: ancient, + })], + }, + {}, // conditional UpdateItem → FAILED + {}, // PutItem task_stranded event + {}, // PutItem task_failed event + {}, // UpdateItem decrement concurrency + { Items: [] }, // Query HYDRATING + ]); + + await handler(); + + // Capture the UpdateItem call that transitions status; assert condition. + const transitionCall = (mockDdbSend.mock.calls as [{ _type: string; input: Record }][]) + .find(([c]) => c._type === 'UpdateItem' && String(c.input.ConditionExpression).includes('= :expected')); + expect(transitionCall).toBeDefined(); + const input = transitionCall![0].input as { + Key: { task_id: { S: string } }; + ExpressionAttributeValues: Record; + }; + expect(input.Key.task_id.S).toBe('t-stranded'); + expect(input.ExpressionAttributeValues[':failed'].S).toBe('FAILED'); + expect(input.ExpressionAttributeValues[':expected'].S).toBe('SUBMITTED'); + + // Events written. + const putCalls = (mockDdbSend.mock.calls as [{ _type: string; input: Record }][]) + .filter(([c]) => c._type === 'PutItem'); + expect(putCalls).toHaveLength(2); + const eventTypes = putCalls.map(([c]) => { + const item = (c.input as { Item: { event_type: { S: string } } }).Item; + return item.event_type.S; + }); + expect(eventTypes).toEqual(expect.arrayContaining(['task_stranded', 'task_failed'])); + + // Concurrency decrement. + const decrementCall = (mockDdbSend.mock.calls as [{ _type: string; input: Record }][]) + .find(([c]) => c._type === 'UpdateItem' && String(c.input.UpdateExpression).includes('active_count')); + expect(decrementCall).toBeDefined(); + }); + + test('task advances during reconcile (ConditionalCheckFailedException) → skipped cleanly', async () => { + const ancient = new Date(Date.now() - 25 * 60 * 1000).toISOString(); + const conditionalErr = Object.assign(new Error('ConditionalCheckFailed'), { + name: 'ConditionalCheckFailedException', + }); + primeResponses([ + { + Items: [mockTaskRow({ + task_id: 't-raced', + user_id: 'u-4', + created_at: ancient, + })], + }, + conditionalErr, // UpdateItem transition rejected (task already advanced) + { Items: [] }, // HYDRATING query + ]); + + // Must NOT throw; no events written, no concurrency decrement. + await handler(); + + const writes = (mockDdbSend.mock.calls as [{ _type: string; input: Record }][]) + .filter(([c]) => c._type === 'PutItem') + .length; + expect(writes).toBe(0); + }); + + test('HYDRATING status also scanned (both SUBMITTED + HYDRATING queries run)', async () => { + primeResponses([ + { Items: [] }, // SUBMITTED + { Items: [] }, // HYDRATING + ]); + + await handler(); + + const queryCalls = (mockDdbSend.mock.calls as [{ _type: string; input: Record }][]) + .filter(([c]) => c._type === 'Query'); + expect(queryCalls).toHaveLength(2); + const statusValues = queryCalls.map(([c]) => { + const values = (c.input as { ExpressionAttributeValues: Record }).ExpressionAttributeValues; + return values[':status'].S; + }); + expect(statusValues).toEqual(expect.arrayContaining(['SUBMITTED', 'HYDRATING'])); + }); + + describe('final log severity escalation', () => { + // Spy on the logger module used by the handler. We import the logger + // directly and replace the three level methods with jest.fn before + // each test so we can assert exactly which level was called. + // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports + const loggerModule = require('../../src/handlers/shared/logger') as { + logger: { + info: (m: string, d?: Record) => void; + warn: (m: string, d?: Record) => void; + error: (m: string, d?: Record) => void; + }; + }; + + let infoSpy: jest.SpyInstance; + let warnSpy: jest.SpyInstance; + let errorSpy: jest.SpyInstance; + + beforeEach(() => { + infoSpy = jest.spyOn(loggerModule.logger, 'info').mockImplementation(() => { /* silence */ }); + warnSpy = jest.spyOn(loggerModule.logger, 'warn').mockImplementation(() => { /* silence */ }); + errorSpy = jest.spyOn(loggerModule.logger, 'error').mockImplementation(() => { /* silence */ }); + }); + + afterEach(() => { + infoSpy.mockRestore(); + warnSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + /** + * Find the final reconciler log line (i.e. the one whose message + * starts with 'Stranded-task reconciler finished') across all spies + * and return its [level, message, payload] triple. + */ + function findFinalLog(): { level: 'INFO' | 'WARN' | 'ERROR'; message: string; payload: Record } { + const match = (spy: jest.SpyInstance, level: 'INFO' | 'WARN' | 'ERROR') => { + const call = spy.mock.calls.find( + (c: unknown[]) => typeof c[0] === 'string' && (c[0] as string).startsWith('Stranded-task reconciler finished'), + ); + return call ? { level, message: call[0] as string, payload: (call[1] ?? {}) as Record } : null; + }; + return match(errorSpy, 'ERROR') ?? match(warnSpy, 'WARN') ?? match(infoSpy, 'INFO') + ?? (() => { throw new Error('No final reconciler log line found'); })(); + } + + test('test_logs_ERROR_with_RECONCILER_TOTAL_FAILURE_error_id_when_every_task_fails', async () => { + // Two candidates both hit an exception on the first DDB write + // (UpdateItem transition). None transition cleanly, so totalFailed=0, + // totalStranded=2, totalErrors=2 → systemic failure path. + const ancient = new Date(Date.now() - 25 * 60 * 1000).toISOString(); + const ddbErr = Object.assign(new Error('DDB blew up'), { name: 'InternalServerError' }); + primeResponses([ + // SUBMITTED query → two candidates. + { + Items: [ + mockTaskRow({ task_id: 't-fail-1', user_id: 'u-1', created_at: ancient }), + mockTaskRow({ task_id: 't-fail-2', user_id: 'u-2', created_at: ancient }), + ], + }, + ddbErr, // UpdateItem for t-fail-1 → throws + ddbErr, // UpdateItem for t-fail-2 → throws + { Items: [] }, // HYDRATING query + ]); + + await handler(); + + const final = findFinalLog(); + expect(final.level).toBe('ERROR'); + expect(final.payload.error_id).toBe('RECONCILER_TOTAL_FAILURE'); + expect(final.payload.stranded).toBe(2); + expect(final.payload.failed).toBe(0); + expect(final.payload.errors).toBe(2); + }); + + test('test_logs_WARN_with_RECONCILER_PARTIAL_FAILURE_when_some_tasks_fail', async () => { + // One success (4 writes), one failure (throws on UpdateItem). + const ancient = new Date(Date.now() - 25 * 60 * 1000).toISOString(); + const ddbErr = Object.assign(new Error('DDB throttled'), { name: 'ProvisionedThroughputExceededException' }); + primeResponses([ + // SUBMITTED query → two candidates. + { + Items: [ + mockTaskRow({ task_id: 't-ok', user_id: 'u-a', created_at: ancient }), + mockTaskRow({ task_id: 't-fail', user_id: 'u-b', created_at: ancient }), + ], + }, + {}, // UpdateItem t-ok (transition) → success + {}, // PutItem task_stranded event + {}, // PutItem task_failed event + {}, // UpdateItem decrement concurrency + ddbErr, // UpdateItem t-fail (transition) → throws + { Items: [] }, // HYDRATING query + ]); + + await handler(); + + const final = findFinalLog(); + expect(final.level).toBe('WARN'); + expect(final.payload.error_id).toBe('RECONCILER_PARTIAL_FAILURE'); + expect(final.payload.stranded).toBe(2); + expect(final.payload.failed).toBe(1); + expect(final.payload.errors).toBe(1); + }); + + test('test_logs_INFO_on_full_success', async () => { + // Two candidates, both transition cleanly. + const ancient = new Date(Date.now() - 25 * 60 * 1000).toISOString(); + primeResponses([ + { + Items: [ + mockTaskRow({ task_id: 't-1', user_id: 'u-a', created_at: ancient }), + mockTaskRow({ task_id: 't-2', user_id: 'u-b', created_at: ancient }), + ], + }, + {}, {}, {}, {}, // t-1: transition + 2 events + decrement + {}, {}, {}, {}, // t-2: transition + 2 events + decrement + { Items: [] }, // HYDRATING + ]); + + await handler(); + + const final = findFinalLog(); + expect(final.level).toBe('INFO'); + expect(final.payload.error_id).toBeUndefined(); + expect(final.payload.stranded).toBe(2); + expect(final.payload.failed).toBe(2); + expect(final.payload.errors).toBe(0); + }); + + test('test_no_stranded_tasks_logs_INFO_not_ERROR', async () => { + // Empty-query case: totalStranded=0. Must NOT alarm. + primeResponses([ + { Items: [] }, // SUBMITTED + { Items: [] }, // HYDRATING + ]); + + await handler(); + + const final = findFinalLog(); + expect(final.level).toBe('INFO'); + expect(final.payload.stranded).toBe(0); + expect(final.payload.errors).toBe(0); + expect(errorSpy).not.toHaveBeenCalled(); + }); + }); + + test('query paginates with ExclusiveStartKey when LastEvaluatedKey present', async () => { + const ancient = new Date(Date.now() - 25 * 60 * 1000).toISOString(); + // findStrandedCandidates paginates internally and returns ALL rows + // before the handler starts writing. So the call order is: + // Query SUBMITTED page1 (with LEK) → Query SUBMITTED page2 (no LEK) + // → 4 writes for page1 candidate → 4 writes for page2 candidate + // → Query HYDRATING (empty). + primeResponses([ + // SUBMITTED page 1 + { + Items: [mockTaskRow({ + task_id: 't-page1', + user_id: 'u-a', + created_at: ancient, + })], + LastEvaluatedKey: { task_id: { S: 't-page1' } }, + }, + // SUBMITTED page 2 + { + Items: [mockTaskRow({ + task_id: 't-page2', + user_id: 'u-b', + created_at: ancient, + })], + }, + // Writes for both candidates (4 each = 8 total). + {}, {}, {}, {}, + {}, {}, {}, {}, + // HYDRATING + { Items: [] }, + ]); + + await handler(); + + const failedIds = (mockDdbSend.mock.calls as [{ _type: string; input: Record }][]) + .filter(([c]) => c._type === 'UpdateItem' && String(c.input.ConditionExpression).includes('= :expected')) + .map(([c]) => (c.input as { Key: { task_id: { S: string } } }).Key.task_id.S); + expect(failedIds).toEqual(expect.arrayContaining(['t-page1', 't-page2'])); + }); +}); diff --git a/cdk/test/handlers/shared/create-task-core.test.ts b/cdk/test/handlers/shared/create-task-core.test.ts index 83a7f58..f0acbf1 100644 --- a/cdk/test/handlers/shared/create-task-core.test.ts +++ b/cdk/test/handlers/shared/create-task-core.test.ts @@ -364,4 +364,76 @@ describe('createTaskCore', () => { expect(result.statusCode).toBe(400); expect(result.body).toContain('pr_number is only allowed'); }); + + // -- trace flag (design §10.1) -------------------------------------- + + test('trace: true persists on the task record and surfaces in the response', async () => { + const result = await createTaskCore( + { repo: 'org/repo', task_description: 'deep debug', trace: true }, + makeContext(), + 'req-trace-1', + ); + expect(result.statusCode).toBe(201); + const body = JSON.parse(result.body); + expect(body.data.trace).toBe(true); + + // Verify the PutCommand carried trace on the record. + const putCall = mockSend.mock.calls.find( + c => (c[0] as { _type?: string; input?: { Item?: { trace?: unknown } } })._type === 'Put' + && (c[0] as { input?: { Item?: unknown } }).input?.Item !== undefined, + ); + expect(putCall).toBeDefined(); + const item = (putCall![0] as { input: { Item: Record } }).input.Item; + expect(item.trace).toBe(true); + }); + + test('trace omitted or false does NOT persist a trace field (slim wire payload)', async () => { + const result = await createTaskCore( + { repo: 'org/repo', task_description: 'normal' }, + makeContext(), + 'req-trace-2', + ); + expect(result.statusCode).toBe(201); + const body = JSON.parse(result.body); + expect(body.data.trace).toBe(false); + + const putCall = mockSend.mock.calls.find( + c => (c[0] as { _type?: string })._type === 'Put' + && (c[0] as { input?: { Item?: unknown } }).input?.Item !== undefined, + ); + const item = (putCall![0] as { input: { Item: Record } }).input.Item; + expect(item).not.toHaveProperty('trace'); + }); + + test('trace with non-boolean type returns 400 (strict boolean validation)', async () => { + // Prevents a misbehaving client from accidentally enabling trace + // with ``"trace": "false"`` (truthy string). + const result = await createTaskCore( + { repo: 'org/repo', task_description: 'x', trace: 'true' } as any, + makeContext(), + 'req-trace-3', + ); + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.message).toContain('trace'); + }); + + test.each([ + ['"false"', 'false'], + ['numeric 0', 0], + ['numeric 1', 1], + ['null', null], + ['empty object', {}], + ])('trace as %s is rejected with 400', async (_label, value) => { + // Adversarial inputs: the strict ``typeof === 'boolean'`` check + // must reject every non-boolean shape, not just the obvious string + // case. A future refactor that switches to a truthy test would + // pass the single "'true'" test above but break on these. + const result = await createTaskCore( + { repo: 'org/repo', task_description: 'x', trace: value } as any, + makeContext(), + `req-trace-adv-${String(value)}`, + ); + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body).error.message).toContain('trace'); + }); }); diff --git a/cdk/test/handlers/shared/error-classifier.test.ts b/cdk/test/handlers/shared/error-classifier.test.ts index d8acee9..4818c27 100644 --- a/cdk/test/handlers/shared/error-classifier.test.ts +++ b/cdk/test/handlers/shared/error-classifier.test.ts @@ -223,6 +223,49 @@ describe('classifyError', () => { expect(result!.retryable).toBe(false); }); + test('classifies error_max_turns as TIMEOUT with specific title (ordered before generic catch-all)', () => { + // Regression guard: pre-fix, the agent's specific + // ``agent_status='error_max_turns'`` signal was swallowed by the + // generic "Agent task did not succeed" title, leaving users + // without a clear remedy. The specific pattern must match first. + const result = classifyError( + "Task did not succeed (agent_status='error_max_turns', build_ok=False)", + ); + expect(result!.category).toBe(ErrorCategory.TIMEOUT); + expect(result!.title).toBe('Exceeded max turns'); + expect(result!.retryable).toBe(true); + expect(result!.remedy).toMatch(/--max-turns/); + }); + + test('classifies error_max_budget_usd as TIMEOUT with specific title', () => { + const result = classifyError( + "Task did not succeed (agent_status='error_max_budget_usd', build_ok=False)", + ); + expect(result!.category).toBe(ErrorCategory.TIMEOUT); + expect(result!.title).toBe('Exceeded max budget'); + expect(result!.retryable).toBe(true); + expect(result!.remedy).toMatch(/--max-budget/); + }); + + test('classifies error_during_execution with a mid-turn-error title', () => { + const result = classifyError( + "Task did not succeed (agent_status='error_during_execution', build_ok=False)", + ); + expect(result!.category).toBe(ErrorCategory.AGENT); + expect(result!.title).toBe('Agent errored during execution'); + expect(result!.retryable).toBe(true); + }); + + test('matches agent_status with or without quotes around the literal', () => { + // Defensive: the agent writer currently emits single-quoted + // repr values (``agent_status='error_max_turns'``) but a future + // refactor could drop the quotes. The pattern must match both. + const quoted = classifyError("Task did not succeed (agent_status='error_max_turns', build_ok=False)"); + const unquoted = classifyError('Task did not succeed (agent_status=error_max_turns, build_ok=False)'); + expect(quoted!.title).toBe('Exceeded max turns'); + expect(unquoted!.title).toBe('Exceeded max turns'); + }); + test('classifies receive_response failure', () => { const result = classifyError( 'receive_response() failed: Connection reset by peer', @@ -377,7 +420,7 @@ describe('classifyError', () => { repo: 'owner/repo', task_type: 'new_task', branch_name: 'bgagent/task-1/fix', - channel_source: 'cli', + channel_source: 'api', status_created_at: 'FAILED#2026-01-01T00:00:00Z', created_at: '2026-01-01T00:00:00Z', updated_at: '2026-01-01T00:00:00Z', @@ -403,5 +446,65 @@ describe('classifyError', () => { expect(detail.error_classification).not.toBeNull(); expect(detail.error_classification!.category).toBe('unknown'); }); + + // Regression: all numeric fields coerce through ``coerceNumericOrNull`` + // so the DDB Document-client's string-typed Number deserialization + // cannot leak into downstream consumers (same bug class as the + // ``costUsd.toFixed`` crash fixed in commit ``c09bfd7``). The cast + // to ``unknown as TaskRecord`` simulates a record produced by the + // Document client where ``Number`` attributes came back as strings. + test('coerces string-typed numeric DDB fields to numbers on output', () => { + const record = { + ...baseRecord, + duration_s: '12.5', + cost_usd: '0.0042', + max_turns: '30', + max_budget_usd: '1.50', + turns_attempted: '7', + turns_completed: '6', + } as unknown as TaskRecord; + const detail = toTaskDetail(record); + expect(typeof detail.duration_s).toBe('number'); + expect(detail.duration_s).toBe(12.5); + expect(typeof detail.cost_usd).toBe('number'); + expect(detail.cost_usd).toBe(0.0042); + expect(typeof detail.max_turns).toBe('number'); + expect(detail.max_turns).toBe(30); + expect(typeof detail.max_budget_usd).toBe('number'); + expect(detail.max_budget_usd).toBe(1.5); + expect(typeof detail.turns_attempted).toBe('number'); + expect(detail.turns_attempted).toBe(7); + expect(typeof detail.turns_completed).toBe('number'); + expect(detail.turns_completed).toBe(6); + }); + + test('coerces unparseable numeric strings to null (does not crash)', () => { + const record = { + ...baseRecord, + turns_attempted: 'not-a-number', + turns_completed: 'NaN', + } as unknown as TaskRecord; + const detail = toTaskDetail(record); + expect(detail.turns_attempted).toBeNull(); + expect(detail.turns_completed).toBeNull(); + }); + + // Compile-time regression for Finding #10 — ``ChannelSource`` is a + // literal union, not ``string``. The ``satisfies`` assertions below + // exercise the valid members; the ``@ts-expect-error`` comments pin + // the narrowing — if someone widens ``ChannelSource`` to ``string`` + // these will become un-erroring and fail the build. + test('channel_source narrows to the literal union', () => { + const apiRecord: TaskRecord = { ...baseRecord, channel_source: 'api' }; + const webhookRecord: TaskRecord = { ...baseRecord, channel_source: 'webhook' }; + expect(toTaskDetail(apiRecord).channel_source).toBe('api'); + expect(toTaskDetail(webhookRecord).channel_source).toBe('webhook'); + + // @ts-expect-error — 'slack' is not a valid ChannelSource + const invalid: TaskRecord = { ...baseRecord, channel_source: 'slack' }; + // Keep ``invalid`` used so the block doesn't get DCE'd and the + // ``@ts-expect-error`` above remains anchored to a real assignment. + expect(invalid.channel_source).toBeDefined(); + }); }); }); diff --git a/cdk/test/handlers/shared/github-comment.test.ts b/cdk/test/handlers/shared/github-comment.test.ts new file mode 100644 index 0000000..deb65f3 --- /dev/null +++ b/cdk/test/handlers/shared/github-comment.test.ts @@ -0,0 +1,682 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { + BGAGENT_COMMENT_MARKER_PREFIX, + GitHubCommentError, + renderCommentBody, + sanitizeMarkdownLinkTarget, + upsertTaskComment, +} from '../../../src/handlers/shared/github-comment'; +import { logger } from '../../../src/handlers/shared/logger'; + +// ``fetch`` is the global transport; each test installs its own mock. +const originalFetch = global.fetch; + +function mockResponse(opts: { + status: number; + ok?: boolean; + etag?: string | null; + body?: unknown; + rateLimitRemaining?: string; + rateLimitReset?: string; +}): Response { + const headers = new Headers(); + if (opts.etag !== null && opts.etag !== undefined) { + headers.set('etag', opts.etag); + } + if (opts.rateLimitRemaining !== undefined) { + headers.set('x-ratelimit-remaining', opts.rateLimitRemaining); + } + if (opts.rateLimitReset !== undefined) { + headers.set('x-ratelimit-reset', opts.rateLimitReset); + } + return { + ok: opts.ok ?? (opts.status >= 200 && opts.status < 300), + status: opts.status, + headers, + json: async () => opts.body ?? {}, + } as unknown as Response; +} + +afterEach(() => { + global.fetch = originalFetch; + jest.restoreAllMocks(); +}); + +describe('github-comment: upsertTaskComment — POST', () => { + test('creates a new comment when existingCommentId is undefined', async () => { + const fetchMock = jest.fn().mockResolvedValue( + mockResponse({ + status: 201, + etag: '"abc123"', + body: { id: 999, body: 'body' }, + }), + ); + global.fetch = fetchMock as unknown as typeof fetch; + + const result = await upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 42, + body: '# body', + token: 'ghp_xxx', + existingCommentId: undefined, + }); + + expect(result).toEqual({ commentId: 999, created: true }); + // Exactly one POST — no fallback GET/PATCH on first publish. + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe('https://api.github.com/repos/owner/repo/issues/42/comments'); + expect((init as RequestInit).method).toBe('POST'); + const headers = (init as RequestInit).headers as Record; + expect(headers.Authorization).toBe('token ghp_xxx'); + // Defense: GitHub's PATCH endpoint rejects ``If-Match`` with HTTP 400 + // ("Conditional request headers are not allowed in unsafe requests"). + // No write path on this helper should ever emit that header. (Scenario + // 7-ext deploy validation caught this in production.) + expect(headers['If-Match']).toBeUndefined(); + }); + + test('throws GitHubCommentError with status on POST failure', async () => { + global.fetch = jest.fn().mockResolvedValue( + mockResponse({ status: 422, ok: false, etag: '"x"' }), + ) as unknown as typeof fetch; + + await expect( + upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 1, + body: 'b', + token: 't', + existingCommentId: undefined, + }), + ).rejects.toMatchObject({ name: 'GitHubCommentError', httpStatus: 422 }); + }); + + test('POST response without an ETag header is accepted (ETag is no longer load-bearing)', async () => { + // Pre-fix, a missing ETag header threw because the caller needed + // it as ``If-Match`` on the next PATCH. After dropping the + // conditional-PATCH path, ETag is merely informational — absence + // must not fail the dispatch. + global.fetch = jest.fn().mockResolvedValue( + mockResponse({ status: 201, etag: null, body: { id: 42, body: 'b' } }), + ) as unknown as typeof fetch; + + await expect( + upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 1, + body: 'b', + token: 't', + existingCommentId: undefined, + }), + ).resolves.toEqual({ commentId: 42, created: true }); + }); +}); + +describe('github-comment: upsertTaskComment — PATCH', () => { + test('PATCHes the existing comment directly (one call, no GET, no If-Match header)', async () => { + // Design §6.4 post-fix: a single PATCH call per event. GitHub's + // REST API does not support ``If-Match`` on ``PATCH /issues/ + // comments/{id}`` — every conditional PATCH returns HTTP 400 + // ("Conditional request headers are not allowed in unsafe requests + // unless supported by the endpoint"). Concurrency is instead + // handled upstream by DDB Stream ordering. See file header. + const fetchMock = jest.fn().mockResolvedValueOnce( + mockResponse({ status: 200, etag: '"after"', body: { id: 7, body: 'new' } }), + ); + global.fetch = fetchMock as unknown as typeof fetch; + + const result = await upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 42, + body: 'new', + token: 't', + existingCommentId: 7, + }); + + expect(result).toEqual({ commentId: 7, created: false }); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe('https://api.github.com/repos/owner/repo/issues/comments/7'); + expect((init as RequestInit).method).toBe('PATCH'); + const headers = (init as RequestInit).headers as Record; + // BLOCKER regression guard: no conditional headers on PATCH. + expect(headers['If-Match']).toBeUndefined(); + expect(headers['If-None-Match']).toBeUndefined(); + }); + + test('on 404 (comment deleted upstream): falls back to POSTing a fresh comment', async () => { + const fetchMock = jest.fn() + // PATCH returns 404 + .mockResolvedValueOnce(mockResponse({ status: 404, ok: false, etag: null })) + // fallback POST + .mockResolvedValueOnce( + mockResponse({ status: 201, etag: '"new"', body: { id: 8, body: 'body' } }), + ); + global.fetch = fetchMock as unknown as typeof fetch; + + const result = await upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 42, + body: 'new', + token: 't', + existingCommentId: 7, + }); + + // NEW comment id, created=true so the caller persists the new id. + expect(result).toEqual({ commentId: 8, created: true }); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect((fetchMock.mock.calls[0][1] as RequestInit).method).toBe('PATCH'); + expect((fetchMock.mock.calls[1][1] as RequestInit).method).toBe('POST'); + }); + + test('non-404 error (500) propagates without retry', async () => { + const fetchMock = jest.fn() + .mockResolvedValueOnce(mockResponse({ status: 500, ok: false, etag: null })); + global.fetch = fetchMock as unknown as typeof fetch; + + await expect( + upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 42, + body: 'new', + token: 't', + existingCommentId: 7, + }), + ).rejects.toMatchObject({ name: 'GitHubCommentError', httpStatus: 500 }); + // No retry on generic 5xx — caller's batch-level dispatcher log is + // the right layer to see the failure. + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + test('non-404 error (400) propagates without retry (guards against the If-Match regression reappearing silently)', async () => { + // If a future refactor re-adds a conditional header and GitHub + // returns 400, the error should bubble up as a GitHubCommentError + // with httpStatus=400 rather than being swallowed. The fallback + // POST must NOT fire on 400 — only 404 (comment deleted) triggers + // the POST retry. + const fetchMock = jest.fn() + .mockResolvedValueOnce(mockResponse({ status: 400, ok: false, etag: null })); + global.fetch = fetchMock as unknown as typeof fetch; + + await expect( + upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 42, + body: 'new', + token: 't', + existingCommentId: 7, + }), + ).rejects.toMatchObject({ name: 'GitHubCommentError', httpStatus: 400 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + test('network error during PATCH is wrapped in GitHubCommentError', async () => { + const fetchMock = jest.fn() + .mockRejectedValueOnce(new TypeError('fetch failed')); + global.fetch = fetchMock as unknown as typeof fetch; + + await expect( + upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 42, + body: 'new', + token: 't', + existingCommentId: 7, + }), + ).rejects.toBeInstanceOf(GitHubCommentError); + }); + + test('PATCH body contains the rendered input verbatim', async () => { + // Locks the payload contract — a regression that stringified the + // wrong object would break every in-place edit silently. + const fetchMock = jest.fn().mockResolvedValueOnce( + mockResponse({ status: 200, etag: '"after"', body: { id: 7 } }), + ); + global.fetch = fetchMock as unknown as typeof fetch; + + await upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 42, + body: '# The Body', + token: 't', + existingCommentId: 7, + }); + + const init = fetchMock.mock.calls[0][1] as RequestInit; + expect(JSON.parse(init.body as string)).toEqual({ body: '# The Body' }); + }); +}); + +describe('github-comment: X-RateLimit-Remaining WARN-below-500 (L3 item 4)', () => { + test('emits a WARN when x-ratelimit-remaining < 500 on POST response', async () => { + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => undefined); + global.fetch = jest.fn().mockResolvedValue( + mockResponse({ + status: 201, + etag: '"abc"', + body: { id: 1, body: 'b' }, + rateLimitRemaining: '450', + rateLimitReset: '1714500000', + }), + ) as unknown as typeof fetch; + + await upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 1, + body: 'b', + token: 't', + existingCommentId: undefined, + }); + + expect(warnSpy).toHaveBeenCalledWith( + 'GitHub rate limit low', + expect.objectContaining({ + event: 'github.rate_limit_low', + remaining: 450, + reset_at: '1714500000', + repo: 'owner/repo', + }), + ); + }); + + test('emits a WARN when x-ratelimit-remaining < 500 on PATCH response', async () => { + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => undefined); + // Single-call PATCH path. + global.fetch = jest.fn().mockResolvedValueOnce( + mockResponse({ + status: 200, + etag: '"after"', + body: { id: 7 }, + rateLimitRemaining: '100', + }), + ) as unknown as typeof fetch; + + await upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 42, + body: 'new', + token: 't', + existingCommentId: 7, + }); + + expect(warnSpy).toHaveBeenCalledWith( + 'GitHub rate limit low', + expect.objectContaining({ remaining: 100, repo: 'owner/repo' }), + ); + }); + + test('does NOT warn when x-ratelimit-remaining is well above the 500 threshold', async () => { + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => undefined); + global.fetch = jest.fn().mockResolvedValue( + mockResponse({ + status: 201, + etag: '"abc"', + body: { id: 1, body: 'b' }, + rateLimitRemaining: '4999', + }), + ) as unknown as typeof fetch; + + await upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 1, + body: 'b', + token: 't', + existingCommentId: undefined, + }); + + // Rate-limit WARN is the only warn site touched by this path; a + // future unrelated warn in the dispatcher would break this. Pin + // specifically on the rate-limit event name. + const rateLimitWarns = warnSpy.mock.calls.filter( + c => (c[1] as Record | undefined)?.event === 'github.rate_limit_low', + ); + expect(rateLimitWarns).toHaveLength(0); + }); + + test('does NOT warn when x-ratelimit-remaining is absent (e.g. GHES variants)', async () => { + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => undefined); + global.fetch = jest.fn().mockResolvedValue( + mockResponse({ + status: 201, + etag: '"abc"', + body: { id: 1, body: 'b' }, + // No rateLimitRemaining set — header absent on the response. + }), + ) as unknown as typeof fetch; + + await upsertTaskComment({ + repo: 'owner/repo', + issueOrPrNumber: 1, + body: 'b', + token: 't', + existingCommentId: undefined, + }); + + const rateLimitWarns = warnSpy.mock.calls.filter( + c => (c[1] as Record | undefined)?.event === 'github.rate_limit_low', + ); + expect(rateLimitWarns).toHaveLength(0); + }); +}); + +describe('github-comment: renderCommentBody', () => { + test('renders a stable Markdown body with the bgagent marker and all fields', () => { + const body = renderCommentBody({ + taskId: 'abc123', + status: 'RUNNING', + repo: 'owner/repo', + latestEventType: 'agent_milestone', + latestEventAt: '2026-04-30T12:00:00Z', + prUrl: 'https://github.com/owner/repo/pull/42', + durationS: 90, + costUsd: 0.25, + }); + + // Leading HTML marker so future lookups can grep the comment thread. + expect(body.startsWith('')).toBe(true); + expect(body).toContain('| Task | `abc123` |'); + expect(body).toContain('| Status | **RUNNING** |'); + expect(body).toContain('agent_milestone'); + expect(body).toContain('[link](https://github.com/owner/repo/pull/42)'); + expect(body).toContain('| Duration | 90s |'); + expect(body).toContain('| Cost | $0.2500 |'); + }); + + test('sanitizes event types that contain Markdown-breaking characters', () => { + // Defensive against future writers emitting freer-form event + // strings — today all event types are snake_case enum values. + const body = renderCommentBody({ + taskId: 'abc', + status: 'RUNNING', + repo: 'o/r', + latestEventType: 'agent`|break\nline', + latestEventAt: '2026-04-30T12:00:00Z', + prUrl: null, + durationS: null, + costUsd: null, + }); + expect(body).toContain('agentbreakline'); + // Ensure the injection characters never made it into the rendered body. + expect(body).not.toMatch(/agent`/); + }); + + test('truncates bodies that would exceed the 65 536 GitHub ceiling', () => { + // Repeat a long line many times to cross the 60k cap. + const hugeStatus = 'RUNNING'.repeat(10_000); // 70k chars + const body = renderCommentBody({ + taskId: 'abc', + status: hugeStatus, + repo: 'o/r', + latestEventType: 'task_created', + latestEventAt: '2026-04-30T12:00:00Z', + prUrl: null, + durationS: null, + costUsd: null, + }); + expect(body.length).toBeLessThanOrEqual(65_536); + expect(body).toContain('(truncated'); + }); + + test('exports the BGAGENT marker prefix constant for downstream callers', () => { + // The marker prefix is the public convention for identifying + // bgagent-owned comments in PR threads. Exporting it keeps a + // Chunk K reconciliation / forensics caller from re-inventing + // the regex. + expect(BGAGENT_COMMENT_MARKER_PREFIX).toBe('bgagent:task-id='); + const body = renderCommentBody({ + taskId: 'T1', + status: 'COMPLETED', + repo: 'o/r', + latestEventType: 'task_completed', + latestEventAt: '2026-04-30T12:00:00Z', + prUrl: null, + durationS: null, + costUsd: null, + }); + expect(body).toContain(``); + }); + + test('omits optional rows when fields are null', () => { + const body = renderCommentBody({ + taskId: 'abc', + status: 'SUBMITTED', + repo: 'o/r', + latestEventType: 'task_created', + latestEventAt: '2026-04-30T12:00:00Z', + prUrl: null, + durationS: null, + costUsd: null, + }); + + expect(body).not.toContain('Pull request'); + expect(body).not.toContain('Duration'); + expect(body).not.toContain('Cost'); + // Required rows still present. + expect(body).toContain('| Task | `abc` |'); + expect(body).toContain('| Status | **SUBMITTED** |'); + }); +}); + +// --------------------------------------------------------------------------- +// Krokoko code review finding #9 — renderCommentBody self-defends against +// uncoerced DDB string numerics +// --------------------------------------------------------------------------- + +describe('github-comment: renderCommentBody numeric self-defense (finding #9)', () => { + // The fanout dispatcher coerces ``cost_usd`` / ``duration_s`` at its + // own boundary, but that coverage is brittle: a future caller (a + // Chunk K reconciler, a Phase 3 rehydration path) that forgets the + // step would hit the same ``TypeError: toFixed is not a function`` + // bug commit 9fe704e fixed in the fanout dispatcher. ``renderCommentBody`` + // now coerces again so the crash surface is closed at the render site. + + test('string-typed costUsd from an uncoerced DDB Item does NOT throw', () => { + // Direct repro of the Scenario-7-ext symptom at the render boundary. + // The body must render a valid Cost row, not throw TypeError. + const body = renderCommentBody({ + taskId: 'abc', + status: 'COMPLETED', + repo: 'o/r', + latestEventType: 'task_completed', + latestEventAt: '2026-05-05T00:00:00Z', + prUrl: null, + // DynamoDB Document-client deserialization: Number → string. + durationS: '96.0' as unknown as number, + costUsd: '0.20939010000000002' as unknown as number, + }); + expect(body).toContain('| Cost | $0.2094 |'); + expect(body).toContain('| Duration | 96s |'); + }); + + test('non-finite string costUsd collapses to null and omits the Cost row', () => { + // A corrupt writer emitting ``'NaN'`` or ``'Infinity'`` must NOT + // produce a ``$NaN`` row. Coercion returns null, row is omitted, + // and a ``numeric.coercion_failed`` warn fires via the shared + // coercion helper so the writer bug surfaces in CloudWatch. + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => undefined); + try { + const body = renderCommentBody({ + taskId: 'abc', + status: 'COMPLETED', + repo: 'o/r', + latestEventType: 'task_completed', + latestEventAt: '2026-05-05T00:00:00Z', + prUrl: null, + durationS: null, + costUsd: 'not-a-number' as unknown as number, + }); + expect(body).not.toContain('$NaN'); + expect(body).not.toContain('| Cost |'); + const coercionWarn = warnSpy.mock.calls.find( + c => (c[1] as Record | undefined)?.event === 'numeric.coercion_failed', + ); + expect(coercionWarn).toBeDefined(); + expect((coercionWarn?.[1] as Record).field).toBe('cost_usd'); + } finally { + warnSpy.mockRestore(); + } + }); + + test('null cost / duration render no row (unchanged behavior — absent is not corrupt)', () => { + // Regression guard: the self-defense must NOT start warning on the + // legitimate "absent" case. Only non-finite coercions warn. + const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => undefined); + try { + renderCommentBody({ + taskId: 'abc', + status: 'RUNNING', + repo: 'o/r', + latestEventType: 'agent_turn', + latestEventAt: '2026-05-05T00:00:00Z', + prUrl: null, + durationS: null, + costUsd: null, + }); + const coercionWarns = warnSpy.mock.calls.filter( + c => (c[1] as Record | undefined)?.event === 'numeric.coercion_failed', + ); + expect(coercionWarns).toHaveLength(0); + } finally { + warnSpy.mockRestore(); + } + }); +}); + +// --------------------------------------------------------------------------- +// Krokoko code review finding #12 — Markdown injection via prUrl +// --------------------------------------------------------------------------- + +describe('github-comment: sanitizeMarkdownLinkTarget (finding #12)', () => { + // The helper is exported so callers outside renderCommentBody (e.g. + // future Slack / email renderers that may interpolate into markdown) + // can share the same validation surface. + + test('accepts a well-formed https GitHub PR URL unchanged', () => { + const ok = 'https://github.com/owner/repo/pull/42'; + expect(sanitizeMarkdownLinkTarget(ok)).toBe(ok); + }); + + test('accepts a plain http URL unchanged', () => { + // Enterprise / self-hosted GitHub may serve over plain HTTP in + // internal networks; we still allow it. Non-http(s) schemes are + // rejected below. + const ok = 'http://github.internal/owner/repo/pull/7'; + expect(sanitizeMarkdownLinkTarget(ok)).toBe(ok); + }); + + test.each([ + // Each of these, if interpolated into ``[link]()`` verbatim, + // would break the Markdown table layout or inject trailing content. + ['close-paren', 'https://evil.example.com/a)|injected'], + ['pipe', 'https://evil.example.com/a|new-col'], + ['newline', 'https://evil.example.com/a\nnew line'], + ['carriage-return', 'https://evil.example.com/a\rnew line'], + ['bracket', 'https://evil.example.com/a]extra'], + ['quote', 'https://evil.example.com/a"title"'], + ['space', 'https://evil.example.com/a b'], + ['tab', 'https://evil.example.com/a\tb'], + ['backtick', 'https://evil.example.com/a`b'], + ])('rejects %s injection attempt: %s', (_label, crafted) => { + expect(sanitizeMarkdownLinkTarget(crafted)).toBeNull(); + }); + + test.each([ + ['javascript', 'javascript:alert(1)'], + ['data', 'data:text/html,'], + ['file', 'file:///etc/passwd'], + ['ftp', 'ftp://evil.example.com/x'], + ])('rejects non-http(s) scheme: %s', (_label, bad) => { + expect(sanitizeMarkdownLinkTarget(bad)).toBeNull(); + }); + + test('rejects a malformed URL that cannot be parsed', () => { + expect(sanitizeMarkdownLinkTarget('not a url at all')).toBeNull(); + }); + + test('null / undefined pass through as null (omits the row)', () => { + expect(sanitizeMarkdownLinkTarget(null)).toBeNull(); + expect(sanitizeMarkdownLinkTarget(undefined)).toBeNull(); + }); +}); + +describe('github-comment: renderCommentBody Markdown-link injection guard (finding #12)', () => { + test('crafted prUrl with ) | ] does not break the Markdown table', () => { + // End-to-end: what happens at the render boundary when the PR URL + // is hostile. Pre-fix, the body contained ``[link](evil)|injected)`` + // which rendered a broken link AND started a new table column. + // Post-fix, the row is omitted entirely. + const body = renderCommentBody({ + taskId: 'abc', + status: 'COMPLETED', + repo: 'o/r', + latestEventType: 'task_completed', + latestEventAt: '2026-05-05T00:00:00Z', + prUrl: 'evil)|injected' as unknown as string, + durationS: null, + costUsd: null, + }); + // The Pull-request row is omitted because the URL failed validation. + expect(body).not.toContain('Pull request'); + // Defense-in-depth: none of the injection characters appear in a + // link-like context. Specifically, no ``[link](`` with a trailing + // pipe or close-paren that could close the link and open a new + // column. + expect(body).not.toMatch(/\[link\]\([^)]*[|)]/); + }); + + test('javascript: scheme prUrl is rejected (omits the row rather than rendering)', () => { + // An attacker who controlled ``pr_url`` (e.g. via a future webhook + // field) could supply ``javascript:...``. Browsers don't execute + // clicks on Markdown links in GitHub comments (GitHub rewrites + // targets) but the row would still display the attacker-chosen + // label. Safer to omit entirely. + const body = renderCommentBody({ + taskId: 'abc', + status: 'RUNNING', + repo: 'o/r', + latestEventType: 'task_created', + latestEventAt: '2026-05-05T00:00:00Z', + prUrl: 'javascript:alert(1)' as unknown as string, + durationS: null, + costUsd: null, + }); + expect(body).not.toContain('javascript:'); + expect(body).not.toContain('Pull request'); + }); + + test('legitimate https PR URL still renders the link row unchanged', () => { + // Regression guard: the sanitization must NOT reject real GitHub + // PR links — that would mask terminal comments silently. + const prUrl = 'https://github.com/owner/repo/pull/42'; + const body = renderCommentBody({ + taskId: 'abc', + status: 'COMPLETED', + repo: 'owner/repo', + latestEventType: 'task_completed', + latestEventAt: '2026-05-05T00:00:00Z', + prUrl, + durationS: null, + costUsd: null, + }); + expect(body).toContain(`| Pull request | [link](${prUrl}) |`); + }); +}); diff --git a/cdk/test/handlers/shared/numeric.test.ts b/cdk/test/handlers/shared/numeric.test.ts new file mode 100644 index 0000000..d51abc1 --- /dev/null +++ b/cdk/test/handlers/shared/numeric.test.ts @@ -0,0 +1,86 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { coerceNumericOrNull, type CoerceLogger } from '../../../src/handlers/shared/numeric'; + +function mkLogger(): { logger: CoerceLogger; warnCalls: Array<{ message: string; meta?: Record }> } { + const warnCalls: Array<{ message: string; meta?: Record }> = []; + const logger: CoerceLogger = { + warn: (message, meta) => { warnCalls.push({ message, meta }); }, + }; + return { logger, warnCalls }; +} + +describe('coerceNumericOrNull', () => { + test('passes real numbers through unchanged', () => { + const { logger, warnCalls } = mkLogger(); + expect(coerceNumericOrNull(42, { field: 'x' }, logger)).toBe(42); + expect(coerceNumericOrNull(0, { field: 'x' }, logger)).toBe(0); + expect(coerceNumericOrNull(-1.5, { field: 'x' }, logger)).toBe(-1.5); + expect(warnCalls).toHaveLength(0); + }); + + test('parses numeric strings (DDB Document-client shape)', () => { + // The actual shape observed in Scenario 7-extended deploy + // validation: ``duration_s: "96.0"`` and + // ``cost_usd: "0.20939010000000002"``. + const { logger, warnCalls } = mkLogger(); + expect(coerceNumericOrNull('96.0', { field: 'duration_s' }, logger)).toBe(96); + expect(coerceNumericOrNull('0.20939010000000002', { field: 'cost_usd' }, logger)) + .toBeCloseTo(0.2094, 4); + expect(warnCalls).toHaveLength(0); + }); + + test('treats null / undefined / empty-string as absent (no warn)', () => { + const { logger, warnCalls } = mkLogger(); + expect(coerceNumericOrNull(null, { field: 'x' }, logger)).toBeNull(); + expect(coerceNumericOrNull(undefined, { field: 'x' }, logger)).toBeNull(); + expect(coerceNumericOrNull('', { field: 'x' }, logger)).toBeNull(); + expect(warnCalls).toHaveLength(0); + }); + + test('non-finite coercion collapses to null AND emits a warn', () => { + // Corrupt input (non-numeric string, real NaN, Infinity) must + // surface in CloudWatch so writer bugs are visible rather than + // silently dropping data from the consumer's render. + const { logger, warnCalls } = mkLogger(); + + expect(coerceNumericOrNull('not-a-number', { field: 'cost_usd', task_id: 't-1' }, logger)) + .toBeNull(); + expect(coerceNumericOrNull(NaN, { field: 'x' }, logger)).toBeNull(); + expect(coerceNumericOrNull(Infinity, { field: 'x' }, logger)).toBeNull(); + expect(coerceNumericOrNull(-Infinity, { field: 'x' }, logger)).toBeNull(); + + expect(warnCalls).toHaveLength(4); + for (const call of warnCalls) { + expect(call.meta?.event).toBe('numeric.coercion_failed'); + expect(call.meta?.field).toBeDefined(); + expect(call.meta?.raw).toBeDefined(); + } + // The task_id context propagates through. + expect(warnCalls[0].meta?.task_id).toBe('t-1'); + }); + + test('warn payload preserves the raw input for operator triage', () => { + const { logger, warnCalls } = mkLogger(); + coerceNumericOrNull('oops', { field: 'cost_usd', task_id: 't-1', event_id: 'e-1' }, logger); + expect(warnCalls[0].meta?.raw).toBe('oops'); + expect(warnCalls[0].meta?.event_id).toBe('e-1'); + }); +}); diff --git a/cdk/test/handlers/shared/validation.test.ts b/cdk/test/handlers/shared/validation.test.ts index 21a4389..4aecf4e 100644 --- a/cdk/test/handlers/shared/validation.test.ts +++ b/cdk/test/handlers/shared/validation.test.ts @@ -26,6 +26,7 @@ import { isValidRepo, isValidTaskDescriptionLength, isValidTaskType, + isValidUlid, isValidWebhookName, MAX_TASK_DESCRIPTION_LENGTH, parseBody, @@ -343,6 +344,49 @@ describe('isValidTaskType', () => { }); }); +describe('isValidUlid', () => { + test('accepts a canonical 26-char Crockford Base32 ULID', () => { + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5FAV')).toBe(true); + }); + + test('accepts ULIDs containing every allowed character', () => { + // timestamp-only characters 0-9 and a spread of allowed letters + expect(isValidUlid('0123456789ABCDEFGHJKMNPQRS')).toBe(true); + expect(isValidUlid('TVWXYZ0123456789ABCDEFGHJK')).toBe(true); + }); + + test('accepts lowercase input (case-insensitive)', () => { + expect(isValidUlid('01arz3ndektsv4rrffq69g5fav')).toBe(true); + }); + + test('rejects wrong length', () => { + expect(isValidUlid('')).toBe(false); + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5FA')).toBe(false); // 25 + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5FAVX')).toBe(false); // 27 + }); + + test('rejects Crockford-excluded letters I, L, O, U', () => { + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5FAI')).toBe(false); + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5FAL')).toBe(false); + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5FAO')).toBe(false); + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5FAU')).toBe(false); + }); + + test('rejects non-Base32 punctuation', () => { + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5F!V')).toBe(false); + expect(isValidUlid('01ARZ3NDEKTSV4RRFFQ69G5F-V')).toBe(false); + }); + + test('rejects non-string input', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect(isValidUlid(42 as any)).toBe(false); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect(isValidUlid(null as any)).toBe(false); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect(isValidUlid(undefined as any)).toBe(false); + }); +}); + describe('validatePrNumber', () => { test('returns the number for valid positive integers', () => { expect(validatePrNumber(1)).toBe(1); diff --git a/cdk/test/stacks/agent.test.ts b/cdk/test/stacks/agent.test.ts index de54c29..54b5294 100644 --- a/cdk/test/stacks/agent.test.ts +++ b/cdk/test/stacks/agent.test.ts @@ -36,8 +36,37 @@ describe('AgentStack', () => { expect(template).toBeDefined(); }); - test('creates exactly 5 DynamoDB tables', () => { - template.resourceCountIs('AWS::DynamoDB::Table', 5); + test('creates exactly 6 DynamoDB tables (including TaskNudgesTable for Phase 2)', () => { + template.resourceCountIs('AWS::DynamoDB::Table', 6); + }); + + test('outputs TaskNudgesTableName', () => { + template.hasOutput('TaskNudgesTableName', { + Description: 'Name of the DynamoDB task nudges table (Phase 2)', + }); + }); + + test('creates TaskNudgesTable with task_id PK and nudge_id SK and no stream', () => { + const tables = template.findResources('AWS::DynamoDB::Table'); + const nudgeTables = Object.values(tables).filter(t => { + const ks = (t as { Properties?: { KeySchema?: Array<{ AttributeName: string }> } }).Properties?.KeySchema ?? []; + return ks.length === 2 && ks[0]!.AttributeName === 'task_id' && ks[1]!.AttributeName === 'nudge_id'; + }); + expect(nudgeTables).toHaveLength(1); + const props = (nudgeTables[0] as { Properties?: { StreamSpecification?: unknown } }).Properties ?? {}; + // No DynamoDB stream on nudges (poll-consumed). + expect(props.StreamSpecification).toBeUndefined(); + }); + + test('runtime receives NUDGES_TABLE_NAME env var', () => { + const runtimes = template.findResources('AWS::BedrockAgentCore::Runtime'); + const runtimeList = Object.values(runtimes); + expect(runtimeList).toHaveLength(1); + for (const rt of runtimeList) { + const envVars = (rt as { Properties?: { EnvironmentVariables?: Record } }) + .Properties?.EnvironmentVariables ?? {}; + expect(envVars).toHaveProperty('NUDGES_TABLE_NAME'); + } }); test('outputs TaskTableName', () => { @@ -71,9 +100,70 @@ describe('AgentStack', () => { }); test('outputs RuntimeArn', () => { - template.hasOutput('RuntimeArn', { - Description: 'ARN of the AgentCore runtime', + template.hasOutput('RuntimeArn', {}); + }); + + test('creates exactly one AgentCore Runtime', () => { + template.resourceCountIs('AWS::BedrockAgentCore::Runtime', 1); + }); + + test('runtime execution role carries ECR pull permissions', () => { + const policies = template.findResources('AWS::IAM::Policy'); + + const rolesWithEcrPull = Object.values(policies).filter(policy => { + const statements = policy.Properties?.PolicyDocument?.Statement ?? []; + return statements.some((s: { Action?: unknown }) => { + const action = s.Action; + const actions = Array.isArray(action) ? action : [action]; + return actions.includes('ecr:BatchGetImage') + && actions.includes('ecr:GetDownloadUrlForLayer') + && actions.includes('ecr:BatchCheckLayerAvailability'); + }); }); + + expect(rolesWithEcrPull.length).toBeGreaterThanOrEqual(1); + }); + + test('runtime has 8-hour lifecycle limits (idle + max)', () => { + const runtimes = template.findResources('AWS::BedrockAgentCore::Runtime'); + const runtimeList = Object.values(runtimes); + expect(runtimeList).toHaveLength(1); + for (const rt of runtimeList) { + expect(rt.Properties?.LifecycleConfiguration).toEqual({ + IdleRuntimeSessionTimeout: 28800, + MaxLifetime: 28800, + }); + } + }); + + test('TaskEventsTable has DynamoDB Streams enabled with NEW_IMAGE', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + KeySchema: [ + { AttributeName: 'task_id', KeyType: 'HASH' }, + { AttributeName: 'event_id', KeyType: 'RANGE' }, + ], + StreamSpecification: { + StreamViewType: 'NEW_IMAGE', + }, + }); + }); + + test('orchestrator IAM policy grants InvokeAgentRuntime on the runtime', () => { + // Find the orchestrator's IAM policy that contains InvokeAgentRuntime. + const policies = template.findResources('AWS::IAM::Policy'); + const invokePolicies = Object.values(policies).filter(p => { + const statements = p.Properties?.PolicyDocument?.Statement ?? []; + return statements.some((s: { Action?: string | string[] }) => { + const actions = Array.isArray(s.Action) ? s.Action : [s.Action]; + return actions.includes('bedrock-agentcore:InvokeAgentRuntime'); + }); + }); + expect(invokePolicies.length).toBeGreaterThanOrEqual(1); + + // The policy must reference the runtime's ARN (via Fn::GetAtt on the + // Runtime* logical id). + const serialized = JSON.stringify(invokePolicies); + expect(serialized).toMatch(/"Fn::GetAtt":\["Runtime[0-9A-F]+","AgentRuntimeArn"\]/); }); test('outputs ApiUrl', () => { diff --git a/cli/src/api-client.ts b/cli/src/api-client.ts index 94939de..fec20f6 100644 --- a/cli/src/api-client.ts +++ b/cli/src/api-client.ts @@ -27,11 +27,14 @@ import { CreateWebhookRequest, CreateWebhookResponse, ErrorResponse, + NudgeRequest, + NudgeResponse, PaginatedResponse, SuccessResponse, TaskDetail, TaskEvent, TaskSummary, + TraceUrlResponse, WebhookDetail, } from './types'; @@ -48,7 +51,13 @@ export class ApiClient { return this.baseUrl; } - private async request(method: string, path: string, body?: unknown, headers?: Record): Promise { + private async request( + method: string, + path: string, + body?: unknown, + headers?: Record, + signal?: AbortSignal, + ): Promise { const token = await getAuthToken(); const url = `${this.getBaseUrl()}${path}`; @@ -65,29 +74,54 @@ export class ApiClient { ...headers, }, body: body ? JSON.stringify(body) : undefined, + signal, }); debug(`Response: ${res.status} ${res.statusText}`); let json: unknown; + let jsonParseOk = true; try { json = await res.json(); } catch { - throw new CliError(`HTTP ${res.status}: ${res.statusText} (non-JSON response)`); + jsonParseOk = false; } - debug(`Response body: ${JSON.stringify(json)}`); + if (jsonParseOk) { + debug(`Response body: ${JSON.stringify(json)}`); + } if (!res.ok) { - const err = json as ErrorResponse; - if (err.error) { + // Keep HTTP-status-carrying errors as ``ApiError`` regardless of + // body shape so callers (e.g. the watch retry loop) can classify + // 4xx-vs-5xx reliably. A WAF / CloudFront / API-GW edge page is + // still a deterministic 4xx from the caller's perspective — + // retrying it would be futile. + if (jsonParseOk && (json as ErrorResponse).error) { + const err = json as ErrorResponse; let message = `${err.error.message} (${err.error.code})`; if (res.status === 401) { message += '\nHint: Run `bgagent login` to re-authenticate.'; } throw new ApiError(res.status, err.error.code, message, err.error.request_id); } - throw new CliError(`HTTP ${res.status}: ${res.statusText}`); + // Non-JSON or envelope-less error body — still an HTTP error, still + // must carry the status so classification works. Code/request_id + // are unavailable at this layer; surface ``HTTP_ERROR`` / empty. + throw new ApiError( + res.status, + 'HTTP_ERROR', + `HTTP ${res.status}: ${res.statusText}${jsonParseOk ? '' : ' (non-JSON response)'}`, + '', + ); + } + + if (!jsonParseOk) { + // 2xx with an unparseable body is a server contract violation — + // neither transient (5xx) nor user-recoverable (4xx). Fail hard + // with ``CliError`` so the retry loop does NOT treat it as + // transient. + throw new CliError(`HTTP ${res.status}: ${res.statusText} (non-JSON response)`); } return json as T; @@ -122,8 +156,14 @@ export class ApiClient { } /** GET /tasks/{task_id} — get task detail. */ - async getTask(taskId: string): Promise { - const res = await this.request>('GET', `/tasks/${encodeURIComponent(taskId)}`); + async getTask(taskId: string, opts?: { signal?: AbortSignal }): Promise { + const res = await this.request>( + 'GET', + `/tasks/${encodeURIComponent(taskId)}`, + undefined, + undefined, + opts?.signal, + ); return res.data; } @@ -133,18 +173,131 @@ export class ApiClient { return res.data; } - /** GET /tasks/{task_id}/events — get task events. */ + /** + * POST /tasks/{task_id}/nudge — send a steering message to a running task (Phase 2). + * + * The server guardrail-screens and rate-limits the nudge before enqueuing it + * for the agent to pick up at the next between-turns seam. Returns HTTP 202 + * with the generated `nudge_id` on success. + */ + async nudgeTask(taskId: string, message: string): Promise { + const body: NudgeRequest = { message }; + const res = await this.request>( + 'POST', + `/tasks/${encodeURIComponent(taskId)}/nudge`, + body, + ); + return res.data; + } + + /** + * GET /tasks/{task_id}/events — fetch one page of task events. + * + * Supports two alternative pagination cursors: + * - ``after`` — a ULID event_id. Server returns events with + * ``event_id > after``. + * - ``nextToken`` — an opaque DynamoDB pagination token for normal + * forward pagination. + * + * If both are passed, the server prefers ``after`` and logs a warning. + * Prefer {@link catchUpEvents} when you want all events after a known + * id drained across pagination (the watch loop uses this). + */ async getTaskEvents(taskId: string, opts?: { limit?: number; nextToken?: string; + after?: string; + /** Request newest-first ordering — mutually exclusive with ``after`` on the server. */ + desc?: boolean; + /** Abort an in-flight request (SIGINT during ``bgagent watch``, etc.). */ + signal?: AbortSignal; }): Promise> { const params = new URLSearchParams(); if (opts?.limit) params.set('limit', String(opts.limit)); if (opts?.nextToken) params.set('next_token', opts.nextToken); + if (opts?.after) params.set('after', opts.after); + if (opts?.desc) params.set('desc', '1'); const qs = params.toString(); const path = `/tasks/${encodeURIComponent(taskId)}/events${qs ? `?${qs}` : ''}`; - return this.request>('GET', path); + return this.request>('GET', path, undefined, undefined, opts?.signal); + } + + /** + * Fetch the combined task + most-recent-events payload that backs the + * deterministic ``bgagent status`` snapshot (design §5.2). + * + * Runs the ``GET /tasks/{id}`` and ``GET /tasks/{id}/events?desc=1&limit=N`` + * calls in parallel so the snapshot is a single round-trip in wall-clock + * terms. The event page is intentionally small (default 20) — the + * formatter only needs the latest tool call, turn, milestone, and cost + * update, which are always recent in a well-behaved event stream. + * + * @param taskId - the task to summarize. + * @param recentEventLimit - how many recent events to pull (default 20). + */ + async getStatusSnapshot( + taskId: string, + recentEventLimit = 20, + ): Promise<{ task: TaskDetail; recentEvents: TaskEvent[] }> { + const [task, eventsPage] = await Promise.all([ + this.getTask(taskId), + this.getTaskEvents(taskId, { limit: recentEventLimit, desc: true }), + ]); + return { task, recentEvents: eventsPage.data }; + } + + /** + * Fetch every event with ``event_id > afterEventId``, paginating through + * the server's ``next_token`` internally. + * + * Paginates forward from a known event_id cursor. Returns events in + * ascending order (oldest first), matching the server's + * ``ScanIndexForward: true``. + * + * @param taskId - the task whose events to fetch. + * @param afterEventId - the ULID cursor; events strictly greater than + * this id are returned. + * @param pageSize - page size passed to the server (default 100, max 100). + * @returns all events after the cursor, in chronological order. + */ + async catchUpEvents( + taskId: string, + afterEventId: string, + pageSize = 100, + opts?: { signal?: AbortSignal }, + ): Promise { + const collected: TaskEvent[] = []; + const signal = opts?.signal; + // First page uses ``after``; subsequent pages use the opaque ``next_token``. + let page = await this.getTaskEvents(taskId, { after: afterEventId, limit: pageSize, signal }); + collected.push(...page.data); + while (page.pagination.has_more && page.pagination.next_token) { + page = await this.getTaskEvents(taskId, { + nextToken: page.pagination.next_token, + limit: pageSize, + signal, + }); + collected.push(...page.data); + } + return collected; + } + + /** + * GET /tasks/{task_id}/trace — get a presigned S3 URL for the + * ``--trace`` trajectory dump (design §10.1). + * + * Returns a short-lived (15-minute) presigned URL the CLI can + * stream directly from S3. The endpoint 404s with code + * ``TRACE_NOT_AVAILABLE`` when the task did not run with + * ``--trace`` or the upload has not yet completed. + */ + async getTraceUrl(taskId: string): Promise { + const res = await this.request>( + 'GET', + `/tasks/${encodeURIComponent(taskId)}/trace`, + ); + return res.data; } /** POST /webhooks — create a new webhook. */ diff --git a/cli/src/auth.ts b/cli/src/auth.ts index e870e91..13c7ba2 100644 --- a/cli/src/auth.ts +++ b/cli/src/auth.ts @@ -57,20 +57,39 @@ export async function login(username: string, password: string): Promise { }); } -/** Get a valid auth token, refreshing automatically if needed. */ +/** Get a valid auth token, refreshing automatically if needed. + * + * The REST API Gateway's Cognito authorizer validates **ID tokens** (checks + * the `aud` claim against the app client ID). All CLI calls go through the + * REST path, so this is the only token we need. + */ export async function getAuthToken(): Promise { + return getIdToken(); +} + +/** Get the Cognito ID token — for REST API Gateway calls. */ +export async function getIdToken(): Promise { + const creds = await ensureFreshCredentials(); + return creds.id_token; +} + +/** Internal: return non-expired credentials, refreshing if needed. */ +async function ensureFreshCredentials(): Promise { const creds = loadCredentials(); if (!creds) { throw new CliError('Not authenticated. Run `bgagent login` first.'); } - if (!isExpired(creds)) { - debug('Using cached token (not expired)'); - return creds.id_token; + debug('Using cached tokens (not expired)'); + return creds; } - - debug('Token expired or near expiry, refreshing...'); - return refreshToken(creds); + debug('Tokens expired or near expiry, refreshing...'); + await refreshToken(creds); + const fresh = loadCredentials(); + if (!fresh) { + throw new CliError('Credentials vanished after refresh. Run `bgagent login`.'); + } + return fresh; } function isExpired(creds: Credentials): boolean { @@ -78,7 +97,7 @@ function isExpired(creds: Credentials): boolean { return Date.now() >= expiryMs - TOKEN_REFRESH_BUFFER_MS; } -async function refreshToken(creds: Credentials): Promise { +async function refreshToken(creds: Credentials): Promise { const config = loadConfig(); const client = new CognitoIdentityProviderClient({ region: config.region }); @@ -102,8 +121,6 @@ async function refreshToken(creds: Credentials): Promise { refresh_token: creds.refresh_token, token_expiry: expiry, }); - - return auth.IdToken; } catch (err) { if (err instanceof CliError) throw err; throw new CliError('Session expired. Run `bgagent login` to re-authenticate.'); diff --git a/cli/src/bin/bgagent.ts b/cli/src/bin/bgagent.ts index 982b1ab..dd8a318 100644 --- a/cli/src/bin/bgagent.ts +++ b/cli/src/bin/bgagent.ts @@ -25,8 +25,11 @@ import { makeConfigureCommand } from '../commands/configure'; import { makeEventsCommand } from '../commands/events'; import { makeListCommand } from '../commands/list'; import { makeLoginCommand } from '../commands/login'; +import { makeNudgeCommand } from '../commands/nudge'; import { makeStatusCommand } from '../commands/status'; import { makeSubmitCommand } from '../commands/submit'; +import { makeTraceCommand } from '../commands/trace'; +import { makeWatchCommand } from '../commands/watch'; import { makeWebhookCommand } from '../commands/webhook'; import { setVerbose } from '../debug'; import { ApiError, CliError } from '../errors'; @@ -52,16 +55,48 @@ program.addCommand(makeSubmitCommand()); program.addCommand(makeListCommand()); program.addCommand(makeStatusCommand()); program.addCommand(makeCancelCommand()); +program.addCommand(makeNudgeCommand()); program.addCommand(makeEventsCommand()); +program.addCommand(makeWatchCommand()); +program.addCommand(makeTraceCommand()); program.addCommand(makeWebhookCommand()); -program.parseAsync(process.argv).catch((err: unknown) => { - if (err instanceof CliError || err instanceof ApiError) { - console.error(`Error: ${err.message}`); - } else if (err instanceof Error) { - console.error(`Error: ${err.message}`); - } else { - console.error('An unexpected error occurred.'); - } - process.exitCode = 1; -}); +// Execute the CLI only when run directly. Importing this module (e.g. +// from a test harness or a wrapper) must not parse the importer's +// ``process.argv`` nor schedule a ``process.exit`` on the importer's +// event loop. Keeping the side-effect behind ``require.main === module`` +// preserves both properties without forcing callers to mock the +// program object. Commands under ``cli/src/commands/*`` already export +// ``makeXxxCommand()`` factories for direct invocation in tests. +if (require.main === module) { + program + .parseAsync(process.argv) + .catch((err: unknown) => { + if (err instanceof CliError || err instanceof ApiError) { + console.error(`Error: ${err.message}`); + } else if (err instanceof Error) { + console.error(`Error: ${err.message}`); + } else { + console.error('An unexpected error occurred.'); + } + process.exitCode = 1; + }) + .finally(() => { + // Node's global ``fetch`` (undici) keeps TCP sockets alive in a + // connection pool by default. After a long-running command like + // ``bgagent watch`` finishes its logical work, those sockets keep + // the event loop open for the pool's idle timeout, leaving the + // process hanging long past task terminal. We set ``exitCode`` + // (so the natural drain path uses it) and schedule a deferred + // ``process.exit`` as a fallback: an ``unref``'d 50 ms timer + // gives async ``stderr`` / ``stdout`` flushes and ``on('exit')`` + // handlers a chance to complete, while still guaranteeing a + // bounded exit time instead of the pool's multi-second + // keep-alive timeout. Observed in Scenarios 6 and 7-extended + // deploy validation where ``bgagent watch`` had to be ``pkill``-ed + // after the task reached COMPLETED. + setTimeout(() => { + process.exit(process.exitCode ?? 0); + }, 50).unref(); + }); +} diff --git a/cli/src/commands/configure.ts b/cli/src/commands/configure.ts index 1e0f0c0..f994ace 100644 --- a/cli/src/commands/configure.ts +++ b/cli/src/commands/configure.ts @@ -18,22 +18,57 @@ */ import { Command } from 'commander'; -import { saveConfig } from '../config'; +import { saveConfig, tryLoadConfig } from '../config'; +import { CliError } from '../errors'; +import { CliConfig } from '../types'; +/** + * All four core fields (api-url, region, user-pool-id, client-id) are required + * the first time — subsequent invocations may update a subset. + */ export function makeConfigureCommand(): Command { return new Command('configure') .description('Configure the CLI with API endpoint and Cognito settings') - .requiredOption('--api-url ', 'API Gateway base URL') - .requiredOption('--region ', 'AWS region') - .requiredOption('--user-pool-id ', 'Cognito User Pool ID') - .requiredOption('--client-id ', 'Cognito App Client ID') + .option('--api-url ', 'API Gateway base URL') + .option('--region ', 'AWS region') + .option('--user-pool-id ', 'Cognito User Pool ID') + .option('--client-id ', 'Cognito App Client ID') .action((opts) => { - saveConfig({ - api_url: opts.apiUrl, - region: opts.region, - user_pool_id: opts.userPoolId, - client_id: opts.clientId, - }); + const existing = tryLoadConfig(); + const providedFlags = { + ...(opts.apiUrl !== undefined ? { api_url: opts.apiUrl } : {}), + ...(opts.region !== undefined ? { region: opts.region } : {}), + ...(opts.userPoolId !== undefined ? { user_pool_id: opts.userPoolId } : {}), + ...(opts.clientId !== undefined ? { client_id: opts.clientId } : {}), + }; + const merged: Partial = { + ...(existing ?? {}), + ...providedFlags, + }; + + // All four core fields must be present after merge — enforces first-time + // configure requires the full quartet while later updates may be partial. + const missing: string[] = []; + if (!merged.api_url) missing.push('--api-url'); + if (!merged.region) missing.push('--region'); + if (!merged.user_pool_id) missing.push('--user-pool-id'); + if (!merged.client_id) missing.push('--client-id'); + if (missing.length > 0) { + throw new CliError( + `Missing required configuration: ${missing.join(', ')}. ` + + 'Provide all four core fields on the first `bgagent configure` call.', + ); + } + + // If the user ran `bgagent configure` with no flags while a complete + // config already existed, there is nothing to save — don't print the + // misleading "Configuration saved." message. + if (existing !== null && Object.keys(providedFlags).length === 0) { + console.log('No configuration changes — all flags were omitted.'); + return; + } + + saveConfig(merged as CliConfig); console.log('Configuration saved.'); }); } diff --git a/cli/src/commands/nudge.ts b/cli/src/commands/nudge.ts new file mode 100644 index 0000000..a1c3d7d --- /dev/null +++ b/cli/src/commands/nudge.ts @@ -0,0 +1,103 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { Command } from 'commander'; +import { ApiClient } from '../api-client'; +import { ApiError, CliError } from '../errors'; +import { formatJson } from '../format'; +import { NUDGE_MAX_MESSAGE_LENGTH } from '../types'; + +/** + * `bgagent nudge ` — send a steering message to a + * running task (Phase 2). The message argument should be quoted in the + * shell if it contains spaces, e.g.: + * + * bgagent nudge TASK-123 "also update the README" + */ +export function makeNudgeCommand(): Command { + return new Command('nudge') + .description('Send a steering message to a running task') + .argument('', 'Task ID to nudge') + .argument('', 'Steering message (quote it if it contains spaces, e.g. "also update the README")') + .option('--output ', 'Output format (text or json)', 'text') + .addHelpText( + 'after', + '\nExamples:\n' + + ' $ bgagent nudge TASK-123 "also update the README"\n' + + ' $ bgagent nudge TASK-123 "focus on the auth module" --output json\n' + + '\nNote: wrap the message in quotes so the shell passes it as a single argument.', + ) + .action(async (taskId: string, message: string, opts) => { + const trimmed = message.trim(); + if (trimmed.length === 0) { + throw new CliError('Nudge message cannot be empty.'); + } + if (trimmed.length > NUDGE_MAX_MESSAGE_LENGTH) { + throw new CliError( + `Nudge message exceeds maximum length of ${NUDGE_MAX_MESSAGE_LENGTH} characters (got ${trimmed.length}).`, + ); + } + + const client = new ApiClient(); + try { + const result = await client.nudgeTask(taskId, trimmed); + + if (opts.output === 'json') { + console.log(formatJson(result)); + } else { + console.log(`Nudge ${result.nudge_id} submitted for task ${result.task_id} at ${result.submitted_at}.`); + } + } catch (err: unknown) { + if (err instanceof ApiError) { + throw mapNudgeError(err); + } + throw err; + } + }); +} + +/** Map nudge-specific API error codes to friendlier CLI messages. */ +function mapNudgeError(err: ApiError): CliError { + switch (err.statusCode) { + case 400: + // Guardrail-blocked or validation error. Pass the server's message + // through verbatim so guardrail reasons are visible to the user. + return new CliError(`Nudge rejected: ${err.message}`); + case 401: + return new CliError( + `Not authenticated (${err.errorCode}). Run \`bgagent login\` to re-authenticate.`, + ); + case 403: + return new CliError( + `Forbidden (${err.errorCode}): this task belongs to another user.`, + ); + case 404: + return new CliError(`Task not found (${err.errorCode}).`); + case 429: + return new CliError( + `Rate limit exceeded (${err.errorCode}). Slow down — nudges are limited per task; try again shortly.`, + ); + case 503: + return new CliError( + `Nudge service temporarily unavailable (${err.errorCode}): ${err.message} Please retry in a moment.`, + ); + default: + return new CliError(err.message); + } +} diff --git a/cli/src/commands/status.ts b/cli/src/commands/status.ts index 1322edc..bd296c2 100644 --- a/cli/src/commands/status.ts +++ b/cli/src/commands/status.ts @@ -19,26 +19,46 @@ import { Command } from 'commander'; import { ApiClient } from '../api-client'; -import { formatJson, formatTaskDetail } from '../format'; +import { formatJson, formatStatusSnapshot } from '../format'; import { exitCodeForStatus, waitForTask } from '../wait'; export function makeStatusCommand(): Command { return new Command('status') - .description('Get task status') + .description('Get a deterministic status snapshot of a task') .argument('', 'Task ID') - .option('--wait', 'Wait for task to reach terminal status') + .option('--wait', 'Block until the task reaches a terminal status, then print the final snapshot and exit with a status-derived code') .option('--output ', 'Output format (text or json)', 'text') .action(async (taskId: string, opts) => { const client = new ApiClient(); + // ``--wait`` is a pure blocking flag: it polls until terminal, + // then renders the SAME snapshot the default path would. There + // is no "rich vs compact" split — the snapshot is the status + // surface, ``--wait`` just delays it until there is a final + // answer. JSON output follows the same rule: same shape, later. if (opts.wait) { const task = await waitForTask(client, taskId); process.stderr.write('\n'); - console.log(opts.output === 'json' ? formatJson(task) : formatTaskDetail(task)); + if (opts.output === 'json') { + console.log(formatJson(task)); + } else { + // Fetch recent events so the snapshot renders consistently + // with the no-wait path. Cheap follow-up call; the task has + // already terminated, so the event window is stable. + const { recentEvents } = await client.getStatusSnapshot(taskId); + console.log(formatStatusSnapshot(task, recentEvents)); + } process.exitCode = exitCodeForStatus(task.status); - } else { + return; + } + + if (opts.output === 'json') { const task = await client.getTask(taskId); - console.log(opts.output === 'json' ? formatJson(task) : formatTaskDetail(task)); + console.log(formatJson(task)); + return; } + + const { task, recentEvents } = await client.getStatusSnapshot(taskId); + console.log(formatStatusSnapshot(task, recentEvents)); }); } diff --git a/cli/src/commands/submit.ts b/cli/src/commands/submit.ts index a09c356..3ebc4e8 100644 --- a/cli/src/commands/submit.ts +++ b/cli/src/commands/submit.ts @@ -35,6 +35,7 @@ export function makeSubmitCommand(): Command { .option('--pr ', 'PR number to iterate on (sets task_type to pr_iteration)', parseInt) .option('--review-pr ', 'PR number to review (sets task_type to pr_review)', parseInt) .option('--idempotency-key ', 'Idempotency key for deduplication') + .option('--trace', 'Capture 4 KB debug previews (design §10.1). Opt-in per task; not routine observability.') .option('--wait', 'Wait for task to complete') .option('--output ', 'Output format (text or json)', 'text') .action(async (opts) => { @@ -74,6 +75,7 @@ export function makeSubmitCommand(): Command { // Note: --pr and --review-pr are mutually exclusive (validated above). ...(opts.pr !== undefined && { task_type: 'pr_iteration' as const, pr_number: opts.pr }), ...(opts.reviewPr !== undefined && { task_type: 'pr_review' as const, pr_number: opts.reviewPr }), + ...(opts.trace && { trace: true }), }; const task = await client.createTask(body, opts.idempotencyKey); diff --git a/cli/src/commands/trace.ts b/cli/src/commands/trace.ts new file mode 100644 index 0000000..8324b7e --- /dev/null +++ b/cli/src/commands/trace.ts @@ -0,0 +1,191 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { createWriteStream, existsSync } from 'node:fs'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import type { ReadableStream as WebReadableStream } from 'node:stream/web'; +import { createGunzip } from 'node:zlib'; +import { Command } from 'commander'; +import { ApiClient } from '../api-client'; +import { ApiError, CliError } from '../errors'; + +/** + * Wall-clock timeout for the S3 fetch (L3 item 1). 2 minutes is + * generous for multi-MB artifacts on slow links and well under the + * 15-minute presigned-URL TTL. A stalled fetch otherwise wedges the + * CLI with no recovery signal. + */ +const TRACE_DOWNLOAD_TIMEOUT_MS = 120_000; + +/** Detect an ``AbortError`` across Node's fetch (``DOMException``) and + * older Error.name='AbortError' conventions. */ +function isAbortError(err: unknown): boolean { + return err instanceof Error && err.name === 'AbortError'; +} + +/** Detect a zlib-decode error from ``createGunzip()``. Node's zlib + * surfaces these as ``Error`` with ``code`` matching ``Z_*_ERROR`` or + * ``errno`` set — match loosely so both Node 20 and 24 flavors catch. + * Duck-typed rather than ``instanceof Error`` because Jest's module + * isolation can (rarely) load ``Error`` from a different realm, making + * ``instanceof`` return false for a perfectly valid error object. */ +function isZlibError(err: unknown): boolean { + if (err === null || typeof err !== 'object') return false; + const e = err as { code?: unknown; message?: unknown }; + if (typeof e.code === 'string' && e.code.startsWith('Z_')) return true; + if (typeof e.message === 'string' && + /zlib|gzip|incorrect header check|invalid stored block/i.test(e.message)) { + return true; + } + return false; +} + +/** + * ``bgagent trace download `` — fetch the ``--trace`` + * trajectory dump for a task (design §10.1). + * + * Output contract: + * * Default (stdout): gunzipped JSONL (pipe-friendly for ``jq -s .``) + * * ``-o `` (file): raw gzipped bytes (preserves the artifact as-is) + * + * The server returns a 15-minute presigned URL; we stream from S3 + * directly so multi-MB artifacts don't buffer in CLI memory. + */ +export function makeTraceCommand(): Command { + const trace = new Command('trace').description('--trace artifact commands (design §10.1)'); + + trace + .command('download') + .description('Download the --trace trajectory dump for a task') + .argument('', 'Task ID') + .option( + '-o, --output ', + 'Write raw gzipped bytes to instead of gunzipped to stdout. Use --force to overwrite an existing file.', + ) + .option('-f, --force', 'Overwrite the output file if it already exists') + .action(async (taskId: string, opts: { output?: string; force?: boolean }) => { + // L4 item 2: refuse to overwrite an existing ``-o `` + // without an explicit ``--force``. Previously the CLI silently + // clobbered existing files, which is a footgun when a user + // re-runs ``bgagent trace download`` and accidentally blows + // away an earlier artifact they wanted to keep. Check is done + // BEFORE the presigned-URL fetch so we also skip the S3 round + // trip on the refusal path. + if (opts.output && !opts.force && existsSync(opts.output)) { + throw new CliError( + `Refusing to overwrite existing file ${opts.output}. Pass --force to overwrite.`, + ); + } + + const client = new ApiClient(); + + let urlInfo; + try { + urlInfo = await client.getTraceUrl(taskId); + } catch (err) { + if (err instanceof ApiError && err.statusCode === 404 && err.errorCode === 'TRACE_NOT_AVAILABLE') { + // Friendlier message than the raw API body — users typically + // don't know which of "did not run with --trace" vs. "not yet + // uploaded" applies, and both have the same remedy. + throw new CliError( + `No trace artifact for task ${taskId}. Either the task did not run with --trace or the upload has not completed. Re-submit with 'bgagent submit --trace ...' to capture a new trace.`, + ); + } + throw err; + } + + // L3 item 1: fetch timeout + SIGINT abort. A stalled S3 download + // (TCP dead-peer, NAT reaping, etc.) otherwise wedges the CLI + // with no recovery signal other than the user killing the shell. + // 2 minutes is generous for multi-MB artifacts on slow links and + // well under the 15-minute presigned-URL TTL. + const ac = new AbortController(); + const timer = setTimeout( + () => ac.abort(new Error('Trace download timed out after 2 minutes')), + TRACE_DOWNLOAD_TIMEOUT_MS, + ); + const onSigint = (): void => ac.abort(new Error('Cancelled by user')); + process.on('SIGINT', onSigint); + + try { + let s3Response: Response; + try { + s3Response = await fetch(urlInfo.url, { signal: ac.signal }); + } catch (err) { + // AbortError surfaces as a DOMException with name='AbortError' + // on Node's fetch (undici). Reason carries our thrown Error. + if (isAbortError(err)) { + const reason = ac.signal.reason; + const reasonMsg = reason instanceof Error ? reason.message : String(reason ?? 'aborted'); + throw new CliError(`Trace download aborted: ${reasonMsg}`); + } + throw err; + } + + if (!s3Response.ok) { + throw new CliError( + `S3 download failed: HTTP ${s3Response.status} ${s3Response.statusText}. ` + + 'The presigned URL may have expired (15-minute TTL). Try \'bgagent trace download\' again.', + ); + } + if (!s3Response.body) { + throw new CliError('S3 response had no body.'); + } + + // ``ReadableStream`` from fetch -> Node Readable -> consumer. + // ``fromWeb`` typing in Node's types expects a WHATWG stream; the + // fetch response body matches. + const nodeReadable = Readable.fromWeb(s3Response.body as unknown as WebReadableStream); + + if (opts.output) { + // -o : write raw gzipped bytes as-is. Preserves the + // artifact for archival / re-inspection with standard tools + // (``zcat file | jq -s .``). No gunzip → no zlib errors to wrap. + await pipeline(nodeReadable, createWriteStream(opts.output)); + // Status line on stderr so it does not pollute stdout (which + // users may be piping through other tools). + console.error(`Wrote ${opts.output}`); + return; + } + + // Default: gunzip to stdout so the pipe contract is ``jq -s .``- + // friendly. A raw ``Z_DATA_ERROR`` stack is actively misleading — + // it looks like a CLI bug rather than a corrupt artifact. Wrap + // zlib failures in a ``CliError`` pointing at the real cause + // (L3 item 1). + try { + await pipeline(nodeReadable, createGunzip(), process.stdout); + } catch (err) { + if (isZlibError(err)) { + throw new CliError( + `Trace artifact is corrupt or not gzipped (${(err as Error).message}). ` + + `Re-download with 'bgagent trace download ${taskId}' or inspect the raw bytes with '-o '.`, + ); + } + throw err; + } + } finally { + clearTimeout(timer); + process.off('SIGINT', onSigint); + } + }); + + return trace; +} diff --git a/cli/src/commands/watch.ts b/cli/src/commands/watch.ts new file mode 100644 index 0000000..4b8af2e --- /dev/null +++ b/cli/src/commands/watch.ts @@ -0,0 +1,653 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { Command } from 'commander'; +import { ApiClient } from '../api-client'; +import { debug, isVerbose } from '../debug'; +import { ApiError } from '../errors'; +import { formatJson } from '../format'; +import { TERMINAL_STATUSES, TaskDetail, TaskEvent } from '../types'; + +/** + * Adaptive polling cadence (design INTERACTIVE_AGENTS.md §5.3). + * + * While events are arriving we stay at ``POLL_FAST_INTERVAL_MS``. When a + * poll returns zero events we back off through the ``BACKOFF_INTERVALS_MS`` + * ladder, resetting to fast on the next poll that delivers events. The + * ladder caps at 5 s to keep status freshness bounded during idle + * stretches without hammering DDB. + */ +const POLL_FAST_INTERVAL_MS = 500; +const BACKOFF_INTERVALS_MS: readonly number[] = [1_000, 2_000, 5_000]; + +/** Adaptive polling state, threaded through the poll loop. */ +interface PollCadenceState { + intervalMs: number; + consecutiveEmptyPolls: number; +} + +/** Compute the next cadence from whether the last poll delivered events. + * Pure so the state machine is test-coverable without timers. */ +export function nextCadence(state: PollCadenceState, sawEvents: boolean): PollCadenceState { + if (sawEvents) { + return { intervalMs: POLL_FAST_INTERVAL_MS, consecutiveEmptyPolls: 0 }; + } + const nextEmpty = state.consecutiveEmptyPolls + 1; + // Ladder index is ``nextEmpty - 1`` (first empty poll picks slot 0 = + // 1 s). After the ladder is exhausted we pin at the cap. + const idx = Math.min(nextEmpty - 1, BACKOFF_INTERVALS_MS.length - 1); + return { intervalMs: BACKOFF_INTERVALS_MS[idx], consecutiveEmptyPolls: nextEmpty }; +} + +/** Retry budget for transient 5xx / network failures. Exhausting it exits + * the watch loop with a clear "rerun to resume" message. 4xx errors are + * deterministic and never retried. */ +const MAX_TRANSIENT_RETRIES = 5; + +/** + * Session-level retry counter (L3 item 5). ``withTransientRetry`` resets + * its per-op ``attempt`` counter on every successful poll, which means a + * flapping upstream at ~50% success rate never trips the 5-retry budget + * even though the user is watching a degraded stream for minutes on end. + * The session counter accumulates across all retries for the life of the + * watch process so a ``SESSION_FLAP_THRESHOLD`` crossing can surface the + * "upstream is flapping" stderr signal exactly once. + * + * Not exposed on any public surface — underscore-prefixed getter is for + * tests only (module-level state makes this awkward to inject, and the + * tradeoff isn't worth a full dependency-injection refactor). + */ +let sessionRetries = 0; +let flapWarnEmitted = false; + +/** Emit-once threshold for the "upstream is flapping" warning. Picked so + * a sustained ~30% failure rate over a few-minute poll window lands + * above it without a transient 2-failure blip crossing. */ +const SESSION_FLAP_THRESHOLD = 10; + +/** Test-only accessor for the module-level retry counter. Prefixed with + * ``_`` to signal "not part of the stable API". */ +export function _getSessionRetries(): number { + return sessionRetries; +} + +/** Test-only reset of the module-level state. Tests that exercise the + * flap warning need a clean slate because the counter is process-lived + * and Jest's module reset does not apply to values captured at import. */ +export function _resetSessionRetries(): void { + sessionRetries = 0; + flapWarnEmitted = false; +} + +/** Exponential backoff with **equal-jitter** (AWS Architecture Blog + * variant): half of the base delay is fixed, the other half is + * randomized. This prevents the degenerate case where ``Math.random()`` + * rolls near-zero on every retry and the CLI retry-spams a degraded + * service with no wait between attempts. Bounded at the ladder cap so + * a retry storm never walks longer than the adaptive poll ceiling. */ +export function transientRetryDelayMs(attempt: number): number { + const base = Math.min(5_000, POLL_FAST_INTERVAL_MS * 2 ** attempt); + const half = Math.floor(base / 2); + return half + Math.floor(Math.random() * (base - half)); +} + +/** Classify an error into retryable vs. terminal. We use a **whitelist** + * rather than a blacklist: only conditions we specifically recognize as + * transient retry. Everything else (programmer errors, JSON parse + * failures, auth-token-expired, CliError) propagates immediately so + * users see an actionable message instead of "re-run to resume" that + * would never succeed. + * + * Transient: + * - ``ApiError`` with status 5xx (server-side hiccup) + * - Network failures surfaced by ``fetch`` as a ``TypeError`` — + * Node's undici implementation reports connect refused / reset / + * DNS failure this way on Node 22+. + * + * Non-transient (propagates with its original message): + * - ``ApiError`` with status 4xx (including 401 auth-expired — the + * ``bgagent login`` hint is already in the message) + * - ``CliError`` (our own deterministic contract-violation signal) + * - Anything else (``TypeError`` that is *not* a fetch failure, + * ``SyntaxError`` from a bad code path, etc.) — a real bug. + */ +function isTransientError(err: unknown): boolean { + if (err instanceof ApiError) { + return err.statusCode >= 500 && err.statusCode < 600; + } + // Node 22+ fetch surfaces network failures as a ``TypeError`` with a + // "fetch failed" message (undici wraps the underlying cause). Match + // loosely so we tolerate both direct ``TypeError`` and DOMException + // lookalikes without retrying genuine programmer ``TypeError``s. + if (err instanceof TypeError && /fetch failed|network/i.test(err.message)) { + return true; + } + return false; +} + +/** Exit code 130 is the conventional POSIX code for "terminated by + * SIGINT". Using it lets shell scripts distinguish Ctrl+C from a failed + * task run. */ +const EXIT_CODE_SIGINT = 130; +/** Size of the initial snapshot fetch used to detect already-terminal tasks + * and seed the catch-up cursor. */ +const SNAPSHOT_PAGE_SIZE = 100; + +/** Progress event types emitted by the agent ProgressWriter. */ +const PROGRESS_EVENT_TYPES = new Set([ + 'agent_turn', + 'agent_tool_call', + 'agent_tool_result', + 'agent_milestone', + 'agent_cost_update', + 'agent_error', +]); + +/** Format an event timestamp to a short local time string. */ +function formatTime(isoTimestamp: string): string { + try { + const date = new Date(isoTimestamp); + return date.toLocaleTimeString(); + } catch { + return isoTimestamp; + } +} + +/** Render a single progress event as a human-readable line. */ +export function renderEvent(event: TaskEvent): string { + const time = formatTime(event.timestamp); + const meta = event.metadata; + + switch (event.event_type) { + case 'agent_turn': { + const turn = meta.turn ?? '?'; + const model = meta.model ?? ''; + const tools = meta.tool_calls_count ?? 0; + let line = `[${time}] Turn #${turn} (${model}, ${tools} tool call${tools === 1 ? '' : 's'})`; + if (meta.thinking_preview) { + line += `\n Thinking: ${meta.thinking_preview}`; + } + if (meta.text_preview) { + line += `\n Text: ${meta.text_preview}`; + } + return line; + } + case 'agent_tool_call': { + const tool = meta.tool_name ?? 'unknown'; + const preview = meta.tool_input_preview ?? ''; + return `[${time}] ▶ ${tool}: ${preview}`; + } + case 'agent_tool_result': { + const tool = meta.tool_name ?? ''; + const isError = meta.is_error ? ' [ERROR]' : ''; + const preview = meta.content_preview ?? ''; + return `[${time}] ◀ ${tool}${isError}: ${preview}`; + } + case 'agent_milestone': { + const milestone = meta.milestone ?? ''; + const details = meta.details ?? ''; + return `[${time}] ★ ${milestone}${details ? ': ' + details : ''}`; + } + case 'agent_cost_update': { + const cost = meta.cost_usd != null ? `$${Number(meta.cost_usd).toFixed(4)}` : '$?'; + const input = meta.input_tokens ?? 0; + const output = meta.output_tokens ?? 0; + return `[${time}] Cost: ${cost} (${input} in / ${output} out tokens)`; + } + case 'agent_error': { + const errType = meta.error_type ?? 'Error'; + const msg = meta.message_preview ?? ''; + return `[${time}] ✖ ${errType}: ${msg}`; + } + default: + return `[${time}] ${event.event_type}: ${JSON.stringify(meta)}`; + } +} + +/* ------------------------------------------------------------------------ */ +/* Structured logging helpers */ +/* ------------------------------------------------------------------------ */ + +/** Log an INFO-level message to stderr. Stdout stays pure NDJSON in either + * mode because info messages never go there; the ``isJson`` parameter is + * kept for call-site documentation of the mode. */ +function logInfo(_isJson: boolean, message: string): void { + process.stderr.write(`${message}\n`); +} + +/** Log an ERROR-level message to stderr regardless of output mode. */ +function logError(message: string): void { + process.stderr.write(`ERROR: ${message}\n`); +} + +/** + * Render the terminal-line message shown when a watch session ends + * because the task reached a terminal state. Includes the task_id (so + * a user with multiple watches or a scroll-back log can correlate) + * and, for non-COMPLETED terminals, a short failure-classification + * hint so the cause is visible without a separate ``bgagent status`` + * round-trip. + * + * Exported for tests. Safe to call with a bare ``{task_id, status}`` + * shape or a full ``TaskDetail`` — only those fields plus + * ``error_classification`` / ``error_message`` are read. + */ +export function formatTerminalMessage(task: Pick): string { + const status = task.status.toLowerCase(); + const prefix = `Task ${task.task_id} ${status}.`; + if (task.status === 'COMPLETED') return prefix; + // Prefer the structured classification (category + title) when the + // server has computed one — it's both stable and user-oriented. Fall + // back to the raw ``error_message`` so a classifier gap doesn't + // swallow the only signal we have. Never return the whole prefix + // with a trailing empty reason. + const cls = task.error_classification; + if (cls) return `${prefix} ${cls.category}: ${cls.title}`; + const msg = task.error_message?.trim(); + if (msg) return `${prefix} ${msg}`; + return prefix; +} + +/* ------------------------------------------------------------------------ */ +/* Formatter boundary */ +/* ------------------------------------------------------------------------ */ + +/** + * A formatter that accepts `TaskEvent` rows (from REST polling) and + * produces human-readable output (text mode) or NDJSON (json mode). + */ +interface Formatter { + emit(ev: TaskEvent): void; +} + +export function makeFormatter(isJson: boolean): Formatter { + return { + emit(ev: TaskEvent): void { + if (isJson) { + console.log(formatJson(ev)); + return; + } + if (PROGRESS_EVENT_TYPES.has(ev.event_type)) { + console.log(renderEvent(ev)); + } + }, + }; +} + +/* ------------------------------------------------------------------------ */ +/* Polling loop */ +/* ------------------------------------------------------------------------ */ + +interface PollOptions { + readonly signal: AbortSignal; + readonly afterEventId?: string; + readonly onEvent: (ev: TaskEvent) => void; + readonly onTerminal: (finalTask: TaskDetail) => void; +} + +/** + * Poll ``GET /tasks/{id}/events`` and ``GET /tasks/{id}`` on an adaptive + * cadence: 500 ms while events are arriving, backing off through + * 1 s / 2 s / 5 s on consecutive empty polls and resetting to fast on + * the next event. Invokes ``onEvent`` for each new event and + * ``onTerminal`` once the task reaches a terminal status. Resolves when + * the task terminates or the abort signal fires. + * + * Transient 5xx / network errors are retried with jittered exponential + * backoff up to ``MAX_TRANSIENT_RETRIES`` times; 4xx errors propagate + * immediately (the next call would return the same failure). On retry + * exhaustion we throw a ``CliError``-like message that tells the user + * to re-run ``bgagent watch`` — the event cursor is durable, so + * resuming is safe. + */ +async function pollTaskEvents( + apiClient: ApiClient, + taskId: string, + options: PollOptions, +): Promise { + let lastSeenEventId: string | null = options.afterEventId ?? null; + let cadence: PollCadenceState = { intervalMs: POLL_FAST_INTERVAL_MS, consecutiveEmptyPolls: 0 }; + debug(`[watch/poll] starting polling loop afterEventId=${lastSeenEventId ?? ''}`); + + while (!options.signal.aborted) { + // Fetch every event past our cursor. ``catchUpEvents`` seeds with + // ``after=lastSeenEventId`` and drains the server's ``next_token`` + // pagination so we see all events — not just the first 100. + const newEvents = await withTransientRetry( + () => (lastSeenEventId + ? apiClient.catchUpEvents(taskId, lastSeenEventId, 100, { signal: options.signal }) + : apiClient.getTaskEvents(taskId, { limit: 100, signal: options.signal }) + .then(r => r.data)), + options.signal, + 'getTaskEvents', + ); + + if (options.signal.aborted) return; + + if (newEvents.length > 0) { + lastSeenEventId = newEvents[newEvents.length - 1].event_id; + debug(`[watch/poll] emitting ${newEvents.length} new events, advanced cursor to ${lastSeenEventId}`); + for (const ev of newEvents) { + options.onEvent(ev); + } + } + + const task = await withTransientRetry( + () => apiClient.getTask(taskId, { signal: options.signal }), + options.signal, + 'getTask', + ); + + if (options.signal.aborted) return; + + if ((TERMINAL_STATUSES as readonly string[]).includes(task.status)) { + debug(`[watch/poll] task reached terminal status=${task.status}`); + options.onTerminal(task); + return; + } + + cadence = nextCadence(cadence, newEvents.length > 0); + debug(`[watch/poll] cadence=${cadence.intervalMs}ms emptyPolls=${cadence.consecutiveEmptyPolls}`); + await abortableSleep(cadence.intervalMs, options.signal); + } +} + +/** + * Execute an API call with retry-on-transient semantics: + * - 5xx / network errors → retry after jittered backoff, up to + * ``MAX_TRANSIENT_RETRIES`` total attempts. + * - 4xx errors → rethrow immediately (deterministic; retrying is futile). + * - Exhausted retries → throw with a "re-run to resume" hint. + * - Abort during retry sleep → throw the original error up (caller will + * check ``signal.aborted`` and exit cleanly). + * + * ``label`` is used only for debug logging so operators can see *which* + * call is retrying during a degraded poll window. + */ +async function withTransientRetry( + op: () => Promise, + signal: AbortSignal, + label: string, +): Promise { + let attempt = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + try { + return await op(); + } catch (err) { + if (signal.aborted) throw err; + if (!isTransientError(err)) { + debug(`[watch/retry] ${label}: non-transient error, propagating: ${String(err)}`); + throw err; + } + attempt += 1; + // Session-level counter (L3 item 5). ``attempt`` resets on every + // successful op; ``sessionRetries`` does not, so a flapping upstream + // that never exhausts the per-op budget still accumulates here. + sessionRetries += 1; + if (sessionRetries >= SESSION_FLAP_THRESHOLD && !flapWarnEmitted) { + flapWarnEmitted = true; + process.stderr.write( + `[watch] upstream is flapping — ${sessionRetries} retries so far; results may be delayed\n`, + ); + } + if (attempt > MAX_TRANSIENT_RETRIES) { + const e = err instanceof Error ? err : new Error(String(err)); + throw new Error( + `Exceeded retry budget after ${MAX_TRANSIENT_RETRIES} transient failures ` + + `(${label}): ${e.message}. Re-run \`bgagent watch \` to resume.`, + ); + } + const delayMs = transientRetryDelayMs(attempt); + debug(`[watch/retry] ${label}: attempt ${attempt}/${MAX_TRANSIENT_RETRIES} after ${delayMs}ms`); + await abortableSleep(delayMs, signal); + } + } +} + +/** Sleep that honours an AbortSignal — resolves on abort instead of rejecting, + * so the polling loop can check ``signal.aborted`` and exit cleanly. */ +function abortableSleep(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal.aborted) { + resolve(); + return; + } + const timer = setTimeout(() => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + resolve(); + }; + signal.addEventListener('abort', onAbort, { once: true }); + }); +} + +/* ------------------------------------------------------------------------ */ +/* Initial snapshot — detect already-terminal tasks and seed cursor */ +/* ------------------------------------------------------------------------ */ + +interface SnapshotResult { + readonly latestEventId: string | null; + readonly events: TaskEvent[]; + readonly taskStatus: string; +} + +/** Fetch the latest events + current task status. Used both to detect a + * task that already terminated before ``bgagent watch`` connected, and to + * seed the polling cursor so we don't re-emit the snapshot's contents on + * the first poll iteration. + * + * Both API calls are wrapped in ``withTransientRetry`` so a cold-start + * hiccup on the Lambda (``fetch failed`` / 5xx / network transients) + * does not crash the watch command before the polling loop gets a + * chance to stabilise. The polling loop itself wraps every subsequent + * call; without the same wrap here, the first request was the weakest + * link (observed Scenario 2 deploy validation, where a cold-start + * failed once then succeeded on re-run). + * + * ``signal`` is required so callers commit to a concrete abort + * controller — otherwise SIGINT during the snapshot's retry backoff + * could never abort a retrying call. The production watch command + * always passes its shared ``AbortController`` signal; tests that + * exercise this path do the same via ``makeWatchCommand``. + * + * Emitted event ordering: events are returned in ascending event_id + * order (REST contract). */ +export async function fetchInitialSnapshot( + apiClient: ApiClient, + taskId: string, + opts: { signal: AbortSignal }, +): Promise { + debug(`[watch/snapshot] fetching initial snapshot task=${taskId}`); + const { signal } = opts; + const [eventsPage, task] = await Promise.all([ + withTransientRetry( + () => apiClient.getTaskEvents(taskId, { limit: SNAPSHOT_PAGE_SIZE, signal }), + signal, + 'initialSnapshot.getTaskEvents', + ), + withTransientRetry( + () => apiClient.getTask(taskId, { signal }), + signal, + 'initialSnapshot.getTask', + ), + ]); + const events = eventsPage.data; + const latestEventId = events.length > 0 ? events[events.length - 1].event_id : null; + debug( + `[watch/snapshot] events=${events.length} latestEventId=${latestEventId ?? ''} ` + + `status=${task.status}`, + ); + return { latestEventId, events, taskStatus: task.status }; +} + +/* ------------------------------------------------------------------------ */ +/* Command definition */ +/* ------------------------------------------------------------------------ */ + +export function makeWatchCommand(): Command { + return new Command('watch') + .description('Watch task progress in real-time') + .argument('', 'Task ID') + .option('--output ', 'Output format (text or json)', 'text') + .action(async (taskId: string, opts) => { + const isJson = opts.output === 'json'; + const apiClient = new ApiClient(); + + debug(`[watch] task=${taskId} isJson=${isJson} verbose=${isVerbose()}`); + + // Abort controller for SIGINT / SIGTERM. + const abortController = new AbortController(); + const onSignal = (): void => { + debug('[watch] SIGINT/SIGTERM received, aborting'); + abortController.abort(); + }; + process.on('SIGINT', onSignal); + process.on('SIGTERM', onSignal); + + try { + // -------- Snapshot: detect already-terminal tasks, seed cursor. -- + let snapshot: SnapshotResult; + try { + snapshot = await fetchInitialSnapshot(apiClient, taskId, { signal: abortController.signal }); + } catch (err) { + // Capture the pre-abort state so the SIGINT-vs-real-error + // disambiguation below works. Then abort the shared controller + // so any sibling ``Promise.all`` leg still inside + // ``withTransientRetry`` stops backing off and burning retries + // against the API. Idempotent — calling ``abort`` on an + // already-aborted controller is a no-op. + const wasUserAborted = abortController.signal.aborted; + abortController.abort(); + + // Only exit 130 if the error IS the abort — i.e., an AbortError + // from our signal THAT WAS ALREADY ABORTED when the error fired. + // Checking post-abort state would swallow a real 401 from an + // expired token that happens to throw at the same moment the + // user Ctrl+Cs as a clean interrupt, and the user would miss + // the ``bgagent login`` hint. + const isAbortError = err instanceof Error && err.name === 'AbortError'; + if (isAbortError && wasUserAborted) { + process.exitCode = EXIT_CODE_SIGINT; + return; + } + const e = err instanceof Error ? err : new Error(String(err)); + logError(`Initial snapshot failed: ${e.message}`); + throw e; + } + + const formatter = makeFormatter(isJson); + + // Task already terminated — print the snapshot tail and exit. + if ((TERMINAL_STATUSES as readonly string[]).includes(snapshot.taskStatus)) { + debug(`[watch] task already terminal status=${snapshot.taskStatus} — printing tail`); + for (const ev of snapshot.events) { + formatter.emit(ev); + } + if (!isJson) { + // Fetch the current task detail so the terminal-line can + // include the error classification (``guardrail: PR context + // blocked``, ``timeout: Exceeded max turns``, etc.). The + // snapshot only carried ``taskStatus``. Best-effort: if the + // GET fails transiently we still print a minimal message + // rather than erroring out after already streaming the tail. + let terminalTask: Pick = { + task_id: taskId, + status: snapshot.taskStatus, + error_classification: null, + error_message: null, + }; + try { + terminalTask = await withTransientRetry( + () => apiClient.getTask(taskId, { signal: abortController.signal }), + abortController.signal, + 'alreadyTerminal.getTask', + ); + } catch (err) { + debug(`[watch] already-terminal getTask failed — printing minimal message: ${String(err)}`); + } + logInfo(isJson, formatTerminalMessage(terminalTask)); + } + process.exitCode = snapshot.taskStatus === 'COMPLETED' ? 0 : 1; + return; + } + + // Emit the snapshot events first so the user sees history before + // live events start flowing. + for (const ev of snapshot.events) { + formatter.emit(ev); + } + const seedCursor = snapshot.latestEventId ?? ''; + + if (!isJson) { + logInfo(isJson, `Watching task ${taskId}... (Ctrl+C to stop)`); + } + + await runPolling(apiClient, taskId, seedCursor, formatter, abortController.signal, isJson); + } finally { + process.removeListener('SIGINT', onSignal); + process.removeListener('SIGTERM', onSignal); + } + }); +} + +/* ------------------------------------------------------------------------ */ +/* Polling runner */ +/* ------------------------------------------------------------------------ */ + +async function runPolling( + apiClient: ApiClient, + taskId: string, + seedCursor: string, + formatter: Formatter, + signal: AbortSignal, + isJson: boolean, +): Promise { + debug(`[watch/poll] runPolling seedCursor=${seedCursor || ''}`); + let finalTask: TaskDetail | null = null; + + await pollTaskEvents(apiClient, taskId, { + signal, + afterEventId: seedCursor || undefined, + onEvent: (ev) => formatter.emit(ev), + onTerminal: (task) => { finalTask = task; }, + }); + + // SIGINT always wins. Check ``signal.aborted`` BEFORE ``finalTask`` + // so a user who Ctrl+C's between ``onTerminal`` firing and this block + // evaluating still gets exit 130 — their intent to interrupt is the + // load-bearing signal, not the coincidental terminal status. POSIX: + // 128 + SIGINT (2) = 130. + if (signal.aborted) { + logInfo(isJson, 'Aborted.'); + process.exitCode = EXIT_CODE_SIGINT; + return; + } + + if (finalTask !== null) { + const task = finalTask as TaskDetail; + if (!isJson) { + logInfo(isJson, formatTerminalMessage(task)); + } + process.exitCode = task.status === 'COMPLETED' ? 0 : 1; + } +} diff --git a/cli/src/config.ts b/cli/src/config.ts index 043fcb7..9ee0f48 100644 --- a/cli/src/config.ts +++ b/cli/src/config.ts @@ -62,6 +62,19 @@ export function saveConfig(config: CliConfig): void { fs.writeFileSync(configPath(), JSON.stringify(config, null, 2) + '\n', { mode: 0o644 }); } +/** Load existing CLI config if present, else return null (no error). */ +export function tryLoadConfig(): CliConfig | null { + const p = configPath(); + if (!fs.existsSync(p)) { + return null; + } + try { + return JSON.parse(fs.readFileSync(p, 'utf-8')) as CliConfig; + } catch { + return null; + } +} + /** Load cached credentials. Returns null if no credentials file exists. */ export function loadCredentials(): Credentials | null { const p = credentialsPath(); diff --git a/cli/src/format.ts b/cli/src/format.ts index 154317e..9ebd8e2 100644 --- a/cli/src/format.ts +++ b/cli/src/format.ts @@ -17,7 +17,7 @@ * SOFTWARE. */ -import { CreateWebhookResponse, TaskDetail, TaskEvent, TaskSummary, WebhookDetail } from './types'; +import { CreateWebhookResponse, TaskDetail, TaskEvent, TaskSummary, TERMINAL_STATUSES, WebhookDetail } from './types'; /** Format a TaskDetail as a key-value detail view. */ export function formatTaskDetail(task: TaskDetail): string { @@ -51,6 +51,9 @@ export function formatTaskDetail(task: TaskDetail): string { if (task.pr_url) { lines.push(`PR: ${task.pr_url}`); } + if (task.trace_s3_uri) { + lines.push(`Trace S3: ${task.trace_s3_uri}`); + } if (task.error_message) { lines.push(...formatErrorLines(task)); } @@ -97,6 +100,154 @@ export function formatTaskList(tasks: TaskSummary[]): string { return formatTable(headers, rows); } +/** + * Render the deterministic ``bgagent status`` snapshot described in + * ``docs/design/INTERACTIVE_AGENTS.md`` §5.2. + * + * Pure function: takes the task detail, a small window of recent events + * (newest first), and an anchor ``now`` so callers can freeze time in + * tests. Never calls an LLM and never fabricates state — every rendered + * field is either read directly from ``task`` / ``events`` or is a + * simple relative-time derivation. + * + * Degrades gracefully when fields are missing (just-submitted task, no + * events yet, no cost recorded) by emitting a placeholder (``—``) rather + * than ``undefined`` or ``NaN``. This is the contract users rely on when + * calling ``status`` repeatedly during a task's lifetime. + * + * @param task - the task detail from ``GET /tasks/{id}``. + * @param events - up to N recent events, ordered newest-first. + * @param now - the reference time for relative durations (epoch ms). + * Defaults to ``Date.now()`` in production; tests pass a fixed value. + */ +export function formatStatusSnapshot( + task: TaskDetail, + events: readonly TaskEvent[], + now: number = Date.now(), +): string { + // Defensive sort. The server contract (``?desc=1`` on + // ``GET /tasks/{id}/events``) returns newest-first, and every helper + // below relies on ``events[0]`` being the most recent event. If that + // invariant is ever violated upstream — a GSI reconfig, a middleware + // reorder, a caller wiring the formatter to a different endpoint — a + // front-to-back walk would silently render the *oldest* tool call as + // "Current" with no user-visible signal. ULIDs are lexicographically + // time-sortable, so a descending ``localeCompare`` is always correct. + const sorted = [...events].sort((a, b) => b.event_id.localeCompare(a.event_id)); + + const header = `Task ${task.task_id} — ${task.status} (${elapsedDescription(task, now)})`; + + const milestoneEvent = findLatest(sorted, 'agent_milestone'); + const lastCostEvent = findLatest(sorted, 'agent_cost_update'); + const lastTurnEvent = findLatest(sorted, 'agent_turn'); + const lastActivityEvent = findLatestActivity(sorted); + + // ``TaskEvent.timestamp`` is typed ``string``, but the event table is + // weakly typed at the storage layer — an agent regression could write + // a row without a valid timestamp. Guard so a missing field renders + // as the placeholder rather than the literal ``undefined``. + const lastEventTs = sorted[0]?.timestamp; + const lastEventLine = typeof lastEventTs === 'string' && lastEventTs.length > 0 + ? lastEventTs + : PLACEHOLDER; + + const lines: string[] = [ + header, + ` Repo: ${task.repo}`, + // Channel provenance — ``api`` for CLI / Cognito submits, + // ``webhook`` for HMAC-signed inbound webhook submits. Shown on + // every task so a user looking at a surprising task's status can + // immediately tell whether it was triggered by an automation / CI + // webhook vs. a manual submission. + ` Channel: ${task.channel_source || PLACEHOLDER}`, + ]; + // Non-default task types carry meaningful context for the default + // snapshot (a pr_iteration against #42 is a different mental model + // than a new_task). Mirrors the ``formatTaskDetail`` treatment. + if (task.task_type && task.task_type !== 'new_task') { + const prSuffix = task.pr_number !== null ? ` (PR #${task.pr_number})` : ''; + lines.push(` Type: ${task.task_type}${prSuffix}`); + } + // Render the task description under its own heading with wrapped + // continuation lines so long prompts stay readable in a ~80-column + // terminal without truncating information the user already typed. + if (task.task_description) { + lines.push(...formatDescriptionLines(task.task_description)); + } + lines.push( + ` Turn: ${describeTurn(task, lastTurnEvent)}`, + ` Last milestone: ${describeMilestone(milestoneEvent, now)}`, + ` Current: ${describeCurrent(task, lastActivityEvent)}`, + ` Cost: ${describeCost(task, lastCostEvent)}`, + ); + // Non-COMPLETED terminal statuses should show the reason inline so + // users do not have to chase it through ``status --wait`` or an + // ``events`` log grep. Prefer the structured classification when the + // API computed one; fall back to the raw ``error_message`` so a + // classifier gap does not swallow the only signal we have. Never + // emit a trailing empty line. + const reasonLine = describeReason(task); + if (reasonLine !== null) { + lines.push(` Reason: ${reasonLine}`); + } + if (task.trace_s3_uri) { + lines.push(` Trace S3: ${task.trace_s3_uri}`); + } + lines.push(` Last event: ${lastEventLine}`); + + return lines.join('\n'); +} + +/** Word-wrap column width used for the ``Description:`` block in the + * status snapshot. Keeps the rendered snapshot readable at the + * conventional 80-column terminal width while leaving headroom for + * the 2-space indent + 15-char label gutter (`` Description: ``) + * that the other snapshot lines use. */ +const DESCRIPTION_WRAP_WIDTH = 60; + +/** Render the task description across one or more lines with a + * dedicated label on the first line and continuation padding on the + * rest. Preserves the user's intent: no truncation, no + * reflowing inside the paragraph beyond whitespace word-wrap. */ +function formatDescriptionLines(description: string): string[] { + const label = ' Description: '; + const indent = ' '.repeat(label.length); + const words = description.trim().split(/\s+/); + if (words.length === 0 || (words.length === 1 && words[0] === '')) return []; + + const wrapped: string[] = []; + let current = ''; + for (const word of words) { + if (current.length === 0) { + current = word; + continue; + } + if (current.length + 1 + word.length <= DESCRIPTION_WRAP_WIDTH) { + current += ' ' + word; + } else { + wrapped.push(current); + current = word; + } + } + if (current.length > 0) wrapped.push(current); + + return wrapped.map((line, i) => (i === 0 ? label + line : indent + line)); +} + +/** Render the terminal-failure reason for the status snapshot. Returns + * ``null`` for COMPLETED / still-running tasks so the caller can skip + * the whole line. Prefers ``error_classification.{category, title}``; + * falls back to trimmed ``error_message``; otherwise returns ``null``. */ +function describeReason(task: TaskDetail): string | null { + if (task.status === 'COMPLETED') return null; + if (!(TERMINAL_STATUSES as readonly string[]).includes(task.status)) return null; + const cls = task.error_classification; + if (cls) return `${cls.category}: ${cls.title}`; + const msg = task.error_message?.trim(); + if (msg) return msg; + return null; +} + /** Format task events as a timeline. */ export function formatEvents(events: TaskEvent[]): string { if (events.length === 0) { @@ -194,3 +345,128 @@ function truncate(text: string, maxLen: number): string { if (text.length <= maxLen) return text; return text.slice(0, maxLen - 3) + '...'; } + +// -- status-snapshot helpers -------------------------------------------------- + +const PLACEHOLDER = '—'; + +function isTerminalStatus(status: string): boolean { + return (TERMINAL_STATUSES as readonly string[]).includes(status); +} + +function elapsedDescription(task: TaskDetail, now: number): string { + // Prefer the authoritative SDK-reported duration once the task has + // landed terminal — it accounts for clock drift between the orchestrator + // and agent. Fall back to ``completed_at - started_at`` so the status + // snapshot still renders something sensible if ``duration_s`` is missing. + if (isTerminalStatus(task.status) && task.duration_s != null) { + return `${humanizeSeconds(task.duration_s)} total`; + } + const start = task.started_at ?? task.created_at; + const startMs = Date.parse(start); + if (Number.isNaN(startMs)) return PLACEHOLDER; + const endMs = task.completed_at ? Date.parse(task.completed_at) : now; + if (Number.isNaN(endMs)) return PLACEHOLDER; + const diffS = Math.max(0, Math.round((endMs - startMs) / 1000)); + return `${humanizeSeconds(diffS)} elapsed`; +} + +function describeTurn(task: TaskDetail, turnEvent: TaskEvent | null): string { + // Prefer the live ``turn`` from the most recent ``agent_turn`` event + // over the persisted ``turns_attempted`` — the former updates mid-task + // (the latter is written on terminal completion in most paths). + const liveTurn = readNumberField(turnEvent?.metadata, 'turn'); + const currentTurn = liveTurn ?? task.turns_attempted ?? null; + const maxTurns = task.max_turns ?? null; + if (currentTurn == null && maxTurns == null) return PLACEHOLDER; + const left = currentTurn == null ? PLACEHOLDER : String(currentTurn); + const right = maxTurns == null ? PLACEHOLDER : String(maxTurns); + return `${left} / ~${right}`; +} + +function describeMilestone(milestoneEvent: TaskEvent | null, now: number): string { + if (!milestoneEvent) return PLACEHOLDER; + const name = readStringField(milestoneEvent.metadata, 'milestone') ?? 'milestone'; + const ago = relativeTime(milestoneEvent.timestamp, now); + return ago ? `${name} (${ago} ago)` : name; +} + +function describeCurrent(task: TaskDetail, activity: TaskEvent | null): string { + if (isTerminalStatus(task.status)) { + return `task ${task.status.toLowerCase()}`; + } + if (!activity) return PLACEHOLDER; + if (activity.event_type === 'agent_tool_call') { + const toolName = readStringField(activity.metadata, 'tool_name') ?? 'tool'; + return `${toolName} tool call`; + } + if (activity.event_type === 'agent_turn') { + const turn = readNumberField(activity.metadata, 'turn'); + return turn != null ? `agent turn ${turn}` : 'agent turn'; + } + return activity.event_type; +} + +function describeCost(task: TaskDetail, costEvent: TaskEvent | null): string { + const liveCost = readNumberField(costEvent?.metadata, 'cost_usd'); + const cost = liveCost ?? task.cost_usd ?? null; + const budget = task.max_budget_usd ?? null; + const costStr = cost == null ? PLACEHOLDER : `$${cost.toFixed(2)}`; + const budgetStr = budget == null ? PLACEHOLDER : `$${budget.toFixed(2)}`; + return `${costStr} / budget ${budgetStr}`; +} + +function findLatest(events: readonly TaskEvent[], eventType: string): TaskEvent | null { + // ``events`` is newest-first; the first match is the latest one. + for (const e of events) { + if (e.event_type === eventType) return e; + } + return null; +} + +function findLatestActivity(events: readonly TaskEvent[]): TaskEvent | null { + for (const e of events) { + if (e.event_type === 'agent_tool_call' || e.event_type === 'agent_turn') { + return e; + } + } + return null; +} + +// ``TaskEvent.metadata`` is non-optional, but callers routinely pass +// ``event?.metadata`` where ``event`` may itself be ``null`` (no matching +// event found). The ``!meta`` guard handles that path — do not remove as +// "dead" without also auditing every callsite. +function readStringField(meta: Record | undefined, key: string): string | null { + if (!meta) return null; + const v = meta[key]; + return typeof v === 'string' ? v : null; +} + +function readNumberField(meta: Record | undefined, key: string): number | null { + if (!meta) return null; + const v = meta[key]; + return typeof v === 'number' && Number.isFinite(v) ? v : null; +} + +/** + * Compact relative time like "42s", "3m 14s", "1h 02m". Returns null if + * the timestamp does not parse — callers fall back to a placeholder. + */ +function relativeTime(isoTimestamp: string, now: number): string | null { + const t = Date.parse(isoTimestamp); + if (Number.isNaN(t)) return null; + const diffS = Math.max(0, Math.round((now - t) / 1000)); + return humanizeSeconds(diffS); +} + +function humanizeSeconds(totalSeconds: number): string { + const s = Math.max(0, Math.floor(totalSeconds)); + if (s < 60) return `${s}s`; + const minutes = Math.floor(s / 60); + const seconds = s % 60; + if (minutes < 60) return `${minutes}m ${String(seconds).padStart(2, '0')}s`; + const hours = Math.floor(minutes / 60); + const remMin = minutes % 60; + return `${hours}h ${String(remMin).padStart(2, '0')}m`; +} diff --git a/cli/src/types.ts b/cli/src/types.ts index 34821ab..f473bbb 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -20,6 +20,15 @@ /** Valid task types for task creation. */ export type TaskType = 'new_task' | 'pr_iteration' | 'pr_review'; +/** + * Provenance of a task's submission. ``api`` covers CLI / Cognito-authenticated + * submissions; ``webhook`` covers HMAC-signed inbound webhook submissions. + * Mirrors ``cdk/src/handlers/shared/types.ts::ChannelSource`` per the CLI + * types-sync contract so downstream switches/predicates get exhaustiveness + * checking on both sides of the wire. + */ +export type ChannelSource = 'api' | 'webhook'; + /** Error categories produced by the runtime error classifier. */ export type ErrorCategoryType = 'auth' | 'network' | 'concurrency' | 'compute' | 'agent' | 'guardrail' | 'config' | 'timeout' | 'unknown'; @@ -46,6 +55,11 @@ export interface TaskDetail { readonly pr_url: string | null; readonly error_message: string | null; readonly error_classification: ErrorClassification | null; + /** Provenance of the task's submission — ``api`` for CLI / Cognito + * submissions, ``webhook`` for HMAC-signed inbound webhooks. + * Mirrors ``cdk/src/handlers/shared/types.ts::TaskDetail``; kept + * in sync per the CLI types-sync contract. */ + readonly channel_source: ChannelSource; readonly created_at: string; readonly updated_at: string; readonly started_at: string | null; @@ -55,6 +69,37 @@ export interface TaskDetail { readonly build_passed: boolean | null; readonly max_turns: number | null; readonly max_budget_usd: number | null; + /** Rev-5 DATA-1: attempts counter from the SDK (may be `max_turns + 1` + * when `agent_status='error_max_turns'` — the aborted attempt is + * counted). Required to match ``cdk/src/handlers/shared/types.ts`` + * (server always emits the field, defaulted to ``null`` in + * ``toTaskDetail`` when absent on the record). */ + readonly turns_attempted: number | null; + /** Rev-5 DATA-1: turns that actually completed (clamped to + * `max_turns` when the cap tripped). Required; see + * ``turns_attempted`` above. */ + readonly turns_completed: number | null; + /** Whether the task was submitted with ``--trace``. Surfaces in + * ``bgagent status --output json`` so scripts can confirm trace + * capture is active. Non-optional because the server always + * emits the field (defaulted to ``false`` in ``toTaskDetail`` on + * the CDK side) — mirrors the CDK guarantee. */ + readonly trace: boolean; + /** S3 URI of the ``--trace`` trajectory dump, or ``null`` when the + * task did not run with ``--trace`` or the agent has not yet + * uploaded. ``bgagent trace download`` reads the presigned URL from + * ``GET /v1/tasks/{id}/trace`` rather than this field, but surfacing + * the URI in ``status --output json`` lets users / scripts detect + * completion without an extra round trip. */ + readonly trace_s3_uri: string | null; +} + +/** Response body of ``GET /v1/tasks/{task_id}/trace`` (design §10.1). */ +export interface TraceUrlResponse { + /** Short-lived presigned S3 URL for the gzipped JSONL trajectory. */ + readonly url: string; + /** ISO-8601 timestamp when ``url`` expires (15 min from issuance). */ + readonly expires_at: string; } /** Task summary returned by GET /v1/tasks list responses. */ @@ -80,6 +125,25 @@ export interface TaskEvent { readonly metadata: Record; } +/** + * Query parameters accepted by GET /v1/tasks/{task_id}/events. + * + * ``after`` and ``next_token`` are mutually exclusive — if both are sent the + * server prefers ``after`` (and logs a warning). ``after`` is a ULID event_id + * cursor used by the CLI to catch up on the next polling iteration. Keep in + * sync with ``cdk/src/handlers/shared/types.ts``. + */ +export interface GetTaskEventsQuery { + readonly limit?: number; + readonly next_token?: string; + readonly after?: string; + /** + * When ``"1"``, requests events in descending ``event_id`` order + * (newest first). Mutually exclusive with ``after`` on the server. + */ + readonly desc?: string; +} + /** Create task request body for POST /v1/tasks. */ export interface CreateTaskRequest { readonly repo: string; @@ -89,6 +153,40 @@ export interface CreateTaskRequest { readonly max_budget_usd?: number; readonly task_type?: TaskType; readonly pr_number?: number; + /** + * Enable the ``--trace`` debug path (design §10.1). When true, the + * agent's ProgressWriter raises its preview-truncation cap from 200 + * chars to 4 KB so debug captures aren't silently clipped mid-field. + * Trace is opt-in per task — routine observability goes through + * ``bgagent watch`` / notifications. + */ + readonly trace?: boolean; +} + +/** + * Maximum length (after trim) of a nudge message. Mirrors + * `cdk/src/handlers/shared/types.ts` so the CLI can reject oversized + * input client-side without an API round-trip. + */ +export const NUDGE_MAX_MESSAGE_LENGTH = 2000; + +/** + * Nudge request body for POST /v1/tasks/{task_id}/nudge (Phase 2). + * + * A short steering message sent mid-task. The server guardrail-screens, + * rate-limits (configurable, default 10/min/task), and stores the nudge; + * the agent picks it up at the next between-turns seam. Keep in sync + * with `cdk/src/handlers/shared/types.ts`. + */ +export interface NudgeRequest { + readonly message: string; +} + +/** Nudge response from POST /v1/tasks/{task_id}/nudge (HTTP 202). */ +export interface NudgeResponse { + readonly task_id: string; + readonly nudge_id: string; + readonly submitted_at: string; } /** Cancel task response from DELETE /v1/tasks/{task_id}. */ @@ -155,7 +253,12 @@ export interface CliConfig { readonly client_id: string; } -/** Cached credentials stored in ~/.bgagent/credentials.json. */ +/** Cached credentials stored in ~/.bgagent/credentials.json. + * + * The Cognito ID token is sent on the Authorization header for REST API + * Gateway calls (API Gateway's Cognito authorizer validates the `aud` + * claim against the app client ID). + */ export interface Credentials { readonly id_token: string; readonly refresh_token: string; diff --git a/cli/test/api-client.test.ts b/cli/test/api-client.test.ts index ce33925..c922a55 100644 --- a/cli/test/api-client.test.ts +++ b/cli/test/api-client.test.ts @@ -113,6 +113,36 @@ describe('ApiClient', () => { }); }); + describe('getTraceUrl', () => { + test('sends GET to /tasks/{id}/trace and returns the presigned URL envelope', async () => { + const payload = { + url: 'https://s3.example/trace?sig=abc', + expires_at: '2026-04-30T20:15:00Z', + }; + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ data: payload }), + }); + + const result = await client.getTraceUrl('abc'); + expect(result).toEqual(payload); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.example.com/tasks/abc/trace', + expect.objectContaining({ method: 'GET' }), + ); + }); + + test('URL-encodes task_id', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ data: { url: 'x', expires_at: 'y' } }), + }); + await client.getTraceUrl('weird/id with space'); + const calledUrl = (mockFetch.mock.calls[0] as [string, unknown])[0]; + expect(calledUrl).toContain(encodeURIComponent('weird/id with space')); + }); + }); + describe('cancelTask', () => { test('sends DELETE', async () => { const cancelResponse = { task_id: 'abc', status: 'CANCELLED', cancelled_at: '2026-01-01T00:00:00Z' }; @@ -145,6 +175,283 @@ describe('ApiClient', () => { expect.anything(), ); }); + + test('passes ?after= when provided', async () => { + const response = { data: [], pagination: { next_token: null, has_more: false } }; + mockFetch.mockResolvedValue({ + ok: true, + json: async () => response, + }); + + const ulid = '01ARZ3NDEKTSV4RRFFQ69G5FAV'; + await client.getTaskEvents('abc', { after: ulid }); + + const url = mockFetch.mock.calls[0][0] as string; + expect(url).toContain('/tasks/abc/events'); + expect(url).toContain(`after=${ulid}`); + // Must not silently send next_token when only after was provided. + expect(url).not.toContain('next_token='); + }); + + test('existing next_token path is preserved', async () => { + const response = { data: [], pagination: { next_token: null, has_more: false } }; + mockFetch.mockResolvedValue({ + ok: true, + json: async () => response, + }); + + await client.getTaskEvents('abc', { nextToken: 'opaque-token', limit: 25 }); + + const url = mockFetch.mock.calls[0][0] as string; + expect(url).toContain('next_token=opaque-token'); + expect(url).toContain('limit=25'); + expect(url).not.toContain('after='); + }); + + test('passes ?desc=1 when desc=true is provided', async () => { + const response = { data: [], pagination: { next_token: null, has_more: false } }; + mockFetch.mockResolvedValue({ + ok: true, + json: async () => response, + }); + + await client.getTaskEvents('abc', { limit: 20, desc: true }); + + const url = mockFetch.mock.calls[0][0] as string; + expect(url).toContain('desc=1'); + expect(url).toContain('limit=20'); + // ``desc: false`` MUST NOT leak as ``desc=0`` or ``desc=false`` — + // the server treats anything truthy-looking as opt-in. + }); + + test('omits desc when desc is falsy or absent', async () => { + const response = { data: [], pagination: { next_token: null, has_more: false } }; + mockFetch.mockResolvedValue({ + ok: true, + json: async () => response, + }); + + await client.getTaskEvents('abc', { limit: 5, desc: false }); + + const url = mockFetch.mock.calls[0][0] as string; + expect(url).not.toContain('desc='); + }); + + test('both after and nextToken are sent verbatim to the server', async () => { + // The client does not arbitrate — the server prefers ``after`` and logs + // a warning. This test just locks in the transport behaviour. + const response = { data: [], pagination: { next_token: null, has_more: false } }; + mockFetch.mockResolvedValue({ + ok: true, + json: async () => response, + }); + + await client.getTaskEvents('abc', { + after: '01ARZ3NDEKTSV4RRFFQ69G5FAV', + nextToken: 'opaque', + }); + + const url = mockFetch.mock.calls[0][0] as string; + expect(url).toContain('after=01ARZ3NDEKTSV4RRFFQ69G5FAV'); + expect(url).toContain('next_token=opaque'); + }); + }); + + describe('catchUpEvents', () => { + test('returns first page when server says no more', async () => { + const events = [ + { event_id: '01ARZ3NDEKTSV4RRFFQ69G5FB0', event_type: 'agent_turn', timestamp: 't1', metadata: {} }, + { event_id: '01ARZ3NDEKTSV4RRFFQ69G5FB1', event_type: 'agent_turn', timestamp: 't2', metadata: {} }, + ]; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: events, + pagination: { next_token: null, has_more: false }, + }), + }); + + const result = await client.catchUpEvents('abc', '01ARZ3NDEKTSV4RRFFQ69G5FAV'); + expect(result).toEqual(events); + expect(mockFetch).toHaveBeenCalledTimes(1); + const url = mockFetch.mock.calls[0][0] as string; + expect(url).toContain('after=01ARZ3NDEKTSV4RRFFQ69G5FAV'); + }); + + test('paginates internally across multiple next_token hops', async () => { + const pageA = [ + { event_id: 'E1', event_type: 'agent_turn', timestamp: 't1', metadata: {} }, + ]; + const pageB = [ + { event_id: 'E2', event_type: 'agent_turn', timestamp: 't2', metadata: {} }, + ]; + const pageC = [ + { event_id: 'E3', event_type: 'agent_turn', timestamp: 't3', metadata: {} }, + ]; + + mockFetch + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ data: pageA, pagination: { next_token: 'tok-1', has_more: true } }), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ data: pageB, pagination: { next_token: 'tok-2', has_more: true } }), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ data: pageC, pagination: { next_token: null, has_more: false } }), + }); + + const result = await client.catchUpEvents('abc', '01ARZ3NDEKTSV4RRFFQ69G5FAV'); + expect(result.map(e => e.event_id)).toEqual(['E1', 'E2', 'E3']); + expect(mockFetch).toHaveBeenCalledTimes(3); + + // First call: uses after + const url1 = mockFetch.mock.calls[0][0] as string; + expect(url1).toContain('after=01ARZ3NDEKTSV4RRFFQ69G5FAV'); + // Second and third: use next_token (no after) + const url2 = mockFetch.mock.calls[1][0] as string; + expect(url2).toContain('next_token=tok-1'); + expect(url2).not.toContain('after='); + const url3 = mockFetch.mock.calls[2][0] as string; + expect(url3).toContain('next_token=tok-2'); + expect(url3).not.toContain('after='); + }); + + test('returns empty array when server reports no events after cursor', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + data: [], + pagination: { next_token: null, has_more: false }, + }), + }); + + const result = await client.catchUpEvents('abc', '01ARZ3NDEKTSV4RRFFQ69G5FAV'); + expect(result).toEqual([]); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + }); + + describe('AbortSignal propagation', () => { + test('threads signal through request() into fetch()', async () => { + // Regression guard: if a refactor ever drops ``signal`` from the + // fetch options, ``bgagent watch`` Ctrl+C becomes unresponsive + // because in-flight requests would have to time out before the + // loop could exit. This test proves the plumbing end-to-end at + // the HTTP boundary. + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ data: { task_id: 'abc' } }), + }); + + const controller = new AbortController(); + await client.getTask('abc', { signal: controller.signal }); + + expect(mockFetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ signal: controller.signal }), + ); + }); + + test('threads signal through getTaskEvents() and catchUpEvents() into fetch()', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ data: [], pagination: { next_token: null, has_more: false } }), + }); + + const controller = new AbortController(); + await client.getTaskEvents('abc', { signal: controller.signal }); + await client.catchUpEvents('abc', '01ARZ3NDEKTSV4RRFFQ69G5FAV', 100, { signal: controller.signal }); + + // Every fetch the client issued must carry the same signal. + const calls = mockFetch.mock.calls; + expect(calls.length).toBeGreaterThanOrEqual(2); + for (const [, init] of calls) { + expect((init as RequestInit).signal).toBe(controller.signal); + } + }); + + test('throws ApiError (not CliError) for non-JSON 4xx body so callers can classify', async () => { + // Regression guard for Chunk H: WAF / CloudFront HTML error pages + // used to come back as CliError without a status, defeating the + // watch retry loop's 4xx-vs-5xx classification. Non-JSON HTTP + // errors must still be ApiError so ``isTransientError`` can see + // the 4xx status and NOT retry. + mockFetch.mockResolvedValue({ + ok: false, + status: 403, + statusText: 'Forbidden', + json: async () => { throw new SyntaxError('Unexpected token <'); }, + }); + + try { + await client.getTask('abc'); + fail('expected an error'); + } catch (err) { + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).statusCode).toBe(403); + } + }); + }); + + describe('getStatusSnapshot', () => { + test('runs getTask and getTaskEvents(desc=1) in parallel and returns both', async () => { + const taskDetail = { task_id: 'abc', status: 'RUNNING' }; + const events = [ + { event_id: 'E2', event_type: 'agent_tool_call', timestamp: 't2', metadata: { tool_name: 'Bash' } }, + { event_id: 'E1', event_type: 'agent_turn', timestamp: 't1', metadata: { turn: 3 } }, + ]; + mockFetch + .mockResolvedValueOnce({ ok: true, json: async () => ({ data: taskDetail }) }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ data: events, pagination: { next_token: null, has_more: false } }), + }); + + const result = await client.getStatusSnapshot('abc'); + + expect(result.task).toEqual(taskDetail); + expect(result.recentEvents).toEqual(events); + + // Two HTTP calls; the events call must carry ``desc=1`` and a bounded limit. + expect(mockFetch).toHaveBeenCalledTimes(2); + const urls = mockFetch.mock.calls.map(c => c[0] as string); + expect(urls.some(u => u === 'https://api.example.com/tasks/abc')).toBe(true); + const eventsUrl = urls.find(u => u.includes('/events')); + expect(eventsUrl).toBeDefined(); + expect(eventsUrl).toContain('desc=1'); + expect(eventsUrl).toContain('limit=20'); + }); + + test('surfaces a getTask failure from the parallel pair', async () => { + // Regression guard against a future refactor to ``Promise.allSettled`` + // that would silently render a broken snapshot. The current contract + // is fail-fast: if either leg errors, the CLI surfaces the error. + mockFetch + .mockRejectedValueOnce(new Error('network down')) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ data: [], pagination: { next_token: null, has_more: false } }), + }); + + await expect(client.getStatusSnapshot('abc')).rejects.toThrow('network down'); + }); + + test('honors a custom recentEventLimit', async () => { + mockFetch + .mockResolvedValueOnce({ ok: true, json: async () => ({ data: { task_id: 'abc' } }) }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ data: [], pagination: { next_token: null, has_more: false } }), + }); + + await client.getStatusSnapshot('abc', 5); + + const eventsUrl = mockFetch.mock.calls.map(c => c[0] as string).find(u => u.includes('/events')); + expect(eventsUrl).toContain('limit=5'); + }); }); describe('createWebhook', () => { @@ -250,7 +557,10 @@ describe('ApiClient', () => { await expect(client.getTask('abc')).rejects.toThrow('bgagent login'); }); - test('throws CliError on non-JSON response', async () => { + test('throws ApiError with HTTP status for non-JSON error response', async () => { + // A non-JSON body on an HTTP error (WAF HTML page, edge proxy, + // 5xx with a plaintext reason) must still carry the status as an + // ``ApiError`` so the watch retry loop can classify 4xx vs 5xx. mockFetch.mockResolvedValue({ ok: false, status: 502, @@ -259,6 +569,14 @@ describe('ApiClient', () => { }); await expect(client.getTask('abc')).rejects.toThrow('non-JSON response'); + // ApiError (not CliError) so isTransientError can see the 502. + try { + await client.getTask('abc'); + fail('expected throw'); + } catch (err) { + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).statusCode).toBe(502); + } }); }); }); diff --git a/cli/test/auth.test.ts b/cli/test/auth.test.ts index e842344..cd2f74e 100644 --- a/cli/test/auth.test.ts +++ b/cli/test/auth.test.ts @@ -56,6 +56,7 @@ describe('auth', () => { mockSend.mockResolvedValue({ AuthenticationResult: { IdToken: 'id-token-123', + AccessToken: 'access-token-123', RefreshToken: 'refresh-token-123', ExpiresIn: 3600, }, @@ -81,33 +82,34 @@ describe('auth', () => { test('returns cached token when not expired', async () => { const futureExpiry = new Date(Date.now() + 60 * 60 * 1000).toISOString(); saveCredentials({ - id_token: 'cached-token', + id_token: 'cached-id', refresh_token: 'refresh-token', token_expiry: futureExpiry, }); + // getAuthToken returns the ID token used by the REST API. const token = await getAuthToken(); - expect(token).toBe('cached-token'); + expect(token).toBe('cached-id'); expect(mockSend).not.toHaveBeenCalled(); }); test('refreshes expired token', async () => { const pastExpiry = new Date(Date.now() - 1000).toISOString(); saveCredentials({ - id_token: 'old-token', + id_token: 'old-id', refresh_token: 'refresh-token', token_expiry: pastExpiry, }); mockSend.mockResolvedValue({ AuthenticationResult: { - IdToken: 'new-token', + IdToken: 'new-id', ExpiresIn: 3600, }, }); const token = await getAuthToken(); - expect(token).toBe('new-token'); + expect(token).toBe('new-id'); }); test('throws when no credentials exist', async () => { diff --git a/cli/test/commands/configure.test.ts b/cli/test/commands/configure.test.ts index c235622..b9319ec 100644 --- a/cli/test/commands/configure.test.ts +++ b/cli/test/commands/configure.test.ts @@ -57,4 +57,61 @@ describe('configure command', () => { expect(config.client_id).toBe('client-xyz'); expect(consoleSpy).toHaveBeenCalledWith('Configuration saved.'); }); + + test('partial update: new field value merges onto existing config', async () => { + const cmd1 = makeConfigureCommand(); + await cmd1.parseAsync([ + 'node', 'test', + '--api-url', 'https://api.example.com', + '--region', 'us-east-1', + '--user-pool-id', 'us-east-1_xyz', + '--client-id', 'client-123', + ]); + + // Update only --region; other fields should persist. + const cmd2 = makeConfigureCommand(); + await cmd2.parseAsync(['node', 'test', '--region', 'us-west-1']); + + const config = JSON.parse( + fs.readFileSync(path.join(tmpDir, 'config.json'), 'utf-8'), + ); + expect(config.api_url).toBe('https://api.example.com'); + expect(config.region).toBe('us-west-1'); + expect(config.user_pool_id).toBe('us-east-1_xyz'); + expect(config.client_id).toBe('client-123'); + }); + + test('first-time configure without all required fields → CliError', async () => { + const cmd = makeConfigureCommand(); + await expect( + cmd.parseAsync([ + 'node', 'test', + '--api-url', 'https://api.example.com', + // missing --region, --user-pool-id, --client-id + ]), + ).rejects.toThrow(/Missing required configuration/); + }); + + test('no flags with complete existing config → reports "No configuration changes" without re-saving', async () => { + // Seed a complete config. + const cmd1 = makeConfigureCommand(); + await cmd1.parseAsync([ + 'node', 'test', + '--api-url', 'https://api.example.com', + '--region', 'us-east-1', + '--user-pool-id', 'us-east-1_abc', + '--client-id', 'client-123', + ]); + const initialMtime = fs.statSync(path.join(tmpDir, 'config.json')).mtimeMs; + + // Run configure again with no flags. + const cmd2 = makeConfigureCommand(); + await cmd2.parseAsync(['node', 'test']); + + // File was not rewritten. + expect(fs.statSync(path.join(tmpDir, 'config.json')).mtimeMs).toBe(initialMtime); + // User-facing message is honest about the no-op. + expect(consoleSpy).toHaveBeenCalledWith('No configuration changes — all flags were omitted.'); + expect(consoleSpy).not.toHaveBeenLastCalledWith('Configuration saved.'); + }); }); diff --git a/cli/test/commands/login.test.ts b/cli/test/commands/login.test.ts index 37b7162..a4bd952 100644 --- a/cli/test/commands/login.test.ts +++ b/cli/test/commands/login.test.ts @@ -57,6 +57,7 @@ describe('login command', () => { mockSend.mockResolvedValue({ AuthenticationResult: { IdToken: 'id-tok', + AccessToken: 'access-tok', RefreshToken: 'ref-tok', ExpiresIn: 3600, }, @@ -73,6 +74,7 @@ describe('login command', () => { fs.readFileSync(path.join(tmpDir, 'credentials.json'), 'utf-8'), ); expect(creds.id_token).toBe('id-tok'); + expect(creds.access_token).toBeUndefined(); expect(consoleSpy).toHaveBeenCalledWith('Login successful. Credentials saved.'); }); }); diff --git a/cli/test/commands/nudge.test.ts b/cli/test/commands/nudge.test.ts new file mode 100644 index 0000000..da8a8b9 --- /dev/null +++ b/cli/test/commands/nudge.test.ts @@ -0,0 +1,214 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { ApiClient } from '../../src/api-client'; +import { makeNudgeCommand } from '../../src/commands/nudge'; +import { ApiError, CliError } from '../../src/errors'; + +jest.mock('../../src/api-client'); + +describe('nudge command', () => { + let consoleSpy: jest.SpiedFunction; + const mockNudgeTask = jest.fn(); + + beforeEach(() => { + consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + mockNudgeTask.mockReset(); + (ApiClient as jest.MockedClass).mockImplementation(() => ({ + createTask: jest.fn(), + listTasks: jest.fn(), + getTask: jest.fn(), + cancelTask: jest.fn(), + nudgeTask: mockNudgeTask, + getTaskEvents: jest.fn(), + createWebhook: jest.fn(), + listWebhooks: jest.fn(), + revokeWebhook: jest.fn(), + }) as unknown as ApiClient); + }); + + afterEach(() => { + consoleSpy.mockRestore(); + }); + + test('sends a nudge and prints confirmation on 202 success', async () => { + mockNudgeTask.mockResolvedValue({ + task_id: 'TASK-123', + nudge_id: 'NUDGE-abc', + submitted_at: '2026-04-22T10:00:00Z', + }); + + const cmd = makeNudgeCommand(); + await cmd.parseAsync(['node', 'test', 'TASK-123', 'also update the README']); + + expect(mockNudgeTask).toHaveBeenCalledWith('TASK-123', 'also update the README'); + const output = consoleSpy.mock.calls[0][0] as string; + expect(output).toContain('NUDGE-abc'); + expect(output).toContain('TASK-123'); + expect(output).toContain('2026-04-22T10:00:00Z'); + }); + + test('trims whitespace from the message before sending', async () => { + mockNudgeTask.mockResolvedValue({ + task_id: 'TASK-123', + nudge_id: 'NUDGE-abc', + submitted_at: '2026-04-22T10:00:00Z', + }); + + const cmd = makeNudgeCommand(); + await cmd.parseAsync(['node', 'test', 'TASK-123', ' focus on auth ']); + + expect(mockNudgeTask).toHaveBeenCalledWith('TASK-123', 'focus on auth'); + }); + + test('outputs JSON when --output json', async () => { + const nudgeData = { + task_id: 'TASK-123', + nudge_id: 'NUDGE-abc', + submitted_at: '2026-04-22T10:00:00Z', + }; + mockNudgeTask.mockResolvedValue(nudgeData); + + const cmd = makeNudgeCommand(); + await cmd.parseAsync(['node', 'test', 'TASK-123', 'hello', '--output', 'json']); + + const output = consoleSpy.mock.calls[0][0] as string; + // Should be valid JSON matching the payload. + expect(JSON.parse(output)).toEqual(nudgeData); + }); + + test('refuses empty message client-side without hitting the server', async () => { + const cmd = makeNudgeCommand(); + cmd.exitOverride(); + await expect( + cmd.parseAsync(['node', 'test', 'TASK-123', ' ']), + ).rejects.toThrow(CliError); + expect(mockNudgeTask).not.toHaveBeenCalled(); + }); + + test('401 unauthenticated → points user to `bgagent login`', async () => { + mockNudgeTask.mockRejectedValue( + new ApiError(401, 'UNAUTHORIZED', 'Missing token', 'req-1'), + ); + + const cmd = makeNudgeCommand(); + cmd.exitOverride(); + await expect( + cmd.parseAsync(['node', 'test', 'TASK-123', 'hello']), + ).rejects.toMatchObject({ + name: 'CliError', + message: expect.stringContaining('bgagent login'), + }); + }); + + test('400 guardrail blocked → shows reason verbatim', async () => { + mockNudgeTask.mockRejectedValue( + new ApiError( + 400, + 'GUARDRAIL_BLOCKED', + 'Nudge blocked by guardrail: policy-violation (GUARDRAIL_BLOCKED)', + 'req-2', + ), + ); + + const cmd = makeNudgeCommand(); + cmd.exitOverride(); + await expect( + cmd.parseAsync(['node', 'test', 'TASK-123', 'bad message']), + ).rejects.toMatchObject({ + name: 'CliError', + message: expect.stringContaining('policy-violation'), + }); + }); + + test('429 rate limit → tells user to slow down', async () => { + mockNudgeTask.mockRejectedValue( + new ApiError(429, 'RATE_LIMITED', 'Too many nudges', 'req-3'), + ); + + const cmd = makeNudgeCommand(); + cmd.exitOverride(); + await expect( + cmd.parseAsync(['node', 'test', 'TASK-123', 'hello']), + ).rejects.toMatchObject({ + name: 'CliError', + message: expect.stringMatching(/rate limit|slow down/i), + }); + }); + + test('404 not found → clear "task not found" message', async () => { + mockNudgeTask.mockRejectedValue( + new ApiError(404, 'NOT_FOUND', 'Task does not exist', 'req-4'), + ); + + const cmd = makeNudgeCommand(); + cmd.exitOverride(); + await expect( + cmd.parseAsync(['node', 'test', 'TASK-missing', 'hello']), + ).rejects.toMatchObject({ + name: 'CliError', + message: expect.stringMatching(/task not found/i), + }); + }); + + test('403 forbidden → "not your task" message', async () => { + mockNudgeTask.mockRejectedValue( + new ApiError(403, 'FORBIDDEN', 'Access denied', 'req-5'), + ); + + const cmd = makeNudgeCommand(); + cmd.exitOverride(); + await expect( + cmd.parseAsync(['node', 'test', 'TASK-123', 'hello']), + ).rejects.toMatchObject({ + name: 'CliError', + message: expect.stringMatching(/another user|not your/i), + }); + }); + + test('503 service unavailable → retry hint, no API call retry loop', async () => { + mockNudgeTask.mockRejectedValue( + new ApiError(503, 'SERVICE_UNAVAILABLE', 'Content screening is temporarily unavailable.', 'req-6'), + ); + + const cmd = makeNudgeCommand(); + cmd.exitOverride(); + await expect( + cmd.parseAsync(['node', 'test', 'TASK-123', 'hello']), + ).rejects.toMatchObject({ + name: 'CliError', + message: expect.stringMatching(/unavailable|retry/i), + }); + expect(mockNudgeTask).toHaveBeenCalledTimes(1); + }); + + test('over-limit message rejected client-side without API call', async () => { + const oversized = 'x'.repeat(2001); + + const cmd = makeNudgeCommand(); + cmd.exitOverride(); + await expect( + cmd.parseAsync(['node', 'test', 'TASK-123', oversized]), + ).rejects.toMatchObject({ + name: 'CliError', + message: expect.stringMatching(/maximum length|2000/i), + }); + expect(mockNudgeTask).not.toHaveBeenCalled(); + }); +}); diff --git a/cli/test/commands/status.test.ts b/cli/test/commands/status.test.ts index 6450619..6d7ded2 100644 --- a/cli/test/commands/status.test.ts +++ b/cli/test/commands/status.test.ts @@ -25,16 +25,19 @@ jest.mock('../../src/api-client'); describe('status command', () => { let consoleSpy: jest.SpiedFunction; const mockGetTask = jest.fn(); + const mockGetStatusSnapshot = jest.fn(); beforeEach(() => { consoleSpy = jest.spyOn(console, 'log').mockImplementation(); mockGetTask.mockReset(); + mockGetStatusSnapshot.mockReset(); (ApiClient as jest.MockedClass).mockImplementation(() => ({ createTask: jest.fn(), listTasks: jest.fn(), getTask: mockGetTask, cancelTask: jest.fn(), getTaskEvents: jest.fn(), + getStatusSnapshot: mockGetStatusSnapshot, createWebhook: jest.fn(), listWebhooks: jest.fn(), revokeWebhook: jest.fn(), @@ -45,44 +48,113 @@ describe('status command', () => { consoleSpy.mockRestore(); }); - test('shows task detail', async () => { - mockGetTask.mockResolvedValue({ + test('renders the deterministic snapshot from a combined task + events payload', async () => { + mockGetStatusSnapshot.mockResolvedValue({ + task: { + task_id: 'abc', + status: 'RUNNING', + repo: 'owner/repo', + issue_number: null, + task_type: 'new_task', + pr_number: null, + task_description: 'Fix bug', + branch_name: 'bgagent/abc/fix', + session_id: null, + pr_url: null, + error_message: null, + error_classification: null, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + started_at: '2026-01-01T00:00:00Z', + completed_at: null, + duration_s: null, + cost_usd: null, + build_passed: null, + max_turns: 12, + max_budget_usd: null, + turns_attempted: null, + turns_completed: null, + }, + recentEvents: [], + }); + + const cmd = makeStatusCommand(); + await cmd.parseAsync(['node', 'test', 'abc']); + + expect(mockGetStatusSnapshot).toHaveBeenCalledWith('abc'); + // The raw ``getTask`` path is only used by ``--output json``. + expect(mockGetTask).not.toHaveBeenCalled(); + const output = consoleSpy.mock.calls[0][0] as string; + expect(output).toContain('Task abc — RUNNING'); + expect(output).toContain('Repo: owner/repo'); + }); + + test('outputs raw TaskDetail JSON when --output json', async () => { + const taskData = { task_id: 'abc', status: 'RUNNING' }; + mockGetTask.mockResolvedValue(taskData); + + const cmd = makeStatusCommand(); + await cmd.parseAsync(['node', 'test', 'abc', '--output', 'json']); + + expect(consoleSpy).toHaveBeenCalledWith(JSON.stringify(taskData, null, 2)); + // JSON consumers keep the existing ``TaskDetail`` contract — no snapshot fetch. + expect(mockGetStatusSnapshot).not.toHaveBeenCalled(); + }); + + test('--wait renders the SAME snapshot layout as the default path (no format bifurcation)', async () => { + // PR #52 UX carry-forward: pre-fix, ``--wait`` rendered a completely + // different ``formatTaskDetail`` view, confusing users who wondered + // why they had to add a blocking flag to see a richer output. + // The new contract: ``--wait`` is a pure blocking flag; same + // snapshot layout renders whether or not it was passed. + const terminalTask = { task_id: 'abc', - status: 'RUNNING', + status: 'COMPLETED', repo: 'owner/repo', issue_number: null, + task_type: 'new_task', + pr_number: null, task_description: 'Fix bug', branch_name: 'bgagent/abc/fix', session_id: null, - pr_url: null, + pr_url: 'https://github.com/owner/repo/pull/1', error_message: null, + error_classification: null, created_at: '2026-01-01T00:00:00Z', - updated_at: '2026-01-01T00:00:00Z', - started_at: null, - completed_at: null, - duration_s: null, - cost_usd: null, - build_passed: null, - max_turns: null, - }); + updated_at: '2026-01-01T00:01:00Z', + started_at: '2026-01-01T00:00:00Z', + completed_at: '2026-01-01T00:01:00Z', + duration_s: 60, + cost_usd: 0.05, + build_passed: true, + max_turns: 12, + max_budget_usd: null, + turns_attempted: 5, + turns_completed: 5, + }; + mockGetTask.mockResolvedValue(terminalTask); + mockGetStatusSnapshot.mockResolvedValue({ task: terminalTask, recentEvents: [] }); const cmd = makeStatusCommand(); - await cmd.parseAsync(['node', 'test', 'abc']); + await cmd.parseAsync(['node', 'test', 'abc', '--wait']); - expect(mockGetTask).toHaveBeenCalledWith('abc'); - expect(consoleSpy).toHaveBeenCalled(); + // ``waitForTask`` polled the task (at least once) and the snapshot + // formatter was invoked — not the old ``formatTaskDetail`` split. const output = consoleSpy.mock.calls[0][0] as string; - expect(output).toContain('abc'); - expect(output).toContain('RUNNING'); + expect(output).toContain('Task abc — COMPLETED'); + expect(output).toContain('Repo: owner/repo'); + // Exit code reflects the terminal status. + expect(process.exitCode).toBe(0); }); - test('outputs JSON when --output json', async () => { - const taskData = { task_id: 'abc', status: 'RUNNING' }; - mockGetTask.mockResolvedValue(taskData); + test('--wait with --output json still returns raw TaskDetail (unchanged for scripting)', async () => { + const terminal = { task_id: 'abc', status: 'FAILED' }; + mockGetTask.mockResolvedValue(terminal); const cmd = makeStatusCommand(); - await cmd.parseAsync(['node', 'test', 'abc', '--output', 'json']); + await cmd.parseAsync(['node', 'test', 'abc', '--wait', '--output', 'json']); - expect(consoleSpy).toHaveBeenCalledWith(JSON.stringify(taskData, null, 2)); + expect(consoleSpy).toHaveBeenCalledWith(JSON.stringify(terminal, null, 2)); + expect(process.exitCode).toBe(1); }); }); diff --git a/cli/test/commands/submit.test.ts b/cli/test/commands/submit.test.ts index 25768ff..a62736a 100644 --- a/cli/test/commands/submit.test.ts +++ b/cli/test/commands/submit.test.ts @@ -283,6 +283,39 @@ describe('submit command', () => { ).rejects.toThrow('--pr and --review-pr cannot be used together'); }); + test('--trace sets trace:true in the create-task request body', async () => { + mockCreateTask.mockResolvedValue({ task_id: 't-trace', status: 'SUBMITTED' }); + + const cmd = makeSubmitCommand(); + await cmd.parseAsync([ + 'node', 'test', + '--repo', 'owner/repo', + '--task', 'deep debugging', + '--trace', + ]); + + expect(mockCreateTask).toHaveBeenCalledWith( + { repo: 'owner/repo', task_description: 'deep debugging', trace: true }, + undefined, + ); + }); + + test('--trace is opt-in — absent flag omits the field entirely (not false)', async () => { + // Keeping the wire payload slim: omit rather than send ``trace: + // false`` so the server's default-false branch is the common path. + mockCreateTask.mockResolvedValue({ task_id: 't-normal', status: 'SUBMITTED' }); + + const cmd = makeSubmitCommand(); + await cmd.parseAsync([ + 'node', 'test', + '--repo', 'owner/repo', + '--task', 'normal task', + ]); + + const [body] = mockCreateTask.mock.calls[0]; + expect(body).not.toHaveProperty('trace'); + }); + test('submits a pr_iteration task with --pr and --task', async () => { mockCreateTask.mockResolvedValue({ task_id: 'pr-abc', diff --git a/cli/test/commands/trace.test.ts b/cli/test/commands/trace.test.ts new file mode 100644 index 0000000..324738a --- /dev/null +++ b/cli/test/commands/trace.test.ts @@ -0,0 +1,375 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { ApiClient } from '../../src/api-client'; +import { makeTraceCommand } from '../../src/commands/trace'; +import { ApiError } from '../../src/errors'; + +jest.mock('../../src/api-client'); + +function mockApiClientWith(getTraceUrl: jest.Mock): void { + (ApiClient as jest.MockedClass).mockImplementation( + () => + ({ + createTask: jest.fn(), + listTasks: jest.fn(), + getTask: jest.fn(), + cancelTask: jest.fn(), + nudgeTask: jest.fn(), + getTaskEvents: jest.fn(), + getStatusSnapshot: jest.fn(), + catchUpEvents: jest.fn(), + getTraceUrl, + createWebhook: jest.fn(), + listWebhooks: jest.fn(), + revokeWebhook: jest.fn(), + }) as unknown as ApiClient, + ); +} + +/** Build a fetch response whose ``body`` is a WHATWG ReadableStream of *bytes*. */ +function makeFetchResponse(ok: boolean, status: number, statusText: string, bytes?: Uint8Array): Response { + const body = bytes !== undefined + ? new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }) + : null; + return { ok, status, statusText, body } as unknown as Response; +} + +describe('trace download command', () => { + const originalFetch = global.fetch; + let tmpDir: string; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'trace-test-')); + }); + + afterEach(() => { + global.fetch = originalFetch; + rmSync(tmpDir, { recursive: true, force: true }); + jest.clearAllMocks(); + }); + + test('writes raw gzipped bytes to -o ', async () => { + const payload = gzipSync(Buffer.from('{"event":"TURN","turn":1}\n', 'utf-8')); + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/trace?sig=abc', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + + global.fetch = jest.fn().mockResolvedValue(makeFetchResponse(true, 200, 'OK', payload)) as typeof global.fetch; + + const outFile = join(tmpDir, 'trace.jsonl.gz'); + const consoleErr = jest.spyOn(console, 'error').mockImplementation(); + try { + const cmd = makeTraceCommand(); + await cmd.parseAsync(['node', 'test', 'download', 'task-1', '-o', outFile]); + + // File exists and contains the raw gzipped payload exactly. + const written = readFileSync(outFile); + expect(Buffer.compare(written, payload)).toBe(0); + // Status message goes to stderr (not stdout). + expect(consoleErr).toHaveBeenCalledWith(`Wrote ${outFile}`); + } finally { + consoleErr.mockRestore(); + } + + expect(getTraceUrl).toHaveBeenCalledWith('task-1'); + // L3 item 1: fetch is invoked with an AbortSignal for timeout / SIGINT. + expect(global.fetch).toHaveBeenCalledWith( + 'https://s3.example/trace?sig=abc', + expect.objectContaining({ signal: expect.anything() }), + ); + }); + + test('streams gunzipped JSONL to stdout by default', async () => { + const jsonl = '{"event":"TURN","turn":1}\n{"event":"TURN","turn":2}\n'; + const payload = gzipSync(Buffer.from(jsonl, 'utf-8')); + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/trace?sig=abc', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + + global.fetch = jest.fn().mockResolvedValue(makeFetchResponse(true, 200, 'OK', payload)) as typeof global.fetch; + + // Capture writes to process.stdout rather than the inherited FD. + const written: Buffer[] = []; + const writeSpy = jest.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => { + written.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk))); + return true; + }) as typeof process.stdout.write); + + try { + const cmd = makeTraceCommand(); + await cmd.parseAsync(['node', 'test', 'download', 'task-1']); + } finally { + writeSpy.mockRestore(); + } + + const actual = Buffer.concat(written).toString('utf-8'); + expect(actual).toBe(jsonl); + }); + + test('friendly 404 message when TRACE_NOT_AVAILABLE', async () => { + const getTraceUrl = jest.fn().mockRejectedValue( + new ApiError(404, 'TRACE_NOT_AVAILABLE', 'Task did not run with --trace.', 'req-1'), + ); + mockApiClientWith(getTraceUrl); + global.fetch = jest.fn() as typeof global.fetch; + + const cmd = makeTraceCommand(); + await expect(cmd.parseAsync(['node', 'test', 'download', 'task-nope'])).rejects.toThrow( + /No trace artifact for task task-nope/, + ); + // Should NOT have attempted to fetch the S3 URL when the API returned 404. + expect(global.fetch).not.toHaveBeenCalled(); + }); + + test('propagates non-404 API errors without reframing', async () => { + const getTraceUrl = jest.fn().mockRejectedValue( + new ApiError(403, 'FORBIDDEN', 'You do not have access to this task.', 'req-2'), + ); + mockApiClientWith(getTraceUrl); + global.fetch = jest.fn() as typeof global.fetch; + + const cmd = makeTraceCommand(); + await expect(cmd.parseAsync(['node', 'test', 'download', 'task-x'])).rejects.toThrow( + /You do not have access/, + ); + expect(global.fetch).not.toHaveBeenCalled(); + }); + + test('surfaces expired-URL 403 from S3 with actionable hint', async () => { + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/expired?sig=stale', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + global.fetch = jest.fn().mockResolvedValue(makeFetchResponse(false, 403, 'Forbidden')) as typeof global.fetch; + + const cmd = makeTraceCommand(); + await expect(cmd.parseAsync(['node', 'test', 'download', 'task-1'])).rejects.toThrow( + /S3 download failed: HTTP 403[^\n]*15-minute TTL/, + ); + }); + + test('rejects with CliError "corrupt or not gzipped" when stdout pipeline hits bad bytes (L3 item 1)', async () => { + // Bytes are NOT a valid gzip stream (magic number 0x1f 0x8b is missing). + // The default (no ``-o``) path pipes through ``createGunzip()``; L3 + // wraps the raw zlib ``Z_DATA_ERROR`` in a ``CliError`` that names + // the real cause (corrupt / not gzipped) rather than surfacing an + // internal stack that looks like a CLI bug. + const junk = new Uint8Array([0x00, 0x01, 0x02, 0x03]); + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/trace?sig=abc', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + + global.fetch = jest.fn().mockResolvedValue( + makeFetchResponse(true, 200, 'OK', junk), + ) as typeof global.fetch; + + // Silence the stdout writes the pipeline attempts before rejecting. + const writeSpy = jest + .spyOn(process.stdout, 'write') + .mockImplementation((() => true) as typeof process.stdout.write); + + try { + const cmd = makeTraceCommand(); + await expect(cmd.parseAsync(['node', 'test', 'download', 'task-1'])).rejects.toThrow( + /corrupt or not gzipped/, + ); + } finally { + writeSpy.mockRestore(); + } + }); + + test('AbortController aborts the fetch when the 2-minute timeout expires (L3 item 1)', async () => { + // Use fake timers so we can advance past the 2-minute wall clock + // without the Jest suite sleeping for 2 real minutes. The fetch + // mock returns a promise that only rejects when the AbortSignal + // fires — mirroring undici's behavior on a stalled S3 stream. + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/trace?sig=abc', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + + // Fake timers must be installed BEFORE the action runs so the + // setTimeout in the handler uses the fake clock. + jest.useFakeTimers(); + try { + global.fetch = jest.fn((_url: string, init?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + // Reject on abort so the action's AbortError handler runs. + init?.signal?.addEventListener('abort', () => { + const abortErr = new Error('The user aborted a request.'); + abortErr.name = 'AbortError'; + reject(abortErr); + }); + }) as unknown as Promise; + }) as typeof global.fetch; + + const cmd = makeTraceCommand(); + // Attach a catch handler BEFORE advancing timers so the rejection + // is never observed as unhandled (which Jest would treat as a + // test failure even if the final assertion also matches). + const done = cmd.parseAsync(['node', 'test', 'download', 'task-1']); + const assertion = expect(done).rejects.toThrow(/timed out after 2 minutes/); + + // Fast-forward past the 2-minute timeout. The in-action timer + // will fire and abort the AbortController. + await jest.advanceTimersByTimeAsync(121_000); + + await assertion; + } finally { + jest.useRealTimers(); + } + }); + + test('SIGINT during fetch aborts the download and cleans up the listener (L3 item 1)', async () => { + // Verify both the listener attach/detach contract AND that a SIGINT + // actually cancels the pending fetch. Fake timers prevent the + // 2-minute watchdog from racing the SIGINT signal. + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/trace?sig=abc', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + + // Track SIGINT listener count to confirm the action both adds (1) + // and removes (0) its handler on completion. + const listenersBefore = process.listenerCount('SIGINT'); + + jest.useFakeTimers(); + try { + let sigintListenerAttached = false; + global.fetch = jest.fn((_url: string, init?: { signal?: AbortSignal }) => { + sigintListenerAttached = process.listenerCount('SIGINT') > listenersBefore; + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + const abortErr = new Error('The user aborted a request.'); + abortErr.name = 'AbortError'; + reject(abortErr); + }); + // Simulate the user hitting Ctrl+C shortly after fetch starts. + // ``process.emit('SIGINT')`` triggers the action's handler + // which calls ac.abort(). + setImmediate(() => process.emit('SIGINT' as never)); + }) as unknown as Promise; + }) as typeof global.fetch; + + const cmd = makeTraceCommand(); + const done = cmd.parseAsync(['node', 'test', 'download', 'task-1']); + // Drain any pending setImmediate / microtasks. + await Promise.resolve(); + jest.runOnlyPendingTimers(); + + await expect(done).rejects.toThrow(/Cancelled by user|aborted/); + expect(sigintListenerAttached).toBe(true); + // Listener must be detached on both success and error paths. + expect(process.listenerCount('SIGINT')).toBe(listenersBefore); + } finally { + jest.useRealTimers(); + } + }); + + test('refuses to overwrite existing -o without --force (L4 item 2)', async () => { + // Seed an existing file. The CLI must refuse BEFORE touching S3 — + // a user who typed the wrong path should not even see network + // activity, and a stale presigned URL shouldn't be minted for a + // doomed operation. + const outFile = join(tmpDir, 'existing.jsonl.gz'); + writeFileSync(outFile, Buffer.from('keep-me')); + + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/trace?sig=abc', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + global.fetch = jest.fn() as typeof global.fetch; + + const cmd = makeTraceCommand(); + await expect( + cmd.parseAsync(['node', 'test', 'download', 'task-1', '-o', outFile]), + ).rejects.toThrow(/Refusing to overwrite/); + + // Existing file untouched. + expect(readFileSync(outFile).toString()).toBe('keep-me'); + // No S3 fetch should have happened — the refusal is pre-fetch so + // we also skip minting a presigned URL for a doomed operation… + // actually ``getTraceUrl`` runs after the existsSync check; assert + // neither the S3 fetch nor the API call ran. + expect(global.fetch).not.toHaveBeenCalled(); + expect(getTraceUrl).not.toHaveBeenCalled(); + }); + + test('overwrites existing -o with --force (L4 item 2)', async () => { + const outFile = join(tmpDir, 'existing.jsonl.gz'); + writeFileSync(outFile, Buffer.from('old-content')); + + const payload = gzipSync(Buffer.from('{"event":"TURN","turn":7}\n', 'utf-8')); + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/trace?sig=abc', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + global.fetch = jest + .fn() + .mockResolvedValue(makeFetchResponse(true, 200, 'OK', payload)) as typeof global.fetch; + + const consoleErr = jest.spyOn(console, 'error').mockImplementation(); + try { + const cmd = makeTraceCommand(); + await cmd.parseAsync(['node', 'test', 'download', 'task-1', '-o', outFile, '--force']); + + // File was overwritten with the new gzipped bytes. + const written = readFileSync(outFile); + expect(Buffer.compare(written, payload)).toBe(0); + expect(consoleErr).toHaveBeenCalledWith(`Wrote ${outFile}`); + } finally { + consoleErr.mockRestore(); + } + }); + + test('rejects when S3 response has no body', async () => { + const getTraceUrl = jest.fn().mockResolvedValue({ + url: 'https://s3.example/weird', + expires_at: '2026-04-30T20:00:00Z', + }); + mockApiClientWith(getTraceUrl); + global.fetch = jest.fn().mockResolvedValue(makeFetchResponse(true, 200, 'OK')) as typeof global.fetch; + + const cmd = makeTraceCommand(); + await expect(cmd.parseAsync(['node', 'test', 'download', 'task-1'])).rejects.toThrow( + /S3 response had no body/, + ); + }); +}); diff --git a/cli/test/commands/watch.test.ts b/cli/test/commands/watch.test.ts new file mode 100644 index 0000000..ec92d39 --- /dev/null +++ b/cli/test/commands/watch.test.ts @@ -0,0 +1,933 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { ApiClient } from '../../src/api-client'; +import { + _getSessionRetries, + _resetSessionRetries, + formatTerminalMessage, + makeWatchCommand, + nextCadence, + renderEvent, + transientRetryDelayMs, +} from '../../src/commands/watch'; +import { loadConfig as loadConfigMocked } from '../../src/config'; +import { ApiError, CliError } from '../../src/errors'; +import { TaskEvent } from '../../src/types'; + +jest.mock('../../src/api-client'); + +// Config is mocked per-test. +jest.mock('../../src/config', () => ({ + loadConfig: jest.fn(), +})); + +// Auth token fetch is stubbed — the real getAuthToken loads config + credentials. +jest.mock('../../src/auth', () => ({ + getAuthToken: jest.fn().mockResolvedValue('test-id-token'), +})); + +const loadConfig = loadConfigMocked as jest.MockedFunction; + +/** Default config for polling tests. */ +const CONFIG_POLLING = { + api_url: 'https://api.example.com', + region: 'us-east-1', + user_pool_id: 'us-east-1_test', + client_id: 'test-client-id', +}; + +// Helper to create a TaskEvent +function makeEvent(overrides: Partial & { event_type: string }): TaskEvent { + const { event_id, event_type, timestamp, metadata, ...rest } = overrides; + return { + event_id: event_id ?? 'evt-001', + event_type, + timestamp: timestamp ?? '2026-04-16T12:00:00Z', + metadata: metadata ?? {}, + ...rest, + } as TaskEvent; +} + +// --------------------------------------------------------------------------- +// renderEvent — formatting +// --------------------------------------------------------------------------- + +describe('renderEvent', () => { + test('renders agent_turn', () => { + const event = makeEvent({ + event_type: 'agent_turn', + metadata: { turn: 1, model: 'claude-4', tool_calls_count: 2, thinking_preview: 'hmm', text_preview: 'hello' }, + }); + const output = renderEvent(event); + expect(output).toContain('Turn #1'); + expect(output).toContain('claude-4'); + expect(output).toContain('2 tool calls'); + expect(output).toContain('Thinking: hmm'); + expect(output).toContain('Text: hello'); + }); + + test('renders agent_tool_call', () => { + const event = makeEvent({ + event_type: 'agent_tool_call', + metadata: { tool_name: 'Bash', tool_input_preview: 'ls -la', turn: 1 }, + }); + const output = renderEvent(event); + expect(output).toContain('▶ Bash'); + expect(output).toContain('ls -la'); + }); + + test('renders agent_tool_result', () => { + const event = makeEvent({ + event_type: 'agent_tool_result', + metadata: { tool_name: 'Bash', is_error: true, content_preview: 'not found', turn: 1 }, + }); + const output = renderEvent(event); + expect(output).toContain('◀ Bash'); + expect(output).toContain('[ERROR]'); + expect(output).toContain('not found'); + }); + + test('renders agent_tool_result without error flag', () => { + const event = makeEvent({ + event_type: 'agent_tool_result', + metadata: { tool_name: 'Bash', is_error: false, content_preview: 'ok', turn: 1 }, + }); + const output = renderEvent(event); + expect(output).not.toContain('[ERROR]'); + }); + + test('renders agent_milestone', () => { + const event = makeEvent({ + event_type: 'agent_milestone', + metadata: { milestone: 'repo_setup_complete', details: 'branch=main' }, + }); + const output = renderEvent(event); + expect(output).toContain('★ repo_setup_complete'); + expect(output).toContain('branch=main'); + }); + + test('renders agent_cost_update', () => { + const event = makeEvent({ + event_type: 'agent_cost_update', + metadata: { cost_usd: 0.0512, input_tokens: 1000, output_tokens: 500, turn: 5 }, + }); + const output = renderEvent(event); + expect(output).toContain('$0.0512'); + expect(output).toContain('1000 in'); + expect(output).toContain('500 out'); + }); + + test('renders agent_error', () => { + const event = makeEvent({ + event_type: 'agent_error', + metadata: { error_type: 'RuntimeError', message_preview: 'something broke' }, + }); + const output = renderEvent(event); + expect(output).toContain('✖ RuntimeError'); + expect(output).toContain('something broke'); + }); + + test('renders unknown event type with JSON metadata', () => { + const event = makeEvent({ + event_type: 'custom_event', + metadata: { foo: 'bar' }, + }); + const output = renderEvent(event); + expect(output).toContain('custom_event'); + expect(output).toContain('"foo"'); + }); + + test('renders agent_turn with 1 tool call (singular)', () => { + const event = makeEvent({ + event_type: 'agent_turn', + metadata: { turn: 1, model: 'claude-4', tool_calls_count: 1 }, + }); + const output = renderEvent(event); + expect(output).toContain('1 tool call)'); + expect(output).not.toContain('1 tool calls'); + }); +}); + +// --------------------------------------------------------------------------- +// watch command — polling path +// --------------------------------------------------------------------------- + +describe('watch command — polling', () => { + let consoleSpy: jest.SpiedFunction; + let stderrSpy: jest.SpiedFunction; + const mockGetTaskEvents = jest.fn(); + const mockGetTask = jest.fn(); + + beforeEach(() => { + consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + stderrSpy = jest.spyOn(process.stderr, 'write').mockImplementation(() => true); + mockGetTaskEvents.mockReset(); + mockGetTask.mockReset(); + loadConfig.mockReset(); + loadConfig.mockReturnValue(CONFIG_POLLING); + process.exitCode = undefined; + // L3 item 5: module-level retry counter is process-lived; reset between + // tests so the flap warn fires deterministically in the dedicated test. + _resetSessionRetries(); + + (ApiClient as jest.MockedClass).mockImplementation(() => ({ + createTask: jest.fn(), + listTasks: jest.fn(), + getTask: mockGetTask, + cancelTask: jest.fn(), + getTaskEvents: mockGetTaskEvents, + catchUpEvents: jest.fn().mockResolvedValue([]), + createWebhook: jest.fn(), + listWebhooks: jest.fn(), + revokeWebhook: jest.fn(), + }) as unknown as ApiClient); + }); + + afterEach(() => { + consoleSpy.mockRestore(); + stderrSpy.mockRestore(); + }); + + test('polls events and exits on terminal state', async () => { + const events = [ + makeEvent({ event_id: 'evt-001', event_type: 'agent_milestone', metadata: { milestone: 'start', details: '' } }), + ]; + + mockGetTaskEvents.mockResolvedValue({ + data: events, + pagination: { next_token: null, has_more: false }, + }); + mockGetTask.mockResolvedValue({ status: 'COMPLETED' }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-1']); + + expect(mockGetTaskEvents).toHaveBeenCalledWith('task-1', expect.objectContaining({ limit: 100 })); + expect(mockGetTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ signal: expect.anything() })); + expect(process.exitCode).toBe(0); + }); + + test('sets exit code 1 for FAILED task', async () => { + mockGetTaskEvents.mockResolvedValue({ + data: [], + pagination: { next_token: null, has_more: false }, + }); + mockGetTask.mockResolvedValue({ status: 'FAILED' }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-2']); + + expect(process.exitCode).toBe(1); + }); + + test('does not re-display already seen events', async () => { + // Snapshot returns 2 events + status=RUNNING; polling then catches up + // with exactly the 1 new event past the cursor, then COMPLETED. + const snapshotEvents = [ + makeEvent({ event_id: 'evt-001', event_type: 'agent_milestone', metadata: { milestone: 'repo_setup', details: '' } }), + makeEvent({ event_id: 'evt-002', event_type: 'agent_turn', metadata: { turn: 1, model: 'c4', tool_calls_count: 0 } }), + ]; + const newEvent = makeEvent({ + event_id: 'evt-003', + event_type: 'agent_milestone', + metadata: { milestone: 'done', details: '' }, + }); + + mockGetTaskEvents.mockResolvedValue({ + data: snapshotEvents, + pagination: { next_token: null, has_more: false }, + }); + + const mockCatchUpEvents = jest.fn().mockResolvedValue([newEvent]); + (ApiClient as jest.MockedClass).mockImplementation(() => ({ + createTask: jest.fn(), + listTasks: jest.fn(), + getTask: mockGetTask, + cancelTask: jest.fn(), + getTaskEvents: mockGetTaskEvents, + catchUpEvents: mockCatchUpEvents, + createWebhook: jest.fn(), + listWebhooks: jest.fn(), + revokeWebhook: jest.fn(), + }) as unknown as ApiClient); + + let taskPollCount = 0; + mockGetTask.mockImplementation(async () => { + taskPollCount++; + return { status: taskPollCount >= 2 ? 'COMPLETED' : 'RUNNING' }; + }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-dedup']); + + // Snapshot prints 2, catchUp returns 1 → 3 total console.log calls. + expect(consoleSpy.mock.calls.length).toBe(3); + // catchUp must be called with the snapshot's last event_id as the cursor. + // Chunk H added a ``{signal}`` trailing arg for Ctrl+C propagation. + expect(mockCatchUpEvents).toHaveBeenCalledWith( + 'task-dedup', + 'evt-002', + 100, + expect.objectContaining({ signal: expect.anything() }), + ); + }); + + test('polling drains all events past the 100-item page limit (regression: BLOCKER silent-stall)', async () => { + // Snapshot returns the first 100 events; server reports has_more=true. + // Polling must call catchUpEvents(taskId, 'evt-100'), which drains the + // tail. Regression guard for the silent-stall bug where watch passed + // {limit: 100} with no after-cursor and the server replayed the same + // oldest 100 events forever. + const snapshotEvents = Array.from({ length: 100 }, (_, i) => + makeEvent({ + event_id: `evt-${String(i + 1).padStart(3, '0')}`, + event_type: 'agent_milestone', + metadata: { milestone: `m${i + 1}`, details: '' }, + }), + ); + const tailEvents = Array.from({ length: 50 }, (_, i) => + makeEvent({ + event_id: `evt-${String(i + 101).padStart(3, '0')}`, + event_type: 'agent_milestone', + metadata: { milestone: `m${i + 101}`, details: '' }, + }), + ); + + mockGetTaskEvents.mockResolvedValue({ + data: snapshotEvents, + pagination: { next_token: 'token-after-100', has_more: true }, + }); + + const mockCatchUpEvents = jest.fn().mockResolvedValue(tailEvents); + (ApiClient as jest.MockedClass).mockImplementation(() => ({ + createTask: jest.fn(), + listTasks: jest.fn(), + getTask: mockGetTask, + cancelTask: jest.fn(), + getTaskEvents: mockGetTaskEvents, + catchUpEvents: mockCatchUpEvents, + createWebhook: jest.fn(), + listWebhooks: jest.fn(), + revokeWebhook: jest.fn(), + }) as unknown as ApiClient); + + let taskPollCount = 0; + mockGetTask.mockImplementation(async () => { + taskPollCount++; + return { status: taskPollCount >= 2 ? 'COMPLETED' : 'RUNNING' }; + }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-big']); + + // Snapshot prints 100 events, catchUp returns the 50-event tail. + expect(consoleSpy.mock.calls.length).toBe(150); + expect(mockCatchUpEvents).toHaveBeenCalledWith( + 'task-big', + 'evt-100', + 100, + expect.objectContaining({ signal: expect.anything() }), + ); + }); + + test('outputs JSON when --output json', async () => { + const event = makeEvent({ event_id: 'evt-001', event_type: 'agent_milestone', metadata: { milestone: 'test', details: '' } }); + mockGetTaskEvents.mockResolvedValue({ + data: [event], + pagination: { next_token: null, has_more: false }, + }); + mockGetTask.mockResolvedValue({ status: 'COMPLETED' }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-json', '--output', 'json']); + + const output = consoleSpy.mock.calls[0][0] as string; + const parsed = JSON.parse(output); + expect(parsed.event_type).toBe('agent_milestone'); + }); + + test('shows stderr message for terminal state', async () => { + mockGetTaskEvents.mockResolvedValue({ + data: [], + pagination: { next_token: null, has_more: false }, + }); + mockGetTask.mockResolvedValue({ status: 'COMPLETED' }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-done']); + + const stderrOutput = stderrSpy.mock.calls.map(c => String(c[0])).join(''); + expect(stderrOutput).toContain('completed'); + }); + + test('prints snapshot tail + exit 0 when task already COMPLETED', async () => { + const tail = [ + makeEvent({ event_id: 'evt-001', event_type: 'agent_milestone', metadata: { milestone: 'done', details: 'ok' } }), + ]; + mockGetTaskEvents.mockResolvedValue({ + data: tail, + pagination: { next_token: null, has_more: false }, + }); + mockGetTask.mockResolvedValue({ status: 'COMPLETED' }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-already-done']); + + // The snapshot event should have been rendered exactly once. + const stdout = consoleSpy.mock.calls.map(c => String(c[0])).join('\n'); + expect(stdout).toContain('done'); + expect(process.exitCode).toBe(0); + }); + + test('prints snapshot tail + exit 1 when task already FAILED', async () => { + mockGetTaskEvents.mockResolvedValue({ + data: [], + pagination: { next_token: null, has_more: false }, + }); + mockGetTask.mockResolvedValue({ status: 'FAILED' }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-already-failed']); + + expect(process.exitCode).toBe(1); + }); + + // -------- Chunk H: transient retry + abort propagation -------- + + test('retries transient 5xx on getTaskEvents and exits cleanly when the next call succeeds', async () => { + // Empty snapshot → polling loop fires getTaskEvents. First call 503s, + // retry succeeds with an empty page, then the task-detail poll returns + // COMPLETED. The command must exit 0, not propagate the 503. + mockGetTaskEvents + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }) // snapshot + .mockRejectedValueOnce(new ApiError(503, 'SERVICE_UNAVAILABLE', 'svc down', 'req-1')) + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }); + + mockGetTask + .mockResolvedValueOnce({ status: 'RUNNING' }) // snapshot + .mockResolvedValueOnce({ status: 'COMPLETED' }); // after retry succeeded + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-retry-5xx']); + + expect(process.exitCode).toBe(0); + // Three getTaskEvents calls total: snapshot, failed, retry-succeeded. + expect(mockGetTaskEvents).toHaveBeenCalledTimes(3); + }); + + test('does not retry on 4xx — deterministic errors propagate immediately', async () => { + // Snapshot succeeds with RUNNING; the first poll returns a 403 which is + // deterministic. The command must surface it without retrying. + mockGetTaskEvents + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }) + .mockRejectedValueOnce(new ApiError(403, 'FORBIDDEN', 'nope', 'req-1')); + + mockGetTask.mockResolvedValueOnce({ status: 'RUNNING' }); + + const cmd = makeWatchCommand(); + await expect(cmd.parseAsync(['node', 'test', 'task-403'])).rejects.toThrow(); + + // Exactly one failing poll after the snapshot; no retries on 4xx. + expect(mockGetTaskEvents).toHaveBeenCalledTimes(2); + }); + + test('does not retry on 401 auth-expired — surfaces login hint immediately', async () => { + // A token that expires mid-session previously got silently retried 5 + // times then presented with a misleading "re-run to resume" message. + // The retry classifier must treat 401 as non-transient so the user + // sees the real ``bgagent login`` hint on the first failure. + mockGetTaskEvents + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }) + .mockRejectedValueOnce(new ApiError(401, 'UNAUTHORIZED', 'token expired', 'req-1')); + + mockGetTask.mockResolvedValueOnce({ status: 'RUNNING' }); + + const cmd = makeWatchCommand(); + await expect(cmd.parseAsync(['node', 'test', 'task-401'])).rejects.toThrow(/token expired/); + expect(mockGetTaskEvents).toHaveBeenCalledTimes(2); + }); + + test('does not retry on CliError (programmer / contract violation) — propagates first failure', async () => { + // Whitelist contract: only 5xx ApiError + TypeError('fetch failed') + // retry. Everything else (including our own CliError) is terminal so + // real bugs surface immediately instead of hiding behind 5 silent + // retries. + mockGetTaskEvents + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }) + .mockRejectedValueOnce(new CliError('bad response shape')); + + mockGetTask.mockResolvedValueOnce({ status: 'RUNNING' }); + + const cmd = makeWatchCommand(); + await expect(cmd.parseAsync(['node', 'test', 'task-cli-err'])).rejects.toThrow(/bad response shape/); + expect(mockGetTaskEvents).toHaveBeenCalledTimes(2); + }); + + test('SIGINT mid-poll sets exit code 130 (POSIX convention)', async () => { + // Snapshot succeeds with RUNNING; the first poll's getTaskEvents is + // set up to fire SIGINT on the *next* event loop tick before it + // resolves. The poll loop should check signal.aborted and exit via + // the aborted branch — process.exitCode must be 130, not 0/1. + mockGetTaskEvents + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }) // snapshot + .mockImplementationOnce(async () => { + // Fire SIGINT just before returning so the poll loop sees + // signal.aborted === true after the await. + process.emit('SIGINT' as never); + return { data: [], pagination: { next_token: null, has_more: false } }; + }); + + mockGetTask.mockResolvedValue({ status: 'RUNNING' }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-sigint']); + + expect(process.exitCode).toBe(130); + }); + + test('session-level retry counter surfaces a "flapping" stderr warn exactly once (L3 item 5)', async () => { + // Regression guard: before L3, ``withTransientRetry`` reset the + // per-op attempt counter on every successful poll. A 50% flapping + // upstream would retry-and-recover forever without ever surfacing + // a signal to the user. The session counter accumulates across all + // retries; crossing 10 emits the stderr warn once. + _resetSessionRetries(); + const realSetTimeout = global.setTimeout; + global.setTimeout = ((fn: () => void) => { + queueMicrotask(fn); + return 0 as unknown as NodeJS.Timeout; + }) as unknown as typeof setTimeout; + + try { + // Snapshot (1 call to each), then a flapping pattern: fail-then-succeed + // alternating for getTaskEvents so each poll incurs exactly 1 retry. + // We need >= 10 retries to cross the threshold, so run 12 flap cycles + // followed by a terminal poll. + mockGetTaskEvents.mockResolvedValueOnce({ + data: [], + pagination: { next_token: null, has_more: false }, + }); // snapshot + + for (let i = 0; i < 12; i += 1) { + mockGetTaskEvents.mockRejectedValueOnce( + new ApiError(503, 'SERVICE_UNAVAILABLE', 'flap', `req-${i}`), + ); + mockGetTaskEvents.mockResolvedValueOnce({ + data: [], + pagination: { next_token: null, has_more: false }, + }); + } + + // Snapshot task poll + per-loop polls. Return COMPLETED after enough + // flaps to cross the threshold (12 retries → well past the 10 threshold). + let taskCallCount = 0; + mockGetTask.mockImplementation(async () => { + taskCallCount += 1; + // 1 snapshot + 12 poll iterations = terminal on the 13th task call + return { status: taskCallCount >= 13 ? 'COMPLETED' : 'RUNNING' }; + }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-flapping']); + + // Session counter must have accumulated past the threshold. + expect(_getSessionRetries()).toBeGreaterThanOrEqual(10); + + // Warn must have fired EXACTLY once despite crossing threshold + // multiple times (threshold 10, saw 12 retries). + const stderrOutput = stderrSpy.mock.calls.map(c => String(c[0])).join(''); + const warnMatches = stderrOutput.match(/upstream is flapping/g) ?? []; + expect(warnMatches).toHaveLength(1); + expect(stderrOutput).toMatch(/retries so far; results may be delayed/); + } finally { + global.setTimeout = realSetTimeout; + _resetSessionRetries(); + } + }); + + test('exhausted retry budget throws a "re-run to resume" message', async () => { + // Snapshot succeeds; then every subsequent events call fails with 503. + // Budget is 5 retries, so the command must reject with a clear message + // pointing the user back at ``bgagent watch``. We stub ``setTimeout`` + // globally to run synchronously so the jittered backoff sleeps don't + // blow the Jest timeout. + const realSetTimeout = global.setTimeout; + // Run every scheduled timer on the next microtask — retry sleeps + + // cadence sleeps both resolve promptly so the poll loop can churn + // through the failure budget without blowing the Jest timeout. Using + // ``queueMicrotask`` (rather than a synchronous ``fn()``) preserves + // the callback/handler ordering the real implementation expects + // inside ``abortableSleep``. + global.setTimeout = ((fn: () => void) => { + queueMicrotask(fn); + return 0 as unknown as NodeJS.Timeout; + }) as unknown as typeof setTimeout; + try { + mockGetTaskEvents.mockResolvedValueOnce({ + data: [], + pagination: { next_token: null, has_more: false }, + }); + // MAX_TRANSIENT_RETRIES = 5 → 1 initial failure + 5 retries = 6 total. + // Queuing exactly 6 makes the test's intent obvious. + for (let i = 0; i < 6; i += 1) { + mockGetTaskEvents.mockRejectedValueOnce(new ApiError(503, 'SERVICE_UNAVAILABLE', 'svc down', `req-${i}`)); + } + + mockGetTask.mockResolvedValueOnce({ status: 'RUNNING' }); + + const cmd = makeWatchCommand(); + await expect(cmd.parseAsync(['node', 'test', 'task-503-storm'])).rejects.toThrow( + /Exceeded retry budget.*Re-run .bgagent watch/, + ); + } finally { + global.setTimeout = realSetTimeout; + } + }); + + test('SIGINT during initial snapshot exits 130 without logging a failure', async () => { + // Snapshot-level abort must surface as exit 130, NOT as an + // "Initial snapshot failed: The operation was aborted" error log. + // The snapshot mock throws an AbortError after the user interrupt. + mockGetTaskEvents.mockImplementationOnce(async () => { + process.emit('SIGINT' as never); + const err = new Error('The operation was aborted'); + err.name = 'AbortError'; + throw err; + }); + mockGetTask.mockResolvedValue({ status: 'RUNNING' }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-snap-abort']); + + expect(process.exitCode).toBe(130); + const stderrOutput = stderrSpy.mock.calls.map(c => String(c[0])).join(''); + expect(stderrOutput).not.toMatch(/Initial snapshot failed/); + }); + + test('non-abort error during snapshot still propagates with its original message', async () => { + // Race guard: a real error (e.g. token expired) during snapshot + // must NOT be swallowed by the exit-130 path even if the user + // coincidentally hits Ctrl+C at the same moment. The snapshot + // catch branch only honors 130 for actual AbortError from our + // signal; other errors carry their ``bgagent login`` hint through. + mockGetTaskEvents.mockRejectedValueOnce( + new ApiError(401, 'UNAUTHORIZED', 'token expired (UNAUTHORIZED)\nHint: Run `bgagent login`', 'req-1'), + ); + mockGetTask.mockRejectedValueOnce( + new ApiError(401, 'UNAUTHORIZED', 'token expired (UNAUTHORIZED)\nHint: Run `bgagent login`', 'req-2'), + ); + + const cmd = makeWatchCommand(); + await expect(cmd.parseAsync(['node', 'test', 'task-real-err'])).rejects.toThrow(/token expired/); + // Must NOT have exited 130 — real errors win over coincidental abort. + expect(process.exitCode).not.toBe(130); + }); + + test('initial snapshot retries transient errors before giving up (cold-start hardening)', async () => { + // Regression guard: Chunk H wrapped the polling loop's API calls + // in ``withTransientRetry`` but left ``fetchInitialSnapshot`` + // making raw calls. A single cold-start ``fetch failed`` / 5xx on + // the snapshot would crash the watch command before the polling + // loop got a chance to stabilise (observed Scenario 2 deploy + // validation). The snapshot must now retry transient errors too. + _resetSessionRetries(); + const realSetTimeout = global.setTimeout; + global.setTimeout = ((fn: () => void) => { + queueMicrotask(fn); + return 0 as unknown as NodeJS.Timeout; + }) as unknown as typeof setTimeout; + + try { + // getTaskEvents: 503 → 503 → success. getTask: succeeds cleanly. + mockGetTaskEvents + .mockRejectedValueOnce(new ApiError(503, 'SERVICE_UNAVAILABLE', 'cold-start 1', 'req-1')) + .mockRejectedValueOnce(new ApiError(503, 'SERVICE_UNAVAILABLE', 'cold-start 2', 'req-2')) + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }) // snapshot success + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }); // first poll + + mockGetTask.mockResolvedValueOnce({ status: 'RUNNING' }) // snapshot + .mockResolvedValueOnce({ status: 'COMPLETED' }); // first poll + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-cold-start']); + + // All three snapshot getTaskEvents attempts fired (2 retries + 1 success). + // Plus 1 polling-loop call after snapshot completes. + expect(mockGetTaskEvents).toHaveBeenCalledTimes(4); + expect(process.exitCode).toBe(0); + // The session retry counter recorded the 2 snapshot retries. + expect(_getSessionRetries()).toBeGreaterThanOrEqual(2); + } finally { + global.setTimeout = realSetTimeout; + } + }); + + test('initial snapshot exhausts retry budget on persistent 5xx and surfaces a "re-run" hint', async () => { + // Budget-exhaustion path: the retry wrapper gives up after + // MAX_TRANSIENT_RETRIES (5) attempts and throws a message that + // tells the user to re-run. The cursor is durable, so resumption + // is safe. + _resetSessionRetries(); + const realSetTimeout = global.setTimeout; + global.setTimeout = ((fn: () => void) => { + queueMicrotask(fn); + return 0 as unknown as NodeJS.Timeout; + }) as unknown as typeof setTimeout; + + try { + // 6 attempts will be made (attempt=1..5 all throw, attempt=6 + // crosses the budget). getTaskEvents is the only leg that fails; + // its sibling getTask succeeds once, and after SIG #2's abort + // plumbing kicks in, getTask's retry loop is short-circuited by + // the aborted signal rather than burning its own budget. + mockGetTaskEvents.mockRejectedValue( + new ApiError(503, 'SERVICE_UNAVAILABLE', 'persistent flap', 'req-x'), + ); + mockGetTask.mockResolvedValue({ status: 'RUNNING' }); + + const cmd = makeWatchCommand(); + await expect(cmd.parseAsync(['node', 'test', 'task-exhaust'])).rejects.toThrow( + /Exceeded retry budget .* Re-run `bgagent watch/, + ); + // 6 attempts: 5 retries + the initial call. + expect(mockGetTaskEvents).toHaveBeenCalledTimes(6); + } finally { + global.setTimeout = realSetTimeout; + } + }); + + test('snapshot error aborts the shared controller so a sibling retry loop terminates (resource-leak guard)', async () => { + // SIG #2 regression guard: the two snapshot calls run under + // ``Promise.all`` with independent retry wrappers. If one leg + // throws a non-transient error (401), the sibling leg must NOT + // keep retrying a flaky 503 in the background — that would pollute + // CloudWatch metrics, burn sessionRetries, and hit rate limits + // after the command has already decided to fail. + _resetSessionRetries(); + const realSetTimeout = global.setTimeout; + global.setTimeout = ((fn: () => void) => { + queueMicrotask(fn); + return 0 as unknown as NodeJS.Timeout; + }) as unknown as typeof setTimeout; + + try { + // getTaskEvents: 401 (non-transient, rethrows immediately). + // getTask: would retry 503 indefinitely if not aborted. We + // verify the abort cancels the second call before it burns + // the full retry budget. + mockGetTaskEvents.mockRejectedValueOnce( + new ApiError(401, 'UNAUTHORIZED', 'token expired', 'req-e'), + ); + let taskAttempts = 0; + mockGetTask.mockImplementation(async () => { + taskAttempts += 1; + throw new ApiError(503, 'SERVICE_UNAVAILABLE', 'slow flap', 'req-t'); + }); + + const cmd = makeWatchCommand(); + await expect(cmd.parseAsync(['node', 'test', 'task-abort'])).rejects.toThrow(/token expired/); + + // Without the abort in the snapshot catch, getTask would retry + // MAX_TRANSIENT_RETRIES times before giving up. With the abort, + // it should stop at most a handful of attempts in (the exact + // count depends on Promise.all race timing with queueMicrotask, + // but it must be strictly less than 6 = initial + 5 retries). + expect(taskAttempts).toBeLessThan(6); + } finally { + global.setTimeout = realSetTimeout; + } + }); + + test('initial snapshot does NOT retry 4xx errors (auth failures should surface immediately)', async () => { + // 4xx errors are deterministic — retrying would be futile and + // would delay the user's ``bgagent login`` hint. The retry wrapper + // classifies 401 as non-transient and rethrows immediately. + mockGetTaskEvents.mockRejectedValueOnce( + new ApiError(401, 'UNAUTHORIZED', 'token expired', 'req-1'), + ); + mockGetTask.mockRejectedValueOnce( + new ApiError(401, 'UNAUTHORIZED', 'token expired', 'req-2'), + ); + + const cmd = makeWatchCommand(); + await expect(cmd.parseAsync(['node', 'test', 'task-401'])).rejects.toThrow(/token expired/); + // Exactly one attempt — no retries on 4xx. + expect(mockGetTaskEvents).toHaveBeenCalledTimes(1); + }); + + test('SIGINT after terminal status lands still honors exit 130 (POSIX contract)', async () => { + // If the user Ctrl+Cs between onTerminal firing and the command + // resolving, their intent to interrupt is the load-bearing signal. + // The ``signal.aborted`` check must come before ``finalStatus`` so + // shells see 130, not 0. + mockGetTaskEvents + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }) // snapshot + .mockResolvedValueOnce({ data: [], pagination: { next_token: null, has_more: false } }); // first poll + + // Task returns COMPLETED on first poll; we abort during the + // task-detail call so ``finalStatus`` AND ``signal.aborted`` are + // both set by the time runPolling evaluates its exit-code block. + mockGetTask + .mockResolvedValueOnce({ status: 'RUNNING' }) // snapshot + .mockImplementationOnce(async () => { + process.emit('SIGINT' as never); + return { status: 'COMPLETED' }; + }); + + const cmd = makeWatchCommand(); + await cmd.parseAsync(['node', 'test', 'task-sigint-vs-terminal']); + + expect(process.exitCode).toBe(130); + }); +}); + +// --------------------------------------------------------------------------- +// Chunk H: transient retry jitter — pure function +// --------------------------------------------------------------------------- + +describe('transientRetryDelayMs (equal-jitter backoff)', () => { + test('never returns 0 — equal-jitter floor prevents retry storms', () => { + // Self-DOS guard: a full-jitter impl (``Math.random() * base``) + // can produce 0 delays and tight-loop a degraded service. Equal + // jitter pins at least half the base delay as a fixed floor. + // Sample enough attempts and values to catch any leak. + for (let attempt = 1; attempt <= 5; attempt += 1) { + for (let i = 0; i < 200; i += 1) { + const ms = transientRetryDelayMs(attempt); + expect(ms).toBeGreaterThan(0); + } + } + }); + + test('respects the 5000 ms ladder ceiling', () => { + for (let i = 0; i < 200; i += 1) { + // Attempt 10 would produce base = 500 * 1024 if unbounded. + expect(transientRetryDelayMs(10)).toBeLessThanOrEqual(5_000); + } + }); +}); + +// --------------------------------------------------------------------------- +// Chunk H: adaptive cadence state machine — pure function +// --------------------------------------------------------------------------- + +describe('nextCadence (adaptive polling state)', () => { + test('stays at 500 ms when events are arriving', () => { + const s0 = { intervalMs: 500, consecutiveEmptyPolls: 0 }; + const s1 = nextCadence(s0, true); + expect(s1).toEqual({ intervalMs: 500, consecutiveEmptyPolls: 0 }); + }); + + test('climbs the backoff ladder on consecutive empty polls', () => { + // Ladder is 1 s → 2 s → 5 s and caps at 5 s. + let s = { intervalMs: 500, consecutiveEmptyPolls: 0 }; + s = nextCadence(s, false); + expect(s).toEqual({ intervalMs: 1_000, consecutiveEmptyPolls: 1 }); + s = nextCadence(s, false); + expect(s).toEqual({ intervalMs: 2_000, consecutiveEmptyPolls: 2 }); + s = nextCadence(s, false); + expect(s).toEqual({ intervalMs: 5_000, consecutiveEmptyPolls: 3 }); + // Further empty polls stay pinned at the cap — don't escalate beyond 5 s. + s = nextCadence(s, false); + expect(s).toEqual({ intervalMs: 5_000, consecutiveEmptyPolls: 4 }); + }); + + test('resets to fast cadence on the next event, regardless of how deep the backoff was', () => { + const deepBackoff = { intervalMs: 5_000, consecutiveEmptyPolls: 7 }; + const reset = nextCadence(deepBackoff, true); + expect(reset).toEqual({ intervalMs: 500, consecutiveEmptyPolls: 0 }); + }); +}); + +// --------------------------------------------------------------------------- +// formatTerminalMessage — carry-forward from Scenario 7-ext take 3 polish +// --------------------------------------------------------------------------- + +describe('formatTerminalMessage', () => { + // Pre-fix, watch printed ``Task completed.`` / ``Task failed.`` with + // no task_id and no failure classification — a user watching multiple + // tasks (or scrolling back through a log) couldn't tell which task + // ended or why. The formatter now includes the task_id always, and + // the error classification (or raw message) on non-COMPLETED + // terminals. + + test('COMPLETED renders task_id + status without an error clause', () => { + expect(formatTerminalMessage({ + task_id: '01KQ...XXX', + status: 'COMPLETED', + error_classification: null, + error_message: null, + })).toBe('Task 01KQ...XXX completed.'); + }); + + test('FAILED with structured classification renders category + title', () => { + expect(formatTerminalMessage({ + task_id: 'T1', + status: 'FAILED', + error_classification: { + category: 'guardrail', + title: 'PR context blocked', + description: 'Bedrock Guardrail flagged the PR context', + remedy: 'Tune the guardrail or redact the triggering content', + retryable: false, + }, + error_message: 'Guardrail blocked: PR context blocked by content policy: CONTENT/PROMPT_ATTACK (LOW)', + })).toBe('Task T1 failed. guardrail: PR context blocked'); + }); + + test('FAILED without classification falls back to error_message', () => { + // Classifier gap / older records / transient: the raw + // ``error_message`` is the only signal. Trim whitespace so the + // fallback doesn't leak leading/trailing newlines into the TTY. + expect(formatTerminalMessage({ + task_id: 'T2', + status: 'FAILED', + error_classification: null, + error_message: ' raw server message with whitespace\n', + })).toBe('Task T2 failed. raw server message with whitespace'); + }); + + test('FAILED with neither classification nor message degrades to bare prefix', () => { + // Defense-in-depth: never emit a trailing space / orphan colon. + expect(formatTerminalMessage({ + task_id: 'T3', + status: 'FAILED', + error_classification: null, + error_message: null, + })).toBe('Task T3 failed.'); + }); + + test('CANCELLED / TIMED_OUT non-COMPLETED terminals also include classification when present', () => { + // Regression guard: the ``status === 'COMPLETED'`` check must be + // exact so CANCELLED / TIMED_OUT still render the classification. + expect(formatTerminalMessage({ + task_id: 'T4', + status: 'CANCELLED', + error_classification: { + category: 'unknown', + title: 'User cancelled', + description: 'Task cancelled by user', + remedy: '', + retryable: true, + }, + error_message: null, + })).toBe('Task T4 cancelled. unknown: User cancelled'); + }); +}); diff --git a/cli/test/format-status-snapshot.test.ts b/cli/test/format-status-snapshot.test.ts new file mode 100644 index 0000000..18b2a5d --- /dev/null +++ b/cli/test/format-status-snapshot.test.ts @@ -0,0 +1,497 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { formatStatusSnapshot } from '../src/format'; +import { ChannelSource, TaskDetail, TaskEvent } from '../src/types'; + +const NOW = Date.parse('2026-04-29T15:30:20Z'); + +/** + * Build a TaskDetail with sensible defaults for status-snapshot tests. + * Callers override only the fields relevant to the scenario under test. + */ +function buildTask(overrides: Partial = {}): TaskDetail { + return { + task_id: 'abc123', + status: 'RUNNING', + repo: 'org/repo', + issue_number: null, + task_type: 'new_task', + pr_number: null, + task_description: 'fix bug', + branch_name: 'bgagent/abc123/fix', + session_id: null, + pr_url: null, + error_message: null, + error_classification: null, + channel_source: 'api', + created_at: '2026-04-29T15:27:00Z', + updated_at: '2026-04-29T15:30:00Z', + started_at: '2026-04-29T15:27:06Z', // 3m 14s before NOW + completed_at: null, + duration_s: null, + cost_usd: null, + build_passed: null, + max_turns: 12, + max_budget_usd: 2.0, + turns_attempted: null, + turns_completed: null, + trace: false, + trace_s3_uri: null, + ...overrides, + }; +} + +function mkEvent(overrides: Partial): TaskEvent { + return { + event_id: '01ARZ3NDEKTSV4RRFFQ69G5FAV', + event_type: 'agent_turn', + timestamp: '2026-04-29T15:30:00Z', + metadata: {}, + ...overrides, + }; +} + +describe('formatStatusSnapshot', () => { + test('happy path renders the full template', () => { + const task = buildTask(); + // Events are newest-first per the ``?desc=1`` contract. ULIDs are + // lexicographically time-sortable; event_ids are chosen so the + // ascending lexical order matches the ascending timestamp order. + const events: TaskEvent[] = [ + mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F04', + event_type: 'agent_tool_call', + timestamp: '2026-04-29T15:30:12Z', + metadata: { tool_name: 'Bash', tool_input_preview: 'pytest tests/', turn: 7 }, + }), + mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F03', + event_type: 'agent_cost_update', + timestamp: '2026-04-29T15:30:11Z', + metadata: { cost_usd: 0.18, input_tokens: 1000, output_tokens: 200, turn: 7 }, + }), + mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F02', + event_type: 'agent_turn', + timestamp: '2026-04-29T15:30:10Z', + metadata: { turn: 7 }, + }), + mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F01', + event_type: 'agent_milestone', + timestamp: '2026-04-29T15:29:38Z', // 42s before NOW + metadata: { milestone: 'nudge_acknowledged' }, + }), + ]; + + const rendered = formatStatusSnapshot(task, events, NOW); + + expect(rendered).toBe( + [ + 'Task abc123 — RUNNING (3m 14s elapsed)', + ' Repo: org/repo', + ' Channel: api', + ' Description: fix bug', + ' Turn: 7 / ~12', + ' Last milestone: nudge_acknowledged (42s ago)', + ' Current: Bash tool call', + ' Cost: $0.18 / budget $2.00', + ' Last event: 2026-04-29T15:30:12Z', + ].join('\n'), + ); + }); + + test('just-submitted task degrades to placeholders', () => { + const task = buildTask({ + status: 'SUBMITTED', + started_at: null, + created_at: '2026-04-29T15:30:18Z', // 2s before NOW + max_turns: null, + max_budget_usd: null, + turns_attempted: null, + }); + + const rendered = formatStatusSnapshot(task, [], NOW); + + expect(rendered).toContain('Task abc123 — SUBMITTED (2s elapsed)'); + expect(rendered).toContain('Turn: —'); + expect(rendered).toContain('Last milestone: —'); + expect(rendered).toContain('Current: —'); + expect(rendered).toContain('Cost: — / budget —'); + expect(rendered).toContain('Last event: —'); + }); + + test('terminal task reports SDK duration and "task completed" current state', () => { + const task = buildTask({ + status: 'COMPLETED', + completed_at: '2026-04-29T15:29:50Z', + duration_s: 164, // 2m 44s — authoritative SDK value + cost_usd: 0.44, + turns_attempted: 11, + }); + + const rendered = formatStatusSnapshot(task, [], NOW); + + expect(rendered).toContain('Task abc123 — COMPLETED (2m 44s total)'); + expect(rendered).toContain('Current: task completed'); + // With no live cost event, falls back to task.cost_usd. + expect(rendered).toContain('Cost: $0.44 / budget $2.00'); + // With no live turn event, falls back to task.turns_attempted. + expect(rendered).toContain('Turn: 11 / ~12'); + }); + + test('events without a milestone show the placeholder', () => { + const task = buildTask(); + const events: TaskEvent[] = [ + mkEvent({ + event_type: 'agent_turn', + timestamp: '2026-04-29T15:30:00Z', + metadata: { turn: 5 }, + }), + ]; + + const rendered = formatStatusSnapshot(task, events, NOW); + + expect(rendered).toContain('Last milestone: —'); + expect(rendered).toContain('Current: agent turn 5'); + }); + + test('tool_call takes priority over turn for "Current"', () => { + // Design contract: the newest agent_tool_call OR agent_turn wins — + // whichever appears first in the newest-first list. A tool call + // mid-turn is the most useful "what is the agent doing right now". + const task = buildTask(); + const events: TaskEvent[] = [ + mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F0B', + event_type: 'agent_tool_call', + timestamp: '2026-04-29T15:30:14Z', + metadata: { tool_name: 'Write', turn: 9 }, + }), + mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F0A', + event_type: 'agent_turn', + timestamp: '2026-04-29T15:30:13Z', + metadata: { turn: 9 }, + }), + ]; + + const rendered = formatStatusSnapshot(task, events, NOW); + expect(rendered).toContain('Current: Write tool call'); + }); + + test('malformed timestamps fall back to placeholders without crashing', () => { + const task = buildTask({ + started_at: 'not-a-date', + created_at: 'also-not-a-date', + }); + + const rendered = formatStatusSnapshot(task, [], NOW); + // Header still renders; elapsed becomes a placeholder. + expect(rendered).toContain(`Task abc123 — RUNNING (${'—'})`); + }); + + test('formats hours for long-running tasks', () => { + const task = buildTask({ + started_at: '2026-04-29T12:25:05Z', // ~3h 5m before NOW + }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toMatch(/\(3h 05m elapsed\)/); + }); + + test('defensively resorts events so ascending input still renders the newest', () => { + // Invariant lock: a future upstream regression (handler, GSI, proxy, + // or caller wiring) could pass events ascending by mistake. The + // formatter must still identify the newest milestone by event_id so + // the snapshot never silently renders a stale tool call as "current". + const task = buildTask(); + const older = mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F01', + event_type: 'agent_milestone', + timestamp: '2026-04-29T15:28:00Z', + metadata: { milestone: 'older' }, + }); + const newer = mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F09', + event_type: 'agent_milestone', + timestamp: '2026-04-29T15:29:50Z', + metadata: { milestone: 'newer' }, + }); + // Both orderings must resolve to "newer" as the latest milestone. + expect(formatStatusSnapshot(task, [newer, older], NOW)).toContain( + 'Last milestone: newer', + ); + expect(formatStatusSnapshot(task, [older, newer], NOW)).toContain( + 'Last milestone: newer', + ); + }); + + test('missing / non-string timestamp degrades "Last event" to placeholder', () => { + // The event table is weakly typed at the storage layer: a malformed + // agent write could produce a row without ``timestamp``. Without the + // guard this line would render the literal ``undefined``. + const task = buildTask(); + const brokenEvent = { + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F10', + event_type: 'agent_turn', + metadata: { turn: 4 }, + } as unknown as TaskEvent; + const rendered = formatStatusSnapshot(task, [brokenEvent], NOW); + expect(rendered).toContain('Last event: —'); + expect(rendered).not.toContain('undefined'); + }); + + test('live cost and turn events override persisted TaskDetail values', () => { + // Contract: a running task may have a fresher ``agent_cost_update`` / + // ``agent_turn`` than what was last persisted on the TaskRecord. The + // snapshot prefers the live event so the user sees the current state, + // not the stale DB row. + const task = buildTask({ + cost_usd: 0.10, + turns_attempted: 3, // stale — the live turn event below is more recent + }); + const events: TaskEvent[] = [ + mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F20', + event_type: 'agent_cost_update', + timestamp: '2026-04-29T15:30:12Z', + metadata: { cost_usd: 0.25, input_tokens: 10, output_tokens: 5, turn: 7 }, + }), + mkEvent({ + event_id: '01ARZ3NDEKTSV4RRFFQ69G5F21', + event_type: 'agent_turn', + timestamp: '2026-04-29T15:30:10Z', + metadata: { turn: 7 }, + }), + ]; + const rendered = formatStatusSnapshot(task, events, NOW); + expect(rendered).toContain('Cost: $0.25 / budget $2.00'); + expect(rendered).toContain('Turn: 7 / ~12'); + }); + + test('renders Trace S3 line when trace_s3_uri is non-null', () => { + // Contract parity with ``formatTaskDetail``: trace-enabled tasks must + // surface the S3 URI in the default ``bgagent status `` snapshot so + // terminal users don't need to fall back to ``--output json`` to + // discover where the trajectory was uploaded. + const task = buildTask({ + trace: true, + trace_s3_uri: 's3://trace-bucket/tenants/u1/tasks/abc123/trace.jsonl.gz', + }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain( + 'Trace S3: s3://trace-bucket/tenants/u1/tasks/abc123/trace.jsonl.gz', + ); + }); + + test('omits Trace S3 line when trace_s3_uri is null', () => { + // Zero-diff for non-trace tasks — matches the conditional rendering of + // ``PR:`` / ``Cost:`` in ``formatTaskDetail``. + const task = buildTask(); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).not.toContain('Trace S3:'); + }); + + // ---- Type + Reason (PR #52 CLI UX carry-forward) ---- + + test('renders Type line for pr_iteration tasks with PR number', () => { + const task = buildTask({ task_type: 'pr_iteration', pr_number: 42 }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Type: pr_iteration (PR #42)'); + }); + + test('renders Type line for pr_review tasks', () => { + const task = buildTask({ task_type: 'pr_review', pr_number: 7 }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Type: pr_review (PR #7)'); + }); + + test('omits Type line for new_task (the compact default path)', () => { + const task = buildTask({ task_type: 'new_task' }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).not.toContain('Type:'); + }); + + test('omits PR-number suffix on Type line when pr_number is absent', () => { + // Defensive: a pr_iteration task without a pr_number would be a + // server-side data shape oddity, but the renderer must not emit a + // dangling "PR #undefined". + const task = buildTask({ task_type: 'pr_iteration', pr_number: null }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Type: pr_iteration\n'); + expect(rendered).not.toContain('PR #'); + }); + + test('FAILED status with structured classification renders Reason line', () => { + const task = buildTask({ + status: 'FAILED', + error_message: 'Agent exceeded max turns', + error_classification: { + category: 'timeout', + title: 'Exceeded max turns', + description: 'The agent hit the configured turn limit.', + remedy: 'Raise --max-turns or simplify the task.', + retryable: true, + }, + }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Reason: timeout: Exceeded max turns'); + }); + + test('FAILED without classification falls back to trimmed error_message', () => { + const task = buildTask({ + status: 'FAILED', + error_message: ' Guardrail blocked: task_description rejected\n', + error_classification: null, + }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Reason: Guardrail blocked: task_description rejected'); + }); + + test('FAILED with neither classification nor message omits Reason line (no trailing colon)', () => { + const task = buildTask({ + status: 'FAILED', + error_message: null, + error_classification: null, + }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).not.toContain('Reason:'); + }); + + test('CANCELLED and TIMED_OUT terminals also render Reason when available', () => { + // Regression guard: the ``=== COMPLETED`` check must be exact so + // other terminals still surface their cause. + const cancelled = buildTask({ + status: 'CANCELLED', + error_classification: { + category: 'unknown', + title: 'User cancelled', + description: '', + remedy: '', + retryable: true, + }, + }); + expect(formatStatusSnapshot(cancelled, [], NOW)).toContain('Reason: unknown: User cancelled'); + + const timedOut = buildTask({ + status: 'TIMED_OUT', + error_classification: { + category: 'timeout', + title: 'Wall-clock budget exceeded', + description: '', + remedy: '', + retryable: false, + }, + }); + expect(formatStatusSnapshot(timedOut, [], NOW)).toContain('Reason: timeout: Wall-clock budget exceeded'); + }); + + test('COMPLETED status never renders a Reason line (even if stale classification lingers)', () => { + const task = buildTask({ + status: 'COMPLETED', + error_message: 'should-never-render', + error_classification: { + category: 'timeout', + title: 'should-never-render', + description: '', + remedy: '', + retryable: false, + }, + }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).not.toContain('Reason:'); + expect(rendered).not.toContain('should-never-render'); + }); + + test('RUNNING status never renders a Reason line', () => { + // Non-terminal. An error_message on a running task would be + // in-flight noise — do not render it at snapshot time. + const task = buildTask({ + status: 'RUNNING', + error_message: 'transient', + error_classification: null, + }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).not.toContain('Reason:'); + }); + + // ---- Channel + Description (PR #52 CLI UX carry-forward) ---- + + test('Channel line shows api for default task records', () => { + const task = buildTask({ channel_source: 'api' }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Channel: api'); + }); + + test('Channel line shows webhook for webhook-submitted tasks', () => { + const task = buildTask({ channel_source: 'webhook' }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Channel: webhook'); + }); + + test('Channel line is always present (even when channel_source is an unexpected value)', () => { + // Defence-in-depth: even though ``ChannelSource`` narrows the type to + // ``api | webhook``, a corrupt DDB record could still arrive at the + // formatter. The snapshot degrades to the placeholder rather than + // omitting the line — consistent with other always-present rows. + const task = buildTask({ channel_source: '' as unknown as ChannelSource }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Channel: —'); + }); + + test('Description line renders the user prompt on short inputs', () => { + const task = buildTask({ task_description: 'Make a small tweak to README.md' }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).toContain('Description: Make a small tweak to README.md'); + }); + + test('Description wraps to continuation lines on long inputs (~60 char cap, word boundaries)', () => { + const long = 'This is a much longer description of a task that the user submitted and needs to be wrapped across multiple lines rather than truncated or shoved onto one very long row that overflows a normal terminal window'; + const task = buildTask({ task_description: long }); + const rendered = formatStatusSnapshot(task, [], NOW); + const lines = rendered.split('\n'); + const descStart = lines.findIndex(l => l.startsWith(' Description:')); + expect(descStart).toBeGreaterThan(-1); + // At least two physical lines rendered for the description. + const continuation = lines[descStart + 1]; + expect(continuation).toMatch(/^ {17}\S/); // 2-space indent + 15-char gutter + non-space + // Every rendered physical line should be <= 80 chars (the snapshot + // target terminal width). + for (let i = descStart; i < lines.length && lines[i].startsWith(' '); i++) { + if (!lines[i].match(/^ {2}[A-Z]/)) break; // next labeled row + expect(lines[i].length).toBeLessThanOrEqual(80); + } + }); + + test('Description is omitted when task_description is null (webhook / minimal record)', () => { + const task = buildTask({ task_description: null }); + const rendered = formatStatusSnapshot(task, [], NOW); + expect(rendered).not.toContain('Description:'); + }); + + test('Description trims leading/trailing whitespace but preserves inner spacing', () => { + const task = buildTask({ task_description: ' Fix the bug ' }); + const rendered = formatStatusSnapshot(task, [], NOW); + // Trimmed at the ends; words inside get single-space-joined because + // the wrapper splits on whitespace. + expect(rendered).toContain('Description: Fix the bug'); + expect(rendered).not.toContain(' Description: '); + }); +}); diff --git a/cli/test/format.test.ts b/cli/test/format.test.ts index 304bef2..9691e06 100644 --- a/cli/test/format.test.ts +++ b/cli/test/format.test.ts @@ -34,6 +34,7 @@ describe('format', () => { pr_url: 'https://github.com/owner/repo/pull/1', error_message: null, error_classification: null, + channel_source: 'api', created_at: '2026-01-01T00:00:00Z', updated_at: '2026-01-01T01:00:00Z', started_at: '2026-01-01T00:01:00Z', @@ -43,6 +44,10 @@ describe('format', () => { build_passed: true, max_turns: 100, max_budget_usd: null, + turns_attempted: null, + turns_completed: null, + trace: false, + trace_s3_uri: null, }; describe('formatTaskDetail', () => { @@ -103,6 +108,23 @@ describe('format', () => { expect(output).not.toContain('PR #:'); }); + test('renders Trace S3 line when trace_s3_uri is non-null', () => { + const traced: TaskDetail = { + ...task, + trace: true, + trace_s3_uri: 's3://trace-bucket/tenants/u1/tasks/abc123/trace.jsonl.gz', + }; + const output = formatTaskDetail(traced); + expect(output).toContain( + 'Trace S3: s3://trace-bucket/tenants/u1/tasks/abc123/trace.jsonl.gz', + ); + }); + + test('omits Trace S3 line when trace_s3_uri is null', () => { + const output = formatTaskDetail(task); + expect(output).not.toContain('Trace S3:'); + }); + test('shows classified error with raw detail when error_classification is present', () => { const failedTask: TaskDetail = { ...task, diff --git a/docs/backlog/CLI_COGNITO_NEW_PASSWORD_CHALLENGE.md b/docs/backlog/CLI_COGNITO_NEW_PASSWORD_CHALLENGE.md new file mode 100644 index 0000000..58d6efb --- /dev/null +++ b/docs/backlog/CLI_COGNITO_NEW_PASSWORD_CHALLENGE.md @@ -0,0 +1,177 @@ +# Backlog: CLI doesn't handle Cognito `NEW_PASSWORD_REQUIRED` challenge + +> **Status:** Backlog — to be worked in a separate session +> **Severity:** Medium — blocks first-time user login without admin intervention +> **Discovered:** 2026-04-16 during Phase 1a deployment to account `169728770098` + +--- + +## Problem + +When a user is created in Cognito via the `AdminCreateUser` flow (the default for ABCA — the stack creates the first admin user with a temporary password), Cognito puts the user in `FORCE_CHANGE_PASSWORD` state. On first login, Cognito's `InitiateAuth` call returns a `ChallengeName: "NEW_PASSWORD_REQUIRED"` response instead of `AuthenticationResult`. + +The ABCA CLI's `login` command (`cli/src/auth.ts`) calls `InitiateAuthCommand` with `AuthFlow: USER_PASSWORD_AUTH`, expects an `AuthenticationResult` in the response, and throws `"Unexpected authentication response from Cognito"` when it encounters the challenge instead. + +## Reproduction + +1. Deploy the ABCA stack to a fresh AWS account +2. The stack auto-creates a user in the Cognito pool with a temporary password +3. `bgagent configure ...` (works) +4. `bgagent login --username ` → enter temporary password → **"Error: Unexpected authentication response from Cognito."** + +Verify the user state: + +```bash +aws cognito-idp list-users \ + --user-pool-id \ + --region \ + --query 'Users[].[Username,UserStatus]' \ + --output table +``` + +Expected: `UserStatus: FORCE_CHANGE_PASSWORD` + +## Current workaround + +Admin-set a permanent password to bypass the challenge: + +```bash +aws cognito-idp admin-set-user-password \ + --user-pool-id \ + --username \ + --password 'NewPassword123!' \ + --permanent \ + --region +``` + +This works but requires AWS admin access — not acceptable for users onboarding onto a shared ABCA deployment. + +## Root cause + +File: `cli/src/auth.ts` (login flow) + +The current implementation (as of `feature/interactive-background-agents`) only handles the `AuthenticationResult` response path from `InitiateAuthCommand`. The `ChallengeName` path (specifically `NEW_PASSWORD_REQUIRED`, but also potentially `SMS_MFA`, `SOFTWARE_TOKEN_MFA`, `MFA_SETUP`) is not handled, triggering a generic error. + +Test coverage for `login.ts` is currently 37% (per `jest --coverage` output) — the challenge paths are uncovered. + +## Design for the fix + +### Scope + +**In scope:** +- `NEW_PASSWORD_REQUIRED` challenge — most common first-login flow +- User-visible error messaging when password doesn't meet the pool's password policy + +**Out of scope (separate backlog items):** +- `SMS_MFA` / `SOFTWARE_TOKEN_MFA` — deferred to MFA feature (mentioned in Iteration 4) +- `MFA_SETUP` — ditto +- `DEVICE_SRP_AUTH` — not used by this CLI +- `SELECT_MFA_TYPE` — ditto + +### Proposed behavior + +When `bgagent login` receives a `NEW_PASSWORD_REQUIRED` challenge: + +1. Print an informational message: `"Your password must be changed on first login."` +2. Prompt for a new password (hidden input, using the existing `readPassword` helper) +3. Prompt to confirm the new password (reject if mismatched) +4. Call `RespondToAuthChallengeCommand` with: + - `ChallengeName: "NEW_PASSWORD_REQUIRED"` + - `ChallengeResponses: { USERNAME: , NEW_PASSWORD: }` + - `Session: ` +5. On success: receive `AuthenticationResult`, cache tokens as usual, print "Login successful." +6. On `InvalidPasswordException`: print the policy requirements (or the Cognito error message) and re-prompt (up to 3 attempts before aborting) + +### Files to modify + +| File | Change | +|------|--------| +| `cli/src/auth.ts` | Handle `ChallengeName === "NEW_PASSWORD_REQUIRED"` branch in the login flow; add `RespondToAuthChallengeCommand` path | +| `cli/src/commands/login.ts` | Add CLI prompts for new password + confirmation; pass through to auth module | +| `cli/test/auth.test.ts` | Add tests for challenge handling (see test cases below) | +| `cli/test/commands/login.test.ts` | Add tests for the command-level prompt flow | + +### API reference + +AWS SDK v3 (already a dependency): + +```typescript +import { + InitiateAuthCommand, + RespondToAuthChallengeCommand, + AuthFlowType, + ChallengeNameType, +} from '@aws-sdk/client-cognito-identity-provider'; + +// After InitiateAuth returns ChallengeName: NEW_PASSWORD_REQUIRED: +const response = await client.send(new RespondToAuthChallengeCommand({ + ClientId: clientId, + ChallengeName: ChallengeNameType.NEW_PASSWORD_REQUIRED, + Session: initiateAuthResponse.Session, + ChallengeResponses: { + USERNAME: username, + NEW_PASSWORD: newPassword, + }, +})); +// response.AuthenticationResult contains IdToken, AccessToken, RefreshToken +``` + +## Test cases + +### Unit tests — `cli/test/auth.test.ts` + +1. **Challenge triggers new-password flow** + - Mock `InitiateAuthCommand` to return `{ ChallengeName: "NEW_PASSWORD_REQUIRED", Session: "sess-123" }` + - Mock `RespondToAuthChallengeCommand` to return `{ AuthenticationResult: { IdToken: "...", AccessToken: "...", RefreshToken: "..." } }` + - Mock password prompt to return a valid new password (both entries matching) + - Assert: `RespondToAuthChallengeCommand` called with correct ChallengeName, Session, USERNAME, NEW_PASSWORD + - Assert: tokens are cached to `~/.bgagent/credentials.json` + - Assert: function resolves successfully + +2. **Mismatched password confirmation fails clearly** + - Mock challenge response path as above + - Mock password prompts to return different values on first call and second call + - Assert: throws/exits with clear "passwords do not match" message (no API call made) + +3. **InvalidPasswordException surfaces policy error** + - Mock `RespondToAuthChallengeCommand` to throw `InvalidPasswordException` with message + - Assert: error message contains the Cognito-returned policy requirement + - Assert: exit code is 1 (or error is re-raised from the async function) + +4. **Normal login unaffected (regression)** + - Mock `InitiateAuthCommand` to return `{ AuthenticationResult: { ... } }` (no challenge) + - Assert: `RespondToAuthChallengeCommand` NOT called + - Assert: tokens cached, function resolves + +5. **Unhandled challenge types still error clearly** + - Mock `InitiateAuthCommand` to return `{ ChallengeName: "SMS_MFA" }` + - Assert: throws with message mentioning the unsupported challenge type (so users know what's happening, vs the current generic "Unexpected response") + +### Integration smoke test (manual, not automated — documented for QA) + +Against a fresh deployment: + +1. Deploy stack to a new AWS account +2. Wait for the admin user to be created by the stack (user status = `FORCE_CHANGE_PASSWORD`) +3. `bgagent configure ...` +4. `bgagent login --username ` — enter temp password, then new password + confirmation +5. Verify: login succeeds, tokens cached +6. Verify: `aws cognito-idp admin-get-user ...` shows `UserStatus: CONFIRMED` +7. Verify: subsequent `bgagent login` with the new password (no challenge) works normally +8. Verify: `bgagent list` (or any authenticated command) works with the cached tokens + +## Acceptance criteria + +- [ ] A user in `FORCE_CHANGE_PASSWORD` state can complete login via `bgagent login` alone (no admin AWS CLI intervention needed) +- [ ] Unit test coverage on `cli/src/auth.ts` rises above 70% (from current ~37% on `login.ts`) +- [ ] All 5 unit test cases above pass +- [ ] Integration smoke test documented in developer guide as a post-deploy verification step +- [ ] Other Cognito challenges (`SMS_MFA`, `MFA_SETUP`, etc.) produce a clear error message identifying the unsupported challenge type, not a generic "unexpected response" +- [ ] No regression to the normal login path (existing tests still pass) + +## Related + +- Current user-visible error: `Error: Unexpected authentication response from Cognito.` +- Error location: `cli/src/auth.ts` (login function — check for `AuthenticationResult` absence) +- AWS docs: [Authentication flows for Amazon Cognito user pools](https://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-authentication-flow.html) +- SDK docs: [`RespondToAuthChallengeCommand`](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/cognito-identity-provider/command/RespondToAuthChallengeCommand/) diff --git a/docs/backlog/EARLY_PROGRESS_MILESTONES.md b/docs/backlog/EARLY_PROGRESS_MILESTONES.md new file mode 100644 index 0000000..72971a0 --- /dev/null +++ b/docs/backlog/EARLY_PROGRESS_MILESTONES.md @@ -0,0 +1,127 @@ +# Backlog: Emit earlier progress milestones during setup phase + +> **Status:** Backlog — to be worked in a separate session +> **Severity:** Low — UX enhancement, no correctness impact +> **Discovered:** 2026-04-17 during Phase 1a E2E testing on account `169728770098` + +--- + +## Problem + +When a user runs `bgagent watch ` on a newly submitted task, the watch display shows nothing for ~2 minutes on a cold start. The first progress event (`agent_milestone: repo_setup_complete`) only fires **after** `setup_repo()` completes — which includes container cold start, git clone, dependency install, and baseline build. + +Observed timeline from a real E2E test (task `01KPCEW4H131EE3EC5SDQWYQHX`, repo `scoropeza/agent-plugins`): + +| Time | Event | Source | +|------|-------|--------| +| 00:53:52 | `task_created` | Orchestrator | +| 00:53:59 | `session_started` | Orchestrator | +| 00:54:00 | Container startup | AgentCore | +| **00:56:09** | **`agent_milestone: repo_setup_complete`** | **Agent (first progress event)** | + +Users see an apparently "stuck" task for ~2 minutes despite meaningful work happening (clone, install, build). + +## Desired behavior + +Emit progress milestones at finer granularity during `run_task()` so users see progress within 1-2 seconds of the agent container starting, and get visibility into each setup sub-phase. + +### Proposed milestones (in order) + +| Milestone | When emitted | Typical latency from session start | +|-----------|-------------|-----------------------------------| +| `agent_started` | First line of `run_task()`, right after `task_state.write_running()` | < 1s | +| `cloning_repo` | Just before `setup_repo()` → `git clone` | ~1s | +| `repo_cloned` | After `git clone` returns | ~5-20s | +| `installing_dependencies` | Before dep install (if detected) | ~5-25s | +| `dependencies_installed` | After dep install | ~30-90s | +| `running_baseline_build` | Before baseline build/verify | ~30-90s | +| `baseline_build_complete` | After baseline build | ~60-120s | +| `repo_setup_complete` | **(existing)** After `setup_repo()` fully complete | ~120-180s | +| `agent_execution_complete` | **(existing)** After `run_agent()` | varies | +| `pr_created` | **(existing)** After `ensure_pr()` | varies | + +### Design notes + +- Use the existing `agent_milestone` event type — no new schema, no CDK changes +- `details` field can include sub-phase specifics (e.g., `details="using npm"`, `details="3.2s"`) +- Milestones must be **fail-open** — a write failure during setup should never block setup from continuing (circuit breaker already handles this) +- If `setup_repo()` is called with skipped dep install (e.g., `task_type: pr_review`), emit `dependencies_skipped` instead + +## Root cause + +`agent/entrypoint.py:run_task()` only emits three milestones: +1. `repo_setup_complete` — after `setup_repo()` +2. `agent_execution_complete` — after `run_agent()` +3. `pr_created` — after `ensure_pr()` (conditional) + +The `setup_repo()` function itself (in `entrypoint.py`) has no progress_writer integration — it logs to stdout (`log("SETUP", ...)` and `log("CMD", ...)`) but doesn't emit structured events. + +## Files to modify + +| File | Change | +|------|--------| +| `agent/entrypoint.py` | Add `progress.write_agent_milestone(...)` calls at the start of `run_task()` and around each sub-phase of `setup_repo()`. Requires passing the `_ProgressWriter` instance into `setup_repo()`, OR calling it from the caller around each step. | +| `agent/tests/test_entrypoint.py` | Add tests verifying milestones are emitted in the expected order (may require splitting `setup_repo` into smaller testable units first). | +| `docs/design/INTERACTIVE_AGENTS.md` | Update the `agent_milestone` metadata table in Section 5 to list the new milestone values. | + +## Test cases + +### Unit tests — `agent/tests/test_entrypoint.py` + +1. **`agent_started` emitted immediately** + - Mock `setup_repo`, `run_agent`, `ensure_pr` to all succeed + - Capture progress_writer calls + - Assert: first call is `agent_milestone` with `milestone="agent_started"` and it occurs before any setup_repo interaction + +2. **Milestones emitted in correct order on happy path** + - Mock all sub-phases with small delays + - Assert: progress_writer receives the 9 milestones in the expected order (see table above) + +3. **Setup failure still emits preceding milestones** + - Mock `git clone` to succeed but `npm install` to fail + - Assert: `cloning_repo`, `repo_cloned`, `installing_dependencies` emitted before the failure + - Assert: `dependencies_installed` NOT emitted + - Assert: task transitions to FAILED, `agent_error` event emitted + +4. **Skipped deps emits `dependencies_skipped`** + - Config with `task_type: pr_review` + - Mock setup to skip dep install + - Assert: `dependencies_skipped` emitted instead of `installing_dependencies`/`dependencies_installed` + +5. **Circuit breaker during setup doesn't block setup** + - Force progress_writer into disabled state (simulate 3 failures) + - Assert: `setup_repo` still completes successfully + - Assert: no exception propagates to `run_task` + +### Integration smoke test (manual, not automated) + +1. Deploy stack with the change +2. Submit a task: `bgagent submit --repo / --task "Simple change"` +3. Run `bgagent watch ` immediately +4. Verify: within 2-5 seconds of the task entering RUNNING, you see `★ agent_started` +5. Verify: cloning_repo → repo_cloned → installing_dependencies → ... milestones appear during the setup phase +6. Verify: `repo_setup_complete` still appears at the end of setup (as before) +7. Verify: no regression in the existing flow (agent completes normally, PR is created) + +## Acceptance criteria + +- [ ] User sees at least one progress event within 5 seconds of the agent container starting +- [ ] At minimum 6 sub-phase milestones are emitted during the setup phase (not just the final `repo_setup_complete`) +- [ ] Milestones are emitted in a consistent order across runs +- [ ] Failure during any sub-phase still emits preceding milestones + a final `agent_error` event +- [ ] All 5 unit test cases pass +- [ ] Integration smoke test documented and verified +- [ ] No regression to Phase 1a baseline milestones (`repo_setup_complete`, `agent_execution_complete`, `pr_created` still emit as expected) +- [ ] `_ProgressWriter` circuit breaker still works during setup (no hard failures) + +## Notes + +- This is purely a UX improvement for the watch command. It does not change the task lifecycle, correctness, or any other behavior. +- A related (larger) improvement is AgentCore pre-warming (Iteration 5 in the roadmap). That addresses the root cause — cold start latency — while this addresses the visibility of that latency. +- Consider whether finer-grained progress during `npm install` itself (e.g., piping install progress) is worthwhile. Probably not — structured milestones at phase boundaries are simpler and sufficient. + +## Related + +- Current milestone emission points: `agent/entrypoint.py` (3 calls in `run_task()`) +- ProgressWriter spec: `agent/progress_writer.py`, design doc Section 5 +- Roadmap item for the root cause: "Environment pre-warming (snapshot-on-schedule)" in `docs/guides/ROADMAP.md` (Iteration 5) diff --git a/docs/design/INTERACTIVE_AGENTS.md b/docs/design/INTERACTIVE_AGENTS.md new file mode 100644 index 0000000..24b2563 --- /dev/null +++ b/docs/design/INTERACTIVE_AGENTS.md @@ -0,0 +1,831 @@ +# Interactive Agents: Async Interaction Design + +> **Status:** Active design +> **Branch:** `feature/interactive-background-agents` +> **Last updated:** 2026-04-29 (rev 6) + +--- + +## Executive summary + +ABCA runs background coding agents that clone a repo, implement a task, run tests, and open a pull request. Tasks run from minutes to hours inside an isolated cloud runtime. The interaction model is **asynchronous by design**: users submit a task and move on; the agent works without supervision; results arrive through notifications (Slack / GitHub comment / email) and as pull requests. + +This document describes the interactivity surfaces layered on top of that model — how users **check in on**, **steer**, and **gate** running agents without requiring a live connection to the compute substrate. + +### Interaction capabilities + +1. **Submit** — `POST /tasks` with a repo and task description. Fire-and-forget by default; the CLI returns a `task_id` and exits. +2. **Status** — `bgagent status ` returns a deterministic, templated snapshot of current state (last milestone, current turn, elapsed time, cost so far). Backed by a Lambda reading `TaskEventsTable`; no LLM, no hallucination, no agent interruption. +3. **Watch** — `bgagent watch ` polls `TaskEventsTable` with an adaptive interval (500 ms when events are arriving, back-off to 5 s when idle). Same endpoint used under the hood for foreground-block UX on `ask` and for HITL approval waits. +4. **Nudge** — `bgagent nudge ""` writes a row into `TaskNudgesTable`. The agent reads pending nudges between turns, acknowledges with a `nudge_acknowledged` milestone event, and integrates the nudge on its next turn. +5. **Ask** — `bgagent ask ""` (Phase 2) writes a question row. The agent answers at the next between-turns boundary; the answer surfaces as a `status_response` event. CLI default is foreground block-and-poll with a spinner; task and answer are both durable if the CLI disconnects. +6. **Approval gates** — Phase 3 Cedar-driven hard gates. Agent emits `approval_requested`, waits for a decision from `bgagent approve` / `bgagent deny` or a Slack button-press. Detailed design in `PHASE3_CEDAR_HITL.md`. + +### Core architectural choices + +- **Single AgentCore Runtime** authenticated via IAM (SigV4) from the orchestrator Lambda. No JWT-authenticated runtime, no direct CLI-to-runtime path. +- **Durable event table (`TaskEventsTable`)** is the one source of truth for agent progress. Every reader — CLI, Slack/GitHub/email dispatchers, status Lambda — reads from this table, never from the live agent. +- **Polling-only CLI.** No SSE, no WebSockets. DDB eventually-consistent reads with an `event_id` cursor are cheap, reliable, and compute-agnostic. +- **Notification plane as first-class.** A FanOutConsumer Lambda subscribes to `TaskEventsTable` DDB Streams and routes per-event-type to per-channel dispatcher Lambdas (Slack, email, GitHub comment). Per-channel defaults ship in v1. +- **Agent interaction via the hook mechanism the Claude Agent SDK provides.** Nudges, asks, and approvals all use `Stop` / between-turns hooks; no mechanism outside the SDK's contract is required. + +--- + +## Revision history + +| Rev | Date | Summary | +|-----|------|---------| +| 6 | 2026-04-29 | Current active design. Async-only interaction model: single runtime, polling-only CLI, notification plane as first-class UX, `bgagent status` + `bgagent watch` + `bgagent nudge` in v1, `bgagent ask` + Phase 3 Cedar HITL layered on top. | + +--- + +## Table of contents + +1. [Design goals](#1-design-goals) +2. [Architecture overview](#2-architecture-overview) +3. [Components](#3-components) +4. [Event model](#4-event-model) +5. [User interactions](#5-user-interactions) +6. [Notification plane](#6-notification-plane) +7. [Security and trust model](#7-security-and-trust-model) +8. [State machine](#8-state-machine) +9. [Error handling and observability](#9-error-handling-and-observability) +10. [Debug escape hatch](#10-debug-escape-hatch) +11. [Architectural decisions](#11-architectural-decisions) +12. [Implementation phases](#12-implementation-phases) +13. [Open questions](#13-open-questions) +14. [Appendix A — Claude Agent SDK reference](#appendix-a--claude-agent-sdk-reference) +15. [Appendix B — AgentCore Runtime reference](#appendix-b--agentcore-runtime-reference) +16. [Appendix C — Competitive landscape](#appendix-c--competitive-landscape) + +--- + +## 1. Design goals + +### Primary goals + +- **Compute-agnostic.** Nothing in the interaction surface depends on a specific compute substrate. The agent could run on AgentCore today and ECS tomorrow with no changes to the CLI or notification plane. +- **Survive disconnect.** Every interaction is durable in DynamoDB. A CLI crash, a closed laptop, or a flaky network never kills a task and never loses a reply. +- **Fire-and-forget by default.** Users submit and move on. Active observation is opt-in through `status`/`watch`. +- **No UX choice at submission time.** There is exactly one submit command and one observation command. Users do not pick between "resilient" and "live" when they submit. +- **Notification as first-class.** When the agent needs a human (approval gate, ask response, task completion), it reaches the user through their configured channel — not by hoping the user is watching a terminal. + +### Explicit non-goals + +- Token-by-token live streaming. Users want to know *what step* the agent is on, not *what character* it's typing. +- Sub-200 ms interaction latency. Human interaction in an async coding workflow is calibrated to seconds, not milliseconds. +- Transactional undo of agent actions. Tool calls are committed; the agent cannot retroactively revert a filesystem change because a user objected after the fact. +- Pair-programming / co-edit modes. A different product shape. + +### Requirements traceability + +| Req | Covered by | +|---|---| +| R1. Users don't pick compute or observability at submission | Single submit command; `TaskEventsTable` is compute-agnostic | +| R2. Fire-and-forget runs independently | Orchestrator path runs without a client connection | +| R3. HITL notification when configured | `approval_requested` event → FanOutConsumer → Slack/email | +| R4. Users can check in + steer any time | `bgagent status` + `bgagent watch` + `bgagent nudge` + (Phase 2) `bgagent ask` | +| R5. Agent updates source context if configured | FanOutConsumer → GitHub issue-comment dispatcher (edit-in-place) | + +--- + +## 2. Architecture overview + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ CLIENT SURFACES │ +│ │ +│ bgagent CLI Slack bot GitHub webhook Web UI (future) │ +│ │ │ │ │ │ +│ └─────────────┴────────────────┴────────────────────┘ │ +│ │ │ +└────────────────────────────────┼────────────────────────────────────────┘ + │ REST (Cognito JWT or HMAC webhook) + ▼ + ┌──────────────────────────────────────────────┐ + │ API Gateway (v1) │ + │ │ + │ POST /tasks submit │ + │ GET /tasks/{id} status-api │ + │ GET /tasks/{id}/events watch │ + │ DELETE /tasks/{id} cancel │ + │ POST /tasks/{id}/nudge nudge │ + │ POST /tasks/{id}/asks ask (P2) │ + │ POST /tasks/{id}/approvals approve P3 │ + │ POST /webhooks/tasks GH webhook │ + └───────────┬──────────────────────────────────┘ + │ + ┌────────────┼───────────────┬───────────────────────┐ + ▼ ▼ ▼ ▼ + SubmitTaskFn CLI-read Fns Nudge/Ask/Approve Webhook Fn + │ (status/events) write Fns │ + │ │ │ │ + │ async │ read │ write │ async + │ invoke │ │ │ invoke + ▼ ▼ ▼ ▼ + OrchestratorFn OrchestratorFn + │ │ + │ admission check │ + │ InvokeAgentRuntime (SigV4) │ + ▼ ▼ + ┌─────────────────────────────────────────────────────────────┐ + │ AgentCore Runtime — single IAM-authed │ + │ (agent container: pipeline, runner, hooks) │ + └──┬────────────────┬───────────────┬──────────────────────┬──┘ + │ writes │ reads │ reads │ reads + ▼ ▼ ▼ ▼ + TaskEvents TaskTable TaskNudges TaskApprovals + Table (state) Table Table (P3) + │ ▲ + │ DDB Stream (NEW_IMAGE) │ + ▼ │ + FanOutConsumer (router) │ + │ │ + ├─→ SlackDispatchFn ──▶ Slack Web API │ + ├─→ EmailDispatchFn ──▶ SES │ + └─→ GitHubDispatchFn ──▶ GitHub REST (edit-in-place) │ + │ │ + │ action-button callback │ + │ (approve/deny) │ + └─────────────────────────┘ +``` + +Key properties: + +- **One write path in, one read path out.** Every durable agent signal lands in `TaskEventsTable` (or `TaskTable` for state transitions). Every consumer reads from there. +- **Orchestrator is the only substrate-aware component.** Replace `InvokeAgentRuntime` with `ecs:RunTask` and the CLI + notification plane don't notice. +- **No client holds a live connection to the agent.** `watch` is a polling loop against `TaskEventsTable`, not a stream from the runtime. + +--- + +## 3. Components + +### 3.1 AgentCore Runtime (IAM-authenticated) + +A single AgentCore Runtime, invoked via `bedrock-agentcore:InvokeAgentRuntime` with SigV4 from the orchestrator Lambda. No JWT authorizer; no direct CLI access. + +- **Input:** task payload from orchestrator (task_id, repo, task_description, optional initial_approvals, optional trace_flag) +- **Output:** none via response stream — the runtime is invoked fire-and-forget. All observable state flows through `TaskEventsTable` and `TaskTable`. +- **Lifecycle:** `idleRuntimeSessionTimeout` and `maxLifetime` both set to 8 hours (AgentCore max). A running task holds the session; an idle runtime is evicted by AgentCore. +- **Compute substitutability:** replacing this with ECS/Fargate is a change confined to the orchestrator + the AgentCore Runtime CDK construct. Nothing else in the system observes the difference. + +### 3.2 OrchestratorFn + +Durable-execution Lambda that owns the task lifecycle from submission to terminal state. + +Responsibilities: +- **Admission control** — atomic DDB conditional update on `UserConcurrencyTable` (`active_count < max`); reject with 429 if over quota. +- **State transition** `SUBMITTED → HYDRATING → RUNNING → FINALIZING → terminal`. +- **Invocation** — calls `InvokeAgentRuntime` with SigV4. +- **Poll loop** — waits for the agent to land a terminal status in `TaskTable`; enforces heartbeat watchdog; transitions to `FAILED` if the container dies. +- **Finalize** — TTL + concurrency decrement + synthesized terminal event. + +Hydration (blueprint merge, repo config, PAT retrieval, prompt assembly) is targeted to live **inside the agent container at startup**, not in the orchestrator. This keeps the orchestrator thin, lets heavy I/O fail inside a durable 8 h runtime rather than a 15 min Lambda, and gives the runtime container the IAM it needs for those reads anyway. + +> **Status (2026-04-30):** the rev-6 PR ships with hydration still in the orchestrator Lambda for scope reasons — moving it is pure architectural relocation with no user-visible delta and a ~2,700 lines porting surface (TypeScript → Python with new boto3 clients and a GraphQL GitHub path). Tracked as AD-11 carry-forward in upstream [issue #53](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/53) — current plan is a hybrid split: keep lightweight preflight in the orchestrator, move heavy I/O hydration to the container. Contract drift during the deferral window is bounded by the `SUPPORTED_HYDRATED_CONTEXT_VERSION` version gate in `agent/src/models.py`. + +### 3.3 SubmitTaskFn + +Validates a submission, writes the `TaskRecord` with `status=SUBMITTED`, emits a `task_created` event, and async-invokes `OrchestratorFn`. + +- Single path for all tasks. No `execution_mode` branching. +- Works identically for CLI-initiated submissions (Cognito JWT) and webhook-initiated submissions (HMAC, after the webhook authorizer). + +### 3.4 TaskEventsTable + +The durable event spine. PK = `task_id`, SK = `event_id` (ULID), TTL enabled, **DDB Streams enabled** (`NEW_IMAGE`). + +Writers: +- `ProgressWriter` (inside the agent container) — per tool call, per turn, per milestone, cost updates, errors. +- `OrchestratorFn` — `task_created`, `hydration_*`, `session_started`, `task_*`, `preflight_*`, `admission_rejected`, `guardrail_blocked`. +- Cancel/reconciler handlers — `task_cancelled`, `task_stranded`. + +Readers: +- `get-task-events` Lambda (backs `bgagent watch` and `bgagent events`). +- `bgagent status` Lambda (templated snapshot). +- `FanOutConsumer` (stream-subscribed; see §6). + +Cost profile is negligible: eventually-consistent queries with a cursor return ~0.5 RCU per page. 50 simultaneous watchers polling every 2 seconds is pennies per active hour. + +### 3.5 TaskTable + +Task state machine: `SUBMITTED → HYDRATING → RUNNING → FINALIZING → {COMPLETED, FAILED, CANCELLED, TIMED_OUT}` with Phase 3 adding `AWAITING_APPROVAL`. + +Writers: create-task, orchestrator, cancel, agent pipeline (terminal write), reconcilers. Transitions are conditional DDB writes; illegal transitions are rejected. + +### 3.6 TaskNudgesTable + +PK = `task_id`, SK = `nudge_id`. A row represents a pending user steering message. + +Producer: `POST /tasks/{id}/nudge` handler (after ownership check, guardrail scan, and rate-limit conditional update). +Consumer: agent between-turns hook reads pending nudges, emits `nudge_acknowledged` milestone, and injects the nudge text into the next turn via `decision: "block"`. + +### 3.7 TaskApprovalsTable (Phase 3) + +Phase 3 approval-request spine. Detailed schema in `PHASE3_CEDAR_HITL.md`. Semantics summary: +- Agent writes an approval row with the request context. +- Agent transitions `RUNNING → AWAITING_APPROVAL` and enters a poll loop. +- User responds via REST (`POST /tasks/{id}/approvals/{request_id}`) or via a Slack button dispatched by the notification plane. +- On decision, agent transitions back to `RUNNING`; denial reasons are injected as Stop-hook steering on the next turn. + +### 3.8 FanOutConsumer (router) + +Lambda subscribed to `TaskEventsTable` DDB Streams (`ParallelizationFactor: 1`, preserving per-`task_id` ordering by shard). Reads per-task notification config (from `TaskTable` metadata or `RepoTable` defaults), filters events by channel subscription, and invokes per-channel dispatcher Lambdas. + +- **SlackDispatchFn** — posts to configured channel / DM. Includes action buttons for `approval_required` events. +- **EmailDispatchFn** — SES. +- **GitHubDispatchFn** — edits a single GitHub issue comment in place via `PATCH /repos/{o}/{r}/issues/comments/{id}`. On 404 (comment deleted upstream) falls back to POSTing a fresh comment. Per-task ordering is guaranteed upstream by DDB Stream `ParallelizationFactor: 1`, so no conditional-request header is needed (and GitHub's REST API does not accept `If-Match` on this endpoint — see §6.4). + +Detailed routing and default filters in §6. + +### 3.9 Reconcilers + +Two scheduled Lambdas that backstop the state machine: + +- **Stranded-task reconciler** (every 5 min) — catches tasks stuck in non-terminal states past a unified timeout (`STRANDED_TIMEOUT_SECONDS=1200` default). Covers `OrchestratorFn` async-invoke crashes and container crashes. Transitions stuck tasks to `FAILED` with a `task_stranded` event. +- **Concurrency reconciler** (every 15 min) — recomputes `active_count` per user by querying the `UserStatusIndex` GSI and corrects drift in `UserConcurrencyTable`. + +### 3.10 CLI (`bgagent`) + +Commands: +- `submit` — fire-and-forget; returns `task_id`. +- `status ` — templated snapshot. +- `watch ` — adaptive polling loop. +- `events ` — raw event stream (debug). +- `nudge ""` — steer. +- `cancel ` — stop the task. +- `ask ""` (Phase 2) — ask the agent a question. +- `approve ` / `deny ` / `pending` / `policies` (Phase 3) — HITL. + +Authentication: Cognito User Pool ID token in `Authorization` header for all REST calls. Token caching in `~/.bgagent/credentials.json` with auto-refresh. + +--- + +## 4. Event model + +### 4.1 Schema + +`TaskEventsTable` row: + +```jsonc +{ + "task_id": "abc123", // PK + "event_id": "01JXY...", // SK, ULID (time-sortable) + "event_type": "agent_tool_call", + "timestamp": "2026-04-29T15:30:12Z", + "ttl": 1735689600, + "metadata": { + "tool_name": "Bash", + "tool_input_preview": "pytest tests/ -x", // ≤200 chars by default; 4KB with --trace + "turn": 7, + "...": "..." + } +} +``` + +### 4.2 Event types + +| Type | Emitted by | Meaning | +|---|---|---| +| `task_created` | SubmitTaskFn | New task accepted | +| `hydration_started` / `hydration_completed` | Agent startup | Blueprint + repo config loaded | +| `session_started` | Orchestrator | AgentCore session established | +| `agent_turn` | Runner | One model-roundtrip completed; includes turn number, model, thinking preview | +| `agent_tool_call` | Runner / PreToolUse hook | About to invoke a tool | +| `agent_tool_result` | Runner / PostToolUse hook | Tool returned | +| `agent_milestone` | Agent code (pipeline, hooks) | Named checkpoint (`repo_cloned`, `pr_opened`, `nudge_acknowledged`, ...) | +| `agent_cost_update` | Runner | Cumulative token + dollar cost | +| `agent_error` | Runner | Handled exception | +| `approval_required` (P3) | PreToolUse Cedar hook | Cedar policy requires user decision | +| `approval_decided` (P3) | Approve/Deny Lambda | User responded | +| `status_response` (P2) | Between-turns hook | Agent answered an `ask` | +| `nudge_acknowledged` | Between-turns hook | Agent saw a nudge before incorporating it | +| `pr_created` | Pipeline | PR opened for the task | +| `task_completed` / `task_failed` / `task_cancelled` / `task_stranded` | Orchestrator / reconciler | Terminal | + +Named milestones (`pr_created`, `nudge_acknowledged`, `repo_setup_complete`, …) are written as `agent_milestone` events with `metadata.milestone` carrying the name. The fan-out router unwraps an allowlisted subset (§6.2) so per-channel default filters can target milestone names directly (e.g. GitHub's default set includes `pr_created`); unlisted milestone names stay wrapped and do not route. The watch CLI renders all milestones regardless of the allowlist. + +### 4.3 Previews and truncation + +Text fields (thinking, tool input, tool output, error details) are truncated to 200 characters by default to keep event rows small. The `--trace` flag raises the cap to 4 KB and additionally writes a full trajectory to S3 (see §10). + +### 4.4 Cursor semantics + +Consumers page `TaskEventsTable` using `event_id` as a cursor: `KeyConditionExpression: task_id = :id AND event_id > :cursor`, `ConsistentRead: true`. ULID sort order is time-monotonic, so lexical comparison gives time ordering. + +--- + +## 5. User interactions + +### 5.1 `bgagent submit` + +``` +$ bgagent submit --repo org/repo "fix the auth timeout bug" +task submitted: abc123 +``` + +Writes `TaskRecord`, fires orchestrator, returns. The CLI does not wait. `--wait` flag is available for scripting (blocks until terminal state, returns non-zero on failure). + +### 5.2 `bgagent status` + +Deterministic, templated snapshot. No LLM. + +``` +$ bgagent status abc123 +Task abc123 — RUNNING (3m 14s elapsed) + Repo: org/repo + Turn: 7 / ~12 + Last milestone: nudge_acknowledged (42s ago) + Current: Bash tool call + Cost: $0.18 / budget $2.00 + Last event: 2026-04-29T15:30:12Z +``` + +Implementation: +- Lambda reads the last N events from `TaskEventsTable` + current `TaskRecord`. +- Renders from a fixed template. Never calls an LLM. Never hallucinates. +- Fast (<200 ms P95), free, safe to call repeatedly. + +### 5.3 `bgagent watch` + +Polling loop against `GET /tasks/{id}/events?after=` with **adaptive interval**: + +- Start at 500 ms. +- If a poll returns ≥1 event, keep at 500 ms. +- If a poll returns 0 events, back off: 1 s, 2 s, 5 s (cap). +- Reset to 500 ms on the next event. + +Renders events as they arrive. Exits on terminal status. Cursor is the last `event_id` seen. + +Cost profile: 50 simultaneous watchers × ~0.5 RCU per empty poll × 5 s intervals when idle ≈ negligible. + +### 5.4 `bgagent nudge` + +``` +$ bgagent nudge abc123 "also fix the logging module, separate commit" +nudge queued: nudge_01JX... +``` + +Flow: +1. CLI `POST /tasks/{id}/nudge` → rate-limit conditional update + `PutItem` in `TaskNudgesTable`. +2. Agent's Stop hook fires between turns. Calls `nudge_reader.read_pending(task_id)` — returns all pending nudges for this task (concatenated into one `` block if multiple). +3. Hook emits `nudge_acknowledged` milestone to `ProgressWriter` **before** returning to the SDK. User sees this event immediately via `watch` or Slack. +4. Hook returns `{"decision": "block", "reason": }`. The SDK treats this as the start of the next user turn; the agent incorporates the nudge on its response. +5. Nudge row is marked consumed via conditional update (`consumed_at` set only if currently null). + +**Cost model — honest:** the nudge burns one turn from the task's `max_turns` budget. The acknowledgment rides in the same turn (the combined-turn ack pattern). This is the only mechanism the Claude Agent SDK exposes for injecting user-visible text mid-run; there is no "append to system prompt mid-conversation" API (see Appendix A). + +### 5.5 `bgagent ask` (Phase 2) + +Ask the agent a natural-language question that requires its own reasoning. Always burns a turn. Always has latency (bounded above by the agent's current turn duration, which can be minutes). + +**CLI default: foreground block-and-poll with a spinner.** + +``` +$ bgagent ask abc123 "why did you change the retry logic?" +⠋ queued as ask_01JX... — waiting for agent +⠙ agent is running tool: Bash (turn 7/~12) — 42s elapsed +✓ agent responded (1m 14s) + +The existing retry used exponential backoff with no jitter, causing thundering +herd under load. Added jitter to spread retries across the window. +``` + +Flow: +1. CLI `POST /tasks/{id}/asks` → `{ask_id, cursor}`. +2. CLI polls `GET /events?after=&type=status_response&correlation_id=` with adaptive interval. +3. Spinner renders last `agent_turn` / `agent_tool_call` so the user sees the agent is alive. +4. Agent's between-turns hook reads the pending ask, injects it as a user turn via `decision: "block"`, agent answers, hook emits `status_response{ask_id, content, turn}`. +5. CLI prints the response and exits. + +Flags: +- default → foreground block +- `--no-wait` → returns `ask_id` immediately; response delivered via Slack/watch +- `--timeout N` → override default 5 min (hard cap 10 min) + +**Durability:** the ask row lives in DDB regardless of CLI state. If the user Ctrl-Cs or the terminal closes, the ask still executes; the response is retrievable via `bgagent asks show `, `bgagent watch`, or Slack. + +**Rate limit:** 1 open ask per task per user (429 otherwise). Forward-compatible with multi-user team scenarios. + +### 5.6 `bgagent approve` / `deny` / `pending` / `policies` (Phase 3) + +HITL approval commands. All flows are REST + DDB; no streaming. Detailed design in `PHASE3_CEDAR_HITL.md`. Summary: + +- Agent emits `approval_required` with the tool context. +- Notification plane dispatches the event (Slack with action buttons, email, GitHub). +- User responds via `bgagent approve `, `bgagent deny --reason "…"`, or Slack button click. +- Agent's poll loop sees the decision and proceeds or deny-steers. + +### 5.7 `bgagent cancel` + +Writes `cancellation_requested` flag on `TaskRecord`; agent's between-turns hook checks it and terminates. Agent's PR-short-circuit logic commits partial work before exit. + +--- + +## 6. Notification plane + +### 6.1 FanOutConsumer as router + +``` +TaskEventsTable ──DDB Stream──▶ FanOutConsumer + │ + │ reads notification config + │ (per-task or per-repo) + │ + ┌─────────────┼─────────────┐ + ▼ ▼ ▼ + SlackDispatch EmailDispatch GitHubDispatch + │ │ │ + Slack Web API SES GitHub REST API +``` + +- Single Lambda subscribes to the DDB Stream. Stateless; fails-forward into SQS DLQ on per-event errors. +- `ParallelizationFactor: 1` on the event-source mapping → per-`task_id` shard ordering preserved for free. +- Router reads per-task notification config (channel enablement + event-type filters), then invokes the relevant dispatcher Lambda(s) per event. +- Dispatchers are separate Lambdas so a GitHub API outage doesn't block Slack notifications. + +### 6.2 Per-channel defaults (v1) + +| Channel | Default subscribed events | Opt-in via `--verbose` | +|---|---|---| +| **Slack** | `task_completed`, `task_failed`, `task_cancelled`, `pr_created`, `agent_error`, `approval_required`, `status_response` | adds `agent_milestone` | +| **Email** | `task_completed`, `task_failed`, `approval_required` | — | +| **GitHub issue comment** | `pr_created`, terminal status (single edit-in-place comment) | — already minimal | + +Rationale: if Slack pings on every milestone, users mute the bot within days. Default to the minimal set that surfaces decision-requiring events and completion; power users opt into verbose streams. + +### 6.3 Slack approval buttons + +`approval_required` events delivered to Slack include `Approve` / `Deny` action buttons. On click, Slack invokes an interaction callback Lambda which writes to `TaskApprovalsTable` via the same `POST /approvals` path the CLI uses. This gives the common case (reviewer in Slack, not at a terminal) a one-click response path. + +### 6.4 GitHub issue comment — edit-in-place + +A single comment per task, edited in place as the agent progresses (terminal states + `pr_created` by default). + +**Concurrency:** Per-`task_id` ordering is guaranteed upstream by DDB Streams on `TaskEventsTable` with `ParallelizationFactor: 1`, and the fanout Lambda is the only writer on its own comment, so concurrent edits of the same comment body are not possible — last-writer-wins is safe because there is no concurrent writer to lose to. The dispatcher issues a single PATCH per event (no GET round-trip, no conditional headers). If the comment has been deleted upstream (404), it falls back to POSTing a fresh comment. + +**Tolerated races (bounded, logged, not silenced):** + +- *Persist failure after successful POST* — if the GitHub POST succeeds but the subsequent `TaskTable` UpdateItem that persists `github_comment_id` fails non-benignly (DDB throttling, IAM deny, etc.), the next event for the same task re-POSTs a second comment. Bounded to at most one duplicate per task per failure window (the per-invocation cap stops runaway). Logged at ERROR with `error_id: FANOUT_GITHUB_PERSIST_FAILED` so operators can alarm and reconcile. A sweeper that matches on the `bgagent:task-id=` marker body prefix is a post-v1 follow-up. +- *404 → POST race between sibling invocations* — if the previously-posted comment was deleted upstream and two consecutive fanout invocations independently re-POST before either persists the new id, both POSTs land. The UpdateItem uses `ConditionExpression: github_comment_id = :prev` so only the first persist wins; the sibling's `saveCommentState` surfaces a benign `ConditionalCheckFailedException` at INFO and the sibling's comment survives on GitHub as an orphan (the `bgagent:` marker makes it reconcilable offline). +- *Transient `loadTaskForComment` failure* — if the task record's GetItem fails transiently, `routeEvent`'s `Promise.allSettled` records the dispatcher as rejected and the batch continues. No write lands. The event is effectively dropped; the next event (e.g. `task_completed` after `pr_created`) will render the current task state. + +**Legacy field:** A previous revision persisted `github_comment_etag` on the TaskRecord. That field is no longer written or read; items that still carry it from earlier deploys are ignored by the DocumentClient (fields not declared on the typed surface pass through untouched). No migration required. + +**Why not ETag / `If-Match`:** An earlier revision attempted optimistic concurrency via GitHub's ETag and `If-Match`. In-account validation (PR #52 Scenario 7-extended) proved this does not work: GitHub's REST API rejects conditional-request headers on `PATCH /issues/comments/{id}` with `HTTP 400 "Conditional request headers are not allowed in unsafe requests unless supported by the endpoint"`. The ETag returned on GET is a cache validator only; the write endpoint does not honor it. Upstream ordering via the DDB-Stream configuration above is sufficient on its own. + +### 6.5 Per-task notification config + +Submitted with the task (optional) or resolved from repo defaults: + +```jsonc +{ + "notifications": { + "slack": { "enabled": true, "channel": "#coding-agents", "events": ["default"] }, + "email": { "enabled": true, "events": ["approval_required", "task_failed"] }, + "github": { "enabled": true, "events": ["default"] } + } +} +``` + +`"default"` resolves to the v1 per-channel defaults above. + +--- + +## 7. Security and trust model + +### 7.1 Authentication surfaces + +| Surface | Auth | Notes | +|---|---|---| +| CLI → REST API (all endpoints) | Cognito JWT (ID token) | Managed by User Pool | +| GitHub webhook → `POST /webhooks/tasks` | HMAC-SHA256 via request authorizer | Shared secret in Secrets Manager | +| OrchestratorFn → AgentCore Runtime | SigV4 (IAM) | Lambda execution role | +| Agent container → AWS APIs (DDB, S3, Bedrock) | SigV4 via runtime's execution role | Scoped per-runtime | +| Slack button → interaction callback | Slack signing secret | Standard Slack pattern | + +### 7.2 Nudge security + +- Ownership check: the Lambda verifies `user_id` (from Cognito claims) matches the task's `user_id` before accepting the nudge. +- Rate limit: 10 nudges per task per minute (conditional update on a `RATE##MINUTE#` row). +- Size cap: 2 KB per nudge. +- Guardrail pre-screen: Bedrock guardrail scans nudge text for prompt-injection patterns before persisting. + +### 7.3 Approval security (Phase 3) + +- Ownership check on approve/deny. +- Atomic state transition via `TransactWriteItems` (approval row + TaskTable status). +- Recent-decision cache (60 s) prevents retry-loop storms. +- Denial reason sanitized by the DenyTaskFn Lambda (Bedrock output scanner) before persisting. + +### 7.4 Event table privacy + +- Previews truncate to 200 chars → low risk of accidental secret capture in common cases. +- Agent-side output scanners redact secrets before calling `ProgressWriter`. +- `--trace` flag opts into larger previews + S3 trajectory dumps; S3 objects are written to a user-scoped prefix with short TTL. + +--- + +## 8. State machine + +### 8.1 Core transitions + +``` +SUBMITTED ──▶ HYDRATING ──▶ RUNNING ──▶ FINALIZING ──▶ COMPLETED + │ │ │ │ + │ │ │ └──▶ FAILED + │ │ │ └──▶ TIMED_OUT + │ │ └──▶ CANCELLED + │ │ └──▶ AWAITING_APPROVAL (P3) + │ └──▶ FAILED (stranded) + └──▶ FAILED (stranded) +``` + +### 8.2 Phase 3 addition: `AWAITING_APPROVAL` + +``` +RUNNING ──▶ AWAITING_APPROVAL ──▶ RUNNING (approve or deny-with-steering) + │ + ├──▶ CANCELLED (explicit cancel) + └──▶ FAILED (stranded reconciler catches abandoned approval) +``` + +The `AWAITING_APPROVAL` state holds the user's concurrency slot (paused but alive). See `PHASE3_CEDAR_HITL.md` for full semantics. + +### 8.3 Write rules + +- Every transition is a conditional DDB write: `#status = :fromStatus`. +- Illegal transitions are rejected at the storage layer (not enforced in code). +- The valid-transition table lives in `cdk/src/handlers/shared/task-status.ts`. + +--- + +## 9. Error handling and observability + +### 9.1 Fail-open vs fail-closed + +| Component | Failure posture | Rationale | +|---|---|---| +| `ProgressWriter` | Fail-open (3-strike circuit breaker) | Event telemetry must never crash the task | +| Nudge/ask rate-limit conditional update | Fail-closed (return 429) | Accurate throttling is a product guarantee | +| Cedar policy evaluation | Fail-closed (treat as DENY) | Security-critical; unknown outcome = deny | +| Approval poll DDB read | Fail-open with tolerance (10 consecutive failures → TIMED_OUT) | Tolerate transient DDB errors; fail closed on sustained | +| Notification dispatcher | Fail-open (log + DLQ) | A Slack outage must not block the agent | + +### 9.2 Unified debugging: event correlation + +Every log line, event, and metric carries `task_id`. CloudWatch Logs Insights queries across all Lambdas on `task_id = "abc123"` give the full cross-component picture. + +### 9.3 OpenTelemetry + +Each component emits OTEL traces with `task_id` as a baggage item. OrchestratorFn starts the root span; AgentCore runtime continues it via env-var propagation; Lambdas downstream of DDB Streams resume from the event's `traceparent` attribute. + +### 9.4 Dashboards + +CloudWatch dashboard shows, per task: +- State transitions timeline +- Event rate by type +- Cost accumulation +- Concurrency slot utilization + +### 9.5 Alarms + +Currently deferred — no operational notification channel exists for this project beyond Slack/email user-facing notifications. When an ops channel is added (SNS/PagerDuty), the alarm plumbing is a small follow-up; the metric data is already flowing. + +--- + +## 10. Debug escape hatch + +### 10.1 `--trace` flag + +Without live streaming, a developer debugging a misbehaving agent needs a richer offline view than the default 200-char event previews. The `--trace` flag: + +``` +$ bgagent submit --trace "fix the auth bug" +``` + +Changes for a trace-enabled task: +- `ProgressWriter` preview truncation raised from 200 chars → 4 KB. +- Full agent trajectory (SDK message log, tool I/O, hook callbacks) written to S3 on task completion. +- A `trajectory_uploaded` milestone event with the S3 URI is emitted; the CLI can surface it at the end of `watch` or `status`. + +Storage: +- S3 prefix: `s3:///traces//.jsonl.gz`. +- TTL: 7 days (lifecycle policy). +- Pre-signed URLs available via `bgagent trace download `. + +### 10.2 When to use it + +- Reproducible failure modes during development. +- Customer-reported "agent did the wrong thing" incidents. +- Reward-hacking / hallucination audits. + +Not intended for routine observability — that's what `watch` and notifications are for. + +--- + +## 11. Architectural decisions + +Short summaries of the load-bearing choices. Each decision is phrased as the chosen option; rationales are concise. + +### AD-1. Single AgentCore Runtime, IAM-authenticated + +Exactly one runtime, invoked via SigV4 from the orchestrator. The CLI never talks directly to the runtime. + +*Why:* Compute-substrate portability (ECS/Fargate swap requires only orchestrator changes); simpler auth; one runtime to operate and observe. Direct CLI-to-runtime paths would reintroduce substrate coupling and force a choice between live-stream and durability at submission time. + +### AD-2. Polling-only CLI + +`bgagent watch` / `bgagent status` / `bgagent ask` all use REST-polling against `TaskEventsTable` with an adaptive interval. No SSE. No WebSockets. + +*Why:* Human-scale interaction latency (seconds) is well-served by polling; DDB costs are trivial; no streaming infrastructure to build, operate, or secure. Cursor, GitHub Copilot coding agent, and Codex all use the same pattern. + +### AD-3. `TaskEventsTable` as the single event spine + +Every durable signal from the agent flows through this table. Every consumer reads from it. + +*Why:* Decouples the agent from every consumer. CLI, Slack bot, GitHub integration, and any future web UI all read the same substrate without touching the runtime. + +### AD-4. Notification plane as first-class + +FanOutConsumer routes events per-channel with sensible defaults shipping in v1. + +*Why:* In an async product, notifications are the primary UX. Shipping without defaults would cause users to mute integrations on day one. + +### AD-5. Nudge acknowledgment via combined-turn ack + +The between-turns hook emits a `nudge_acknowledged` milestone to `ProgressWriter` **before** returning `decision: "block"` with the nudge text. One turn burned (same as today); acknowledgment visible immediately. + +*Why:* The Claude Agent SDK does not expose a mechanism to append to system context mid-conversation. The `HookEvent` enum is fixed; `ClaudeAgentOptions.system_prompt` is construction-time only; `hookSpecificOutput.additionalContext` is user-visible-only (confirmed `not-planned` by Anthropic). One-turn-per-nudge is an architectural constraint of the SDK; we surface it honestly rather than pretending it's free. + +### AD-6. `bgagent status` is deterministic; `bgagent ask` is the agent + +`status` = templated Lambda reading `TaskEventsTable`. `ask` = a real question to the agent, always costs a turn, always has latency. + +*Why:* Users understand dashboard reads vs. questions-to-a-thinking-entity. One command per contract is clearer than one command with a flag that silently changes execution model. + +### AD-7. `bgagent ask` foreground block-and-poll + +Default UX blocks with a spinner showing current agent activity. Durable underneath — CLI disconnect does not cancel the ask or lose the answer. + +*Why:* Matches user expectation of a synchronous CLI call. Survives a closed laptop. Spinner surfaces the bounded-but-non-trivial latency (turns can take minutes) without feeling like a hang. + +### AD-8. HITL: hard gates only in v1 + +Phase 3 ships hard gates. No soft questions, no "proceed with default if no response" semantics. + +*Why:* Soft-question-with-timeout creates a ticking-clock UX that's actively hostile in an async workflow. "Gate or no gate" is the coherent choice. A future `effect: "advise"` tier (non-blocking FYI events, no timeout) is documented in the Phase 3 design as post-v1. + +### AD-9. GitHub edit-in-place via DDB-Stream ordering, not SQS FIFO + +DDB Streams on `TaskEventsTable` with `ParallelizationFactor: 1` give per-`task_id` ordering. The fanout Lambda is the only writer on its own comment, so no concurrent writer exists to race — last-writer-wins is safe. The dispatcher PATCHes directly (no GET-then-PATCH, no conditional headers). + +*Why:* Simpler than SQS FIFO (no queue, no DLQ, no per-group throughput ceiling), and lower latency than a GET-then-PATCH round-trip. + +*Rejected alternative — `If-Match` ETag:* An earlier revision of this design used optimistic concurrency via GitHub's ETag. Deploy-validation (PR #52 Scenario 7-extended) proved that `PATCH /issues/comments/{id}` rejects `If-Match` with HTTP 400 (`"Conditional request headers are not allowed in unsafe requests unless supported by the endpoint"`). The ETag returned on GET is a cache validator only. Upstream DDB-Stream ordering makes the ETag unnecessary anyway. + +### AD-10. Stranded-task reconciler with a unified timeout + +One timeout value covers all stranded cases (orchestrator crash, container crash, general abandonment). + +*Why:* The interactive-specific timeout disappeared along with the interactive path. One reconciler, one threshold, easier to reason about. + +### AD-11. Agent-side hydration (hybrid split; partially deferred) + +Blueprint merging, repo config, PAT retrieval, and prompt assembly are targeted for the agent container at startup, not the orchestrator Lambda. + +*Why:* Hydration artifacts (cloned repos, merged blueprints, rendered prompts) are large and only needed inside the runtime. Failures belong inside the durable 8 h runtime rather than a 15 min Lambda. The runtime already has the IAM it needs for those reads. Industry precedent (Cursor background agents, GitHub Copilot coding agent, Devin, Temporal's activity-worker pattern, LangGraph's queue-worker split) converges on worker-side hydration for long-running async agents. + +*Target shape — hybrid split:* keep the **cheap preflight** in the orchestrator (PAT validity check, repo-existence check, guardrail screen on the raw `task_description`) so we still fail fast before burning an AgentCore compute slot. Move the **heavy I/O hydration** (GitHub issue / PR fetch including review threads, prompt assembly, Memory retrieval, S3 blueprint reads) into the agent container. + +*Status (2026-04-30):* **deferred to a follow-up PR**, tracked at [upstream issue #53](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/53). Rev-6 ships with full hydration still in the orchestrator Lambda. Reasons: (a) pure architectural relocation with no user-visible change, (b) ~2,700 lines porting surface (1,190 LOC of `context-hydration.ts` + 1,514 LOC of tests) requiring new boto3 surfaces in the container and a GraphQL GitHub client, (c) PR #52 already ships 10,000+ lines of changes across the SSE removal — folding in hydration would blur the review narrative. The Pydantic `SUPPORTED_HYDRATED_CONTEXT_VERSION` gate in `agent/src/models.py` bounds drift risk during the deferral window. + +### AD-12. `--trace` as the debug escape hatch + +Opt-in per task: 4 KB previews + full trajectory to S3 with TTL. + +*Why:* Without live streaming, debugging needs a richer offline artifact. Opt-in keeps normal-task storage costs flat. + +--- + +## 12. Implementation phases + +### Phase 1 — v1 PR + +- Single orchestrator path; delete all direct-SSE / two-runtime / interactive-mode infrastructure +- `bgagent status` (deterministic) +- `bgagent watch` with adaptive polling interval +- `bgagent nudge` with combined-turn acknowledgment +- FanOutConsumer router + per-channel default filters +- GitHub edit-in-place dispatcher (DDB-Stream ordering, 404 → POST fallback) +- Stub Slack/email dispatchers (log-only, ready for real integration in Phase 2) +- Unified stranded-task reconciler timeout +- `--trace` debug flag + +### Phase 2 — Ask + first real notifications + +- `bgagent ask` end-to-end (REST, agent-side between-turns hook, foreground block-and-poll CLI, durability-on-disconnect) +- Real Slack dispatcher (webhook + action buttons → approval callback Lambda) +- Per-task notification config + `bgagent notifications configure` + +### Phase 3 — Cedar HITL + +- Hard-gate approval gates with Cedar policy evaluation +- `bgagent approve` / `deny` / `pending` / `policies` +- `AWAITING_APPROVAL` state + orchestrator handling +- Full design in `PHASE3_CEDAR_HITL.md` + +### Phase 4 — Dispatcher polish + +- Real email dispatcher (SES) +- Real GitHub dispatcher (beyond the v1 edit-in-place stub) +- Per-repo default notification config +- `--verbose` opt-in for milestone-level events +- Dashboard widgets for notification delivery health + +### Deferred + +- **LLM-synthesized status summary** — `bgagent ask` without targeting the agent; Lambda calls an LLM to narrate state. Cost + hallucination trade-offs; revisit if v1 feedback warrants. +- **Cedar `effect: "advise"` tier** — non-blocking FYI policy tier for post-v1. Design sketch in `PHASE3_CEDAR_HITL.md`. +- **Outbound WebSocket from agent** — only if a concrete sub-200 ms latency requirement surfaces. Agent-initiated egress avoids dual-auth problems and works on any compute. +- **Multi-user watch** — multiple users attached to the same task's live event stream (teams). + +--- + +## 13. Open questions + +| ID | Question | Owner | +|---|---|---| +| Q1 | Retention policy for `--trace` S3 artifacts — 7 days or 30? Size cap per user? | Design | +| Q2 | Should `bgagent pending` (Phase 3) show all pending approvals across all of a user's tasks, or filter to a single `task_id`? | Phase 3 impl | +| Q3 | Slack action button callbacks — Slack signing secret rotation strategy? | Phase 2 impl | +| Q4 | Per-repo default notification config precedence vs per-task overrides — does per-task always win? Partial overrides? | Phase 4 impl | +| Q5 | `bgagent ask` concurrent limit — do we expose `--queue` semantics to explicitly enqueue vs 429? | Phase 2 impl | + +--- + +## Appendix A — Claude Agent SDK reference + +Pinned version: `claude-agent-sdk==0.1.53` (Python). + +### A.1 Hook surface (v0.1.53) + +`HookEvent` enum: `PreToolUse | PostToolUse | PostToolUseFailure | UserPromptSubmit | Stop | SubagentStart | SubagentStop | PreCompact | PermissionRequest | Notification`. + +Our usage: +- `PreToolUse` → Cedar policy evaluation (Phase 3), `can_use_tool`-style allow/deny. +- `PostToolUse` → output scanner (secret/PII redaction). +- `Stop` (between-turns) → `_cancel_between_turns_hook`, `_nudge_between_turns_hook`, Phase 2 ask hook, Phase 3 approval poll. + +### A.2 Between-turns injection mechanism + +Stop hook return values: +- `{}` → no-op, SDK proceeds to stop or loop. +- `{"decision": "block", "reason": ""}` → SDK emits `reason` as a synthetic user turn; agent responds on its next iteration. + +This is the **only** SDK-supported mechanism to inject agent-visible text mid-conversation. Implications: +- Every nudge, ask, and deny-with-steering burns one turn from `max_turns`. +- No "append to system prompt mid-run" primitive exists. `ClaudeAgentOptions.system_prompt` is set at construction. +- `hookSpecificOutput.additionalContext` on PostToolUse appears in docs but does not reach the model's context; Anthropic has confirmed this as `not-planned` (GitHub issues `claude-code#18427`, `claude-code#19643`). + +### A.3 Mid-run cancellation + +`ClaudeSDKClient.interrupt()` cancels the current turn without rolling back prior tool results. Used in our cancel path along with `cancellation_requested` flag on `TaskRecord`. + +--- + +## Appendix B — AgentCore Runtime reference + +### B.1 Service contract + +- HTTP on port 8080: `/invocations` (JSON + optional SSE response), `/ping` (liveness). +- `/ping` returning `"HealthyBusy"` signals an active session and prevents idle eviction. +- `maxLifetime` and `idleRuntimeSessionTimeout` both configurable up to 8 hours. We set both to the maximum. + +### B.2 Invocation + +`bedrock-agentcore:InvokeAgentRuntime` — SigV4-authenticated API call from the orchestrator. Payload is the task context; response is ignored (fire-and-forget). + +### B.3 Session management + +Same `runtimeSessionId` routes to the same MicroVM **within the same runtime ARN**. We use this property for the agent's own internal resumability (re-invocation with the same session ID lands on the same container if it's still alive), but never for CLI→runtime direct attach (which we don't do). + +--- + +## Appendix C — Competitive landscape + +Products surveyed for interaction patterns (primary sources: product docs, engineering blogs): + +| Product | Interaction model | Notes | +|---|---|---| +| **Devin (Cognition)** | Slack-thread chat during execution; fully async notifications | Closest analog; mid-run Q&A via in-thread messages is a shipped feature | +| **GitHub Copilot coding agent** | Fire-and-forget; progress visible as commits/PR activity | No mid-run steering; notifications via GitHub itself | +| **OpenAI Codex (cloud)** | SSE in web UI; external view is polling; no mid-run course-correction | Explicitly documents inability to steer mid-run | +| **Replit Agent** | Task board UI; user checks progress; no live terminal stream | Novel: automated "Decision-Time Guidance" (internal classifier-driven steering) | +| **Cursor background agents** | Pure fire-and-forget; user manually checks state | No built-in completion notifications (open feature request) | + +Key observations: +- Fire-and-forget + notifications is the dominant pattern for long-running coding agents. +- Mid-run steering exists only where there's a persistent conversation surface (Devin's Slack thread); our `bgagent nudge` + `bgagent ask` is the equivalent. +- No product ships "proceed with default if no response" for approval gates. Hard gates or no gates — that's the shipped landscape. +- Polling-based observation is ubiquitous and well-tolerated at minute-to-hour task durations. diff --git a/docs/design/PHASE3_CEDAR_HITL.md b/docs/design/PHASE3_CEDAR_HITL.md new file mode 100644 index 0000000..e676f89 --- /dev/null +++ b/docs/design/PHASE3_CEDAR_HITL.md @@ -0,0 +1,1883 @@ +# Phase 3 — Cedar-driven HITL Approval Gates + +> **Status:** Detailed design, pre-implementation. +> **Companion:** [`INTERACTIVE_AGENTS.md`](./INTERACTIVE_AGENTS.md) §5.6 (CLI commands), §8.2 (state machine). +> **Visual:** [`../diagrams/phase3-cedar-hitl.drawio`](../diagrams/phase3-cedar-hitl.drawio). +> **Rev:** 3 (2026-04-29 — hard-gate-only v1; notification-plane UX). +> **Implementation:** not started. + +--- + +## 0. Contents + +1. [What we are building, in one paragraph](#1-what-we-are-building-in-one-paragraph) +2. [The three-outcome model and why Cedar alone can't give it](#2-the-three-outcome-model) +3. [Design decisions (locked)](#3-design-decisions-locked) +4. [End-to-end request flow](#4-end-to-end-request-flow) +5. [Cedar policy authoring guide](#5-cedar-policy-authoring-guide) +6. [Engine implementation](#6-engine-implementation) +7. [REST API contract](#7-rest-api-contract) +8. [CLI UX](#8-cli-ux) +9. [State machine + concurrency](#9-state-machine--concurrency) +10. [Data model](#10-data-model) +11. [Observability](#11-observability) +12. [Security model](#12-security-model) +13. [Failure modes + fail-closed posture](#13-failure-modes--fail-closed-posture) +14. [Sample scenarios](#14-sample-scenarios) +15. [Implementation plan](#15-implementation-plan) +16. [Implementation notes (carry-forward tasks)](#16-implementation-notes-carry-forward-tasks) +17. [Deferred / out of scope](#17-deferred--out-of-scope) + +--- + +## 1. What we are building, in one paragraph + +When the agent is about to call a tool (Bash, Write, Edit, WebFetch, etc.), our Cedar policy engine decides **Allow** or **Deny**. Phase 3 adds a third outcome — **Require-approval** — that pauses the tool call, writes an approval request to a DynamoDB table **atomically with the task state transition**, dispatches a notification through the fan-out plane (Slack with action buttons, email, GitHub issue comment), and awaits a human response via a new REST endpoint + CLI command. The agent polls DynamoDB for the user's decision with strongly-consistent reads; on approval it proceeds, on denial (or timeout) the decision text is injected into the agent's context via the Phase 2 Stop-hook mechanism so the agent adapts rather than spinning. At task-submit time the user can also *pre-approve* scopes (specific tools, bash patterns, rule IDs, path patterns, or `all_session`) so low-risk agents run without any interactive gates. Cedar policies are tagged with a `@tier("hard-gate")` annotation to mark rules that should trigger an approval instead of an absolute deny — the same Cedar language, two policy files, one new outcome. + +**v1 ships hard gates only.** The agent pauses indefinitely for a decision (bounded only by the task's `maxLifetime`); on timeout it fail-closed denies with steering. There is no "proceed with default if no response" mode in v1 — see §17 for the deferred `@tier("advise")` semantics that would add non-blocking advisory events post-v1. + +--- + +## 2. The three-outcome model + +### Cedar's native model is binary + +The [Cedar authorization engine](https://www.cedarpolicy.com/) answers one question per call: given a `(principal, action, resource, context)` tuple, is the action **Allowed**, **Denied**, or **NoDecision** (no policy matched)? Our engine treats `NoDecision` as deny (fail-closed) and returns a boolean `allowed` to callers. That's the baseline Phase 3 extends. + +### What we add + +We layer a **three-outcome abstraction** on top of Cedar by running **two evaluations per tool call** against two separate policy sets: + +```text +┌──────────────────────────────────────────────────────────────────────────┐ +│ PolicyEngine.evaluate_tool_use(tool_name, tool_input) │ +│ │ +│ 1. Cedar eval against HARD_DENY_POLICIES │ +│ └─ Deny → return PolicyDecision(outcome=DENY, reason=...) │ +│ Absolute. No allowlist can override. │ +│ │ +│ 2. In-process allowlist fast-path │ +│ └─ match → return PolicyDecision(outcome=ALLOW, reason=...) │ +│ Pre-approved (from --pre-approve) or previously approved │ +│ with scope != this_call. │ +│ │ +│ 2.5. Recent-decision cache (anti-retry-loop) │ +│ └─ cached DENIED/TIMED_OUT for (tool_name, input_sha) within 60s │ +│ → auto-deny with same reason (prevents re-gate storms) │ +│ │ +│ 3. Cedar eval against HARD_GATE_POLICIES │ +│ └─ Deny → return PolicyDecision(outcome=REQUIRE_APPROVAL, │ +│ reason, timeout_s, severity, │ +│ matching_rule_ids) │ +│ Human must approve before the tool runs. Agent waits │ +│ indefinitely (bounded by task maxLifetime); timeout │ +│ fail-closed denies with steering. │ +│ │ +│ 4. Default ALLOW │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +Each evaluation is a Cedar call — sub-millisecond. No network hop. No AWS API. The "approval wait" (step 3's downstream handling) is entirely inside our `PreToolUse` hook coroutine. + +The SDK never sees `REQUIRE_APPROVAL` — after the wait, our hook returns the SDK's native `{"permissionDecision": "allow" | "deny"}` shape. The three-outcome model is an internal engine abstraction. + +### Why two policy sets, not one + +Cedar doesn't have a `require_approval` effect. We encode the tiering as a physical split into two policy files (`hard_deny.cedar`, `hard_gate.cedar`), validated by a `@tier("hard-deny" | "hard-gate")` annotation on each rule. + +Key properties: + +- **Security reviewers can read the hard-deny file alone.** Most review effort lives there because those rules are absolute; hard-gate rules have a human safety net. +- **Rule authors know where a rule lives by which file it's in.** No "forbid-with-marker" patterns that can be accidentally miscategorized. +- **Forward-compatible with a future `@tier("advise")` tier** (see §17) — a third file for non-blocking advisory rules can be added without changing the engine's outer loop. + +--- + +## 3. Design decisions + +| # | Decision | Summary | +|---|---|---| +| 1 | **Cedar encoding: two policy sets** | Physical split into `hard_deny.cedar` and `hard_gate.cedar`, validated via `@tier("hard-deny" \| "hard-gate")` annotation. Forward-compatible with a future `@tier("advise")` set (see §17). | +| 2 | **Hook point: extend `PreToolUse`, not `can_use_tool`** | PreToolUse is already async-compatible, already wired to Cedar, and already owns the tool-governance boundary. | +| 3 | **Wait mechanism: DDB strongly-consistent polling, 2s → 5s backoff** | Initial 2s cadence for the first 30s, then 5s. `ConsistentRead=True` so the agent never misses an approval that already landed. Agent waits indefinitely (bounded by `maxLifetime`). | +| 4 | **Scope allowlist: in-process, seeded from persisted `initial_approvals`** | Runtime escalation lives in the `PolicyEngine` instance. Submit-time `--pre-approve` flags persist on TaskTable and seed the allowlist at container startup. Lost on restart (rare; reconciler fails stranded tasks). | +| 5 | **CLI UX: standalone `bgagent approve/deny` + `--pre-approve ` + `bgagent policies list` + `bgagent pending`** | All REST-polling; no streaming prompts. User discovers pending approvals via `pending` or via the fan-out plane (Slack action buttons, email link). | +| 6 | **Timeouts: per-task default + per-rule Cedar annotation override, min wins, bounded floor + ceiling, fail-closed** | Floor: 30s (engine-enforced on both task default and rule annotations). Ceiling: `min(1h, maxLifetime - remaining_cleanup_margin)` — sized so the TTL on the approval row always covers the decision window. On timeout → fail-closed DENY with steering injected as a user turn. Never auto-approve, never proceed-with-default. | +| 7 | **Concurrency slots: AWAITING_APPROVAL holds the slot** | Matches PAUSED semantics. Container is alive, consuming memory. | +| 8 | **Hard-deny is absolute** | No `--pre-approve` scope can bypass it. CreateTaskFn validates and rejects `rule:`. | +| 9 | **Submit-time scope cap: 20 entries, ≤128 chars each** | Keeps audit trail legible, bounds allowlist check cost, limits abuse-vector damage. | +| 10 | **Cedar annotations** | `@rule_id(...)`, `@tier("hard-deny" \| "hard-gate")`, `@approval_timeout_s(...)`, `@severity(...)`, `@category(...)`. Recoverable via `cedarpy.policies_to_json_str()` → JSON. Multi-match merging: min timeout wins (clamped by floor), max severity wins. | +| 11 | **Atomic state transitions via DDB TransactWriteItems** | The approval-request row write and the TaskTable status transition are a single atomic transaction. No partial-failure states. | +| 12 | **Ownership encoded in ConditionExpression, not fetch-then-check** | `ApproveTaskFn` / `DenyTaskFn` put `user_id = :caller` into the ConditionExpression on TaskApprovalsTable. Authorization and state transition are atomic. | +| 13 | **Per-task approval-gate cap: 50, fail-task on exceed** | Prevents denial-loop storms. Matches Phase 2 nudge cap. | +| 14 | **Per-minute approval-creation rate limit: 20/task** | Agent-side throttle independent of per-task lifetime cap. | +| 15 | **Recent-decision cache: deny an identical (tool, input) for 60s after DENIED/TIMED_OUT** | Prevents retry-loop amplification on the same destructive action. | +| 16 | **Denial reason sanitized in the Lambda, before persisting** | `DenyTaskFn` runs `output_scanner` on the reason before writing to DDB. The agent never sees unscanned text. | +| 17 | **`tool_input_preview` stripped of ANSI/control characters at agent-side write + CLI render** | Defense in depth against approver-confusion attacks where a prompt-injected tool input overwrites the CLI prompt with a different command. | +| 18 | **Deny-as-steering injected via Stop hook `between_turns_hooks`, not via `permissionDecisionReason`** | Reuses the validated Phase 2 nudge mechanism. `` XML block wrapped by the same `_xml_escape` utility. | +| 19 | **`rule:` discovery via new endpoint** | `GET /v1/repos/{repo_id}/policies` + `bgagent policies list` surfaces the rule IDs + annotations + whether each rule is hard-deny or hard-gate. Solves the otherwise-undiscoverable `rule:X` pre-approval scope. | +| 20 | **`write_path:` scope** | Added so users can pre-approve file writes under specific path patterns (e.g., `write_path:docs/**`) without needing to grant all Writes. | +| 21 | **`tool_group:file_write` convenience scope** | Resolves to `{Write, Edit}`. Prevents the surprise of pre-approving `Write` and still getting gated on `Edit`. | +| 22 | **Pre-implementation spike: cedarpy annotation round-trip** | Day 1 of implementation validates that `policies_to_json_str()` returns annotations in the expected shape. If the API has changed, fall back to policy-ID prefix conventions. | +| 23 | **Approval notifications via the fan-out plane** | `approval_required` events flow through `FanOutConsumer` → Slack/email/GitHub dispatchers. Slack messages include `Approve` / `Deny` action buttons that POST to the REST API. No streaming CLI prompts. | + +--- + +## 4. End-to-end request flow + +Narrative walk-through of the happy path. Sequence diagrams in [phase3-cedar-hitl.drawio pages 3-6](../diagrams/phase3-cedar-hitl.drawio). + +### Setup (task start) + +1. User runs `bgagent submit --repo my-org/my-app --task "rebase feature-x onto main and push" --approval-timeout 600 --pre-approve tool_type:Read --pre-approve bash_pattern:"git status*"`. +2. CLI validates each scope string client-side (format, ≤128 chars, cap 20). Rejects invalid syntax without round-trip. +3. CLI POSTs `/v1/tasks` with `{repo, task, initial_approvals: [...], approval_timeout_s: 600}`. +4. `CreateTaskFn` validates `initial_approvals`: + - max 20 entries, ≤128 chars each + - rejects `rule:` where `` names a hard-deny rule (resolved via shared policy-parsing library against the repo's blueprint; see §5.4) + - rejects degenerate `bash_pattern`/`write_path` scopes that match too broadly (see §7.3) + - honors `Blueprint.security.maxPreApprovalScope` (see §7.3) + - normalizes scope strings (trim whitespace; case-sensitive as documented) +5. Task persists. `approval_timeout_s` and `initial_approvals` become DDB attributes on the task row. +6. Container spawns on the AgentCore Runtime. `PolicyEngine.__init__` loads: + - `HARD_DENY_POLICIES` (built-in + repo blueprint's `security.cedarPolicies.hard`) + - `HARD_GATE_POLICIES` (built-in + repo blueprint's `security.cedarPolicies.hard_gate`) + - Annotation lookup table: `{policy_id: {annotation: value}}` built from `cedarpy.policies_to_json_str()` once, cached for the task lifetime + - Rule-ID map: `{rule_id_annotation: policy_id}` to resolve `--pre-approve rule:` → internal Cedar policy ID + - Allowlist seeded from `initial_approvals` + - Annotation validation: `@rule_id` uniqueness enforced (duplicate = task fails to start); `@approval_timeout_s` must be integer ≥ 30 (malformed or below floor = task fails to start) +7. Container emits `agent_milestone("pre_approvals_loaded", {count: 2, scopes: ["tool_type:Read", "bash_pattern:git status*"]})` so Terminal A's stream shows the starting posture. +8. Agent begins normal work. + +### First approval gate (hard-gate hit) + +9. Agent decides to run `Bash(command="git push --force origin feature-x")`. +10. SDK fires `PreToolUse` hook with `tool_name="Bash"`, `tool_input={command: "..."}`. +11. Hook calls `PolicyEngine.evaluate_tool_use`: + - Hard-deny eval: matches nothing → `allowed=True` + - Allowlist fast-path: `tool_type:Bash`? no. `bash_pattern` matches `git push --force ...`? `git status*` doesn't match `git push --force ...` → skip + - Recent-decision cache: no matching `(Bash, sha256(input))` in cache → skip + - Hard-gate eval: policy `push_to_protected_branch` matches. `diagnostics.reasons == ["policy1"]`. Lookup: `policy1` → annotations `{rule_id: "push_to_protected_branch", approval_timeout_s: "300", severity: "medium"}`. + - Returns `PolicyDecision(outcome=REQUIRE_APPROVAL, reason="Cedar hard-gate: push_to_protected_branch", timeout_s=300, severity="medium", matching_rule_ids=["push_to_protected_branch"])`. + + Effective timeout computation: + ``` + effective = max( + FLOOR_30S, + min( + rule_annotation_timeout_s or task_default, # 300 + task_default, # 600 from submit + maxLifetime_remaining_s - CLEANUP_MARGIN_120S # ~7h remaining + ) + ) + → effective = 300s + ``` + If `maxLifetime_remaining_s - CLEANUP_MARGIN_120S < FLOOR_30S`, hook returns DENY immediately with reason `"insufficient lifetime for approval"` (§13.7). + +12. Hook checks per-task approval-gate cap (50) and per-minute rate limit (20/task). If either exceeded → DENY with reason `"approval-gate cap exceeded"` (fail-closed). +13. Hook mints `request_id = _ulid()` (26-char ULID). +14. Hook builds the approval row payload: + ```python + row = { + "task_id": "01KPW...", + "request_id": "01KPR...", + "tool_name": "Bash", + "tool_input_preview": strip_ansi("git push --force origin feature-x")[:256], + "tool_input_sha256": "abc123...", + "reason": "Cedar hard-gate: push_to_protected_branch", + "severity": "medium", + "matching_rule_ids": ["push_to_protected_branch"], # list, not set — supports empty + "status": "PENDING", + "created_at": "2026-04-23T14:00:00Z", + "timeout_s": 300, + "ttl": 1734567890, # created_at + timeout_s + CLEANUP_MARGIN_120S; always covers the decision window + "user_id": "...", + "repo": "my-org/my-app" + } + ``` +15. **Atomic transition** — hook issues `TransactWriteItems` with two operations: + - Put on `TaskApprovalsTable` (new row with status=PENDING) + - ConditionalUpdate on `TaskTable`: `status = :awaiting, awaiting_approval_request_id = :rid WHERE status = :running` + Both succeed or both fail. On `TransactionCanceledException` (most likely the TaskTable condition fails because another process moved the status), the hook emits `approval_write_failed` and returns DENY. +16. Hook emits `agent_milestone("approval_requested", {...})` to both `ProgressWriter` (DDB audit) and `sse_adapter` (live stream). Best-effort emission — transactional write has already committed; milestone failure is observability degradation, not state degradation. +17. Terminal A stream renders: + ``` + [14:00:00] ★ approval_requested: Bash "git push --force origin feature-x" (medium) + reason: Cedar hard-gate: push_to_protected_branch + bgagent approve 01KPR... [--scope ...] + bgagent deny 01KPR... [--reason "..."] + timeout 300s + ``` + Severity colors the line (respecting `NO_COLOR` env var). +18. Hook enters poll loop with strongly-consistent reads: + ```python + async def _poll_for_decision(task_id, request_id, timeout_s): + start = time.monotonic() + interval = 2 + consecutive_failures = 0 + while True: + elapsed = time.monotonic() - start + if elapsed >= timeout_s: + return TimedOut() + if elapsed > 30: + interval = 5 # backoff + try: + row = await _ddb_get_approval(task_id, request_id, ConsistentRead=True) + consecutive_failures = 0 + if row is None: + # Row disappeared between write and poll — treat as stranded + return TimedOut(reason="approval row missing; fail-closed") + if row["status"] != "PENDING": + return Decided(row) + except Exception as exc: + consecutive_failures += 1 + if consecutive_failures == 3: + log("WARN", f"approval poll degraded for {request_id}: {exc}") + emit_milestone("approval_poll_degraded", {...}) + if consecutive_failures >= 10: + return TimedOut(reason="approval poll consecutive failures") + await asyncio.sleep(interval) + ``` +19. The approval CAP and local-timeout paths ALWAYS attempt to write the row to TIMED_OUT (best-effort conditional update `status = :pending`) before returning. This prevents orphan PENDING rows when the agent bails internally. + +### User responds + +20. User in Terminal B runs `bgagent approve --scope tool_type_session`. +21. CLI validates scope syntax client-side. +22. CLI POSTs `/v1/tasks/{task_id}/approve` with `{request_id, decision: "approve", scope: "tool_type_session"}`. +23. `ApproveTaskFn`: + - Validates Cognito JWT, extracts `sub` as `caller_user_id`. + - Single `UpdateItem` on `TaskApprovalsTable` with compound ConditionExpression: + ``` + #status = :pending AND user_id = :caller AND task_id = :task_id + ``` + If all three conditions hold → atomic flip to APPROVED. Ownership + state + existence check in a single call. No TOCTOU. + - On `ConditionalCheckFailedException` with `ReturnValuesOnConditionCheckFailure=ALL_OLD`: distinguishes between (a) row missing (404 `REQUEST_NOT_FOUND`), (b) wrong user (404 `REQUEST_NOT_FOUND` — don't leak existence), (c) wrong status (409 `REQUEST_ALREADY_DECIDED`). + - Records audit event to TaskEventsTable directly (`approval_decision_recorded`) so the 90-day audit trail is owned by the Lambda, not dependent on agent milestones. + - Returns 202 `{task_id, request_id, status: "APPROVED", scope, decided_at}` or error. +24. Agent's poll reads the `APPROVED` row on next tick (within 2-5s). +25. Hook executes decision in this order: + - a. **Atomic resume transition**: `TransactWriteItems` — TaskTable `status = :running, REMOVE awaiting_approval_request_id WHERE status = :awaiting AND awaiting_approval_request_id = :rid`. If this fails (likely because user cancelled during the poll gap), hook skips allowlist mutation and returns DENY with reason `"task no longer awaiting approval"`. + - b. **Allowlist mutation** (only if `scope != "this_call"`): `PolicyEngine._allowlist.add(scope)`. Synchronously logged. + - c. **Milestone emission** (best-effort): `approval_granted` to both writers. + - d. **Return to SDK**: `{"permissionDecision": "allow"}`. +26. SDK runs the tool. Stream shows: + ``` + [14:00:12] ★ approval_granted: request_id=01KPR... scope=tool_type_session + [14:00:12] ▶ Bash: git push --force origin feature-x + [14:00:14] ◀ Bash: remote: Force pushed. New SHA abc123. + ``` + +### Continuation + +27. Agent continues with its turn, hits another `Bash` call (say `git log --oneline -5`). +28. PreToolUse hook → PolicyEngine.evaluate_tool_use: + - Hard-deny: no match + - Allowlist: `tool_type:Bash` ← matches. Returns ALLOW fast-path. +29. No new approval request. Tool runs immediately. +30. Eventually agent reaches task completion, opens PR, writes memory, task → `COMPLETED`. + +### Denial with steering text + +If instead the user runs `bgagent deny --reason "use --force-with-lease instead"`: +- `DenyTaskFn` runs `output_scanner.scan(reason)` to redact any accidental secrets/PII from the reason **before** writing it to DDB. +- Flips row to DENIED with sanitized reason, atomic ownership check. +- Agent's poll reads DENIED row. +- Hook execution order: + - a. Atomic resume transition to RUNNING (same as approve path). + - b. **Inject denial into agent context** via the Phase 2 `between_turns_hooks` registry. The hook appends a synthetic `sanitized reason` block to the next Stop-seam injection. This reuses the validated nudge mechanism (§5 below) — the agent sees the denial as authoritative context on its next turn boundary. + - c. Milestone emission: `approval_denied`. + - d. Return to SDK: `{"permissionDecision": "deny", "permissionDecisionReason": "User denied — see next-turn context for reason"}`. The SDK's denial text is a terse fallback; the steering text arrives via the robust Stop-hook injection, not via `permissionDecisionReason`. + +Why this dual path: the Claude Agent SDK's `permissionDecisionReason` reaches the model as a tool-call-rejected system hint, which the model treats as a reason-to-retry-differently signal but is NOT the guaranteed steering surface the Phase 2 nudge pattern uses. By layering the denial as a between-turns user message, we guarantee the steering text becomes authoritative context. The `permissionDecisionReason` remains there as belt-and-suspenders for model versions that don't emit a Stop between the rejected tool call and the next turn. + +--- + +## 5. Cedar policy authoring guide + +### 5.1 Policy file layout + +Two physical files, each with exactly one tier: + +- `agent/policies/hard_deny.cedar` — contains ONLY `@tier("hard-deny")` policies +- `agent/policies/hard_gate.cedar` — contains ONLY `@tier("hard-gate")` policies + +Per-repo customization lives in `blueprint.yaml`: + +```yaml +security: + cedarPolicies: + hard_deny: | + @tier("hard-deny") + @rule_id("block_prod_writes") + forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like "prod/**" }; + hard_gate: | + @tier("hard-gate") + @rule_id("deploy_staging") + @approval_timeout_s("900") + @severity("high") + @category("destructive") + forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*terraform apply*" }; + maxPreApprovalScope: "tool_type_session" # optional; caps what --pre-approve can grant +``` + +`PolicyEngine.__init__` concatenates built-in + blueprint rules per tier, validates with a probe `cedarpy.is_authorized()` call. **Any** of the following cause task-start failure (not silent-fallback): +- Malformed policy syntax +- Duplicate `@rule_id` values across tiers +- `@approval_timeout_s` below floor (30s) or non-integer +- `@tier` value mismatches the file section (rules whose @tier disagrees with the file they live in) +- Missing `@rule_id` on a hard-gate rule + +Fail-on-error is the right posture for blueprint misconfiguration — silent-fallback would let broken policies slip into production. + +### 5.2 Annotation reference + +| Annotation | Required? | Values | Purpose | +|---|---|---|---| +| `@rule_id("...")` | **Yes on hard-gate**, recommended on hard-deny | Kebab-case or snake_case identifier, unique across both tiers | Stable ID for `--pre-approve rule:X`, for audit trail, and for the `bgagent policies` discovery endpoint. `PolicyEngine.__init__` raises on duplicates. | +| `@tier("hard-deny"\|"hard-gate")` | **Yes** | Exactly one of "hard-deny" or "hard-gate" | Validates policy is in the correct file/section. Engine rejects mismatch at load time. | +| `@approval_timeout_s("N")` | No | Integer seconds ≥ 30 | Per-rule timeout. If absent, uses task default. Has no effect on hard-deny rules. Values below the floor are rejected at load time. | +| `@severity("low"\|"medium"\|"high")` | No | One of the three | Shown in CLI approval prompt, colored by severity. Default: "medium". | +| `@category("...")` | No | "destructive", "network", "filesystem", "auth", or free-form | UX grouping. CLI could filter approvals by category. Not enforced. | + +### 5.3 Common policy patterns + +**Block absolute dangers** (hard-deny): +```cedar +@tier("hard-deny") +@rule_id("rm_slash") +forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*rm -rf /*" }; + +@tier("hard-deny") +@rule_id("write_git_internals") +forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like ".git/*" }; + +@tier("hard-deny") +@rule_id("write_git_internals_nested") +forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like "*/.git/*" }; + +@tier("hard-deny") +@rule_id("drop_table") +forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*DROP TABLE*" }; +``` + +**Absolute deny on destructive git ops** (hard-deny — part of the built-in starter set): +```cedar +@tier("hard-deny") +@rule_id("force_push_main") +@severity("high") +@category("destructive") +forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*git push --force origin main*" + || context.command like "*git push --force origin prod*" + || context.command like "*git push -f origin main*" + || context.command like "*git push -f origin prod*" }; +``` + +Force-pushing to `main` or `prod` is the canonical "you almost certainly don't want this" action. Absolute deny; not bypassable via `--pre-approve`. A repo that legitimately needs this (release automation) adds an override in its blueprint and removes this rule from the policy set. + +**Gate non-force pushes to protected branches** (hard-gate — part of the built-in starter set): +```cedar +@tier("hard-gate") +@rule_id("push_to_protected_branch") +@approval_timeout_s("300") +@severity("medium") +@category("destructive") +forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*git push origin main*" + || context.command like "*git push origin prod*" + || context.command like "*git push origin master*" + || context.command like "*git push origin release/*" }; +``` + +A non-force push to a protected branch gates — catches the case where an agent tries to push directly rather than opening a PR. Low frequency, high impact → worth waiting for a human. + +**Absolute deny on credential writes** (hard-deny — part of the built-in starter set): +```cedar +@tier("hard-deny") +@rule_id("write_credentials") +@severity("high") +@category("auth") +forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like "*credentials*" }; +``` + +Writing a file with "credentials" in the path is a strong signal of accidental secret persistence. Absolute deny. + +**Gate `.env` writes** (hard-gate — part of the built-in starter set): +```cedar +@tier("hard-gate") +@rule_id("write_env_files") +@approval_timeout_s("600") +@severity("high") +@category("filesystem") +forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like "*.env" }; +``` + +`.env` writes are plausibly intentional (template scaffolding, `.env.example` generation) but high-impact enough to warrant a human decision. + +**Optional patterns (not shipped by default — copy into your blueprint if your repo needs them):** +```cedar +// Gate writes under a conventional infrastructure/ directory. Not in the +// built-in set because the "infrastructure/" path is a repo convention, +// not a standard — many repos use cdk/, terraform/, deploy/, etc. Add to +// your blueprint if your repo uses this layout. +// @tier("hard-gate") +// @rule_id("write_infrastructure") +// @approval_timeout_s("900") +// @severity("high") +// @category("filesystem") +// forbid (principal, action == Agent::Action::"write_file", resource) +// when { context.file_path like "infrastructure/*" }; + +// Gate all outbound WebFetch. Not in the built-in set because DNS +// Firewall already restricts egress to an allowlist; gating every +// WebFetch produces high-volume approval requests on doc-heavy tasks. +// Add to your blueprint if your repo wants stricter scrutiny. +// @tier("hard-gate") +// @rule_id("webfetch_any") +// @approval_timeout_s("300") +// @severity("medium") +// @category("network") +// forbid (principal, action == Agent::Action::"invoke_tool", +// resource == Agent::Tool::"WebFetch"); + +// Gate writes to specific CI config. Example — tune paths per repo. +// @tier("hard-gate") +// @rule_id("write_github_workflows") +// @approval_timeout_s("600") +// @severity("high") +// @category("filesystem") +// forbid (principal, action == Agent::Action::"write_file", resource) +// when { context.file_path like ".github/workflows/*" }; +``` + +Per the sentinel trick (see §6.2), `invoke_tool` matches on the real tool-name UID. The other actions (`write_file`, `execute_bash`) use a sentinel UID with the real value in `context`. + +### 5.4 Policy discovery — shared parser + +Because `CreateTaskFn` needs to validate `rule:` pre-approvals against the target repo's actual policy set, we ship a **shared policy-parsing library** used in both places: + +- `cdk/src/handlers/shared/cedar-policy.ts` — thin wrapper around cedarpy's JSON form for TypeScript +- `agent/src/policy.py` — the full engine + +Both consume the blueprint's `security.cedarPolicies` section. `CreateTaskFn` loads the target repo's blueprint (via the existing `RepoTable` store), concatenates with the built-in policies, parses via `cedarpy.policies_to_json_str()`, and extracts `rule_id` + `tier` annotations. `--pre-approve rule:X` is validated: +- `X` exists as some rule's `@rule_id` → ok +- `X` refers to a hard-deny rule → 400 at submit time (hard-deny cannot be bypassed) +- `X` refers to a hard-gate rule → ok; passes through + +Runtime enforcement is still the authoritative layer. Submit-time validation is a UX guard — any drift between submit-time and runtime-loaded policies (possible if blueprint changes between them) causes the task to fail at container start with a clear error, not silently misbehave. + +### 5.5 Gotchas for policy authors + +**`like` is glob, not regex.** Only `*` (zero-or-more) and `?` (exactly-one-char) wildcards. If you need regex, write multiple `forbid` rules. + +**Case sensitivity.** `like` is case-sensitive. `*rm -rf*` won't match `*Rm -Rf*`. If case-insensitivity matters, write both variants. + +**Don't match `resource ==` for user-supplied values.** `Bash` commands and file paths go through the sentinel UID. Always use `context.command` / `context.file_path` in the `when` clause, never `resource == ...`. + +**`@rule_id` must be globally unique.** Including across tiers. `PolicyEngine.__init__` raises on duplicates. + +**Hard-deny rules shouldn't have `@approval_timeout_s`.** It has no effect. Engine logs WARN but doesn't reject (backward compatibility if someone moves a rule between tiers). + +**The default ruleset is shared across all tasks.** Per-task overrides live in the Blueprint and are isolated to tasks on that repo. The engine never allows a task to loosen the default hard-deny set via Blueprint — only add to it. + +**`@approval_timeout_s` values below 30 are rejected at load.** There is no way to configure unusably-short approval windows. + +--- + +## 6. Engine implementation + +### 6.1 Extended `PolicyDecision` shape + +```python +from dataclasses import dataclass +from enum import Enum + +class Outcome(str, Enum): + ALLOW = "allow" + DENY = "deny" # absolute (hard-deny or upstream error or cap-exceeded) + REQUIRE_APPROVAL = "require_approval" # hard-gate hit + +@dataclass(frozen=True) +class PolicyDecision: + outcome: Outcome + reason: str + # Only populated when outcome == REQUIRE_APPROVAL: + timeout_s: int | None = None + severity: str | None = None + matching_rule_ids: tuple[str, ...] = () + duration_ms: float = 0 + + @property + def allowed(self) -> bool: + """Backward-compat shim for Phase 1a/1b callers.""" + return self.outcome == Outcome.ALLOW +``` + +### 6.2 `evaluate_tool_use` skeleton + +```python +def evaluate_tool_use(self, tool_name: str, tool_input: dict) -> PolicyDecision: + start = time.monotonic() + base_context = {"task_type": self._task_type, "repo": self._repo} + input_sha = _sha256(json.dumps(tool_input, sort_keys=True)) + + # STEP 1 — Hard-deny (absolute) + hard = self._eval_tier(self._hard_policies, tool_name, tool_input, base_context) + if hard.decision == "deny": + return PolicyDecision(outcome=Outcome.DENY, + reason=f"Hard-deny: {hard.rule_ids}", + duration_ms=_elapsed(start)) + + # STEP 2 — Allowlist fast-path (covers tool_type, bash_pattern, write_path, all_session) + if self._allowlist.matches(tool_name, tool_input): + return PolicyDecision(outcome=Outcome.ALLOW, + reason="Pre-approved by allowlist", + duration_ms=_elapsed(start)) + + # STEP 2.5 — Recent-decision cache (anti-retry-loop, 60s TTL) + cached = self._recent_decisions.get((tool_name, input_sha)) + if cached is not None: + return PolicyDecision(outcome=Outcome.DENY, + reason=f"Recent decision ({cached.decision}) within 60s: {cached.reason}", + duration_ms=_elapsed(start)) + + # STEP 3 — Hard-gate (require approval) + gate = self._eval_tier(self._hard_gate_policies, tool_name, tool_input, base_context) + if gate.decision == "deny": + # Rule-scope allowlist check happens AFTER hard-gate eval (rule_ids + # aren't known until Cedar tells us which policies matched) + if any(rid in self._allowlist._rule_ids for rid in gate.rule_ids): + return PolicyDecision(outcome=Outcome.ALLOW, + reason=f"Allowlist rule: {gate.rule_ids}", + duration_ms=_elapsed(start)) + + annotations = self._merge_annotations(gate.rule_ids) + return PolicyDecision( + outcome=Outcome.REQUIRE_APPROVAL, + reason=f"Hard-gate: {', '.join(annotations['rule_ids'])}", + timeout_s=annotations["timeout_s"], + severity=annotations["severity"], + matching_rule_ids=tuple(annotations["rule_ids"]), + duration_ms=_elapsed(start), + ) + + # STEP 4 — Default allow + return PolicyDecision(outcome=Outcome.ALLOW, reason="permitted", + duration_ms=_elapsed(start)) +``` + +The recent-decision cache is a simple `dict[(tool_name, input_sha), (decision, reason, inserted_at)]` with a 60-second sliding window. Entries are added by the PreToolUse hook whenever an approval resolves to DENIED or TIMED_OUT — not on APPROVED (we don't want to accidentally auto-deny a tool call the user just approved). Cache is in-process, lost on restart. + +### 6.3 Annotation merging + +When multiple hard-gate rules match a single tool call: + +```python +def _merge_annotations(self, policy_ids: list[str]) -> dict: + rule_ids, timeouts, severities = [], [], [] + for pid in policy_ids: + ann = self._annotations[pid] + rule_ids.append(ann.get("rule_id", pid)) + if "approval_timeout_s" in ann: + try: + t = int(ann["approval_timeout_s"]) + if t >= FLOOR_30S: + timeouts.append(t) + except ValueError: + log("WARN", f"malformed @approval_timeout_s on {ann.get('rule_id', pid)}") + severities.append(ann.get("severity", "medium")) + + # Task default always eligible + timeouts.append(self._task_default_timeout_s) + + raw_min_timeout = min(timeouts) + return { + "rule_ids": rule_ids, + "timeout_s": max(FLOOR_30S, raw_min_timeout), # floor enforcement + "severity": _max_severity(severities), # "high" > "medium" > "low" + } +``` + +**Rationale for min/max choices**: +- **Timeout → min (above floor)**: multiple rules matching means multiple concerns. Users should have *less* time to decide when stakes are higher. Floor prevents unusable 5s windows. +- **Severity → max**: the most severe concern governs the UX coloring. + +### 6.4 Allowlist data structure + +```python +class ApprovalAllowlist: + def __init__(self, initial_scopes: list[str]): + self._all_session = False + self._tool_types: set[str] = set() + self._tool_groups: set[str] = set() # file_write → {Write, Edit} + self._rule_ids: set[str] = set() + self._bash_patterns: list[str] = [] # glob patterns + self._write_path_patterns: list[str] = [] # glob patterns, for Write/Edit file_path + + for scope in initial_scopes: + self.add(scope) + + TOOL_GROUPS = {"file_write": {"Write", "Edit"}} + + def add(self, scope: str) -> None: + if scope == "all_session": + self._all_session = True + elif scope.startswith("tool_type:"): + self._tool_types.add(scope.split(":", 1)[1]) + elif scope.startswith("tool_group:"): + group = scope.split(":", 1)[1] + if group not in self.TOOL_GROUPS: + raise ValueError(f"unknown tool_group: {group!r}") + self._tool_groups.add(group) + elif scope.startswith("rule:"): + self._rule_ids.add(scope.split(":", 1)[1]) + elif scope.startswith("bash_pattern:"): + self._bash_patterns.append(scope.split(":", 1)[1]) + elif scope.startswith("write_path:"): + self._write_path_patterns.append(scope.split(":", 1)[1]) + else: + raise ValueError(f"unknown scope: {scope!r}") + + def matches(self, tool_name: str, tool_input: dict) -> bool: + if self._all_session: + return True + if tool_name in self._tool_types: + return True + for group in self._tool_groups: + if tool_name in self.TOOL_GROUPS[group]: + return True + if tool_name == "Bash": + cmd = tool_input.get("command", "") + if any(fnmatch(cmd, pat) for pat in self._bash_patterns): + return True + if tool_name in ("Write", "Edit"): + path = tool_input.get("file_path", "") + if any(fnmatch(path, pat) for pat in self._write_path_patterns): + return True + # rule_ids matched after hard-gate eval — see evaluate_tool_use + return False +``` + +### 6.5 PreToolUse hook changes + +Phase 3 PreToolUse hook (compressed for doc; implementation will be richer): + +```python +async def pre_tool_use_hook(hook_input, tool_use_id, ctx, *, + engine, task_id, user_id, progress, sse_adapter, + task_default_timeout_s): + tool_name, tool_input = _extract(hook_input) + decision = engine.evaluate_tool_use(tool_name, tool_input) + + if decision.outcome == Outcome.ALLOW: + return _allow() + if decision.outcome == Outcome.DENY: + return _deny(decision.reason) + + # REQUIRE_APPROVAL path. + # Cap + rate-limit check. + if engine.approval_gate_count >= APPROVAL_GATE_CAP_PER_TASK: + return _deny("approval-gate cap exceeded (50/task)") + if engine.approvals_in_last_minute >= APPROVAL_RATE_LIMIT: + return _deny("approval-gate rate limit exceeded (20/min)") + + # Compute effective timeout with floor/ceiling. + remaining = _remaining_maxlifetime_s() + effective_timeout = max( + FLOOR_30S, + min(decision.timeout_s or task_default_timeout_s, + task_default_timeout_s, + remaining - CLEANUP_MARGIN_120S), + ) + if remaining - CLEANUP_MARGIN_120S < FLOOR_30S: + return _deny(f"insufficient maxLifetime remaining ({remaining}s) for approval") + + request_id = _ulid() + engine.approval_gate_count += 1 + + row = { + "task_id": task_id, "request_id": request_id, + "tool_name": tool_name, + "tool_input_preview": _strip_ansi(_preview(tool_input))[:256], + "tool_input_sha256": _sha256(_serialize(tool_input)), + "reason": decision.reason, "severity": decision.severity, + "matching_rule_ids": list(decision.matching_rule_ids), + "status": "PENDING", + "created_at": _iso_now(), + "timeout_s": effective_timeout, + "ttl": int(time.time()) + effective_timeout + CLEANUP_MARGIN_120S, + "user_id": user_id, "repo": engine.repo, + } + + # ATOMIC: put approval row + transition TaskTable status in one transaction. + try: + await _transact_write_approval_request(task_id, request_id, row) + except TransactionCanceledException as exc: + # Either the task was concurrently cancelled, or status wasn't RUNNING. + _emit("approval_write_failed", {"request_id": request_id, "reason": str(exc)}) + return _deny("approval system unavailable") + + _emit("approval_requested", { + "request_id": request_id, "tool_name": tool_name, + "input_preview": row["tool_input_preview"], + "reason": decision.reason, "severity": decision.severity, + "timeout_s": effective_timeout, + "matching_rule_ids": list(decision.matching_rule_ids), + }) + + outcome = await _poll_for_decision(task_id, request_id, effective_timeout) + + # On TIMED_OUT, attempt to write the row to TIMED_OUT so future reads see + # a terminal state (not orphaned PENDING). + if outcome.status == "TIMED_OUT": + await _best_effort_update_status(task_id, request_id, "TIMED_OUT", + reason=outcome.reason) + + # ATOMIC: resume TaskTable status RUNNING, conditional on awaiting_approval_request_id matching. + try: + await _transact_resume(task_id, request_id) + except TransactionCanceledException: + # User cancelled (or some other path) during poll; abandon gracefully. + _emit("approval_resume_failed", {"request_id": request_id}) + return _deny("task no longer awaiting approval") + + if outcome.status == "APPROVED": + if outcome.scope and outcome.scope != "this_call": + engine._allowlist.add(outcome.scope) + _emit("approval_granted", {"request_id": request_id, + "scope": outcome.scope or "this_call", + "decided_at": outcome.decided_at}) + return _allow() + + # DENIED or TIMED_OUT — cache for 60s + inject denial via Stop hook path. + engine._recent_decisions.record( + tool_name, _sha256(_serialize(tool_input)), + decision="DENIED" if outcome.status == "DENIED" else "TIMED_OUT", + reason=outcome.reason, + ) + if outcome.status == "DENIED": + # Queue steering injection via Stop hook's between_turns_hooks. + engine._queue_denial_injection( + request_id=request_id, + reason=outcome.reason, # already sanitized by DenyTaskFn + decided_at=outcome.decided_at, + ) + _emit("approval_denied" if outcome.status == "DENIED" else "approval_timed_out", + {"request_id": request_id, "reason": outcome.reason}) + return _deny(f"User {outcome.status.lower()}: see next turn context for details") +``` + +`engine._queue_denial_injection` appends to a list consumed by a new `_denial_between_turns_hook` — registered alongside `_nudge_between_turns_hook` in the Phase 2 `between_turns_hooks` list. At the next Stop hook fire, the denial is emitted as `…` XML (sanitized via `_xml_escape` from the shared utility introduced with Phase 2). + +--- + +## 7. REST API contract + +### 7.1 `POST /v1/tasks/{task_id}/approve` + +**Request** (CLI → API Gateway → `ApproveTaskFn`): +```http +POST /v1/tasks/01KPW.../approve +Authorization: Bearer +Content-Type: application/json + +{ + "request_id": "01KPR...", + "decision": "approve", + "scope": "tool_type_session" +} +``` + +**Responses**: + +| Status | Code | When | Body | +|---|---|---|---| +| 202 | — | Success | `{task_id, request_id, status: "APPROVED", scope, decided_at}` | +| 400 | `VALIDATION_ERROR` | Bad scope format, missing fields | `{error, message, field}` | +| 401 | `UNAUTHORIZED` | Missing/invalid JWT | — | +| 404 | `REQUEST_NOT_FOUND` | Row missing OR wrong user (both surfaces 404 to prevent enumeration) | — | +| 409 | `REQUEST_ALREADY_DECIDED` | Status != PENDING | `{error, message, current_status}` | +| 409 | `TASK_NOT_AWAITING_APPROVAL` | Task's current status is not AWAITING_APPROVAL | `{error, message, current_status}` | +| 429 | `RATE_LIMIT_EXCEEDED` | Per-user > 30 approve/min | — | +| 503 | `SERVICE_UNAVAILABLE` | DDB throttled or upstream failure | — | + +**Authorization + state + existence check is a single DDB operation**: +```python +response = ddb.update_item( + TableName=TASK_APPROVALS_TABLE, + Key={"task_id": task_id, "request_id": request_id}, + UpdateExpression="SET #s = :approved, decided_at = :now, #sc = :scope", + ConditionExpression="#s = :pending AND user_id = :caller", + ExpressionAttributeNames={"#s": "status", "#sc": "scope"}, + ExpressionAttributeValues={ + ":approved": "APPROVED", ":pending": "PENDING", + ":now": now_iso, ":scope": scope, ":caller": cognito_sub, + }, + ReturnValuesOnConditionCheckFailure="ALL_OLD", +) +``` + +On `ConditionalCheckFailedException`: +- If `OldImage` is absent → row never existed → 404 `REQUEST_NOT_FOUND` +- If `OldImage.user_id != caller` → 404 (same code, prevent existence oracle) +- If `OldImage.status != "PENDING"` → 409 `REQUEST_ALREADY_DECIDED` + +In addition, the Lambda does a separate GetItem on `TaskTable` to check `status == "AWAITING_APPROVAL"` — if the task has already moved (e.g., was cancelled), return 409 `TASK_NOT_AWAITING_APPROVAL` before even attempting the update. This check is belt-and-suspenders; the atomic UpdateItem handles the rest. + +After successful update, `ApproveTaskFn` writes an audit event to `TaskEventsTable` (`approval_decision_recorded` event_type), ensuring the 90-day audit trail is owned by the Lambda path — not dependent on the agent's milestone emission. + +### 7.2 `POST /v1/tasks/{task_id}/deny` + +Identical shape with `decision: "deny"` and optional `reason`: + +```json +{ + "request_id": "01KPR...", + "reason": "use force-with-lease instead; force is too risky" +} +``` + +`DenyTaskFn`: +1. Auth check (Cognito JWT) +2. Run `output_scanner.scan(reason)` — redacts AWS keys, GitHub PATs, API tokens, etc. from the reason text before persisting +3. Truncate sanitized reason to 2000 chars (matches Phase 2 nudge limit for consistency) +4. Atomic conditional update (same shape as approve) +5. Write audit event to TaskEventsTable + +The agent reads the sanitized reason from DDB. It never sees unscanned user text. + +### 7.3 `POST /v1/tasks` — new optional fields + +Extended request shape: + +```json +{ + "repo": "my-org/my-app", + "task": "...", + "task_type": "new_task", + "approval_timeout_s": 600, + "initial_approvals": [ + "tool_type:Read", + "bash_pattern:git status*", + "write_path:docs/**", + "rule:safe_read_config", + "tool_group:file_write" + ] +} +``` + +`CreateTaskFn` validations: +1. Length cap: ≤20 entries +2. Per-entry length cap: ≤128 chars +3. Scope format parsing: normalized to known shape; leading/trailing whitespace trimmed +4. Scope value validation: + - `tool_type:X` — X must be in known tool set (Read, Bash, Write, Edit, Glob, Grep, WebFetch, ...) + - `tool_group:X` — X must be in known group set (currently `file_write`) + - `bash_pattern:X` — X ≤128 chars; reject if X is degenerate (`*`, `**`, `?*`, or patterns where wildcard-char ratio exceeds 50%) — see §7.4 + - `write_path:X` — same rules as bash_pattern + - `rule:X` — X must exist in the (built-in + target repo's blueprint) hard-gate policy set per the shared policy-parsing library; hard-deny rule IDs rejected + - `all_session` — rejected if `Blueprint.security.maxPreApprovalScope` forbids +5. `approval_timeout_s` within `[30, min(3600, maxLifetime - 300)]` — cap at 1 hour OR (maxLifetime - 5min), whichever is smaller. Prevents multi-hour slot-exhaustion attacks and keeps approval windows within the TTL budget. + +### 7.4 Degenerate-pattern detection + +A pattern is considered degenerate if: +- Length ≤ 2, OR +- Consists only of `*`, `?`, and whitespace, OR +- Ratio of wildcard chars (`*` + `?`) to literal chars exceeds 50% + +Degenerate `bash_pattern:` and `write_path:` scopes are rejected at submit with 400 `VALIDATION_ERROR`. Users wanting broad permission must use the explicit `all_session` scope (which is subject to `maxPreApprovalScope` blueprint cap). + +### 7.5 `maxPreApprovalScope` ordering + +Blueprint's `maxPreApprovalScope` is a partial order: + +``` +this_call < { tool_type_session, tool_group, bash_pattern, write_path, rule } < all_session +``` + +If `maxPreApprovalScope: "tool_type_session"`, `all_session` is rejected. All other scopes pass. Setting it to `"this_call"` (meaningless) is rejected at blueprint load. Blueprint absence defaults to unbounded (except `all_session` requires explicit `--yes` on CLI). + +### 7.6 `GET /v1/repos/{repo_id}/policies` + +New read-only endpoint for rule discovery and `bgagent policies list`: + +**Response** (200): +```json +{ + "repo_id": "my-org/my-app", + "policies": { + "hard_deny": [ + {"rule_id": "rm_slash", "category": "destructive", + "summary": "Reject rm -rf / and similar"}, + {"rule_id": "force_push_main", "category": "destructive", + "summary": "Reject force-push to main/prod"}, + {"rule_id": "write_credentials", "category": "auth", + "summary": "Reject writes to paths containing 'credentials'"}, + ... + ], + "hard_gate": [ + {"rule_id": "push_to_protected_branch", "severity": "medium", + "category": "destructive", "approval_timeout_s": 300, + "summary": "Non-force push to a protected branch"}, + {"rule_id": "write_env_files", "severity": "high", + "category": "filesystem", "approval_timeout_s": 600, + "summary": "Write to *.env files"}, + ... + ] + } +} +``` + +Loaded by the Lambda on demand from the target repo's blueprint + built-in policies. `summary` is a human-readable annotation `@summary("...")` if present, else falls back to the first line of the `when` clause rendered as text. + +Rate-limited 30/min/user; cached 5min per repo in-Lambda. + +--- + +## 8. CLI UX + +### 8.1 New commands + +```bash +# Approve a specific pending request +bgagent approve [--scope ] [--output text|json] + +# Deny a specific pending request, optionally with a reason the agent sees (sanitized server-side) +bgagent deny [--reason "..."|--reason-file ] [--output text|json] + +# List all pending approvals across the user's active tasks (solves request-id lookup) +bgagent pending [--output text|json] + +# Discover policies for a repo (solves rule-id lookup) +bgagent policies list --repo [--tier hard-deny|hard-gate] [--output text|json] +bgagent policies show --repo --rule [--output text|json] +``` + +### 8.2 Extended `submit` / `run` flags + +```bash +bgagent submit \ + --repo my-org/my-app \ + --task "..." \ + --approval-timeout 600 \ + --pre-approve tool_type:Read \ + --pre-approve write_path:"docs/**" \ + --pre-approve tool_group:file_write \ + --pre-approve rule:safe_file_read \ + --pre-approve-file ./approvals.yaml + +# Shorthand for no approval gates (requires --yes): +bgagent submit --task "..." --pre-approve all_session --yes +``` + +`--pre-approve-file` reads a YAML/JSON array of scope strings — supports the 20-entry cap without command-line bloat. + +### 8.3 Notification UX + +Approval requests surface through the fan-out plane (see [`INTERACTIVE_AGENTS.md`](./INTERACTIVE_AGENTS.md) §6) — not through a CLI stream. When the agent emits an `approval_required` event to `TaskEventsTable`, `FanOutConsumer` routes it per the user's notification config: + +- **Slack**: posts a message to the configured channel with `Approve` / `Deny` action buttons. Button click invokes an interaction-callback Lambda that writes to `TaskApprovalsTable` via the same path `bgagent approve` uses. +- **Email**: sends a one-line summary with a link that deep-links to the approve/deny REST endpoint (optional authenticated click-through). +- **GitHub issue comment**: appends to the in-place comment that the task is waiting for approval (visible to anyone watching the issue). +- **CLI via `bgagent watch`**: the event shows up in the polling stream as any other event: + +```text +[14:00:00] ★ approval_requested: Bash "git push origin main" (severity=medium) + reason: Cedar hard-gate: push_to_protected_branch + respond: bgagent approve 01KPR... [--scope tool_type_session] + bgagent deny 01KPR... [--reason "..."] + timeout: 300s (or "bgagent pending" to list all) +``` + +`bgagent watch` formats the line with severity color (respecting `NO_COLOR`; emits `[HIGH]` prefix when set). No interactive prompt in the watch stream — approval responses are always explicit commands. + +**Discovery path.** A user who wasn't watching at all finds pending approvals via: + +- `bgagent pending` — lists every open approval across the user's tasks. +- Slack button click — zero commands, one-tap response. +- Inbound from email link → REST API. + +### 8.4 Safety UX + +When `--pre-approve all_session` is passed without `--yes`: + +```bash +$ bgagent submit --task "apply terraform plan" --pre-approve all_session +WARNING: --pre-approve all_session disables Cedar hard-gate approval gates + for this task. Hard-deny policies (rm -rf /, write to .git/, DROP + TABLE, etc.) still apply. + Add --yes to skip this prompt. +Continue? [y/N] +``` + +Hard-deny enforcement is clearly called out so users don't mistake `all_session` for root. + +### 8.5 `bgagent pending` output + +```text +Pending approvals (3): + + 01KPW0...(task) / 01KPR0...(request) + ├─ Bash: git push --force origin feature-x + ├─ severity: high + ├─ reason: Cedar hard-gate: push_to_protected_branch + ├─ timeout: 4m 32s remaining + └─ approve|deny + + 01KPW1.../01KPR1... + ├─ Write: /workspace/.../src/.env + ├─ severity: high + ├─ timeout: 9m 12s remaining + ... +``` + +Picking one (`bgagent approve` or `bgagent deny` with the listed IDs) is straightforward. Shell completion (tab-complete task_id + request_id from `bgagent pending` output) is a Phase 3b enhancement. + +--- + +## 9. State machine + concurrency + +### 9.1 New state: AWAITING_APPROVAL + +Transitions added (extending §7 of INTERACTIVE_AGENTS.md): + +``` +RUNNING → AWAITING_APPROVAL (on REQUIRE_APPROVAL; via TransactWriteItems) +AWAITING_APPROVAL → RUNNING (on approve OR deny OR timeout; via TransactWriteItems) +AWAITING_APPROVAL → CANCELLED (on explicit `bgagent cancel`) +AWAITING_APPROVAL → FAILED (on reconciler detecting stranded approval; new edge) +HYDRATING → AWAITING_APPROVAL (if a hard-gate gate fires during hydration; rare but possible) +``` + +No direct `AWAITING_APPROVAL → COMPLETED/FINALIZING` without RUNNING in between. + +### 9.2 Orchestrator impact + +- `waitStrategy` adds `AWAITING_APPROVAL` as non-terminal. +- `finalizeTask` recognizes `AWAITING_APPROVAL`. +- `ACTIVE_STATUSES` (used by `GET /tasks?status=active` and `reconcile-concurrency.ts`) gains `AWAITING_APPROVAL`. +- `task_state.py::write_terminal` condition expression accepts `AWAITING_APPROVAL` as a valid source state. + +### 9.3 Concurrency slot semantics + +**AWAITING_APPROVAL holds the user's concurrency slot.** + +Rationale: the Docker container is alive. Memory allocated. The AgentCore microVM pool is committed. Releasing the slot while the resource is still held lies to accounting and opens a resource-exhaustion vector. + +Concrete behavior: + +```text +Bob's per-user cap: 10. +t=0: Bob submits 10 tasks. count=10. 11th submit → 429. +t=2m: Task #1 → AWAITING_APPROVAL. count still 10. + Bob's 12th submit → 429. He must approve, cancel, or wait. +t=30m: Bob approves task #1. task → RUNNING. count still 10. +t=45m: Task #1 completes. count → 9. Bob can submit task #11. +``` + +### 9.4 `maxLifetime` clock does not pause + +AgentCore Runtime's `maxLifetime = 28800s` (8h) is an absolute timer from session start. It does NOT pause during `AWAITING_APPROVAL`. + +This has a concrete implication: the hook computes an `effective_timeout` bounded by `maxLifetime - remaining - CLEANUP_MARGIN_120S`. If the task has been running 7h55m and hits a hard-gate gate, the effective timeout might be clamped to a much shorter value than the task default. Below the 30s floor → immediate DENY with reason `"insufficient lifetime"`. + +### 9.5 Stranded-approval reconciliation + +`reconcile-stranded-tasks.ts` gains an AWAITING_APPROVAL-aware branch: + +- Detects tasks in AWAITING_APPROVAL with `age > 2 * timeout_s` +- Best-effort conditional-updates TaskApprovalsTable row → `STRANDED` status +- Transitions TaskTable → `FAILED` with reason `"approval stranded (container eviction)"` +- Emits `approval_stranded` event to TaskEventsTable + +This closes the Phase 3a container-eviction gap. Without this, a container restart mid-approval would leave the task hanging until the user manually cancelled. + +`reconcile-concurrency.ts` (scheduled every 5 min) already scans for orphaned concurrency counters; with `AWAITING_APPROVAL` added to `ACTIVE_STATUSES` it correctly counts awaiting tasks as active. + +### 9.6 Attended vs unattended mode + +The design assumes a human is watching. For truly unattended tasks (scheduled automation, cron-driven runs) the `--pre-approve all_session` path skips hard-gate entirely. No additional mode flag needed — the set of scopes in `initial_approvals` dictates the attendance expectation. + +--- + +## 10. Data model + +### 10.1 New DynamoDB table: `TaskApprovalsTable` + +```typescript +new dynamodb.Table(this, 'Table', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + sortKey: { name: 'request_id', type: dynamodb.AttributeType.STRING }, // ULID + billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, + pointInTimeRecovery: true, + timeToLiveAttribute: 'ttl', + stream: dynamodb.StreamViewType.NEW_AND_OLD_IMAGES, // (evaluated — may drop; see §11) + removalPolicy: RemovalPolicy.RETAIN, +}); +``` + +Attributes: + +| Name | Type | Required | Description | +|---|---|---|---| +| `task_id` | S | Yes | PK; ULID matching TaskTable | +| `request_id` | S | Yes | SK; ULID minted by agent | +| `tool_name` | S | Yes | "Bash", "Write", etc. | +| `tool_input_preview` | S | Yes | First 256 chars of serialized tool input, ANSI/control-stripped | +| `tool_input_sha256` | S | Yes | Full-input hash for audit + recent-decision cache | +| `reason` | S | Yes | Cedar matching rule description | +| `severity` | S | Yes | "low" \| "medium" \| "high" | +| `matching_rule_ids` | L | Yes | List (not Set — can be empty) of hard-gate rule IDs | +| `status` | S | Yes | PENDING \| APPROVED \| DENIED \| TIMED_OUT \| STRANDED | +| `created_at` | S | Yes | ISO8601 | +| `decided_at` | S | No | Set when status != PENDING | +| `scope` | S | No | Set on APPROVED | +| `deny_reason` | S | No | Set on DENIED; sanitized user text | +| `timeout_s` | N | Yes | Resolved timeout for audit | +| `ttl` | N | Yes | `created_at_epoch + timeout_s + CLEANUP_MARGIN_120S` — always covers the decision window | +| `user_id` | S | Yes | Used in ownership check `ConditionExpression` | +| `repo` | S | Yes | Denormalized for fan-out | + +**TTL sizing**: the TTL is always `timeout_s + 120s`, so a 300s approval window has a 420s TTL, a 3600s window has a 3720s TTL. The row never expires during the decision window. After the decision + a short grace period, DDB's eventual-consistency TTL reaper cleans up. + +**Why a list, not a StringSet, for `matching_rule_ids`**: DDB string sets cannot be empty. Pathological no-match hard-gate hits would fail to persist. Lists handle empty gracefully. + +**Why no GSI in v1**: query pattern is always `(task_id, request_id)` for agent polls; the `bgagent pending` listing is implemented as a Scan with FilterExpression `user_id = :caller AND status = :pending` — acceptable at current scale. When pending-approval volume grows, add a GSI on `user_id`. + +### 10.2 `TaskTable` additions + +Four new attributes on the existing task row: + +| Name | Type | Required | Description | +|---|---|---|---| +| `approval_timeout_s` | N | No | Default timeout for hard-gate gates. Default 300. | +| `initial_approvals` | L | No | List of scope strings from submit time | +| `awaiting_approval_request_id` | S | No | Set when status = AWAITING_APPROVAL; cleared on transition back (via joint `UpdateExpression`) | +| `approval_gate_count` | N | No | Running counter of approval gates fired on this task; used to enforce the 50-gate cap | + +Joint updates on AWAITING_APPROVAL transitions always set/clear `awaiting_approval_request_id` in the same `UpdateExpression` as the status change — either within the TransactWriteItems Put+Update, or in the single UpdateItem on resume. + +### 10.3 TaskTable status enum update + +```typescript +export const TASK_STATUSES = [ + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', + 'FINALIZING', 'COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT', +] as const; + +export const ACTIVE_STATUSES = new Set([ + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', +]); + +export const VALID_TRANSITIONS = { + // ...existing... + RUNNING: ['FINALIZING', 'CANCELLED', 'TIMED_OUT', 'FAILED', 'AWAITING_APPROVAL'], + AWAITING_APPROVAL: ['RUNNING', 'CANCELLED', 'FAILED'], // FAILED via reconciler only + HYDRATING: ['RUNNING', 'FAILED', 'CANCELLED', 'AWAITING_APPROVAL'], // rare but possible + // ... +}; +``` + +--- + +## 11. Observability + +### 11.1 New `agent_milestone` event types + +Emitted to both `ProgressWriter` (DDB, 90d) and `sse_adapter` (live stream). Plus audit events emitted by the REST Lambdas directly to TaskEventsTable. + +| Event | Source | Metadata | +|---|---|---| +| `pre_approvals_loaded` | Agent | `{count, scopes[]}` | +| `approval_requested` | Agent | `{request_id, tool_name, input_preview, reason, severity, timeout_s, matching_rule_ids[]}` | +| `approval_granted` | Agent | `{request_id, scope, decided_at}` | +| `approval_denied` | Agent | `{request_id, reason, decided_at}` | +| `approval_timed_out` | Agent | `{request_id, timeout_s}` | +| `approval_stranded` | Reconciler | `{request_id, age_s, reason}` | +| `approval_write_failed` | Agent | `{request_id?, error}` | +| `approval_resume_failed` | Agent | `{request_id, error}` | +| `approval_poll_degraded` | Agent | `{request_id, consecutive_failures}` | +| `approval_timeout_capped` | Agent | `{requested: N, effective: M, reason}` — surfaces when min-wins clips user's requested timeout | +| `approval_cap_exceeded` | Agent | `{request_id, count, cap}` — when 50-gate cap fires | +| `approval_rate_limit_exceeded` | Agent | `{request_id, rate, limit}` | +| `approval_decision_recorded` | ApproveTaskFn / DenyTaskFn | `{request_id, status, scope?, reason?, decided_at, caller_user_id}` — authoritative audit record | + +### 11.2 Fan-out plane — primary UX channel for approvals + +Approval events flow to the `FanOutConsumer` router via `TaskEventsTable` DDB Streams (see [`INTERACTIVE_AGENTS.md`](./INTERACTIVE_AGENTS.md) §6). The router invokes per-channel dispatcher Lambdas (`SlackDispatchFn`, `EmailDispatchFn`, `GitHubDispatchFn`) according to the user's notification config. + +**`TaskApprovalsTable` Streams are NOT consumed by the fan-out router.** The approval row is working state; the audit trail is in `TaskEventsTable`. Enabling Streams on `TaskApprovalsTable` would duplicate events for no benefit. Final design: `TaskApprovalsTable` does not have Streams enabled. + +**Per-channel event routing for approval events:** + +| Channel | Events subscribed by default | Payload notes | +|---|---|---| +| **Slack** | `approval_required`, `approval_decided` (granted/denied), `approval_timed_out` | Messages include `Approve` / `Deny` action buttons on `approval_required`. Button click → Slack interaction-callback Lambda → POSTs to `/v1/tasks/{id}/approvals/{request_id}` via the user's Cognito-mapped identity. | +| **Email (SES)** | `approval_required` with `severity: high` | Deep-link URL to the REST endpoint; user signs in once, decision routed. | +| **GitHub issue comment** | `approval_required` appended to the in-place comment | Visible to anyone watching the originating issue. | + +**Rate-limited per-user**: 10 approval-related fan-out messages per user per minute. Prevents notification spam. Rate-limit counter shared with other approval-related events (requested, stranded, decided); enforced in the router before dispatcher invocation. + +**Slack button security**: `approve` / `deny` button payloads are signed by Slack; the interaction-callback Lambda validates the signing secret before writing. User mapping from Slack user ID → Cognito user ID is configured per workspace via `bgagent notifications configure --workspace `. + +### 11.3 Dashboard additions + +Extend `TaskDashboard` (`cdk/src/constructs/task-dashboard.ts`). These are read-only CloudWatch widgets that surface approval behavior to operators; no notification channel or on-call action required: + +- **Approval request rate** (line, 7d): count of `approval_requested` per hour, across all tasks. +- **Approval response time** (line + p50/p99): `decided_at - created_at`, per decision; plotted for the three outcome types. +- **Outcome distribution** (stacked bar, per hour): granted / denied / timed_out / stranded. Inverts quickly if notifications break. +- **Active AWAITING_APPROVAL tasks** (gauge): current count across the fleet. +- **Per-task approval-gate count distribution** (histogram): spot tasks approaching the 50-gate cap. +- **Top hard-gate rules by match frequency** (table): which rules are firing; informs rule tuning over time. + +### 11.4 OTEL trace integration + +Every `agent_milestone("approval_*")` event carries `trace_id` / `span_id`. A span `hitl.approval_wait` brackets the PreToolUse poll loop: `span.duration = decided_at - created_at`. `hitl.approval_race_loss` emitted when the agent's local timeout fired <5s before a late user decision (useful for tuning). + +### 11.5 CloudWatch alarms — deferred + +Operator-facing CloudWatch alarms that would page on: +- High approval-timeout rate (users not responding, notifications broken) +- Tasks stuck in AWAITING_APPROVAL beyond `timeout_s + 60s` (reconciler failure) +- High approval-write failure rate (DDB throttled or IAM drift) +- Approval-gate cap hit (suspicious retry loop) + +…are **out of scope for Phase 3a** because the project does not yet have a notification channel (Slack / PagerDuty / SNS topic / email distribution list) configured for operational alerts. Adding alarms without a notification channel produces CloudWatch widgets that nobody sees — no safety benefit. + +If / when an operational channel is added to the stack, these alarms become a small follow-up: wire CloudWatch metric filters on the milestone event types already emitted (§11.1), then an alarm + SNS action per threshold. The supporting metric data already flows (decisions 3-15 guarantee it); only the plumbing is deferred. + +--- + +## 12. Security model + +### 12.1 Trust boundaries + +- **Agent container ↔ TaskApprovalsTable**: IAM role on the runtime has `GetItem` / `PutItem` / conditional `UpdateItem` on the table. Agent writes pending, reads decisions, writes TIMED_OUT on internal timeout. +- **User CLI ↔ API Gateway**: Cognito JWT (same authorizer as `/tasks/*`). +- **ApproveTaskFn/DenyTaskFn ↔ TaskApprovalsTable**: Lambda IAM policy allows `UpdateItem` with authorization condition (`user_id = :caller`) built into the ConditionExpression. +- **Blueprint origin**: blueprints are CDK-deployed constructs (see `cdk/src/constructs/blueprint.ts`). Platform operators deploy them. Users cannot upload arbitrary blueprint.yaml from the target repo. This property is load-bearing for the security model — if blueprint origin ever becomes user-uploaded, the blueprint-injection section (§12.4) must be re-evaluated. + +### 12.2 Ownership encoded in ConditionExpression + +No TOCTOU window. The single `UpdateItem` on `TaskApprovalsTable` encodes: + +``` +#status = :pending AND user_id = :caller +``` + +Authorization and state transition are atomic. A compromised internal caller (Lambda with raw DDB access) or a logic bug in a future refactor that forgets the ownership check still can't flip rows without matching the `user_id`. + +### 12.3 Race prevention + +**Race 1 — user approves at T, agent times out at T+ε**: +- Agent's poll loop times out → best-effort conditional update `status = TIMED_OUT WHERE status = :pending` +- User's CLI writes `APPROVED WHERE status = :pending` +- One wins atomically +- The loser: + - If TIMED_OUT wins: user gets 409 `REQUEST_ALREADY_DECIDED`. User sees "approval expired". + - If APPROVED wins: agent's poll reads APPROVED on next tick. Agent proceeds. + +**Race 2 — double-approve**: +- Two concurrent CLI invocations. Second gets 409 `REQUEST_ALREADY_DECIDED`. Idempotent. + +**Race 3 — cancel during AWAITING_APPROVAL**: +- Agent writes `RUNNING WHERE status = :awaiting AND awaiting_approval_request_id = :rid` +- User writes `CANCELLED WHERE status = :awaiting` (via `bgagent cancel`) +- If CANCELLED wins: agent's resume fails with TransactionCanceledException. Hook emits `approval_resume_failed` and returns DENY. Task is already CANCELLED; agent's turn is aborted. +- If RUNNING wins: `bgagent cancel` gets 409 `TASK_ALREADY_RUNNING` (or similar) — user sees "task resumed before cancel landed". + +### 12.4 Blueprint content safety + +The blueprint trust model (§12.1) means blueprint Cedar policies are trusted by construction. Nonetheless the engine enforces: + +- Cedar syntax validation at load → fail-on-error +- Duplicate `@rule_id` → fail-on-error +- `@tier` mismatch with physical file/section → fail-on-error +- `@approval_timeout_s < 30` → fail-on-error +- Missing `@rule_id` on hard-gate rule → fail-on-error + +These guard against blueprint misconfiguration, not malicious intent. If the blueprint model ever changes to user-uploadable, additional safeguards needed: per-blueprint policy count cap (50), total policy text size cap (64KB), per-eval timeout on `is_authorized` (100ms). + +### 12.5 `all_session` does not override hard-deny + +Hard-deny is evaluated FIRST, before the allowlist fast-path (§6.2). No `initial_approvals` scope can bypass it. `CreateTaskFn` rejects `rule:` at submit. + +### 12.6 Denial reason sanitization in the Lambda + +`DenyTaskFn` runs `output_scanner.scan(reason)` — the existing agent-side scanner that redacts AWS keys, GitHub PATs, OAuth tokens, and common secrets — **before** persisting to DDB. + +Sanitization at the Lambda layer means: +- TaskApprovalsTable stores only sanitized text (visible to operators with DDB read) +- TaskEventsTable audit record stores only sanitized text (90d retention) +- Fan-out Slack/email notifications only see sanitized text +- Agent reads sanitized text verbatim; no secondary scanning needed + +Additionally, both CLI and Lambda log `message_length` not `reason` in CloudWatch logs (matching Phase 2 nudge logging discipline). + +### 12.7 `tool_input_preview` terminal-escape sanitization + +`_strip_ansi` removes: +- ANSI CSI sequences (`\x1b[...m`, etc.) +- OSC sequences (`\x1b]...\x07`) +- Control characters below 0x20 except `\t\n` +- DEL (0x7F) + +Applied at two layers: +- **Agent-side at write**: `tool_input_preview` is sanitized before DDB Put +- **CLI-side at render**: `bgagent pending`, `bgagent approve` output, and the live stream renderer all pass preview text through `_strip_ansi` before display + +Defense in depth: rows written before the agent-side sanitization landed (if any) are still rendered safely. + +### 12.8 Recent-decision cache prevents approval-gate storms + +After a DENIED or TIMED_OUT outcome, the engine caches `(tool_name, tool_input_sha256)` for 60s. The agent's next identical tool call auto-denies without a new approval request. A prompt-injected agent cannot burn through approval gates with the same destructive action. + +Cache is NOT populated on APPROVED (don't want to cache-block a just-approved call). + +### 12.9 Per-task + per-rate caps + +- Per-task hard cap: 50 approval gates. Exceeded → task → FAILED with reason `"approval-gate cap exceeded"`. +- Per-minute rate limit: 20 approval-row writes. Exceeded → fail-closed deny on the gate that tripped it. +- Fan-out notification cap: 10 approval-related messages per user per minute. Exceeded → messages dropped (logged). + +These caps bound the worst-case behavior of a compromised account or prompt-injected agent. + +### 12.10 JWT replay + +Cognito JWT with signature + expiry validation on API Gateway. Approval row conditional-update prevents replay from mutating state. + +--- + +## 13. Failure modes + fail-closed posture + +### 13.1 DDB write failure at approval creation + +TransactWriteItems fails → hook emits `approval_write_failed` and returns DENY. No partial-state leakage. + +### 13.2 Poll read failures + +- Single failed GetItem: log WARN, continue polling +- After 3 consecutive failures: emit `approval_poll_degraded` event +- After 10 consecutive failures: treat as TIMED_OUT, best-effort UpdateItem to TIMED_OUT, fail-closed deny to SDK + +### 13.3 Ownership mismatch + +ApproveTaskFn sees JWT whose sub doesn't match row's user_id: atomic conditional-update fails → returns 404 `REQUEST_NOT_FOUND` (no existence oracle). + +### 13.4 Cedar engine crash mid-evaluation + +`evaluate_tool_use` catches all exceptions from `cedarpy.is_authorized` and returns `Outcome.DENY` with reason `"fail-closed: "`. Matches existing behavior. + +### 13.5 Multiple matching rules with conflicting annotations + +Covered in §6.3 (min timeout clamped by floor; max severity). + +### 13.6 Container restart mid-approval + +Detected by `reconcile-stranded-tasks.ts` (§9.5). Transitions task to FAILED with reason `"approval stranded (container eviction)"`. User sees clear failure, can resubmit. No silent hang. + +### 13.7 Insufficient lifetime remaining for approval + +If `remaining_maxLifetime - CLEANUP_MARGIN_120S < FLOOR_30S`, hook immediately returns DENY with reason `"insufficient maxLifetime for approval"`. Task continues without a gate — or, if the gate was load-bearing, fails gracefully in RUNNING state. + +### 13.8 PreToolUse hook itself crashes + +Existing behavior: hook's outer try/except returns fail-closed deny. Extended in Phase 3 to log hook crash with context (request_id if available) for triage. + +### 13.9 Resume transition fails (user cancelled during poll) + +Hook emits `approval_resume_failed` and returns DENY. Task is already in its new state (CANCELLED); hook doesn't attempt to resume. + +--- + +## 14. Sample scenarios + +### 14.1 Scenario A: force-push with per-rule timeout + +Setup: repo `my-org/my-app` blueprint extends hard-gate with `force_push_main` (@approval_timeout_s=600). Task default is 300s. + +```bash +$ bgagent submit --repo my-org/my-app \ + --task "merge feature-x into main and push" \ + --approval-timeout 300 +``` + +Agent runs `git push origin main`. `push_to_protected_branch` matches (non-force push to a protected branch). Annotations: `timeout_s=300`, `severity=medium`. + +``` +[14:00:00] ★ approval_requested: Bash "git push origin main" (severity=medium) + reason: Cedar hard-gate: push_to_protected_branch + respond: bgagent approve 01KPR... [--scope tool_type_session] + timeout: 300s +``` + +User approves with `tool_type_session` (either via `bgagent approve` or a Slack button). Events: + +``` +[14:00:08] ★ approval_granted: request_id=01KPR... scope=tool_type_session +[14:00:08] ▶ Bash: git push origin main +[14:00:10] ◀ Bash: Everything up-to-date +``` + +Later `git status` call → allowlist fast-path → no new approval. + +### 14.2 Scenario B: Force-push to main hits hard-deny + +Agent proposes `Bash: git push --force origin main`. Hard-deny rule `force_push_main` matches → immediate DENY with reason `"Hard-deny: force_push_main"`. No approval request. Task stays in RUNNING. + +Recent-decision cache now has `(Bash, sha256("git push --force origin main"))` for 60s — a retry would auto-deny without re-running Cedar. + +Agent adapts, opens a PR via `gh pr create` instead. No rule matches. Tool runs. + +### 14.3 Scenario C: Trusted automation with `all_session` + +```bash +$ bgagent submit --repo my-org/infra \ + --task "apply approved terraform plan for staging-v2" \ + --pre-approve all_session --yes +``` + +Blueprint on `my-org/infra` allows `maxPreApprovalScope: "all_session"`. Task runs fully autonomously. Zero approval gates. Hard-deny still enforces. + +Stream shows `[14:20:00] ★ pre_approvals_loaded: count=1 scopes=[all_session]` at startup so operators see the starting posture. + +### 14.4 Scenario D: Denying with steering reason + +```bash +$ bgagent submit --repo my-org/my-app \ + --task "Update the deployment scripts to use the new release branch" \ + --approval-timeout 600 +``` + +Agent tries `Bash: git push origin release/v2`. Hard-gate rule `push_to_protected_branch` hits. `approval_requested` → user: + +```bash +$ bgagent deny 01KPW... 01KPR... \ + --reason "move it to src/dashboard/v1.deprecated instead of deleting; we may need to reference it in migrations" +``` + +`DenyTaskFn` sanitizes (no secrets in this reason, passes through unchanged), writes to DDB. Agent's poll reads DENIED. + +Hook executes: atomic resume to RUNNING → queue denial injection via `between_turns_hooks` → return to SDK with fallback deny reason. + +Next Stop seam fires. The between-turns injector emits: + +```xml + +move it to src/dashboard/v1.deprecated instead of deleting; we may need to reference it in migrations + +``` + +Agent reads the denial on its next turn, adapts: + +``` +[14:30:12] ▶ Bash: git mv src/dashboard/v1 src/dashboard/v1.deprecated +[14:30:13] ◀ Bash: (success) +``` + +Task proceeds. Denial-as-steering worked via the same robust path Phase 2 nudges use. + +### 14.5 Scenario E: AI-DLC phased pre-approvals + +Three-phase workflow with escalating trust: + +```bash +# Phase 1 — analysis only +$ bgagent submit --repo my-org/new-feature \ + --task "analyze the existing auth module and produce a design doc" \ + --pre-approve tool_type:Read \ + --pre-approve tool_type:Glob \ + --pre-approve tool_type:Grep \ + --pre-approve bash_pattern:"ls *" \ + --pre-approve bash_pattern:"find *" + +# Phase 2 — documentation writes +$ bgagent submit --repo my-org/new-feature \ + --task "update docs/auth.md per the approved design doc" \ + --pre-approve tool_type:Read \ + --pre-approve write_path:"docs/**" \ + --pre-approve tool_group:file_write \ + --pre-approve bash_pattern:"git add docs/**" \ + --pre-approve bash_pattern:"git commit *" + +# Phase 3 — full implementation +$ bgagent submit --repo my-org/new-feature \ + --task "implement the auth module per approved design + docs" \ + --pre-approve all_session --yes +``` + +Each phase has explicit scope. Matches real-world review workflows. Visible in audit via `pre_approvals_loaded` event. + +--- + +## 15. Implementation plan + +### 15.1 Milestone structure + +**Phase 3a** — core feature (3-4 weeks of work): +- Day 1: commit the cedarpy annotation round-trip test (agent side, `agent/tests/test_cedarpy_annotations_contract.py`) + the `@cedar-policy/cedar-wasm` parse test (Lambda side, `cdk/test/handlers/shared/cedar-policy.test.ts`). Both packages already spiked 2026-04-24: `cedarpy.policies_to_json_str()` returns annotations verbatim under `staticPolicies..annotations`; `@cedar-policy/cedar-wasm/nodejs` exports `policySetTextToParts` + `policyToJson(text)` which together expose the same data (see §15.6). +- Engine refactor (hard-deny + hard-gate + annotations + allowlist + recent-decisions) +- New DDB table, new Lambdas, new CLI commands +- PreToolUse hook extension (atomic transitions) +- `bgagent policies list` + `bgagent pending` (support UX that unblocks real usage) +- Happy path + fail-closed tests +- E2E on `backgroundagent-dev` + +**Phase 3b** — polish (1-2 weeks): +- CLI inline streaming prompt (UX research first) +- `approve --defer` / allowlist revocation (`bgagent revoke-approval`) +- CloudWatch alarm plumbing (§11.5) — deferred until an operational notification channel is available +- More hard-gate policies in the default set based on real usage + +### 15.2 Phase 3a task list + +~35 focused items. Ordered by dependency. + +| # | Package | File | Change | +|---|---|---|---| +| 1 | agent | Spike | Validate cedarpy.policies_to_json_str() returns annotations. Confirm `diagnostics.reasons` shape for multi-match. If API diverges, update §6 before proceeding. | +| 2 | agent | `src/policy.py` | Extend `PolicyDecision` (outcome/timeout_s/severity/matching_rule_ids/allowed-property). Split `_DEFAULT_POLICIES` into hard-deny + hard-gate. Add annotation parsing. Implement `ApprovalAllowlist` + `RecentDecisionCache`. Load-time validation (rule_id uniqueness, tier mismatch, annotation floor). | +| 3 | agent | `policies/hard_deny.cedar` (new) | Migrate current hard-deny rules + add DROP TABLE. Annotations. | +| 4 | agent | `policies/hard_gate.cedar` (new) | force-push, *.env, infrastructure/**, credentials. Annotations. | +| 5 | agent | `tests/test_policy.py` | Three-outcome, annotation merging, allowlist (incl. write_path, tool_group), recent-decision cache, pre-approval seeding, annotation round-trip. | +| 6 | cdk | `src/constructs/task-approvals-table.ts` (new) | Table + TTL + PITR (no Streams). | +| 7 | cdk | `src/handlers/shared/cedar-policy.ts` (new) | Shared policy-parsing library for Lambda-side rule-id validation. | +| 8 | cdk | `src/handlers/approve-task.ts` (new) | POST /approve with ownership-in-condition + audit event. | +| 9 | cdk | `src/handlers/deny-task.ts` (new) | POST /deny with output_scanner sanitization + audit event. | +| 10 | cdk | `src/handlers/get-policies.ts` (new) | GET /v1/repos/{repo}/policies. | +| 11 | cdk | `src/handlers/shared/types.ts` | ApprovalRequest/Response/DenyRequest + Scope union + extended CreateTaskRequest. | +| 12 | cdk | `src/handlers/shared/response.ts` | New error codes (REQUEST_NOT_FOUND, REQUEST_ALREADY_DECIDED, TASK_NOT_AWAITING_APPROVAL). | +| 13 | cdk | `src/constructs/task-api.ts` | Wire /approve, /deny, /repos/{}/policies routes. Grants. | +| 14 | cdk | `src/stacks/agent.ts` | Instantiate TaskApprovalsTable. Env var on runtimes. | +| 15 | cdk | `src/constructs/task-status.ts` | AWAITING_APPROVAL enum + transitions. | +| 16 | cdk | `src/handlers/create-task.ts` | Validate initial_approvals + approval_timeout_s with all safeguards (degenerate patterns, hard-deny rule rejection, maxPreApprovalScope ceiling, blueprint-resolved rule lookup). | +| 17 | cdk | `src/handlers/orchestrate-task.ts` | waitStrategy + finalizeTask handle AWAITING_APPROVAL. | +| 18 | cdk | `src/constructs/stranded-task-reconciler.ts` | Detect + transition stranded AWAITING_APPROVAL tasks. | +| 19 | cdk | `src/handlers/fanout-task-events.ts` | Dispatch rules for approval_* events + per-user notification rate limit. | +| 20 | agent | `src/hooks.py` | PreToolUse REQUIRE_APPROVAL path: atomic transitions, caps, poll, resume, denial-injection queue. | +| 21 | agent | `src/hooks.py` | `_denial_between_turns_hook` registered alongside `_nudge_between_turns_hook`. Shared `_xml_escape`. | +| 22 | agent | `src/task_state.py` | AWAITING_APPROVAL in transition helpers (TransactWriteItems primitive). | +| 23 | agent | `src/progress_writer.py` | `write_approval_*` convenience methods over `write_agent_milestone`. | +| 24 | cli | `src/commands/approve.ts` (new) | + 429 handling, `NO_COLOR` check. | +| 25 | cli | `src/commands/deny.ts` (new) | + `--reason-file` support. | +| 26 | cli | `src/commands/pending.ts` (new) | `bgagent pending` listing across active tasks. | +| 27 | cli | `src/commands/policies.ts` (new) | `bgagent policies list` + `policies show`. | +| 28 | cli | `src/commands/submit.ts` + `run.ts` | --approval-timeout, --pre-approve (repeatable), --pre-approve-file, all_session confirmation with --yes bypass. | +| 29 | cli | `src/api-client.ts` | approveTask, denyTask, listPending, listPolicies, extended createTask. | +| 30 | cli | `src/types.ts` | Mirror CDK types. Scope union + validator. | +| 31 | cdk | `test/handlers/approve-task.test.ts` (new) | Happy path, race, ownership-in-condition, scope validation, 409/404 distinction. | +| 32 | cdk | `test/handlers/deny-task.test.ts` (new) | Same shape + output_scanner integration. | +| 33 | cdk | `test/handlers/get-policies.test.ts` (new) | Discovery endpoint tests. | +| 34 | cdk | `test/handlers/create-task.test.ts` | initial_approvals validation (degenerate patterns, hard-deny rule rejection, blueprint resolution). | +| 35 | cli | `test/commands/*.test.ts` | CLI command tests. | +| 36 | agent | `tests/test_hooks.py` | REQUIRE_APPROVAL path, atomic transitions, caps, recent-decision cache, denial injection. | +| 37 | docs | `docs/design/INTERACTIVE_AGENTS.md` | Confirm §5.6 (approval CLI commands) and §8.2 (state machine) reflect Phase 3 wiring. | + +### 15.3 Testing strategy + +- **Unit**: ~80% coverage target, matching Phase 2. +- **Integration**: + - Cedar annotation round-trip test (write, parse, recover all 5 annotations) + - Full PreToolUse → PolicyDecision → DDB pipeline + - Allowlist seeding from initial_approvals + - Shared policy-parsing library consistency (Lambda side == agent side) +- **E2E** on `backgroundagent-dev`: 5 scenarios (A-E from §14). Both RuntimeJwt and Runtime-IAM paths. +- **Race tests**: + - Approve vs. timeout concurrent + - Deny vs. timeout concurrent + - Double-approve + - Cancel during AWAITING_APPROVAL + - Late approval after TIMED_OUT (expect 409) +- **Chaos tests**: + - Container restart mid-approval (simulated via kill + reconciler) + - DDB throttle during poll (simulated via mock) + - Bash retry loop after DENIED (expect recent-decision cache auto-deny) +- **Security tests**: + - Wrong user JWT → 404 (not 403) + - ANSI-injected tool_input_preview → stripped at both layers + - Malformed Cedar annotations → task fails to start + - Degenerate bash_pattern → 400 at submit + - Sanitizer-removing-secret test (OUTPUT_SCANNER integration) + +### 15.4 Rollout — no feature flag + +Cedar-HITL is shipped as standard functionality — no per-repo enable/disable flag. The safety posture of a given task is determined entirely by the content of the loaded policy set (built-in + blueprint) and the user's `--pre-approve` scopes at submit time. + +Built-in policies shipped with the agent: + +**Hard-deny (absolute, no scope bypasses them)**: +- `rm_slash` — `rm -rf /` +- `write_git_internals`, `write_git_internals_nested` — writes under `.git/` +- `drop_table` — SQL destructive DDL +- `force_push_main` — `git push --force` (or `-f`) to `main`/`prod` +- `write_credentials` — writes to files with `credentials` in the path + +**Hard-gate starter set (require approval by default)**: +- `push_to_protected_branch` — non-force push to `main`/`master`/`prod`/`release/*` — medium, 300s +- `write_env_files` — `like "*.env"` — high, 600s + +Users who want fully autonomous execution (no approval gates) pass `--pre-approve all_session --yes` at submit. Repos that want additional gates add them via `Blueprint.security.cedarPolicies.hard_gate`. Repos that want a different policy set can override specific built-in rules by `@rule_id` via the blueprint's `security.cedarPolicies.disable` list (see §17 for the disable-by-id mechanism, implemented as part of 3a). + +Rollout steps: + +1. **Implement + merge to main.** Built-in policies ship with the hard-deny + hard-gate sets above. No flag, no global kill switch. Any task on any repo instantly has the gate behavior for rules in the starter set; any task with `--pre-approve all_session` bypasses hard-gate rules (hard-deny rules remain enforced regardless). +2. **`backgroundagent-dev` validation.** Deploy merged code. Run E2E scenarios A–E: + - A: force-push gated + approved via CLI + - B: hard-deny path (DROP TABLE blocked, not gated) + - C: `--pre-approve all_session` bypasses hard-gate + - D: deny-with-reason steers agent via `` injection + - E: AI-DLC-style phased pre-approvals + Confirm Phase 1a/1b/2 regressions still pass. Confirm dashboards render. +3. **Pilot period (2 weeks).** Designate `scoropeza/agent-plugins` as the pilot repo (non-critical, active usage). Monitor: + - Any stranded tasks → indicates reconciler gap + - Timeout rate on approval_requested + - Per-task approval-gate count distribution — spot anomalous retry loops + - User-reported friction: "is the gate firing on things it shouldn't?" + If the starter set is too noisy, tune. If reliability is solid, proceed. +4. **Default for all repos.** Once the pilot is stable, the starter set is already live for everyone — no "flip the switch" step because there was no flag. Ongoing tuning happens by modifying built-in policies in code or via repo blueprints. + +**Rollback mechanism.** If the pilot surfaces a bug: remove the problem rule from `hard_gate.cedar` and redeploy (~5 min). No flag to flip. If the bug is more fundamental (engine regression), `git revert` the Phase 3 merge and redeploy — Phase 2 tests continue to pass because the backward-compat shim on `PolicyDecision.allowed` preserves the hook contract. + +**Success criteria for "pilot done":** +- Zero stranded tasks in 2 weeks +- <10% timeout rate on `approval_requested` +- Zero `approval_cap_exceeded` events (if any fire, either the cap is wrong or adversarial traffic to investigate) +- No regressions in Phase 1a/1b/2 tests (CI enforced on every commit) +- User-initiated gates that work: every hard-gate match produces a visible `★ approval_requested` in the stream and a responsive `bgagent approve/deny` cycle + +### 15.5 Backward compatibility + +- Existing tasks without `initial_approvals` → empty list → no pre-approvals, default `approval_timeout_s = 300` +- Existing policies without `@rule_id` / `@tier` → engine fails to start (fail-closed). Blueprint authors must add annotations explicitly during migration. +- `PolicyDecision.allowed` property provides backward compat for existing `if not decision.allowed` callers +- Hook return shape unchanged — Phase 1a/1b tests continue to pass + +### 15.6 Shared Cedar parsing — `@cedar-policy/cedar-wasm` API quickref + +The Lambda side (`CreateTaskFn`, `ApproveTaskFn`, `GetPoliciesFn`) uses [`@cedar-policy/cedar-wasm`](https://www.npmjs.com/package/@cedar-policy/cedar-wasm) — AWS's official WASM-compiled Cedar engine. Same Rust core as the Python `cedarpy` binding we already use in the agent. Spiked + verified 2026-04-24. + +**Package:** `@cedar-policy/cedar-wasm@4.10.0` (or latest major 4.x). +**Size:** 4.1 MB unzipped / ~1.5 MB zipped — well under Lambda limits. +**Import:** `const cedar = require('@cedar-policy/cedar-wasm/nodejs');` — use the CJS nodejs sub-export, NOT the default ESM export (ESM fails with `ERR_UNKNOWN_FILE_EXTENSION` on the `.wasm` file in Node 22). + +**Core functions used by the design:** + +| Function | Purpose | +|---|---| +| `policySetTextToParts(text: string)` | Split a multi-policy Cedar text into an array of individual policy texts. Returns `{type: "success", policies: string[]}` or `{type: "failure", errors: [...]}` | +| `policyToJson(text: string)` | Parse a single policy text into structured JSON. Returns `{type: "success", json: {annotations, effect, principal, action, resource, conditions}}` — annotations preserved verbatim under `json.annotations` as a `Record` | +| `isAuthorized({principal, action, resource, context, policies: {staticPolicies: string}, entities: []})` | Main authorization call. Entity references are `{type, id}` objects, **not** string literals. Returns `{type, response: {decision, diagnostics: {reason: string[]}}}` — `diagnostics.reason` is the list of matching policy IDs (e.g. `["policy1", "policy2"]`) for multi-match | + +**Minimal annotation-extraction pattern (the only thing `CreateTaskFn` needs for rule validation):** + +```typescript +// cdk/src/handlers/shared/cedar-policy.ts (sketch) +import * as cedar from '@cedar-policy/cedar-wasm/nodejs'; + +export interface ParsedRule { + ruleId: string; + tier: 'hard-deny' | 'hard-gate'; + severity?: 'low' | 'medium' | 'high'; + category?: string; + approvalTimeoutS?: number; +} + +export function parseRules(policiesText: string): ParsedRule[] { + const splitResult = cedar.policySetTextToParts(policiesText); + if (splitResult.type !== 'success') { + throw new Error(`Cedar policy parse failed: ${JSON.stringify(splitResult.errors)}`); + } + const rules: ParsedRule[] = []; + for (const policyText of splitResult.policies ?? []) { + const jsonResult = cedar.policyToJson(policyText); + if (jsonResult.type !== 'success') continue; + const annotations = jsonResult.json.annotations ?? {}; + const tier = annotations.tier; + const ruleId = annotations.rule_id; + if (tier !== 'hard-deny' && tier !== 'hard-gate') { + throw new Error(`Missing or invalid @tier annotation on policy (rule_id=${ruleId})`); + } + if (!ruleId) { + throw new Error(`Missing @rule_id annotation on ${tier}-deny policy`); + } + rules.push({ + ruleId, + tier, + severity: annotations.severity as ParsedRule['severity'], + category: annotations.category, + approvalTimeoutS: annotations.approval_timeout_s ? parseInt(annotations.approval_timeout_s, 10) : undefined, + }); + } + return rules; +} + +export function isHardDenyRule(rules: ParsedRule[], ruleId: string): boolean { + return rules.some(r => r.ruleId === ruleId && r.tier === 'hard'); +} +``` + +**API differences from Python cedarpy to be aware of during implementation:** + +1. Results are always wrapped in `{type: "success" | "failure", ...}`. Always check `.type` before accessing payload. +2. `isAuthorized` takes a single call object (not 3 positional args). Entities are `{type, id}` objects. +3. The Lambda cold-start penalty is ~30ms for the first `require()` (WASM module instantiation). Keep the import at module scope — not inside the handler — so subsequent invocations reuse the already-instantiated module. +4. The Node binding is CJS; the Lambda bundler (esbuild) treats the `.wasm` file as an external asset and Lambda's layer mechanism handles it automatically. No custom esbuild loader needed. + +--- + +## 16. Implementation notes (carry-forward tasks) + +Items from the 2026-04-24 design review not captured above as design changes — to be addressed during implementation and removed from this list once completed. These are P1-P2 findings; P0s have been integrated into the main design body. + +**IMPL-1** (data-flow P1-5): Scope string normalization. CLI + Lambda must agree. Document: trim whitespace, preserve case on `tool_type:` (Bash/Read/Write are canonical; reject case-shifted variants). + +**IMPL-2** (data-flow P1-7): Dual-write ordering between `progress_writer` and `sse_adapter` is best-effort; canonical source is TaskEventsTable. Document this in the implementation guide alongside Phase 2. + +**IMPL-3** (data-flow P2-1): Catch `ValueError` in `_merge_annotations` on malformed `@approval_timeout_s`; skip the annotation, log WARN. Engine already fails the task at load time if below floor, so this is a belt-and-suspenders. + +**IMPL-4** (data-flow P2-4): Test constraint — tests MUST NOT assert specific positional Cedar policy IDs. Use `@rule_id` annotations exclusively. + +**IMPL-5** (security SA-11 residual): Both the Lambda (audit event) and the agent (milestone) write approval decisions. The Lambda's write is canonical; the agent's is observational. Tests should verify the Lambda write completes even if agent milestone fails. + +**IMPL-6** (security P1-8): Audit trail ownership. `ApproveTaskFn` / `DenyTaskFn` write `approval_decision_recorded` to TaskEventsTable directly (not via agent milestone). Implement as part of the Lambda request flow. + +**IMPL-7** (security blind-spot #5): PolicyEngine MUST be instantiated per task, NOT per container. Verify in server.py bootstrap that a new instance is created on each task invocation (even when attach-don't-spawn logic reuses the container). + +**IMPL-8** (security blind-spot #6): TaskApprovalsTable Streams — confirmed off (§11.2). Do not subscribe any consumer. + +**IMPL-9** (functional P1-3): Runtime allowlist revocation. Not shipped in 3a. Placeholder: `bgagent revoke-approval ` noted in §17. + +**IMPL-10** (functional P1-12): `approval_timeout_s` default 300 documented consistently in §3 #6, §7.3 table, §10.2 attribute description. + +**IMPL-11** (functional P2-8): CLI `submit.ts` gains `--pre-approve` / `--approval-timeout` flags. + +**IMPL-12** (functional P2-9): Poll cadence in §3 #3 reconciled — describe as "initial 2s for 30s, then 5s" without specific call count math (it varies with timeout_s). + +**IMPL-13** (functional FC-5): `bgagent status --allowlist` — inspects current in-process allowlist state. Useful for debugging "why is this tool being gated again?". Low priority; add to `bgagent status` if cheap. + +**IMPL-14** (functional FC-6): Tool_use_id correlation. SDK handles internally. No hook-side changes needed; tests should verify the hook does not echo tool_use_id in its response. + +**IMPL-15** (functional FC-9): Recent-decision cache 60s window — tune after observation. Default 60s is a reasonable starting point. + +**IMPL-16** (CLI UX): ULID length is 26 chars, not 33. Update all CLI help text and error messages. + +**IMPL-17** (CLI UX): Shell completion (tab-complete task_id + request_id from `bgagent pending`). Deferred to 3b; document in §17. + +**IMPL-18** (FC-7): PolicyEngine freezing is implicit (single `__init__` call, no reload path). Add a test: assert that no code path calls `load_policies` after `__init__` completes. + +--- + +## 17. Deferred / out of scope + +### 17.1 Multi-user approval + +Future: multi-user approval (e.g., two of three reviewers must approve for `rule:deploy_prod`). Scope: §9.8 INTERACTIVE_AGENTS.md, Iteration 5. + +### 17.2 Per-rule auto-approve on timeout + +`@on_timeout("allow")` annotation sketched. Safety footgun. Revisit in 3b if demand. + +### 17.3 `@tier("advise")` — non-blocking advisory rules + +A third policy tier for rules that should surface but not block. Semantics sketch: + +- Cedar matches → emit `agent_milestone("advise_matched", {rule_ids, severity, tool_name, input_preview})` via `ProgressWriter` + fan-out. +- **No block.** Tool call proceeds immediately as if ALLOWED. +- **No timeout, no approval row, no state transition.** The engine never pauses. +- `PolicyDecision` gains `Outcome.ADVISE` but `evaluate_tool_use` returns ALLOW to the hook (internal tier, not a new SDK `permissionDecision`). +- Event framing: past-tense ("agent did X, matched rule Y"). Fan-out to Slack/email is FYI — no action buttons, audit-only. +- File layout: `agent/policies/advise.cedar`. Third file alongside `hard_deny.cedar` + `hard_gate.cedar`. + +Deferred because (a) shipping with gate-or-not is the simpler mental model for v1 users, (b) we want to observe whether hard-gates alone produce acceptable UX before introducing a third outcome, and (c) a concrete "I want to know but not be blocked" use case hasn't surfaced yet. First candidate rule if we ship it: `push_to_protected_branch` (force-push to any branch — informational for feature-branch workflows where force-pushing is routine). + +### 17.4 Interactive streaming prompts + +UX research first. Unlikely to ship — the async-only direction for the platform suggests notification-plane delivery is the right shape. + +### 17.4 Persistent allowlist across container restarts + +Today: in-process; reconciler fails stranded tasks. Phase 3b could persist to TaskTable + hydrate on restart. Not critical given rare restarts. + +### 17.5 `bgagent approve --defer` + +Escape hatch: "cancel + release slot". Clearer than silent timeout. Phase 3b. + +### 17.6 Policy hot-reload + +Today: policies frozen at task start. A long-running task can't benefit from a fresh hard-gate rule added mid-task. Probably fine; submission is the authoritative moment. Not a Phase 3 goal. + +### 17.7 Severity-based routing + +CLI: `bgagent approve --severity high` auto-approves high only, leaves medium/low. Phase 3b. + +### 17.8 Runtime allowlist revocation + +`bgagent revoke-approval `. User realization "oh wait, I didn't mean to approve ALL Bash". Phase 3b — implementation is straightforward (remove from in-process allowlist + emit `approval_revoked` milestone). + +### 17.9 Bulk approve + +`bgagent approve --all-pending` to approve everything pending. Power-user. Low priority; users WILL ask. + +### 17.10 Shell completion for task_id / request_id + +Tab-complete from `bgagent pending`. Deferred to 3b. + +### 17.11 Policy linting + +`bgagent lint-policies --repo ` to validate blueprint Cedar before submission. Catches annotation errors in development rather than at container start. Phase 3b. + +### 17.12 Richer approval annotations + +`@approval_requires_mfa("true")`, `@approval_channel("slack")` for enterprise workflows (step-up auth, audit channel). Good ideas; deferred. + +### 17.13 Cross-task scope inheritance + +"Apply the same pre-approvals I used on my last task." Convenience. Phase 3b. + +--- + +## Appendix A — Key file change map + +See §15.2. Net new files: ~13. Net modified files: ~15. Total LOC estimate: ~3500 production + ~2000 test = ~5500 lines. Larger than Phase 2 (+2950 / -34) because of the new Lambda × 3 + discovery endpoint + shared parser + state machine + reconciler updates. + +## Appendix B — Review checklist (pre-merge) + +- [ ] Day-1 cedarpy spike run; annotation round-trip confirmed +- [ ] All 5 Cedar annotations parse + recover via `policies_to_json_str()` round-trip test +- [ ] Every hard-deny rule has `@tier("hard-deny")` + `@rule_id` +- [ ] Every hard-gate rule has `@tier("hard-gate")` + `@rule_id` + `@severity` (default medium if missing) +- [ ] `@rule_id` uniqueness enforced at engine load (fail-on-error, not fall-back) +- [ ] `@approval_timeout_s < 30` rejected at load +- [ ] Atomic TransactWriteItems for approval-request creation and resume transitions +- [ ] Ownership encoded in ConditionExpression on ApproveTaskFn / DenyTaskFn +- [ ] Scope validation: rejects `rule:`, degenerate patterns, blueprint-maxPreApprovalScope violations +- [ ] ANSI/control-char stripping in `tool_input_preview` (both layers) +- [ ] `output_scanner.scan` runs in DenyTaskFn before persisting `reason` +- [ ] Recent-decision cache blocks 60s retries +- [ ] Per-task cap (50) + per-minute rate limit (20) + per-user notification cap (10/min) +- [ ] Denial injection via Stop hook `between_turns_hooks` (not `permissionDecisionReason` alone) +- [ ] Stranded-task reconciler transitions AWAITING_APPROVAL > 2×timeout_s to FAILED +- [ ] Race tests pass: approve+timeout, deny+timeout, double-approve, cancel-during-awaiting, late-approval-after-TIMED_OUT +- [ ] E2E on `backgroundagent-dev`: Scenarios A-E, both runtime paths +- [ ] `bgagent pending` + `bgagent policies list` functional +- [ ] Dashboard widgets emitting all approval-* metrics +- [ ] `bgagent status --allowlist` (if IMPL-13 shipped) +- [ ] Built-in starter set loaded: hard-deny = {rm_slash, write_git_internals, write_git_internals_nested, drop_table, force_push_main, write_credentials}; hard-gate = {push_to_protected_branch, write_env_files} +- [ ] No feature flag — Cedar-HITL is standard functionality; `--pre-approve all_session --yes` is the opt-out +- [ ] Backward compat: Phase 1a/1b tests pass without modification +- [ ] ULID length references are 26 chars throughout CLI + docs + +--- + +*End of Phase 3 design doc, rev 2.* diff --git a/docs/design/PHASE_1B_REV5_FOLLOWUPS.md b/docs/design/PHASE_1B_REV5_FOLLOWUPS.md new file mode 100644 index 0000000..6ec0528 --- /dev/null +++ b/docs/design/PHASE_1B_REV5_FOLLOWUPS.md @@ -0,0 +1,187 @@ +# Phase 1b rev-5 — follow-up status + +Created 2026-04-21 after the rev-5 multi-agent validation pass. Each item was +surfaced by one of the validators (`[SFH]` silent-failure-hunter, `[CR]` +code-reviewer, `[TDA]` type-design-analyzer, `[PTA]` pr-test-analyzer) or by +the user during review. This document tracks what landed vs what's still +pending, in the order the rev-5 rounds were executed. + +## Round summary + +| Round | Scope | Commit | Status | +|---|---|---|---| +| Rev-5 core | `bgagent run`, RUN_ELSEWHERE guard, execution_mode propagation, hydration | `022fb88`, `2d9d680` | ✅ | +| Pre-push hardening | P0-a, P0-b, P0-d, P0-e + key nits | `fe84de5` | ✅ | +| Stranded-task reconciler + concurrency raise | P0-c follow-up, MAX_CONCURRENT 3→10 | `9af3b50` | ✅ | +| Round 1 | Correctness: P1-3, P1-1, OBS-4 | `fce9d07` | ✅ | +| Round 2 | Error surfacing: P1-2, P1-5 | `bd7b886` | ✅ | +| Round 3 | Observability: OBS-1/2/3, P1-4 | `0d29939` | ✅ | +| Round 4a | Encapsulation: TDA-1, TDA-2, TDA-6 | `bc56731` | ✅ | +| Round 4b | Shared types: TDA-3, TDA-4, TDA-5 | `228c935` | ✅ | +| Round 5 | Design alignment: POLL-1, DATA-1 | `dfe7b84` | ✅ | +| Round 6 | Housekeeping (this commit) | TBD | in progress | + +## ✅ Landed (grouped by round for traceability) + +### Rev-5 final (pre-hardening) + +- `bgagent run` direct-submit interactive path (`cli/src/commands/run.ts`). +- `execution_mode` end-to-end (CreateTaskRequest, TaskRecord, TaskDetail). +- Server-side RUN_ELSEWHERE guard + TaskTable param hydration. +- Two-runtime ECR pull fix (two `AssetImage.fromAsset` instances to dodge + the L2 `AssetImage.bind` double-attach guard — see CDK-1 below). +- Client-side transport decision from `snapshot.execution_mode` (AgentCore + wraps non-2xx as 424; decide on the client instead of parsing the + wrapped response). + +### Pre-push P0 hardening (`fe84de5`) + +- **P0-a** `_SSEAdapter.write_agent_error` latent `_dropped_count` → + `_undelivered_count` fix + regression test. +- **P0-b** `task_state.get_task` distinguishes NotFound (returns `None`, + fail-open) from FetchFailed (raises `TaskFetchError`; server returns + 503). Prevents duplicate pipelines during DDB blips. +- **P0-d** `bgagent run` wraps `runSse` in try/catch; auto-cancels stranded + task + emits `bgagent status ` resume hint + exit non-zero. +- **P0-e** Post-hydration validation returns 500 + `TASK_RECORD_INCOMPLETE` with a list of missing fields. +- Key nits: shared `_stream.ts`, typed `SnapshotResult.executionMode`, + `TaskDetail.execution_mode` required in CLI, `EXECUTION_MODE_*` string + constants in server.py, `_HEARTBEAT_INTERVAL_SECONDS`, `logInfo` + cleanup, v3 diagram. + +### Stranded-task reconciler (`9af3b50`) — P0-c + +- `cdk/src/constructs/stranded-task-reconciler.ts` + handler + `cdk/src/handlers/reconcile-stranded-tasks.ts`. +- EventBridge schedule every 5 min, per-mode timeouts (300 s interactive, + 1200 s orchestrator / legacy). +- Transitions stranded tasks to FAILED with + `STRANDED_NO_HEARTBEAT`, emits `task_stranded` + `task_failed` events, + decrements concurrency. +- `MAX_CONCURRENT_TASKS_PER_USER` default raised 3 → 10. + +### Round 1 — correctness (`fce9d07`) + +- **P1-3** — attach-path `subscribe()` exception no longer falls through + to duplicate-spawn; returns 503 `SSE_ATTACH_RACE` + (`agent/src/server.py`). Duplicate-pipeline risk closed. +- **P1-1** — 409 on the SSE path is always terminal. RUN_ELSEWHERE → + fallback; any other 409 → `CliError` with a 500-byte body excerpt. + Eliminates reconnect-storm on server-side refusals + (`cli/src/sse-client.ts`). +- **OBS-4** — interactive path records `session_id` on TaskTable via + new `task_state.write_session_info`; cancel-task Lambda resolves the + correct runtime ARN from `execution_mode` + two new env vars + (`RUNTIME_IAM_ARN`, `RUNTIME_JWT_ARN`) to sidestep the CFN cycle that + would have been created by runtime-self-ARN injection. + +### Round 2 — error surfacing (`bd7b886`) + +- **P1-2** — post-SSE `getTask` failure now emits `WARN` to stderr with a + `bgagent status ` suggestion and suffixes the terminal line + with `(inferred)`. +- **P1-5** — new `_debug_cw_exc(message, exc, *, task_id)` helper + formats tracebacks into CloudWatch at every rev-5 bare + `except Exception` site. + +### Round 3 — observability (`0d29939`) + +- **OBS-1** — `_emit_sse_route_metric(task_id, route)` writes + `{event: "SSE_ROUTE", route: "attach"|"spawn"}` to CW stream + `sse_routing/`; called from both `_invoke_sse` branches. + Enables attach-vs-spawn ratio alarms. +- **OBS-2** — after hydration, always log `post-hydration params: + populated=[...] origin={k: 'record'|'caller'}`. +- **OBS-3** — structured `event` fields on admission logs + (`task.admitted.orchestrator_skipped`, `...orchestrator_invoked`, + `...orchestrator_invoke_failed`). +- **P1-4** — `_debug_cw_failures` counter bumped on daemon-thread + failures; every 5 failures (and the first) emits + `{event: "DEBUG_CW_WRITE_FAILURES", count, last_error_type}` via the + separate sse-routing code path. + +### Round 4a — encapsulation (`bc56731`) + +- **TDA-1** — `_AdapterRegistry` class owns `_threads_lock` + enforces + identity-checked pop in one place. Four open-coded sites collapsed to + `remove_if_current(task_id, adapter)`. `insert` raises on genuine + conflict. +- **TDA-2** — `_SSEAdapter.subscription()` context manager yields the + queue and auto-unsubscribes on exit (normal + exception paths). Raw + `subscribe()`/`unsubscribe()` retained for the + `_sse_event_stream` handoff to `StreamingResponse`. +- **TDA-6** — Python `ExecutionMode = Literal["orchestrator", + "interactive"]` + `normalize_execution_mode(raw)` helper for safe + coercion from DDB/env. + +### Round 4b — shared types (`228c935`) + +- **TDA-3** — `ApiErrorCode` union + `ApiErrorBody` envelope + + `isApiError(body, code)` type guard, defined in both + `cdk/src/handlers/shared/types.ts` and `cli/src/types.ts`. sse-client + uses the guard in its 409 branch. +- **TDA-4** — cross-file drift detection via + `cli/test/types-sync.test.ts`. Parses the CDK types.ts source and + asserts `ExecutionMode` + `ApiErrorCode` unions match the CLI + canonical list. Bigger `@abca/shared-types` workspace deferred per + scope. +- **TDA-5** — `SemanticEvent` TypedDict union in + `agent/src/sse_adapter.py`. Six event shapes declared, each mirroring + the sibling `ProgressWriter.write_agent_*` dict. + +### Round 5 — design alignment (`dfe7b84`) + +- **POLL-1** — `watch` polling cadence decays 500 ms → 2 s after 3 min. + First 3 min matches design §9.13.1; the decay caps REST cost for + long-running observation. +- **DATA-1** — `TaskResult` gains `turns_attempted` + `turns_completed` + (clamped to `max_turns` when `error_max_turns`). Legacy `turns` field + retained as `turns_attempted` value for back-compat. + `TaskRecord`/`TaskDetail` in CDK + CLI types mirror; `toTaskDetail` + forwards. + +## Non-code follow-ups tracked elsewhere + +### ✅ CDK-1 — Upstream bug filed: aws/aws-cdk#37663 + +`cdk/src/stacks/agent.ts` has a two-artifact workaround: + +```ts +const artifactIam = agentcore.AgentRuntimeArtifact.fromAsset(runnerPath); +const artifactJwt = agentcore.AgentRuntimeArtifact.fromAsset(runnerPath); +``` + +Root cause in `@aws-cdk/aws-bedrock-agentcore-alpha`'s `AssetImage.bind` +method: it guards against double-grant with `this.bound = true`, so +when the same artifact instance is passed to two Runtimes the second +runtime's execution role never receives ECR pull permissions. Image +pull fails with 424 "no basic auth credentials". + +Filed upstream at with +minimal repro, root-cause analysis, and a suggested fix. The code +comment at `cdk/src/stacks/agent.ts:55-68` now links the issue. +Keep the two-artifact workaround until the upstream fix ships (or +remove it when this repo upgrades to a version that includes the +fix). + +### Candidates NOT landed (by design) + +- **Full `@abca/shared-types` workspace (bigger TDA-4)** — deferred in + favour of the drift-detection test. Spin up when a third package + needs the shared types (e.g., a future SDK package, or if the web + console moves in-tree). +- **`SemanticEvent` threaded through adapter signatures** — TDA-5 + landed the types; call-site propagation (`_enqueue(event: + SemanticEvent)` etc.) deferred until we tighten mypy strictness. +- **CLI formatter for `turns_attempted`/`turns_completed`** — DATA-1 + landed the DDB/REST fields; `bgagent status` / `bgagent watch` + formatters still display just `turns`. UX decision for a separate + pass (e.g., "6 turns (7 attempted — hit max_turns cap)"). + +## Status as of this round + +All validator-surfaced P0/P1/OBS/TDA/POLL/DATA items are either landed +or explicitly classified as not-in-scope above. CDK-1 is filed +upstream (aws/aws-cdk#37663); the two-artifact workaround stays until +the upstream fix ships. diff --git a/docs/design/SECURITY.md b/docs/design/SECURITY.md index 7846048..77c9995 100644 --- a/docs/design/SECURITY.md +++ b/docs/design/SECURITY.md @@ -71,6 +71,8 @@ The blueprint framework ([REPO_ONBOARDING.md](./REPO_ONBOARDING.md)) allows per- **Deployment control** - Custom steps are defined in the `Blueprint` CDK construct and deployed via `cdk deploy`. Only principals with CDK deployment permissions can add or modify them. There is no runtime API for custom step CRUD. +The **same deploy-only property extends to `Blueprint.security.cedarPolicies`** — user-authored Cedar policies live in the CDK source, are typed as `readonly string[]` on the construct, and reach `RepoTable` only through a CloudFormation custom resource invoked at deploy time. Phase 3 (Cedar-driven HITL approval gates — see [`PHASE3_CEDAR_HITL.md`](./PHASE3_CEDAR_HITL.md)) is load-bearing on this property: the engine treats Cedar policies loaded at task start as trusted content. If the blueprint model ever changes to accept user-uploaded policy text via an API path, Phase 3's §12 trust model must be re-evaluated (add per-blueprint policy count cap, per-eval timeout, size cap). + **Input filtering** - The framework strips credential ARNs (`github_token_secret_arn`) and networking configuration (`egress_allowlist`) from the config before passing it to custom Lambda steps. If a custom step needs secrets, it must declare them explicitly and the operator must grant IAM permissions. **What a custom step can do:** diff --git a/docs/diagrams/interactive-agents-phases.drawio b/docs/diagrams/interactive-agents-phases.drawio new file mode 100644 index 0000000..9ec15f7 --- /dev/null +++ b/docs/diagrams/interactive-agents-phases.drawio @@ -0,0 +1,1223 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/diagrams/phase3-cedar-hitl.drawio b/docs/diagrams/phase3-cedar-hitl.drawio new file mode 100644 index 0000000..3dbbdc7 --- /dev/null +++ b/docs/diagrams/phase3-cedar-hitl.drawio @@ -0,0 +1,2874 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/guides/DEVELOPER_GUIDE.md b/docs/guides/DEVELOPER_GUIDE.md index 6ef633e..cd5cdfe 100644 --- a/docs/guides/DEVELOPER_GUIDE.md +++ b/docs/guides/DEVELOPER_GUIDE.md @@ -175,6 +175,27 @@ docker stats bgagent-run # CPU, memory usage docker exec -it bgagent-run bash # shell into the container ``` +#### Testing with progress events (DynamoDB Local) + +By default, progress events and task state writes are silently skipped during local runs (the `TASK_EVENTS_TABLE_NAME` and `TASK_TABLE_NAME` env vars are not set). To enable them locally using DynamoDB Local: + +```bash +# 1. Start DynamoDB Local and create tables +cd agent && mise run local:up + +# 2. Run the agent with --local-events +./agent/run.sh --local-events "owner/repo" 42 + +# 4. In another terminal — query progress events +mise run local:events # table format +mise run local:events:json # JSON format + +# 5. When done — tear down DynamoDB Local +mise run local:down +``` + +The `--local-events` flag connects the agent container to DynamoDB Local on the `agent-local` Docker network and sets the appropriate env vars. The agent code writes to DDB Local using the same code path as production — no mocks or alternate implementations. + #### Environment variables | Variable | Default | Description | diff --git a/docs/guides/USER_GUIDE.md b/docs/guides/USER_GUIDE.md index a67c111..23fd361 100644 --- a/docs/guides/USER_GUIDE.md +++ b/docs/guides/USER_GUIDE.md @@ -479,6 +479,20 @@ node lib/bin/bgagent.js events --output json Use **`--output json`** to see the full payload for **`preflight_failed`** (`reason`, `detail`, and per-check metadata). See **Task events** under **Task lifecycle** for how to interpret common `reason` values. +### Watching a task in real time + +Stream progress events (turns, tool calls, tool results, milestones, cost updates) from a running task and exit automatically when it reaches a terminal state. + +```bash +node lib/bin/bgagent.js watch + +# JSON output (one event per line) — useful for scripting +node lib/bin/bgagent.js watch --output json +``` + +Exit codes: `0` on `COMPLETED`, `1` on `FAILED` / `CANCELLED` / `TIMED_OUT`. Press Ctrl+C to exit early without affecting the task. + + ### Cancelling a task ```bash diff --git a/docs/research/agent-streaming-patterns.md b/docs/research/agent-streaming-patterns.md new file mode 100644 index 0000000..3a76eb5 --- /dev/null +++ b/docs/research/agent-streaming-patterns.md @@ -0,0 +1,263 @@ +# Agent Streaming Architectures: Submission vs. Observation + +> Research date: 2026-04-20. Time-sensitive sources (especially AgentCore quotas and LangGraph Platform APIs) are flagged inline. This report surfaces options and published rationale — it does not recommend a specific choice for ABCA. + +## Executive summary + +Across the platforms surveyed, there are three dominant architectural shapes for "submit a long task + stream its progress": + +1. **Same-process streaming** — the agent executes inside the same request/container that holds the SSE/WebSocket connection. Simple; breaks when the client disconnects or when non-interactive consumers (webhooks, Slack, cron) need the same events. Used by **CopilotKit's default `InMemoryAgentRunner`**, **Mastra `agent.stream()`**, and **OpenAI Assistants API streaming runs**. +2. **Orchestrator + Observer (durable run + join stream)** — the agent runs as a background job writing events to a durable log and/or pub/sub channel; any number of clients can attach, detach, and reattach. Used by **LangGraph Platform** (Postgres checkpointer + Redis pubsub, `client.runs.join_stream()`) and **Vercel `resumable-stream`** (Redis pubsub producer/consumer). +3. **Pull/query-based observation** — no streaming primitive; clients poll durable state or issue queries against the running worker. Used by **Temporal** (queries, signals, event history) and — partially — by the **OpenAI Assistants API fallback path** when SSE drops. + +AgentCore is interesting because it sits *between* shapes 1 and 2: multiple `InvokeAgentRuntime` calls with the same `runtimeSessionId` **route to the same microVM**, so a streaming observer can in principle attach to a pipeline kicked off by an earlier sync invocation — but the agent code has to explicitly support that "attach" mode; AgentCore itself does not provide a join-stream primitive the way LangGraph does. + +--- + +## 1. CopilotKit / AG-UI + +**Runtime vs. gateway.** CopilotKit's `CopilotRuntime` is both orchestrator and gateway. It exposes `/agent/:agentId/run` (execution) and `/agent/:agentId/connect` (context), and it streams AG-UI events back over SSE or WebSockets ([DeepWiki: CopilotRuntime Architecture](https://deepwiki.com/CopilotKit/CopilotKit/4.1-copilotruntime-overview), [Self-hosting docs](https://docs.copilotkit.ai/guides/self-hosting)). + +**Execution model.** Each `POST /agent/:agentId/run` triggers a **new agent lifecycle**. The default `InMemoryAgentRunner` is stateless. Persistence options exist (`SqliteAgentRunner` for conversation state, `IntelligenceAgentRunner` for durable threads over WebSocket), but these persist *conversation state*, not the in-flight execution itself — a new client connecting does not attach to an in-progress run; it starts one. + +**AG-UI protocol** ([CopilotKit blog](https://www.copilotkit.ai/blog/introducing-ag-ui-the-protocol-where-agents-meet-users/)) standardises the *wire format* (JSON events: messages, tool calls, state patches, lifecycle signals) but leaves the attach-vs-restart question to the implementation. AG2, Phoenix, and Microsoft Agent Framework ship their own runtimes ([MS Learn AG-UI integration](https://learn.microsoft.com/en-us/agent-framework/integrations/ag-ui/?pivots=programming-language-csharp)). + +**Implication for ABCA:** AG-UI defines *what* flows over the wire, not *where* the agent lives. The protocol is compatible with both the same-process and orchestrator+observer patterns. + +--- + +## 2. LangGraph Cloud / LangGraph Platform + +This is the clearest published example of the orchestrator+observer pattern in the agent space. + +**Architecture** ([LangGraph persistence docs](https://docs.langchain.com/oss/python/langgraph/persistence), [Redis fault-tolerant task execution](https://neuralware.github.io/posts/langgraph-redis/)): + +- **Postgres** is the checkpointer — every node execution writes a state snapshot keyed by thread + checkpoint id. This is the durable log. +- **Redis** handles fault-tolerant task queueing and real-time pub/sub between workers and streaming endpoints. +- **Threads** hold state; **runs** are individual executions against a thread. + +**Stream modes** ([Streaming concepts](https://langchain-ai-langgraph-40.mintlify.app/concepts/streaming)): `values` (full state after each node), `updates` (deltas), `messages` (token-by-token), `events` (detailed lifecycle). Each can be requested per-run. + +**Reliable streaming + `join_stream`** ([Changelog: reliable streaming](https://changelog.langchain.com/announcements/reliable-streaming-and-efficient-state-management-in-langgraph)): + +- `GET /threads/{thread_id}/runs/{run_id}/stream` and the SDK method `client.runs.join_stream()` let a client **attach to an already-running run**, including background runs started earlier with no streaming consumer. +- Clients can navigate away and return; the run keeps executing; reconnection resumes streaming. +- Thread status values (`idle`, `busy`, `error`, `interrupted`) are queryable for non-streaming consumers. + +**How LangGraph avoids double-execution:** the run is kicked off as a background job; the SSE endpoint is purely an observer over Redis pubsub + Postgres checkpoints. The agent does not re-run when a client connects. + +**Time-sensitive:** `join_stream` and background runs are platform features (LangGraph Platform / LangGraph Cloud). Self-hosted OSS `langgraph` gets the primitives (checkpointers, pub/sub interface) but not the hosted endpoint out of the box. + +--- + +## 3. OpenAI Assistants API + +**Streaming runs** ([API reference](https://platform.openai.com/docs/api-reference/assistants-streaming/events.xlsx)): pass `"stream": true` to Create Run, Create Thread and Run, or Submit Tool Outputs; the response is SSE. + +**No resume / no attach.** The OpenAI forum and implementation guides ([production patterns](https://michaeljohnpena.com/blog/2024-01-03-assistants-api-patterns), [community threads](https://community.openai.com/t/how-to-resume-streaming-in-python-after-submitting-function-call-outputs-in-openai-assistants-api/1119902)) confirm: + +- Once an SSE stream ends with `[DONE]`, the connection cannot be reused. +- There is no documented way to attach a second stream to an in-progress run. Disconnected clients must fall back to polling `GET /threads/{thread_id}/runs/{run_id}` for status and reading messages from the thread after completion. +- Submitting tool outputs **ends the current stream**; a fresh stream must be opened for the continuation. +- "Background mode" responses have a **5-minute staleness window** — if the client doesn't reconnect within 5 minutes, streaming is no longer possible and results must be fetched via GET ([community: 5-min limit](https://community.openai.com/t/stream-background-this-response-can-no-longer-be-streamed-because-it-is-more-than-5-minutes-old/1372287)). + +**How OpenAI avoids double-running:** the run is stateful on their side (thread + run entities); the SSE stream is one-shot and disposable. They own the whole stack, so they can guarantee the run keeps going server-side even if the client drops — clients just can't observe live after disconnect. The API is being sunset in favour of the Responses API ([migration guide](https://calstudio.com/blog/responses-api)). + +--- + +## 4. Mastra + +**Architecture** ([Mastra streaming overview](https://mastra.ai/docs/streaming/overview), [Mastra streaming blog](https://mastra.ai/blog/mastra-streaming)): `agent.stream()` runs the agent **synchronously within the request context**. The returned object exposes `textStream`, `text` (full-response promise), `finishReason`, `usage`. Workflows use `run.stream()` for event-based streams. + +- Heavy investment in **nested streaming** — agent-in-tool, agent-in-workflow-step compose correctly. +- No documented background/interactive split; no join-stream primitive. Same-process model. +- Custom protocol layered on AI SDK v5 compatibility ([v5 announcement](https://mastra.ai/blog/announcing-mastra-improved-agent-orchestration-ai-sdk-v5-support)). + +**Implication:** Mastra optimises for the interactive case. Non-interactive fan-out (webhooks, cron) is outside the streaming layer's scope. + +--- + +## 5. AWS Bedrock AgentCore + +Two findings are load-bearing for ABCA. + +### 5.1 Same session ID → same microVM + +From the [Runtime sessions doc](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-sessions.html): + +> "Context is preserved between invocations to the same session … By using the same `runtimeSessionId` for related invocations, you ensure that context is maintained across the conversation … Without a consistent session ID, each request may be routed to a new microVM." + +So AgentCore *does* support multiple concurrent `InvokeAgentRuntime` calls landing on the same container. This is the primitive you need to build an attach pattern — but AgentCore does not provide a managed join-stream. Whether the second invocation observes the first or starts a new pipeline depends entirely on **your agent code**: the container receives a new HTTP invocation on `/invocations` and decides what to do with it. + +The documented recent additions ([filesystem + execute command](https://aws.amazon.com/blogs/machine-learning/persist-session-state-with-filesystem-configuration-and-execute-shell-commands/), [observability](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-get-started.html)) persist *filesystem* and *OTel traces* across invocations, not a running pipeline's live event stream. No native pub/sub on sessions is documented. + +### 5.2 Quotas — the 60-minute streaming cap is real + +From the [AgentCore quotas page](https://docs.aws.amazon.com/fr_fr/bedrock-agentcore/latest/devguide/bedrock-agentcore-limits.html) (fetched 2026-04-20): + +| Limit | Value | +|---|---| +| Streaming max duration (response streaming and WebSocket) | **60 minutes** | +| Asynchronous job max duration | **8 hours** | +| Session max lifetime (`maxLifetime`) | 60s min, 28,800s (8h) max, default 900s (15m) | +| Idle session timeout | 15 min default (configurable) | +| Synchronous request timeout | 15 min | +| Active session workloads | 1,000 (us-east-1/us-west-2), 500 elsewhere, adjustable | +| Hardware per session | 2 vCPU / 8 GB | + +**Contradiction resolved during research.** An initial summary claimed "no maximum time for streaming connections." The authoritative quotas page disagrees — streaming is capped at 60 minutes; the 8-hour figure applies only to **async jobs**. A long-running pipeline (>60 min) cannot be observed through a single AgentCore streaming connection; the observer must reconnect. + +**Time-sensitive:** AgentCore is a young service; quotas and feature surface area change. Verify at deploy time. + +--- + +## 6. Temporal.io (analogy, not a direct fit) + +Temporal deliberately separates "run the workflow" from "observe the workflow" ([message passing encyclopedia](https://docs.temporal.io/encyclopedia/workflow-message-passing)): + +- **Queries** — synchronous, read-only inspection of workflow state. No activities, no mutation. Served from the worker replaying history. +- **Signals** — asynchronous writes delivered into the workflow. +- **Updates** — synchronous validated writes (combine query + signal semantics). +- **Event History** — every workflow produces a durable, replayable log. Retrievable via `temporal workflow show`, `DescribeWorkflowExecution`, or the Web UI. + +**No native event streaming.** The [community thread on listening to event streams](https://community.temporal.io/t/listening-to-event-streams-in-a-workflow/10677/2) recommends: external Kafka/SNS consumer dispatches signals into the workflow, and workflow uses `continue-as-new` to manage history size. The observer-facing UI is effectively a rich front-end over event history + describe APIs. + +**Implication for ABCA:** Temporal's pattern is "durable log + periodic describe/query" rather than live push. Latency for observation is poll-interval bound, not sub-second. But reconnection is trivial — event history is the source of truth and always resumable. + +--- + +## 7. Vercel AI SDK / Next.js + +Vercel's ecosystem has painfully learned that the Next.js request model (serverless, per-connection, time-limited) fights long-running streams: + +- [Issue #230: running as background jobs with QStash](https://github.com/vercel/ai/issues/230) — streaming doesn't compose cleanly with async background execution. +- [Issue #12949: 30s idle timeout drops tool_call streaming](https://github.com/vercel/ai/issues/12949) — edge/serverless idle timeouts kill streams. + +Their answer: **[`vercel/resumable-stream`](https://github.com/vercel/resumable-stream)** — a small library implementing exactly the orchestrator+observer pattern over Redis: + +- The **first** request for a given `streamId` becomes the **producer**: it runs the generator to completion even if the original client disconnects, buffering chunks to Redis. +- **Subsequent** requests for the same `streamId` (with optional `resumeAt` position) publish a message on a Redis pubsub channel; the producer replays buffered messages from the resume point, then forwards new chunks. +- Storage: Redis `INCR` + `SUBSCRIBE` for the common single-consumer case; `SET`/`GET` for buffering; pubsub channels for fan-out. + +**Why it matters for ABCA:** this is the reference implementation of "one producer, many observers, reconnectable, catch-up via log" in a serverless-friendly shape. The primitives map directly to DynamoDB (durable log) + pubsub (SNS/EventBridge/IoT Core) or to ElastiCache Redis if you want the same library. + +--- + +## 8. Named architectural patterns + +Costs, latencies, and AWS mappings for each. + +### 8.1 Same-process streaming (Model A) + +- **Who uses it:** CopilotKit `InMemoryAgentRunner`, Mastra, OpenAI Assistants streaming, current ABCA JWT runtime behaviour. +- **Latency:** sub-100ms; the agent is writing directly to the socket. +- **AWS shape:** one Lambda/container per request; streaming via API Gateway HTTP API response streaming, Lambda Function URL with `RESPONSE_STREAM`, or AgentCore Runtime streaming mode. +- **Trade-offs vs ABCA's current design:** this *is* ABCA's current design. Breaks when the CLI closes the tab (no durable consumer), and does not fan out to webhooks/Slack without duplicating work. + +### 8.2 Orchestrator + Observer (durable run + join stream) + +- **Who uses it:** LangGraph Platform (`join_stream`), Vercel `resumable-stream`. +- **Latency:** sub-100ms live (pubsub hop), few-seconds catch-up (log replay). +- **AWS-native implementations:** + - **DynamoDB log + SNS per-task topic.** Cost: SNS topic-per-task isn't idiomatic (topics are long-lived); better to use **one topic, filter by `task_id` message attribute**. Subscribers (Lambda for webhooks, WebSocket Lambda for CLI) filter via subscription filter policies. + - **DynamoDB log + EventBridge** with event-pattern match on `task_id`. Works but EventBridge has ~0.5s publish latency and rule limits. + - **DynamoDB Streams + Lambda fan-out** — Streams support 2 simultaneous consumers per shard natively; use **EventBridge Pipes** to fan out to more ([DDB → EventBridge Pipes pattern](https://www.boyney.io/blog/2022-11-03-eventbridge-events-with-dynamodb), [transactional outbox with Pipes](https://aws.amazon.com/blogs/compute/implementing-the-transactional-outbox-pattern-with-amazon-eventbridge-pipes/)). + - **IoT Core MQTT** — topic per task (`tasks/{task_id}/events`), clients subscribe via MQTT-over-WSS. Scales to very high fan-out; adds an IoT dependency. + - **API Gateway WebSocket + DynamoDB connection registry** ([AWS tutorial](https://docs.aws.amazon.com/apigateway/latest/developerguide/websocket-api-chat-app.html)): store `(connectionId, task_id)` in DDB with a GSI on `task_id`; on new event, query GSI and `postToConnection` to each. Standard pattern. +- **Trade-offs vs current design:** fixes double-execution; adds one infra component (pubsub) and one write path in the pipeline. Plays perfectly with reconnect/catch-up via DDB log. + +### 8.3 Dual-write / CQRS + +- **Who uses it:** LangGraph (Postgres checkpointer + Redis pubsub), generally the AWS transactional-outbox pattern. +- **Latency:** same as orchestrator+observer for live subscribers; tail-read from log for late joiners. +- **Key concern:** atomicity. If pipeline writes DDB then publishes, a crash between the two loses the pubsub event (live subscribers miss it, but the log is intact — they catch up on reconnect). If pubsub fails, the log is still authoritative. AWS outbox pattern ([EventBridge Pipes + DDB Streams](https://aws.amazon.com/blogs/compute/implementing-the-transactional-outbox-pattern-with-amazon-eventbridge-pipes/)) solves this by making the log the only write and deriving the pubsub from DDB Streams. + +### 8.4 Pull-based with push signal + +- **Who uses it:** OpenAI Assistants fallback (`GET /runs/{id}`), Temporal (describe + query), many CI systems. +- **Latency:** poll interval bound (1–5s typical). +- **AWS shape:** clients call `GET /tasks/{id}/events?after=` against a Lambda backed by DDB. Optionally a lightweight SSE/WS channel sends "new event, poll now" signals to avoid constant polling. +- **Trade-offs:** simplest to build; worst live latency; trivially handles non-interactive consumers (they just GET on their schedule). + +### 8.5 WebSocket/MQTT hub + +- **Who uses it:** IoT platforms, trading floors, collaborative editors. +- **AWS shape:** **API Gateway WebSocket API** (Lambda-backed, good ≤100k concurrent) or **IoT Core MQTT** (millions of concurrent subscribers, topic-based filtering native). +- **Latency:** sub-100ms. +- **Trade-offs:** WebSocket reconnection + backfill is the application's problem; IoT Core requires adopting the IoT SDK/MQTT dependency on the client. + +--- + +## 9. 60-minute streaming cap — reality check + +The quotas page confirms AgentCore streaming is capped at **60 minutes** per connection, while asynchronous jobs can run **8 hours** ([quotas](https://docs.aws.amazon.com/fr_fr/bedrock-agentcore/latest/devguide/bedrock-agentcore-limits.html)). Patterns that match the observation question: + +- **Orchestrator runs as async job (up to 8h); observer opens a fresh 60-min streaming connection and reconnects as needed.** This is exactly the LangGraph `join_stream` shape. Every 60 minutes the observer drops and reopens, picking up from the last event cursor in the durable log — no pipeline interruption. +- **Pipeline outside AgentCore.** If the orchestrator pipeline lives in ECS/Fargate or Step Functions, AgentCore's streaming quota only constrains the *observer* runtime. Pipeline duration is bounded by the executor (Fargate tasks: up to 14 days; Step Functions Standard: 1 year). +- **No documented pattern for >60-min single-connection observation.** All published long-running-agent guidance assumes reconnect + durable log. + +**Time-sensitive:** the 60-min figure was documented 2026-04-20. AgentCore is iterating fast; verify before making long-term decisions. + +--- + +## Comparison matrix + +| Platform | Execution location | Attach to running? | Reconnect / resume | Non-interactive fan-out | Durable log | +|---|---|---|---|---|---| +| CopilotKit `InMemoryAgentRunner` | Same process as stream | No (restart) | No | Not built in | No | +| LangGraph Platform | Background worker | **Yes** (`join_stream`) | **Yes** (Redis + Postgres) | Yes (thread/run APIs) | Postgres checkpointer | +| OpenAI Assistants | OpenAI-hosted | No | No (5-min background window, else poll) | Via polling | Thread/run entities | +| Mastra | Same process as stream | No | No | Not built in | No | +| AWS AgentCore | microVM per session | **Same session → same VM**, but no managed join-stream | Agent code responsibility | Not built in | Filesystem + OTel traces | +| Temporal | Worker | Query / signal (not stream) | N/A (history is authoritative) | Yes (event history) | Event history | +| Vercel `resumable-stream` | First-request producer | **Yes** (same streamId) | **Yes** (Redis pubsub + buffer) | Via polling | Redis buffer | + +--- + +## Sources + +1. CopilotKit AG-UI blog — https://www.copilotkit.ai/blog/introducing-ag-ui-the-protocol-where-agents-meet-users/ +2. CopilotRuntime Architecture (DeepWiki) — https://deepwiki.com/CopilotKit/CopilotKit/4.1-copilotruntime-overview +3. CopilotKit Self-Hosting — https://docs.copilotkit.ai/guides/self-hosting +4. MS Agent Framework AG-UI integration — https://learn.microsoft.com/en-us/agent-framework/integrations/ag-ui/?pivots=programming-language-csharp +5. LangGraph reliable streaming announcement — https://changelog.langchain.com/announcements/reliable-streaming-and-efficient-state-management-in-langgraph +6. LangGraph streaming concepts — https://langchain-ai-langgraph-40.mintlify.app/concepts/streaming +7. LangGraph persistence docs — https://docs.langchain.com/oss/python/langgraph/persistence +8. LangGraph Redis fault-tolerant task execution — https://neuralware.github.io/posts/langgraph-redis/ +9. OpenAI Assistants API reference (streaming) — https://platform.openai.com/docs/api-reference/assistants-streaming/events.xlsx +10. OpenAI Assistants production patterns — https://michaeljohnpena.com/blog/2024-01-03-assistants-api-patterns +11. OpenAI community: resume after tool outputs — https://community.openai.com/t/how-to-resume-streaming-in-python-after-submitting-function-call-outputs-in-openai-assistants-api/1119902 +12. OpenAI community: 5-min background streaming window — https://community.openai.com/t/stream-background-this-response-can-no-longer-be-streamed-because-it-is-more-than-5-minutes-old/1372287 +13. OpenAI → Responses API migration — https://calstudio.com/blog/responses-api +14. Mastra streaming overview — https://mastra.ai/docs/streaming/overview +15. Mastra streaming blog — https://mastra.ai/blog/mastra-streaming +16. AgentCore runtime sessions — https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-sessions.html +17. AgentCore invoke runtime — https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-invoke-agent.html +18. AgentCore filesystem + execute command — https://aws.amazon.com/blogs/machine-learning/persist-session-state-with-filesystem-configuration-and-execute-shell-commands/ +19. AgentCore observability — https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-get-started.html +20. AgentCore quotas — https://docs.aws.amazon.com/fr_fr/bedrock-agentcore/latest/devguide/bedrock-agentcore-limits.html +21. AgentCore WebSocket limits analysis — https://www.paulserban.eu/blog/post/amazon-bedrock-agentcore-websocket-limits-5-critical-bottlenecks-you-need-to-know/ +22. Temporal message passing encyclopedia — https://docs.temporal.io/encyclopedia/workflow-message-passing +23. Temporal community: listening to event streams — https://community.temporal.io/t/listening-to-event-streams-in-a-workflow/10677/2 +24. Vercel AI SDK — QStash background jobs issue #230 — https://github.com/vercel/ai/issues/230 +25. Vercel AI SDK — 30s idle timeout issue #12949 — https://github.com/vercel/ai/issues/12949 +26. vercel/resumable-stream — https://github.com/vercel/resumable-stream +27. DynamoDB + EventBridge integration — https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/eventbridge-for-dynamodb.html +28. DynamoDB Streams → EventBridge (Boyney) — https://www.boyney.io/blog/2022-11-03-eventbridge-events-with-dynamodb +29. Transactional outbox with EventBridge Pipes — https://aws.amazon.com/blogs/compute/implementing-the-transactional-outbox-pattern-with-amazon-eventbridge-pipes/ +30. API Gateway WebSocket chat app tutorial — https://docs.aws.amazon.com/apigateway/latest/developerguide/websocket-api-chat-app.html +31. AWS prescriptive guidance: orchestration models — https://docs.aws.amazon.com/prescriptive-guidance/latest/agentic-ai-serverless/orchestration-models.html + +--- + +## Methodology + +Two rounds of parallel research via `kiro-search.sh`: + +- Round 1: 9 parallel searches (one per topic area). +- Round 1 fetches: 6 parallel URL fetches for depth on CopilotKit runtime, LangGraph reconnect, AgentCore session attach semantics, Vercel resumable-stream internals, AgentCore quotas, and Mastra streaming. +- Round 2: 3 gap-closing searches (LangGraph Platform internals, OpenAI resume limitations, API Gateway WebSocket fan-out). + +**Contradiction surfaced and resolved:** an initial Kiro summary asserted "no maximum time for streaming connections" on AgentCore. The authoritative quotas page (source 20) contradicts this — streaming is capped at 60 minutes, and the 8-hour figure applies only to asynchronous jobs. This report uses the quotas-page figures. + +Date: 2026-04-20. diff --git a/docs/src/content/docs/architecture/Interactive-agents.md b/docs/src/content/docs/architecture/Interactive-agents.md new file mode 100644 index 0000000..1f37995 --- /dev/null +++ b/docs/src/content/docs/architecture/Interactive-agents.md @@ -0,0 +1,835 @@ +--- +title: Interactive agents +--- + +# Interactive Agents: Async Interaction Design + +> **Status:** Active design +> **Branch:** `feature/interactive-background-agents` +> **Last updated:** 2026-04-29 (rev 6) + +--- + +## Executive summary + +ABCA runs background coding agents that clone a repo, implement a task, run tests, and open a pull request. Tasks run from minutes to hours inside an isolated cloud runtime. The interaction model is **asynchronous by design**: users submit a task and move on; the agent works without supervision; results arrive through notifications (Slack / GitHub comment / email) and as pull requests. + +This document describes the interactivity surfaces layered on top of that model — how users **check in on**, **steer**, and **gate** running agents without requiring a live connection to the compute substrate. + +### Interaction capabilities + +1. **Submit** — `POST /tasks` with a repo and task description. Fire-and-forget by default; the CLI returns a `task_id` and exits. +2. **Status** — `bgagent status ` returns a deterministic, templated snapshot of current state (last milestone, current turn, elapsed time, cost so far). Backed by a Lambda reading `TaskEventsTable`; no LLM, no hallucination, no agent interruption. +3. **Watch** — `bgagent watch ` polls `TaskEventsTable` with an adaptive interval (500 ms when events are arriving, back-off to 5 s when idle). Same endpoint used under the hood for foreground-block UX on `ask` and for HITL approval waits. +4. **Nudge** — `bgagent nudge ""` writes a row into `TaskNudgesTable`. The agent reads pending nudges between turns, acknowledges with a `nudge_acknowledged` milestone event, and integrates the nudge on its next turn. +5. **Ask** — `bgagent ask ""` (Phase 2) writes a question row. The agent answers at the next between-turns boundary; the answer surfaces as a `status_response` event. CLI default is foreground block-and-poll with a spinner; task and answer are both durable if the CLI disconnects. +6. **Approval gates** — Phase 3 Cedar-driven hard gates. Agent emits `approval_requested`, waits for a decision from `bgagent approve` / `bgagent deny` or a Slack button-press. Detailed design in `PHASE3_CEDAR_HITL.md`. + +### Core architectural choices + +- **Single AgentCore Runtime** authenticated via IAM (SigV4) from the orchestrator Lambda. No JWT-authenticated runtime, no direct CLI-to-runtime path. +- **Durable event table (`TaskEventsTable`)** is the one source of truth for agent progress. Every reader — CLI, Slack/GitHub/email dispatchers, status Lambda — reads from this table, never from the live agent. +- **Polling-only CLI.** No SSE, no WebSockets. DDB eventually-consistent reads with an `event_id` cursor are cheap, reliable, and compute-agnostic. +- **Notification plane as first-class.** A FanOutConsumer Lambda subscribes to `TaskEventsTable` DDB Streams and routes per-event-type to per-channel dispatcher Lambdas (Slack, email, GitHub comment). Per-channel defaults ship in v1. +- **Agent interaction via the hook mechanism the Claude Agent SDK provides.** Nudges, asks, and approvals all use `Stop` / between-turns hooks; no mechanism outside the SDK's contract is required. + +--- + +## Revision history + +| Rev | Date | Summary | +|-----|------|---------| +| 6 | 2026-04-29 | Current active design. Async-only interaction model: single runtime, polling-only CLI, notification plane as first-class UX, `bgagent status` + `bgagent watch` + `bgagent nudge` in v1, `bgagent ask` + Phase 3 Cedar HITL layered on top. | + +--- + +## Table of contents + +1. [Design goals](#1-design-goals) +2. [Architecture overview](#2-architecture-overview) +3. [Components](#3-components) +4. [Event model](#4-event-model) +5. [User interactions](#5-user-interactions) +6. [Notification plane](#6-notification-plane) +7. [Security and trust model](#7-security-and-trust-model) +8. [State machine](#8-state-machine) +9. [Error handling and observability](#9-error-handling-and-observability) +10. [Debug escape hatch](#10-debug-escape-hatch) +11. [Architectural decisions](#11-architectural-decisions) +12. [Implementation phases](#12-implementation-phases) +13. [Open questions](#13-open-questions) +14. [Appendix A — Claude Agent SDK reference](#appendix-a--claude-agent-sdk-reference) +15. [Appendix B — AgentCore Runtime reference](#appendix-b--agentcore-runtime-reference) +16. [Appendix C — Competitive landscape](#appendix-c--competitive-landscape) + +--- + +## 1. Design goals + +### Primary goals + +- **Compute-agnostic.** Nothing in the interaction surface depends on a specific compute substrate. The agent could run on AgentCore today and ECS tomorrow with no changes to the CLI or notification plane. +- **Survive disconnect.** Every interaction is durable in DynamoDB. A CLI crash, a closed laptop, or a flaky network never kills a task and never loses a reply. +- **Fire-and-forget by default.** Users submit and move on. Active observation is opt-in through `status`/`watch`. +- **No UX choice at submission time.** There is exactly one submit command and one observation command. Users do not pick between "resilient" and "live" when they submit. +- **Notification as first-class.** When the agent needs a human (approval gate, ask response, task completion), it reaches the user through their configured channel — not by hoping the user is watching a terminal. + +### Explicit non-goals + +- Token-by-token live streaming. Users want to know *what step* the agent is on, not *what character* it's typing. +- Sub-200 ms interaction latency. Human interaction in an async coding workflow is calibrated to seconds, not milliseconds. +- Transactional undo of agent actions. Tool calls are committed; the agent cannot retroactively revert a filesystem change because a user objected after the fact. +- Pair-programming / co-edit modes. A different product shape. + +### Requirements traceability + +| Req | Covered by | +|---|---| +| R1. Users don't pick compute or observability at submission | Single submit command; `TaskEventsTable` is compute-agnostic | +| R2. Fire-and-forget runs independently | Orchestrator path runs without a client connection | +| R3. HITL notification when configured | `approval_requested` event → FanOutConsumer → Slack/email | +| R4. Users can check in + steer any time | `bgagent status` + `bgagent watch` + `bgagent nudge` + (Phase 2) `bgagent ask` | +| R5. Agent updates source context if configured | FanOutConsumer → GitHub issue-comment dispatcher (edit-in-place) | + +--- + +## 2. Architecture overview + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ CLIENT SURFACES │ +│ │ +│ bgagent CLI Slack bot GitHub webhook Web UI (future) │ +│ │ │ │ │ │ +│ └─────────────┴────────────────┴────────────────────┘ │ +│ │ │ +└────────────────────────────────┼────────────────────────────────────────┘ + │ REST (Cognito JWT or HMAC webhook) + ▼ + ┌──────────────────────────────────────────────┐ + │ API Gateway (v1) │ + │ │ + │ POST /tasks submit │ + │ GET /tasks/{id} status-api │ + │ GET /tasks/{id}/events watch │ + │ DELETE /tasks/{id} cancel │ + │ POST /tasks/{id}/nudge nudge │ + │ POST /tasks/{id}/asks ask (P2) │ + │ POST /tasks/{id}/approvals approve P3 │ + │ POST /webhooks/tasks GH webhook │ + └───────────┬──────────────────────────────────┘ + │ + ┌────────────┼───────────────┬───────────────────────┐ + ▼ ▼ ▼ ▼ + SubmitTaskFn CLI-read Fns Nudge/Ask/Approve Webhook Fn + │ (status/events) write Fns │ + │ │ │ │ + │ async │ read │ write │ async + │ invoke │ │ │ invoke + ▼ ▼ ▼ ▼ + OrchestratorFn OrchestratorFn + │ │ + │ admission check │ + │ InvokeAgentRuntime (SigV4) │ + ▼ ▼ + ┌─────────────────────────────────────────────────────────────┐ + │ AgentCore Runtime — single IAM-authed │ + │ (agent container: pipeline, runner, hooks) │ + └──┬────────────────┬───────────────┬──────────────────────┬──┘ + │ writes │ reads │ reads │ reads + ▼ ▼ ▼ ▼ + TaskEvents TaskTable TaskNudges TaskApprovals + Table (state) Table Table (P3) + │ ▲ + │ DDB Stream (NEW_IMAGE) │ + ▼ │ + FanOutConsumer (router) │ + │ │ + ├─→ SlackDispatchFn ──▶ Slack Web API │ + ├─→ EmailDispatchFn ──▶ SES │ + └─→ GitHubDispatchFn ──▶ GitHub REST (edit-in-place) │ + │ │ + │ action-button callback │ + │ (approve/deny) │ + └─────────────────────────┘ +``` + +Key properties: + +- **One write path in, one read path out.** Every durable agent signal lands in `TaskEventsTable` (or `TaskTable` for state transitions). Every consumer reads from there. +- **Orchestrator is the only substrate-aware component.** Replace `InvokeAgentRuntime` with `ecs:RunTask` and the CLI + notification plane don't notice. +- **No client holds a live connection to the agent.** `watch` is a polling loop against `TaskEventsTable`, not a stream from the runtime. + +--- + +## 3. Components + +### 3.1 AgentCore Runtime (IAM-authenticated) + +A single AgentCore Runtime, invoked via `bedrock-agentcore:InvokeAgentRuntime` with SigV4 from the orchestrator Lambda. No JWT authorizer; no direct CLI access. + +- **Input:** task payload from orchestrator (task_id, repo, task_description, optional initial_approvals, optional trace_flag) +- **Output:** none via response stream — the runtime is invoked fire-and-forget. All observable state flows through `TaskEventsTable` and `TaskTable`. +- **Lifecycle:** `idleRuntimeSessionTimeout` and `maxLifetime` both set to 8 hours (AgentCore max). A running task holds the session; an idle runtime is evicted by AgentCore. +- **Compute substitutability:** replacing this with ECS/Fargate is a change confined to the orchestrator + the AgentCore Runtime CDK construct. Nothing else in the system observes the difference. + +### 3.2 OrchestratorFn + +Durable-execution Lambda that owns the task lifecycle from submission to terminal state. + +Responsibilities: +- **Admission control** — atomic DDB conditional update on `UserConcurrencyTable` (`active_count < max`); reject with 429 if over quota. +- **State transition** `SUBMITTED → HYDRATING → RUNNING → FINALIZING → terminal`. +- **Invocation** — calls `InvokeAgentRuntime` with SigV4. +- **Poll loop** — waits for the agent to land a terminal status in `TaskTable`; enforces heartbeat watchdog; transitions to `FAILED` if the container dies. +- **Finalize** — TTL + concurrency decrement + synthesized terminal event. + +Hydration (blueprint merge, repo config, PAT retrieval, prompt assembly) is targeted to live **inside the agent container at startup**, not in the orchestrator. This keeps the orchestrator thin, lets heavy I/O fail inside a durable 8 h runtime rather than a 15 min Lambda, and gives the runtime container the IAM it needs for those reads anyway. + +> **Status (2026-04-30):** the rev-6 PR ships with hydration still in the orchestrator Lambda for scope reasons — moving it is pure architectural relocation with no user-visible delta and a ~2,700 lines porting surface (TypeScript → Python with new boto3 clients and a GraphQL GitHub path). Tracked as AD-11 carry-forward in upstream [issue #53](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/53) — current plan is a hybrid split: keep lightweight preflight in the orchestrator, move heavy I/O hydration to the container. Contract drift during the deferral window is bounded by the `SUPPORTED_HYDRATED_CONTEXT_VERSION` version gate in `agent/src/models.py`. + +### 3.3 SubmitTaskFn + +Validates a submission, writes the `TaskRecord` with `status=SUBMITTED`, emits a `task_created` event, and async-invokes `OrchestratorFn`. + +- Single path for all tasks. No `execution_mode` branching. +- Works identically for CLI-initiated submissions (Cognito JWT) and webhook-initiated submissions (HMAC, after the webhook authorizer). + +### 3.4 TaskEventsTable + +The durable event spine. PK = `task_id`, SK = `event_id` (ULID), TTL enabled, **DDB Streams enabled** (`NEW_IMAGE`). + +Writers: +- `ProgressWriter` (inside the agent container) — per tool call, per turn, per milestone, cost updates, errors. +- `OrchestratorFn` — `task_created`, `hydration_*`, `session_started`, `task_*`, `preflight_*`, `admission_rejected`, `guardrail_blocked`. +- Cancel/reconciler handlers — `task_cancelled`, `task_stranded`. + +Readers: +- `get-task-events` Lambda (backs `bgagent watch` and `bgagent events`). +- `bgagent status` Lambda (templated snapshot). +- `FanOutConsumer` (stream-subscribed; see §6). + +Cost profile is negligible: eventually-consistent queries with a cursor return ~0.5 RCU per page. 50 simultaneous watchers polling every 2 seconds is pennies per active hour. + +### 3.5 TaskTable + +Task state machine: `SUBMITTED → HYDRATING → RUNNING → FINALIZING → {COMPLETED, FAILED, CANCELLED, TIMED_OUT}` with Phase 3 adding `AWAITING_APPROVAL`. + +Writers: create-task, orchestrator, cancel, agent pipeline (terminal write), reconcilers. Transitions are conditional DDB writes; illegal transitions are rejected. + +### 3.6 TaskNudgesTable + +PK = `task_id`, SK = `nudge_id`. A row represents a pending user steering message. + +Producer: `POST /tasks/{id}/nudge` handler (after ownership check, guardrail scan, and rate-limit conditional update). +Consumer: agent between-turns hook reads pending nudges, emits `nudge_acknowledged` milestone, and injects the nudge text into the next turn via `decision: "block"`. + +### 3.7 TaskApprovalsTable (Phase 3) + +Phase 3 approval-request spine. Detailed schema in `PHASE3_CEDAR_HITL.md`. Semantics summary: +- Agent writes an approval row with the request context. +- Agent transitions `RUNNING → AWAITING_APPROVAL` and enters a poll loop. +- User responds via REST (`POST /tasks/{id}/approvals/{request_id}`) or via a Slack button dispatched by the notification plane. +- On decision, agent transitions back to `RUNNING`; denial reasons are injected as Stop-hook steering on the next turn. + +### 3.8 FanOutConsumer (router) + +Lambda subscribed to `TaskEventsTable` DDB Streams (`ParallelizationFactor: 1`, preserving per-`task_id` ordering by shard). Reads per-task notification config (from `TaskTable` metadata or `RepoTable` defaults), filters events by channel subscription, and invokes per-channel dispatcher Lambdas. + +- **SlackDispatchFn** — posts to configured channel / DM. Includes action buttons for `approval_required` events. +- **EmailDispatchFn** — SES. +- **GitHubDispatchFn** — edits a single GitHub issue comment in place via `PATCH /repos/{o}/{r}/issues/comments/{id}`. On 404 (comment deleted upstream) falls back to POSTing a fresh comment. Per-task ordering is guaranteed upstream by DDB Stream `ParallelizationFactor: 1`, so no conditional-request header is needed (and GitHub's REST API does not accept `If-Match` on this endpoint — see §6.4). + +Detailed routing and default filters in §6. + +### 3.9 Reconcilers + +Two scheduled Lambdas that backstop the state machine: + +- **Stranded-task reconciler** (every 5 min) — catches tasks stuck in non-terminal states past a unified timeout (`STRANDED_TIMEOUT_SECONDS=1200` default). Covers `OrchestratorFn` async-invoke crashes and container crashes. Transitions stuck tasks to `FAILED` with a `task_stranded` event. +- **Concurrency reconciler** (every 15 min) — recomputes `active_count` per user by querying the `UserStatusIndex` GSI and corrects drift in `UserConcurrencyTable`. + +### 3.10 CLI (`bgagent`) + +Commands: +- `submit` — fire-and-forget; returns `task_id`. +- `status ` — templated snapshot. +- `watch ` — adaptive polling loop. +- `events ` — raw event stream (debug). +- `nudge ""` — steer. +- `cancel ` — stop the task. +- `ask ""` (Phase 2) — ask the agent a question. +- `approve ` / `deny ` / `pending` / `policies` (Phase 3) — HITL. + +Authentication: Cognito User Pool ID token in `Authorization` header for all REST calls. Token caching in `~/.bgagent/credentials.json` with auto-refresh. + +--- + +## 4. Event model + +### 4.1 Schema + +`TaskEventsTable` row: + +```jsonc +{ + "task_id": "abc123", // PK + "event_id": "01JXY...", // SK, ULID (time-sortable) + "event_type": "agent_tool_call", + "timestamp": "2026-04-29T15:30:12Z", + "ttl": 1735689600, + "metadata": { + "tool_name": "Bash", + "tool_input_preview": "pytest tests/ -x", // ≤200 chars by default; 4KB with --trace + "turn": 7, + "...": "..." + } +} +``` + +### 4.2 Event types + +| Type | Emitted by | Meaning | +|---|---|---| +| `task_created` | SubmitTaskFn | New task accepted | +| `hydration_started` / `hydration_completed` | Agent startup | Blueprint + repo config loaded | +| `session_started` | Orchestrator | AgentCore session established | +| `agent_turn` | Runner | One model-roundtrip completed; includes turn number, model, thinking preview | +| `agent_tool_call` | Runner / PreToolUse hook | About to invoke a tool | +| `agent_tool_result` | Runner / PostToolUse hook | Tool returned | +| `agent_milestone` | Agent code (pipeline, hooks) | Named checkpoint (`repo_cloned`, `pr_opened`, `nudge_acknowledged`, ...) | +| `agent_cost_update` | Runner | Cumulative token + dollar cost | +| `agent_error` | Runner | Handled exception | +| `approval_required` (P3) | PreToolUse Cedar hook | Cedar policy requires user decision | +| `approval_decided` (P3) | Approve/Deny Lambda | User responded | +| `status_response` (P2) | Between-turns hook | Agent answered an `ask` | +| `nudge_acknowledged` | Between-turns hook | Agent saw a nudge before incorporating it | +| `pr_created` | Pipeline | PR opened for the task | +| `task_completed` / `task_failed` / `task_cancelled` / `task_stranded` | Orchestrator / reconciler | Terminal | + +Named milestones (`pr_created`, `nudge_acknowledged`, `repo_setup_complete`, …) are written as `agent_milestone` events with `metadata.milestone` carrying the name. The fan-out router unwraps an allowlisted subset (§6.2) so per-channel default filters can target milestone names directly (e.g. GitHub's default set includes `pr_created`); unlisted milestone names stay wrapped and do not route. The watch CLI renders all milestones regardless of the allowlist. + +### 4.3 Previews and truncation + +Text fields (thinking, tool input, tool output, error details) are truncated to 200 characters by default to keep event rows small. The `--trace` flag raises the cap to 4 KB and additionally writes a full trajectory to S3 (see §10). + +### 4.4 Cursor semantics + +Consumers page `TaskEventsTable` using `event_id` as a cursor: `KeyConditionExpression: task_id = :id AND event_id > :cursor`, `ConsistentRead: true`. ULID sort order is time-monotonic, so lexical comparison gives time ordering. + +--- + +## 5. User interactions + +### 5.1 `bgagent submit` + +``` +$ bgagent submit --repo org/repo "fix the auth timeout bug" +task submitted: abc123 +``` + +Writes `TaskRecord`, fires orchestrator, returns. The CLI does not wait. `--wait` flag is available for scripting (blocks until terminal state, returns non-zero on failure). + +### 5.2 `bgagent status` + +Deterministic, templated snapshot. No LLM. + +``` +$ bgagent status abc123 +Task abc123 — RUNNING (3m 14s elapsed) + Repo: org/repo + Turn: 7 / ~12 + Last milestone: nudge_acknowledged (42s ago) + Current: Bash tool call + Cost: $0.18 / budget $2.00 + Last event: 2026-04-29T15:30:12Z +``` + +Implementation: +- Lambda reads the last N events from `TaskEventsTable` + current `TaskRecord`. +- Renders from a fixed template. Never calls an LLM. Never hallucinates. +- Fast (<200 ms P95), free, safe to call repeatedly. + +### 5.3 `bgagent watch` + +Polling loop against `GET /tasks/{id}/events?after=` with **adaptive interval**: + +- Start at 500 ms. +- If a poll returns ≥1 event, keep at 500 ms. +- If a poll returns 0 events, back off: 1 s, 2 s, 5 s (cap). +- Reset to 500 ms on the next event. + +Renders events as they arrive. Exits on terminal status. Cursor is the last `event_id` seen. + +Cost profile: 50 simultaneous watchers × ~0.5 RCU per empty poll × 5 s intervals when idle ≈ negligible. + +### 5.4 `bgagent nudge` + +``` +$ bgagent nudge abc123 "also fix the logging module, separate commit" +nudge queued: nudge_01JX... +``` + +Flow: +1. CLI `POST /tasks/{id}/nudge` → rate-limit conditional update + `PutItem` in `TaskNudgesTable`. +2. Agent's Stop hook fires between turns. Calls `nudge_reader.read_pending(task_id)` — returns all pending nudges for this task (concatenated into one `` block if multiple). +3. Hook emits `nudge_acknowledged` milestone to `ProgressWriter` **before** returning to the SDK. User sees this event immediately via `watch` or Slack. +4. Hook returns `{"decision": "block", "reason": }`. The SDK treats this as the start of the next user turn; the agent incorporates the nudge on its response. +5. Nudge row is marked consumed via conditional update (`consumed_at` set only if currently null). + +**Cost model — honest:** the nudge burns one turn from the task's `max_turns` budget. The acknowledgment rides in the same turn (the combined-turn ack pattern). This is the only mechanism the Claude Agent SDK exposes for injecting user-visible text mid-run; there is no "append to system prompt mid-conversation" API (see Appendix A). + +### 5.5 `bgagent ask` (Phase 2) + +Ask the agent a natural-language question that requires its own reasoning. Always burns a turn. Always has latency (bounded above by the agent's current turn duration, which can be minutes). + +**CLI default: foreground block-and-poll with a spinner.** + +``` +$ bgagent ask abc123 "why did you change the retry logic?" +⠋ queued as ask_01JX... — waiting for agent +⠙ agent is running tool: Bash (turn 7/~12) — 42s elapsed +✓ agent responded (1m 14s) + +The existing retry used exponential backoff with no jitter, causing thundering +herd under load. Added jitter to spread retries across the window. +``` + +Flow: +1. CLI `POST /tasks/{id}/asks` → `{ask_id, cursor}`. +2. CLI polls `GET /events?after=&type=status_response&correlation_id=` with adaptive interval. +3. Spinner renders last `agent_turn` / `agent_tool_call` so the user sees the agent is alive. +4. Agent's between-turns hook reads the pending ask, injects it as a user turn via `decision: "block"`, agent answers, hook emits `status_response{ask_id, content, turn}`. +5. CLI prints the response and exits. + +Flags: +- default → foreground block +- `--no-wait` → returns `ask_id` immediately; response delivered via Slack/watch +- `--timeout N` → override default 5 min (hard cap 10 min) + +**Durability:** the ask row lives in DDB regardless of CLI state. If the user Ctrl-Cs or the terminal closes, the ask still executes; the response is retrievable via `bgagent asks show `, `bgagent watch`, or Slack. + +**Rate limit:** 1 open ask per task per user (429 otherwise). Forward-compatible with multi-user team scenarios. + +### 5.6 `bgagent approve` / `deny` / `pending` / `policies` (Phase 3) + +HITL approval commands. All flows are REST + DDB; no streaming. Detailed design in `PHASE3_CEDAR_HITL.md`. Summary: + +- Agent emits `approval_required` with the tool context. +- Notification plane dispatches the event (Slack with action buttons, email, GitHub). +- User responds via `bgagent approve `, `bgagent deny --reason "…"`, or Slack button click. +- Agent's poll loop sees the decision and proceeds or deny-steers. + +### 5.7 `bgagent cancel` + +Writes `cancellation_requested` flag on `TaskRecord`; agent's between-turns hook checks it and terminates. Agent's PR-short-circuit logic commits partial work before exit. + +--- + +## 6. Notification plane + +### 6.1 FanOutConsumer as router + +``` +TaskEventsTable ──DDB Stream──▶ FanOutConsumer + │ + │ reads notification config + │ (per-task or per-repo) + │ + ┌─────────────┼─────────────┐ + ▼ ▼ ▼ + SlackDispatch EmailDispatch GitHubDispatch + │ │ │ + Slack Web API SES GitHub REST API +``` + +- Single Lambda subscribes to the DDB Stream. Stateless; fails-forward into SQS DLQ on per-event errors. +- `ParallelizationFactor: 1` on the event-source mapping → per-`task_id` shard ordering preserved for free. +- Router reads per-task notification config (channel enablement + event-type filters), then invokes the relevant dispatcher Lambda(s) per event. +- Dispatchers are separate Lambdas so a GitHub API outage doesn't block Slack notifications. + +### 6.2 Per-channel defaults (v1) + +| Channel | Default subscribed events | Opt-in via `--verbose` | +|---|---|---| +| **Slack** | `task_completed`, `task_failed`, `task_cancelled`, `pr_created`, `agent_error`, `approval_required`, `status_response` | adds `agent_milestone` | +| **Email** | `task_completed`, `task_failed`, `approval_required` | — | +| **GitHub issue comment** | `pr_created`, terminal status (single edit-in-place comment) | — already minimal | + +Rationale: if Slack pings on every milestone, users mute the bot within days. Default to the minimal set that surfaces decision-requiring events and completion; power users opt into verbose streams. + +### 6.3 Slack approval buttons + +`approval_required` events delivered to Slack include `Approve` / `Deny` action buttons. On click, Slack invokes an interaction callback Lambda which writes to `TaskApprovalsTable` via the same `POST /approvals` path the CLI uses. This gives the common case (reviewer in Slack, not at a terminal) a one-click response path. + +### 6.4 GitHub issue comment — edit-in-place + +A single comment per task, edited in place as the agent progresses (terminal states + `pr_created` by default). + +**Concurrency:** Per-`task_id` ordering is guaranteed upstream by DDB Streams on `TaskEventsTable` with `ParallelizationFactor: 1`, and the fanout Lambda is the only writer on its own comment, so concurrent edits of the same comment body are not possible — last-writer-wins is safe because there is no concurrent writer to lose to. The dispatcher issues a single PATCH per event (no GET round-trip, no conditional headers). If the comment has been deleted upstream (404), it falls back to POSTing a fresh comment. + +**Tolerated races (bounded, logged, not silenced):** + +- *Persist failure after successful POST* — if the GitHub POST succeeds but the subsequent `TaskTable` UpdateItem that persists `github_comment_id` fails non-benignly (DDB throttling, IAM deny, etc.), the next event for the same task re-POSTs a second comment. Bounded to at most one duplicate per task per failure window (the per-invocation cap stops runaway). Logged at ERROR with `error_id: FANOUT_GITHUB_PERSIST_FAILED` so operators can alarm and reconcile. A sweeper that matches on the `bgagent:task-id=` marker body prefix is a post-v1 follow-up. +- *404 → POST race between sibling invocations* — if the previously-posted comment was deleted upstream and two consecutive fanout invocations independently re-POST before either persists the new id, both POSTs land. The UpdateItem uses `ConditionExpression: github_comment_id = :prev` so only the first persist wins; the sibling's `saveCommentState` surfaces a benign `ConditionalCheckFailedException` at INFO and the sibling's comment survives on GitHub as an orphan (the `bgagent:` marker makes it reconcilable offline). +- *Transient `loadTaskForComment` failure* — if the task record's GetItem fails transiently, `routeEvent`'s `Promise.allSettled` records the dispatcher as rejected and the batch continues. No write lands. The event is effectively dropped; the next event (e.g. `task_completed` after `pr_created`) will render the current task state. + +**Legacy field:** A previous revision persisted `github_comment_etag` on the TaskRecord. That field is no longer written or read; items that still carry it from earlier deploys are ignored by the DocumentClient (fields not declared on the typed surface pass through untouched). No migration required. + +**Why not ETag / `If-Match`:** An earlier revision attempted optimistic concurrency via GitHub's ETag and `If-Match`. In-account validation (PR #52 Scenario 7-extended) proved this does not work: GitHub's REST API rejects conditional-request headers on `PATCH /issues/comments/{id}` with `HTTP 400 "Conditional request headers are not allowed in unsafe requests unless supported by the endpoint"`. The ETag returned on GET is a cache validator only; the write endpoint does not honor it. Upstream ordering via the DDB-Stream configuration above is sufficient on its own. + +### 6.5 Per-task notification config + +Submitted with the task (optional) or resolved from repo defaults: + +```jsonc +{ + "notifications": { + "slack": { "enabled": true, "channel": "#coding-agents", "events": ["default"] }, + "email": { "enabled": true, "events": ["approval_required", "task_failed"] }, + "github": { "enabled": true, "events": ["default"] } + } +} +``` + +`"default"` resolves to the v1 per-channel defaults above. + +--- + +## 7. Security and trust model + +### 7.1 Authentication surfaces + +| Surface | Auth | Notes | +|---|---|---| +| CLI → REST API (all endpoints) | Cognito JWT (ID token) | Managed by User Pool | +| GitHub webhook → `POST /webhooks/tasks` | HMAC-SHA256 via request authorizer | Shared secret in Secrets Manager | +| OrchestratorFn → AgentCore Runtime | SigV4 (IAM) | Lambda execution role | +| Agent container → AWS APIs (DDB, S3, Bedrock) | SigV4 via runtime's execution role | Scoped per-runtime | +| Slack button → interaction callback | Slack signing secret | Standard Slack pattern | + +### 7.2 Nudge security + +- Ownership check: the Lambda verifies `user_id` (from Cognito claims) matches the task's `user_id` before accepting the nudge. +- Rate limit: 10 nudges per task per minute (conditional update on a `RATE##MINUTE#` row). +- Size cap: 2 KB per nudge. +- Guardrail pre-screen: Bedrock guardrail scans nudge text for prompt-injection patterns before persisting. + +### 7.3 Approval security (Phase 3) + +- Ownership check on approve/deny. +- Atomic state transition via `TransactWriteItems` (approval row + TaskTable status). +- Recent-decision cache (60 s) prevents retry-loop storms. +- Denial reason sanitized by the DenyTaskFn Lambda (Bedrock output scanner) before persisting. + +### 7.4 Event table privacy + +- Previews truncate to 200 chars → low risk of accidental secret capture in common cases. +- Agent-side output scanners redact secrets before calling `ProgressWriter`. +- `--trace` flag opts into larger previews + S3 trajectory dumps; S3 objects are written to a user-scoped prefix with short TTL. + +--- + +## 8. State machine + +### 8.1 Core transitions + +``` +SUBMITTED ──▶ HYDRATING ──▶ RUNNING ──▶ FINALIZING ──▶ COMPLETED + │ │ │ │ + │ │ │ └──▶ FAILED + │ │ │ └──▶ TIMED_OUT + │ │ └──▶ CANCELLED + │ │ └──▶ AWAITING_APPROVAL (P3) + │ └──▶ FAILED (stranded) + └──▶ FAILED (stranded) +``` + +### 8.2 Phase 3 addition: `AWAITING_APPROVAL` + +``` +RUNNING ──▶ AWAITING_APPROVAL ──▶ RUNNING (approve or deny-with-steering) + │ + ├──▶ CANCELLED (explicit cancel) + └──▶ FAILED (stranded reconciler catches abandoned approval) +``` + +The `AWAITING_APPROVAL` state holds the user's concurrency slot (paused but alive). See `PHASE3_CEDAR_HITL.md` for full semantics. + +### 8.3 Write rules + +- Every transition is a conditional DDB write: `#status = :fromStatus`. +- Illegal transitions are rejected at the storage layer (not enforced in code). +- The valid-transition table lives in `cdk/src/handlers/shared/task-status.ts`. + +--- + +## 9. Error handling and observability + +### 9.1 Fail-open vs fail-closed + +| Component | Failure posture | Rationale | +|---|---|---| +| `ProgressWriter` | Fail-open (3-strike circuit breaker) | Event telemetry must never crash the task | +| Nudge/ask rate-limit conditional update | Fail-closed (return 429) | Accurate throttling is a product guarantee | +| Cedar policy evaluation | Fail-closed (treat as DENY) | Security-critical; unknown outcome = deny | +| Approval poll DDB read | Fail-open with tolerance (10 consecutive failures → TIMED_OUT) | Tolerate transient DDB errors; fail closed on sustained | +| Notification dispatcher | Fail-open (log + DLQ) | A Slack outage must not block the agent | + +### 9.2 Unified debugging: event correlation + +Every log line, event, and metric carries `task_id`. CloudWatch Logs Insights queries across all Lambdas on `task_id = "abc123"` give the full cross-component picture. + +### 9.3 OpenTelemetry + +Each component emits OTEL traces with `task_id` as a baggage item. OrchestratorFn starts the root span; AgentCore runtime continues it via env-var propagation; Lambdas downstream of DDB Streams resume from the event's `traceparent` attribute. + +### 9.4 Dashboards + +CloudWatch dashboard shows, per task: +- State transitions timeline +- Event rate by type +- Cost accumulation +- Concurrency slot utilization + +### 9.5 Alarms + +Currently deferred — no operational notification channel exists for this project beyond Slack/email user-facing notifications. When an ops channel is added (SNS/PagerDuty), the alarm plumbing is a small follow-up; the metric data is already flowing. + +--- + +## 10. Debug escape hatch + +### 10.1 `--trace` flag + +Without live streaming, a developer debugging a misbehaving agent needs a richer offline view than the default 200-char event previews. The `--trace` flag: + +``` +$ bgagent submit --trace "fix the auth bug" +``` + +Changes for a trace-enabled task: +- `ProgressWriter` preview truncation raised from 200 chars → 4 KB. +- Full agent trajectory (SDK message log, tool I/O, hook callbacks) written to S3 on task completion. +- A `trajectory_uploaded` milestone event with the S3 URI is emitted; the CLI can surface it at the end of `watch` or `status`. + +Storage: +- S3 prefix: `s3:///traces//.jsonl.gz`. +- TTL: 7 days (lifecycle policy). +- Pre-signed URLs available via `bgagent trace download `. + +### 10.2 When to use it + +- Reproducible failure modes during development. +- Customer-reported "agent did the wrong thing" incidents. +- Reward-hacking / hallucination audits. + +Not intended for routine observability — that's what `watch` and notifications are for. + +--- + +## 11. Architectural decisions + +Short summaries of the load-bearing choices. Each decision is phrased as the chosen option; rationales are concise. + +### AD-1. Single AgentCore Runtime, IAM-authenticated + +Exactly one runtime, invoked via SigV4 from the orchestrator. The CLI never talks directly to the runtime. + +*Why:* Compute-substrate portability (ECS/Fargate swap requires only orchestrator changes); simpler auth; one runtime to operate and observe. Direct CLI-to-runtime paths would reintroduce substrate coupling and force a choice between live-stream and durability at submission time. + +### AD-2. Polling-only CLI + +`bgagent watch` / `bgagent status` / `bgagent ask` all use REST-polling against `TaskEventsTable` with an adaptive interval. No SSE. No WebSockets. + +*Why:* Human-scale interaction latency (seconds) is well-served by polling; DDB costs are trivial; no streaming infrastructure to build, operate, or secure. Cursor, GitHub Copilot coding agent, and Codex all use the same pattern. + +### AD-3. `TaskEventsTable` as the single event spine + +Every durable signal from the agent flows through this table. Every consumer reads from it. + +*Why:* Decouples the agent from every consumer. CLI, Slack bot, GitHub integration, and any future web UI all read the same substrate without touching the runtime. + +### AD-4. Notification plane as first-class + +FanOutConsumer routes events per-channel with sensible defaults shipping in v1. + +*Why:* In an async product, notifications are the primary UX. Shipping without defaults would cause users to mute integrations on day one. + +### AD-5. Nudge acknowledgment via combined-turn ack + +The between-turns hook emits a `nudge_acknowledged` milestone to `ProgressWriter` **before** returning `decision: "block"` with the nudge text. One turn burned (same as today); acknowledgment visible immediately. + +*Why:* The Claude Agent SDK does not expose a mechanism to append to system context mid-conversation. The `HookEvent` enum is fixed; `ClaudeAgentOptions.system_prompt` is construction-time only; `hookSpecificOutput.additionalContext` is user-visible-only (confirmed `not-planned` by Anthropic). One-turn-per-nudge is an architectural constraint of the SDK; we surface it honestly rather than pretending it's free. + +### AD-6. `bgagent status` is deterministic; `bgagent ask` is the agent + +`status` = templated Lambda reading `TaskEventsTable`. `ask` = a real question to the agent, always costs a turn, always has latency. + +*Why:* Users understand dashboard reads vs. questions-to-a-thinking-entity. One command per contract is clearer than one command with a flag that silently changes execution model. + +### AD-7. `bgagent ask` foreground block-and-poll + +Default UX blocks with a spinner showing current agent activity. Durable underneath — CLI disconnect does not cancel the ask or lose the answer. + +*Why:* Matches user expectation of a synchronous CLI call. Survives a closed laptop. Spinner surfaces the bounded-but-non-trivial latency (turns can take minutes) without feeling like a hang. + +### AD-8. HITL: hard gates only in v1 + +Phase 3 ships hard gates. No soft questions, no "proceed with default if no response" semantics. + +*Why:* Soft-question-with-timeout creates a ticking-clock UX that's actively hostile in an async workflow. "Gate or no gate" is the coherent choice. A future `effect: "advise"` tier (non-blocking FYI events, no timeout) is documented in the Phase 3 design as post-v1. + +### AD-9. GitHub edit-in-place via DDB-Stream ordering, not SQS FIFO + +DDB Streams on `TaskEventsTable` with `ParallelizationFactor: 1` give per-`task_id` ordering. The fanout Lambda is the only writer on its own comment, so no concurrent writer exists to race — last-writer-wins is safe. The dispatcher PATCHes directly (no GET-then-PATCH, no conditional headers). + +*Why:* Simpler than SQS FIFO (no queue, no DLQ, no per-group throughput ceiling), and lower latency than a GET-then-PATCH round-trip. + +*Rejected alternative — `If-Match` ETag:* An earlier revision of this design used optimistic concurrency via GitHub's ETag. Deploy-validation (PR #52 Scenario 7-extended) proved that `PATCH /issues/comments/{id}` rejects `If-Match` with HTTP 400 (`"Conditional request headers are not allowed in unsafe requests unless supported by the endpoint"`). The ETag returned on GET is a cache validator only. Upstream DDB-Stream ordering makes the ETag unnecessary anyway. + +### AD-10. Stranded-task reconciler with a unified timeout + +One timeout value covers all stranded cases (orchestrator crash, container crash, general abandonment). + +*Why:* The interactive-specific timeout disappeared along with the interactive path. One reconciler, one threshold, easier to reason about. + +### AD-11. Agent-side hydration (hybrid split; partially deferred) + +Blueprint merging, repo config, PAT retrieval, and prompt assembly are targeted for the agent container at startup, not the orchestrator Lambda. + +*Why:* Hydration artifacts (cloned repos, merged blueprints, rendered prompts) are large and only needed inside the runtime. Failures belong inside the durable 8 h runtime rather than a 15 min Lambda. The runtime already has the IAM it needs for those reads. Industry precedent (Cursor background agents, GitHub Copilot coding agent, Devin, Temporal's activity-worker pattern, LangGraph's queue-worker split) converges on worker-side hydration for long-running async agents. + +*Target shape — hybrid split:* keep the **cheap preflight** in the orchestrator (PAT validity check, repo-existence check, guardrail screen on the raw `task_description`) so we still fail fast before burning an AgentCore compute slot. Move the **heavy I/O hydration** (GitHub issue / PR fetch including review threads, prompt assembly, Memory retrieval, S3 blueprint reads) into the agent container. + +*Status (2026-04-30):* **deferred to a follow-up PR**, tracked at [upstream issue #53](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/53). Rev-6 ships with full hydration still in the orchestrator Lambda. Reasons: (a) pure architectural relocation with no user-visible change, (b) ~2,700 lines porting surface (1,190 LOC of `context-hydration.ts` + 1,514 LOC of tests) requiring new boto3 surfaces in the container and a GraphQL GitHub client, (c) PR #52 already ships 10,000+ lines of changes across the SSE removal — folding in hydration would blur the review narrative. The Pydantic `SUPPORTED_HYDRATED_CONTEXT_VERSION` gate in `agent/src/models.py` bounds drift risk during the deferral window. + +### AD-12. `--trace` as the debug escape hatch + +Opt-in per task: 4 KB previews + full trajectory to S3 with TTL. + +*Why:* Without live streaming, debugging needs a richer offline artifact. Opt-in keeps normal-task storage costs flat. + +--- + +## 12. Implementation phases + +### Phase 1 — v1 PR + +- Single orchestrator path; delete all direct-SSE / two-runtime / interactive-mode infrastructure +- `bgagent status` (deterministic) +- `bgagent watch` with adaptive polling interval +- `bgagent nudge` with combined-turn acknowledgment +- FanOutConsumer router + per-channel default filters +- GitHub edit-in-place dispatcher (DDB-Stream ordering, 404 → POST fallback) +- Stub Slack/email dispatchers (log-only, ready for real integration in Phase 2) +- Unified stranded-task reconciler timeout +- `--trace` debug flag + +### Phase 2 — Ask + first real notifications + +- `bgagent ask` end-to-end (REST, agent-side between-turns hook, foreground block-and-poll CLI, durability-on-disconnect) +- Real Slack dispatcher (webhook + action buttons → approval callback Lambda) +- Per-task notification config + `bgagent notifications configure` + +### Phase 3 — Cedar HITL + +- Hard-gate approval gates with Cedar policy evaluation +- `bgagent approve` / `deny` / `pending` / `policies` +- `AWAITING_APPROVAL` state + orchestrator handling +- Full design in `PHASE3_CEDAR_HITL.md` + +### Phase 4 — Dispatcher polish + +- Real email dispatcher (SES) +- Real GitHub dispatcher (beyond the v1 edit-in-place stub) +- Per-repo default notification config +- `--verbose` opt-in for milestone-level events +- Dashboard widgets for notification delivery health + +### Deferred + +- **LLM-synthesized status summary** — `bgagent ask` without targeting the agent; Lambda calls an LLM to narrate state. Cost + hallucination trade-offs; revisit if v1 feedback warrants. +- **Cedar `effect: "advise"` tier** — non-blocking FYI policy tier for post-v1. Design sketch in `PHASE3_CEDAR_HITL.md`. +- **Outbound WebSocket from agent** — only if a concrete sub-200 ms latency requirement surfaces. Agent-initiated egress avoids dual-auth problems and works on any compute. +- **Multi-user watch** — multiple users attached to the same task's live event stream (teams). + +--- + +## 13. Open questions + +| ID | Question | Owner | +|---|---|---| +| Q1 | Retention policy for `--trace` S3 artifacts — 7 days or 30? Size cap per user? | Design | +| Q2 | Should `bgagent pending` (Phase 3) show all pending approvals across all of a user's tasks, or filter to a single `task_id`? | Phase 3 impl | +| Q3 | Slack action button callbacks — Slack signing secret rotation strategy? | Phase 2 impl | +| Q4 | Per-repo default notification config precedence vs per-task overrides — does per-task always win? Partial overrides? | Phase 4 impl | +| Q5 | `bgagent ask` concurrent limit — do we expose `--queue` semantics to explicitly enqueue vs 429? | Phase 2 impl | + +--- + +## Appendix A — Claude Agent SDK reference + +Pinned version: `claude-agent-sdk==0.1.53` (Python). + +### A.1 Hook surface (v0.1.53) + +`HookEvent` enum: `PreToolUse | PostToolUse | PostToolUseFailure | UserPromptSubmit | Stop | SubagentStart | SubagentStop | PreCompact | PermissionRequest | Notification`. + +Our usage: +- `PreToolUse` → Cedar policy evaluation (Phase 3), `can_use_tool`-style allow/deny. +- `PostToolUse` → output scanner (secret/PII redaction). +- `Stop` (between-turns) → `_cancel_between_turns_hook`, `_nudge_between_turns_hook`, Phase 2 ask hook, Phase 3 approval poll. + +### A.2 Between-turns injection mechanism + +Stop hook return values: +- `{}` → no-op, SDK proceeds to stop or loop. +- `{"decision": "block", "reason": ""}` → SDK emits `reason` as a synthetic user turn; agent responds on its next iteration. + +This is the **only** SDK-supported mechanism to inject agent-visible text mid-conversation. Implications: +- Every nudge, ask, and deny-with-steering burns one turn from `max_turns`. +- No "append to system prompt mid-run" primitive exists. `ClaudeAgentOptions.system_prompt` is set at construction. +- `hookSpecificOutput.additionalContext` on PostToolUse appears in docs but does not reach the model's context; Anthropic has confirmed this as `not-planned` (GitHub issues `claude-code#18427`, `claude-code#19643`). + +### A.3 Mid-run cancellation + +`ClaudeSDKClient.interrupt()` cancels the current turn without rolling back prior tool results. Used in our cancel path along with `cancellation_requested` flag on `TaskRecord`. + +--- + +## Appendix B — AgentCore Runtime reference + +### B.1 Service contract + +- HTTP on port 8080: `/invocations` (JSON + optional SSE response), `/ping` (liveness). +- `/ping` returning `"HealthyBusy"` signals an active session and prevents idle eviction. +- `maxLifetime` and `idleRuntimeSessionTimeout` both configurable up to 8 hours. We set both to the maximum. + +### B.2 Invocation + +`bedrock-agentcore:InvokeAgentRuntime` — SigV4-authenticated API call from the orchestrator. Payload is the task context; response is ignored (fire-and-forget). + +### B.3 Session management + +Same `runtimeSessionId` routes to the same MicroVM **within the same runtime ARN**. We use this property for the agent's own internal resumability (re-invocation with the same session ID lands on the same container if it's still alive), but never for CLI→runtime direct attach (which we don't do). + +--- + +## Appendix C — Competitive landscape + +Products surveyed for interaction patterns (primary sources: product docs, engineering blogs): + +| Product | Interaction model | Notes | +|---|---|---| +| **Devin (Cognition)** | Slack-thread chat during execution; fully async notifications | Closest analog; mid-run Q&A via in-thread messages is a shipped feature | +| **GitHub Copilot coding agent** | Fire-and-forget; progress visible as commits/PR activity | No mid-run steering; notifications via GitHub itself | +| **OpenAI Codex (cloud)** | SSE in web UI; external view is polling; no mid-run course-correction | Explicitly documents inability to steer mid-run | +| **Replit Agent** | Task board UI; user checks progress; no live terminal stream | Novel: automated "Decision-Time Guidance" (internal classifier-driven steering) | +| **Cursor background agents** | Pure fire-and-forget; user manually checks state | No built-in completion notifications (open feature request) | + +Key observations: +- Fire-and-forget + notifications is the dominant pattern for long-running coding agents. +- Mid-run steering exists only where there's a persistent conversation surface (Devin's Slack thread); our `bgagent nudge` + `bgagent ask` is the equivalent. +- No product ships "proceed with default if no response" for approval gates. Hard gates or no gates — that's the shipped landscape. +- Polling-based observation is ubiquitous and well-tolerated at minute-to-hour task durations. diff --git a/docs/src/content/docs/architecture/Phase-1b-rev5-followups.md b/docs/src/content/docs/architecture/Phase-1b-rev5-followups.md new file mode 100644 index 0000000..c26bf45 --- /dev/null +++ b/docs/src/content/docs/architecture/Phase-1b-rev5-followups.md @@ -0,0 +1,191 @@ +--- +title: Phase 1b rev5 followups +--- + +# Phase 1b rev-5 — follow-up status + +Created 2026-04-21 after the rev-5 multi-agent validation pass. Each item was +surfaced by one of the validators (`[SFH]` silent-failure-hunter, `[CR]` +code-reviewer, `[TDA]` type-design-analyzer, `[PTA]` pr-test-analyzer) or by +the user during review. This document tracks what landed vs what's still +pending, in the order the rev-5 rounds were executed. + +## Round summary + +| Round | Scope | Commit | Status | +|---|---|---|---| +| Rev-5 core | `bgagent run`, RUN_ELSEWHERE guard, execution_mode propagation, hydration | `022fb88`, `2d9d680` | ✅ | +| Pre-push hardening | P0-a, P0-b, P0-d, P0-e + key nits | `fe84de5` | ✅ | +| Stranded-task reconciler + concurrency raise | P0-c follow-up, MAX_CONCURRENT 3→10 | `9af3b50` | ✅ | +| Round 1 | Correctness: P1-3, P1-1, OBS-4 | `fce9d07` | ✅ | +| Round 2 | Error surfacing: P1-2, P1-5 | `bd7b886` | ✅ | +| Round 3 | Observability: OBS-1/2/3, P1-4 | `0d29939` | ✅ | +| Round 4a | Encapsulation: TDA-1, TDA-2, TDA-6 | `bc56731` | ✅ | +| Round 4b | Shared types: TDA-3, TDA-4, TDA-5 | `228c935` | ✅ | +| Round 5 | Design alignment: POLL-1, DATA-1 | `dfe7b84` | ✅ | +| Round 6 | Housekeeping (this commit) | TBD | in progress | + +## ✅ Landed (grouped by round for traceability) + +### Rev-5 final (pre-hardening) + +- `bgagent run` direct-submit interactive path (`cli/src/commands/run.ts`). +- `execution_mode` end-to-end (CreateTaskRequest, TaskRecord, TaskDetail). +- Server-side RUN_ELSEWHERE guard + TaskTable param hydration. +- Two-runtime ECR pull fix (two `AssetImage.fromAsset` instances to dodge + the L2 `AssetImage.bind` double-attach guard — see CDK-1 below). +- Client-side transport decision from `snapshot.execution_mode` (AgentCore + wraps non-2xx as 424; decide on the client instead of parsing the + wrapped response). + +### Pre-push P0 hardening (`fe84de5`) + +- **P0-a** `_SSEAdapter.write_agent_error` latent `_dropped_count` → + `_undelivered_count` fix + regression test. +- **P0-b** `task_state.get_task` distinguishes NotFound (returns `None`, + fail-open) from FetchFailed (raises `TaskFetchError`; server returns + 503). Prevents duplicate pipelines during DDB blips. +- **P0-d** `bgagent run` wraps `runSse` in try/catch; auto-cancels stranded + task + emits `bgagent status ` resume hint + exit non-zero. +- **P0-e** Post-hydration validation returns 500 + `TASK_RECORD_INCOMPLETE` with a list of missing fields. +- Key nits: shared `_stream.ts`, typed `SnapshotResult.executionMode`, + `TaskDetail.execution_mode` required in CLI, `EXECUTION_MODE_*` string + constants in server.py, `_HEARTBEAT_INTERVAL_SECONDS`, `logInfo` + cleanup, v3 diagram. + +### Stranded-task reconciler (`9af3b50`) — P0-c + +- `cdk/src/constructs/stranded-task-reconciler.ts` + handler + `cdk/src/handlers/reconcile-stranded-tasks.ts`. +- EventBridge schedule every 5 min, per-mode timeouts (300 s interactive, + 1200 s orchestrator / legacy). +- Transitions stranded tasks to FAILED with + `STRANDED_NO_HEARTBEAT`, emits `task_stranded` + `task_failed` events, + decrements concurrency. +- `MAX_CONCURRENT_TASKS_PER_USER` default raised 3 → 10. + +### Round 1 — correctness (`fce9d07`) + +- **P1-3** — attach-path `subscribe()` exception no longer falls through + to duplicate-spawn; returns 503 `SSE_ATTACH_RACE` + (`agent/src/server.py`). Duplicate-pipeline risk closed. +- **P1-1** — 409 on the SSE path is always terminal. RUN_ELSEWHERE → + fallback; any other 409 → `CliError` with a 500-byte body excerpt. + Eliminates reconnect-storm on server-side refusals + (`cli/src/sse-client.ts`). +- **OBS-4** — interactive path records `session_id` on TaskTable via + new `task_state.write_session_info`; cancel-task Lambda resolves the + correct runtime ARN from `execution_mode` + two new env vars + (`RUNTIME_IAM_ARN`, `RUNTIME_JWT_ARN`) to sidestep the CFN cycle that + would have been created by runtime-self-ARN injection. + +### Round 2 — error surfacing (`bd7b886`) + +- **P1-2** — post-SSE `getTask` failure now emits `WARN` to stderr with a + `bgagent status ` suggestion and suffixes the terminal line + with `(inferred)`. +- **P1-5** — new `_debug_cw_exc(message, exc, *, task_id)` helper + formats tracebacks into CloudWatch at every rev-5 bare + `except Exception` site. + +### Round 3 — observability (`0d29939`) + +- **OBS-1** — `_emit_sse_route_metric(task_id, route)` writes + `{event: "SSE_ROUTE", route: "attach"|"spawn"}` to CW stream + `sse_routing/`; called from both `_invoke_sse` branches. + Enables attach-vs-spawn ratio alarms. +- **OBS-2** — after hydration, always log `post-hydration params: + populated=[...] origin={k: 'record'|'caller'}`. +- **OBS-3** — structured `event` fields on admission logs + (`task.admitted.orchestrator_skipped`, `...orchestrator_invoked`, + `...orchestrator_invoke_failed`). +- **P1-4** — `_debug_cw_failures` counter bumped on daemon-thread + failures; every 5 failures (and the first) emits + `{event: "DEBUG_CW_WRITE_FAILURES", count, last_error_type}` via the + separate sse-routing code path. + +### Round 4a — encapsulation (`bc56731`) + +- **TDA-1** — `_AdapterRegistry` class owns `_threads_lock` + enforces + identity-checked pop in one place. Four open-coded sites collapsed to + `remove_if_current(task_id, adapter)`. `insert` raises on genuine + conflict. +- **TDA-2** — `_SSEAdapter.subscription()` context manager yields the + queue and auto-unsubscribes on exit (normal + exception paths). Raw + `subscribe()`/`unsubscribe()` retained for the + `_sse_event_stream` handoff to `StreamingResponse`. +- **TDA-6** — Python `ExecutionMode = Literal["orchestrator", + "interactive"]` + `normalize_execution_mode(raw)` helper for safe + coercion from DDB/env. + +### Round 4b — shared types (`228c935`) + +- **TDA-3** — `ApiErrorCode` union + `ApiErrorBody` envelope + + `isApiError(body, code)` type guard, defined in both + `cdk/src/handlers/shared/types.ts` and `cli/src/types.ts`. sse-client + uses the guard in its 409 branch. +- **TDA-4** — cross-file drift detection via + `cli/test/types-sync.test.ts`. Parses the CDK types.ts source and + asserts `ExecutionMode` + `ApiErrorCode` unions match the CLI + canonical list. Bigger `@abca/shared-types` workspace deferred per + scope. +- **TDA-5** — `SemanticEvent` TypedDict union in + `agent/src/sse_adapter.py`. Six event shapes declared, each mirroring + the sibling `ProgressWriter.write_agent_*` dict. + +### Round 5 — design alignment (`dfe7b84`) + +- **POLL-1** — `watch` polling cadence decays 500 ms → 2 s after 3 min. + First 3 min matches design §9.13.1; the decay caps REST cost for + long-running observation. +- **DATA-1** — `TaskResult` gains `turns_attempted` + `turns_completed` + (clamped to `max_turns` when `error_max_turns`). Legacy `turns` field + retained as `turns_attempted` value for back-compat. + `TaskRecord`/`TaskDetail` in CDK + CLI types mirror; `toTaskDetail` + forwards. + +## Non-code follow-ups tracked elsewhere + +### ✅ CDK-1 — Upstream bug filed: aws/aws-cdk#37663 + +`cdk/src/stacks/agent.ts` has a two-artifact workaround: + +```ts +const artifactIam = agentcore.AgentRuntimeArtifact.fromAsset(runnerPath); +const artifactJwt = agentcore.AgentRuntimeArtifact.fromAsset(runnerPath); +``` + +Root cause in `@aws-cdk/aws-bedrock-agentcore-alpha`'s `AssetImage.bind` +method: it guards against double-grant with `this.bound = true`, so +when the same artifact instance is passed to two Runtimes the second +runtime's execution role never receives ECR pull permissions. Image +pull fails with 424 "no basic auth credentials". + +Filed upstream at with +minimal repro, root-cause analysis, and a suggested fix. The code +comment at `cdk/src/stacks/agent.ts:55-68` now links the issue. +Keep the two-artifact workaround until the upstream fix ships (or +remove it when this repo upgrades to a version that includes the +fix). + +### Candidates NOT landed (by design) + +- **Full `@abca/shared-types` workspace (bigger TDA-4)** — deferred in + favour of the drift-detection test. Spin up when a third package + needs the shared types (e.g., a future SDK package, or if the web + console moves in-tree). +- **`SemanticEvent` threaded through adapter signatures** — TDA-5 + landed the types; call-site propagation (`_enqueue(event: + SemanticEvent)` etc.) deferred until we tighten mypy strictness. +- **CLI formatter for `turns_attempted`/`turns_completed`** — DATA-1 + landed the DDB/REST fields; `bgagent status` / `bgagent watch` + formatters still display just `turns`. UX decision for a separate + pass (e.g., "6 turns (7 attempted — hit max_turns cap)"). + +## Status as of this round + +All validator-surfaced P0/P1/OBS/TDA/POLL/DATA items are either landed +or explicitly classified as not-in-scope above. CDK-1 is filed +upstream (aws/aws-cdk#37663); the two-artifact workaround stays until +the upstream fix ships. diff --git a/docs/src/content/docs/architecture/Phase3-cedar-hitl.md b/docs/src/content/docs/architecture/Phase3-cedar-hitl.md new file mode 100644 index 0000000..5aac12d --- /dev/null +++ b/docs/src/content/docs/architecture/Phase3-cedar-hitl.md @@ -0,0 +1,1887 @@ +--- +title: Phase3 cedar hitl +--- + +# Phase 3 — Cedar-driven HITL Approval Gates + +> **Status:** Detailed design, pre-implementation. +> **Companion:** [`INTERACTIVE_AGENTS.md`](/architecture/interactive-agents) §5.6 (CLI commands), §8.2 (state machine). +> **Visual:** [`/sample-autonomous-cloud-coding-agents/diagrams/phase3-cedar-hitl.drawio`](/sample-autonomous-cloud-coding-agents/diagrams/phase3-cedar-hitl.drawio). +> **Rev:** 3 (2026-04-29 — hard-gate-only v1; notification-plane UX). +> **Implementation:** not started. + +--- + +## 0. Contents + +1. [What we are building, in one paragraph](#1-what-we-are-building-in-one-paragraph) +2. [The three-outcome model and why Cedar alone can't give it](#2-the-three-outcome-model) +3. [Design decisions (locked)](#3-design-decisions-locked) +4. [End-to-end request flow](#4-end-to-end-request-flow) +5. [Cedar policy authoring guide](#5-cedar-policy-authoring-guide) +6. [Engine implementation](#6-engine-implementation) +7. [REST API contract](#7-rest-api-contract) +8. [CLI UX](#8-cli-ux) +9. [State machine + concurrency](#9-state-machine--concurrency) +10. [Data model](#10-data-model) +11. [Observability](#11-observability) +12. [Security model](#12-security-model) +13. [Failure modes + fail-closed posture](#13-failure-modes--fail-closed-posture) +14. [Sample scenarios](#14-sample-scenarios) +15. [Implementation plan](#15-implementation-plan) +16. [Implementation notes (carry-forward tasks)](#16-implementation-notes-carry-forward-tasks) +17. [Deferred / out of scope](#17-deferred--out-of-scope) + +--- + +## 1. What we are building, in one paragraph + +When the agent is about to call a tool (Bash, Write, Edit, WebFetch, etc.), our Cedar policy engine decides **Allow** or **Deny**. Phase 3 adds a third outcome — **Require-approval** — that pauses the tool call, writes an approval request to a DynamoDB table **atomically with the task state transition**, dispatches a notification through the fan-out plane (Slack with action buttons, email, GitHub issue comment), and awaits a human response via a new REST endpoint + CLI command. The agent polls DynamoDB for the user's decision with strongly-consistent reads; on approval it proceeds, on denial (or timeout) the decision text is injected into the agent's context via the Phase 2 Stop-hook mechanism so the agent adapts rather than spinning. At task-submit time the user can also *pre-approve* scopes (specific tools, bash patterns, rule IDs, path patterns, or `all_session`) so low-risk agents run without any interactive gates. Cedar policies are tagged with a `@tier("hard-gate")` annotation to mark rules that should trigger an approval instead of an absolute deny — the same Cedar language, two policy files, one new outcome. + +**v1 ships hard gates only.** The agent pauses indefinitely for a decision (bounded only by the task's `maxLifetime`); on timeout it fail-closed denies with steering. There is no "proceed with default if no response" mode in v1 — see §17 for the deferred `@tier("advise")` semantics that would add non-blocking advisory events post-v1. + +--- + +## 2. The three-outcome model + +### Cedar's native model is binary + +The [Cedar authorization engine](https://www.cedarpolicy.com/) answers one question per call: given a `(principal, action, resource, context)` tuple, is the action **Allowed**, **Denied**, or **NoDecision** (no policy matched)? Our engine treats `NoDecision` as deny (fail-closed) and returns a boolean `allowed` to callers. That's the baseline Phase 3 extends. + +### What we add + +We layer a **three-outcome abstraction** on top of Cedar by running **two evaluations per tool call** against two separate policy sets: + +```text +┌──────────────────────────────────────────────────────────────────────────┐ +│ PolicyEngine.evaluate_tool_use(tool_name, tool_input) │ +│ │ +│ 1. Cedar eval against HARD_DENY_POLICIES │ +│ └─ Deny → return PolicyDecision(outcome=DENY, reason=...) │ +│ Absolute. No allowlist can override. │ +│ │ +│ 2. In-process allowlist fast-path │ +│ └─ match → return PolicyDecision(outcome=ALLOW, reason=...) │ +│ Pre-approved (from --pre-approve) or previously approved │ +│ with scope != this_call. │ +│ │ +│ 2.5. Recent-decision cache (anti-retry-loop) │ +│ └─ cached DENIED/TIMED_OUT for (tool_name, input_sha) within 60s │ +│ → auto-deny with same reason (prevents re-gate storms) │ +│ │ +│ 3. Cedar eval against HARD_GATE_POLICIES │ +│ └─ Deny → return PolicyDecision(outcome=REQUIRE_APPROVAL, │ +│ reason, timeout_s, severity, │ +│ matching_rule_ids) │ +│ Human must approve before the tool runs. Agent waits │ +│ indefinitely (bounded by task maxLifetime); timeout │ +│ fail-closed denies with steering. │ +│ │ +│ 4. Default ALLOW │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +Each evaluation is a Cedar call — sub-millisecond. No network hop. No AWS API. The "approval wait" (step 3's downstream handling) is entirely inside our `PreToolUse` hook coroutine. + +The SDK never sees `REQUIRE_APPROVAL` — after the wait, our hook returns the SDK's native `{"permissionDecision": "allow" | "deny"}` shape. The three-outcome model is an internal engine abstraction. + +### Why two policy sets, not one + +Cedar doesn't have a `require_approval` effect. We encode the tiering as a physical split into two policy files (`hard_deny.cedar`, `hard_gate.cedar`), validated by a `@tier("hard-deny" | "hard-gate")` annotation on each rule. + +Key properties: + +- **Security reviewers can read the hard-deny file alone.** Most review effort lives there because those rules are absolute; hard-gate rules have a human safety net. +- **Rule authors know where a rule lives by which file it's in.** No "forbid-with-marker" patterns that can be accidentally miscategorized. +- **Forward-compatible with a future `@tier("advise")` tier** (see §17) — a third file for non-blocking advisory rules can be added without changing the engine's outer loop. + +--- + +## 3. Design decisions + +| # | Decision | Summary | +|---|---|---| +| 1 | **Cedar encoding: two policy sets** | Physical split into `hard_deny.cedar` and `hard_gate.cedar`, validated via `@tier("hard-deny" \| "hard-gate")` annotation. Forward-compatible with a future `@tier("advise")` set (see §17). | +| 2 | **Hook point: extend `PreToolUse`, not `can_use_tool`** | PreToolUse is already async-compatible, already wired to Cedar, and already owns the tool-governance boundary. | +| 3 | **Wait mechanism: DDB strongly-consistent polling, 2s → 5s backoff** | Initial 2s cadence for the first 30s, then 5s. `ConsistentRead=True` so the agent never misses an approval that already landed. Agent waits indefinitely (bounded by `maxLifetime`). | +| 4 | **Scope allowlist: in-process, seeded from persisted `initial_approvals`** | Runtime escalation lives in the `PolicyEngine` instance. Submit-time `--pre-approve` flags persist on TaskTable and seed the allowlist at container startup. Lost on restart (rare; reconciler fails stranded tasks). | +| 5 | **CLI UX: standalone `bgagent approve/deny` + `--pre-approve ` + `bgagent policies list` + `bgagent pending`** | All REST-polling; no streaming prompts. User discovers pending approvals via `pending` or via the fan-out plane (Slack action buttons, email link). | +| 6 | **Timeouts: per-task default + per-rule Cedar annotation override, min wins, bounded floor + ceiling, fail-closed** | Floor: 30s (engine-enforced on both task default and rule annotations). Ceiling: `min(1h, maxLifetime - remaining_cleanup_margin)` — sized so the TTL on the approval row always covers the decision window. On timeout → fail-closed DENY with steering injected as a user turn. Never auto-approve, never proceed-with-default. | +| 7 | **Concurrency slots: AWAITING_APPROVAL holds the slot** | Matches PAUSED semantics. Container is alive, consuming memory. | +| 8 | **Hard-deny is absolute** | No `--pre-approve` scope can bypass it. CreateTaskFn validates and rejects `rule:`. | +| 9 | **Submit-time scope cap: 20 entries, ≤128 chars each** | Keeps audit trail legible, bounds allowlist check cost, limits abuse-vector damage. | +| 10 | **Cedar annotations** | `@rule_id(...)`, `@tier("hard-deny" \| "hard-gate")`, `@approval_timeout_s(...)`, `@severity(...)`, `@category(...)`. Recoverable via `cedarpy.policies_to_json_str()` → JSON. Multi-match merging: min timeout wins (clamped by floor), max severity wins. | +| 11 | **Atomic state transitions via DDB TransactWriteItems** | The approval-request row write and the TaskTable status transition are a single atomic transaction. No partial-failure states. | +| 12 | **Ownership encoded in ConditionExpression, not fetch-then-check** | `ApproveTaskFn` / `DenyTaskFn` put `user_id = :caller` into the ConditionExpression on TaskApprovalsTable. Authorization and state transition are atomic. | +| 13 | **Per-task approval-gate cap: 50, fail-task on exceed** | Prevents denial-loop storms. Matches Phase 2 nudge cap. | +| 14 | **Per-minute approval-creation rate limit: 20/task** | Agent-side throttle independent of per-task lifetime cap. | +| 15 | **Recent-decision cache: deny an identical (tool, input) for 60s after DENIED/TIMED_OUT** | Prevents retry-loop amplification on the same destructive action. | +| 16 | **Denial reason sanitized in the Lambda, before persisting** | `DenyTaskFn` runs `output_scanner` on the reason before writing to DDB. The agent never sees unscanned text. | +| 17 | **`tool_input_preview` stripped of ANSI/control characters at agent-side write + CLI render** | Defense in depth against approver-confusion attacks where a prompt-injected tool input overwrites the CLI prompt with a different command. | +| 18 | **Deny-as-steering injected via Stop hook `between_turns_hooks`, not via `permissionDecisionReason`** | Reuses the validated Phase 2 nudge mechanism. `` XML block wrapped by the same `_xml_escape` utility. | +| 19 | **`rule:` discovery via new endpoint** | `GET /v1/repos/{repo_id}/policies` + `bgagent policies list` surfaces the rule IDs + annotations + whether each rule is hard-deny or hard-gate. Solves the otherwise-undiscoverable `rule:X` pre-approval scope. | +| 20 | **`write_path:` scope** | Added so users can pre-approve file writes under specific path patterns (e.g., `write_path:docs/**`) without needing to grant all Writes. | +| 21 | **`tool_group:file_write` convenience scope** | Resolves to `{Write, Edit}`. Prevents the surprise of pre-approving `Write` and still getting gated on `Edit`. | +| 22 | **Pre-implementation spike: cedarpy annotation round-trip** | Day 1 of implementation validates that `policies_to_json_str()` returns annotations in the expected shape. If the API has changed, fall back to policy-ID prefix conventions. | +| 23 | **Approval notifications via the fan-out plane** | `approval_required` events flow through `FanOutConsumer` → Slack/email/GitHub dispatchers. Slack messages include `Approve` / `Deny` action buttons that POST to the REST API. No streaming CLI prompts. | + +--- + +## 4. End-to-end request flow + +Narrative walk-through of the happy path. Sequence diagrams in [phase3-cedar-hitl.drawio pages 3-6](/sample-autonomous-cloud-coding-agents/diagrams/phase3-cedar-hitl.drawio). + +### Setup (task start) + +1. User runs `bgagent submit --repo my-org/my-app --task "rebase feature-x onto main and push" --approval-timeout 600 --pre-approve tool_type:Read --pre-approve bash_pattern:"git status*"`. +2. CLI validates each scope string client-side (format, ≤128 chars, cap 20). Rejects invalid syntax without round-trip. +3. CLI POSTs `/v1/tasks` with `{repo, task, initial_approvals: [...], approval_timeout_s: 600}`. +4. `CreateTaskFn` validates `initial_approvals`: + - max 20 entries, ≤128 chars each + - rejects `rule:` where `` names a hard-deny rule (resolved via shared policy-parsing library against the repo's blueprint; see §5.4) + - rejects degenerate `bash_pattern`/`write_path` scopes that match too broadly (see §7.3) + - honors `Blueprint.security.maxPreApprovalScope` (see §7.3) + - normalizes scope strings (trim whitespace; case-sensitive as documented) +5. Task persists. `approval_timeout_s` and `initial_approvals` become DDB attributes on the task row. +6. Container spawns on the AgentCore Runtime. `PolicyEngine.__init__` loads: + - `HARD_DENY_POLICIES` (built-in + repo blueprint's `security.cedarPolicies.hard`) + - `HARD_GATE_POLICIES` (built-in + repo blueprint's `security.cedarPolicies.hard_gate`) + - Annotation lookup table: `{policy_id: {annotation: value}}` built from `cedarpy.policies_to_json_str()` once, cached for the task lifetime + - Rule-ID map: `{rule_id_annotation: policy_id}` to resolve `--pre-approve rule:` → internal Cedar policy ID + - Allowlist seeded from `initial_approvals` + - Annotation validation: `@rule_id` uniqueness enforced (duplicate = task fails to start); `@approval_timeout_s` must be integer ≥ 30 (malformed or below floor = task fails to start) +7. Container emits `agent_milestone("pre_approvals_loaded", {count: 2, scopes: ["tool_type:Read", "bash_pattern:git status*"]})` so Terminal A's stream shows the starting posture. +8. Agent begins normal work. + +### First approval gate (hard-gate hit) + +9. Agent decides to run `Bash(command="git push --force origin feature-x")`. +10. SDK fires `PreToolUse` hook with `tool_name="Bash"`, `tool_input={command: "..."}`. +11. Hook calls `PolicyEngine.evaluate_tool_use`: + - Hard-deny eval: matches nothing → `allowed=True` + - Allowlist fast-path: `tool_type:Bash`? no. `bash_pattern` matches `git push --force ...`? `git status*` doesn't match `git push --force ...` → skip + - Recent-decision cache: no matching `(Bash, sha256(input))` in cache → skip + - Hard-gate eval: policy `push_to_protected_branch` matches. `diagnostics.reasons == ["policy1"]`. Lookup: `policy1` → annotations `{rule_id: "push_to_protected_branch", approval_timeout_s: "300", severity: "medium"}`. + - Returns `PolicyDecision(outcome=REQUIRE_APPROVAL, reason="Cedar hard-gate: push_to_protected_branch", timeout_s=300, severity="medium", matching_rule_ids=["push_to_protected_branch"])`. + + Effective timeout computation: + ``` + effective = max( + FLOOR_30S, + min( + rule_annotation_timeout_s or task_default, # 300 + task_default, # 600 from submit + maxLifetime_remaining_s - CLEANUP_MARGIN_120S # ~7h remaining + ) + ) + → effective = 300s + ``` + If `maxLifetime_remaining_s - CLEANUP_MARGIN_120S < FLOOR_30S`, hook returns DENY immediately with reason `"insufficient lifetime for approval"` (§13.7). + +12. Hook checks per-task approval-gate cap (50) and per-minute rate limit (20/task). If either exceeded → DENY with reason `"approval-gate cap exceeded"` (fail-closed). +13. Hook mints `request_id = _ulid()` (26-char ULID). +14. Hook builds the approval row payload: + ```python + row = { + "task_id": "01KPW...", + "request_id": "01KPR...", + "tool_name": "Bash", + "tool_input_preview": strip_ansi("git push --force origin feature-x")[:256], + "tool_input_sha256": "abc123...", + "reason": "Cedar hard-gate: push_to_protected_branch", + "severity": "medium", + "matching_rule_ids": ["push_to_protected_branch"], # list, not set — supports empty + "status": "PENDING", + "created_at": "2026-04-23T14:00:00Z", + "timeout_s": 300, + "ttl": 1734567890, # created_at + timeout_s + CLEANUP_MARGIN_120S; always covers the decision window + "user_id": "...", + "repo": "my-org/my-app" + } + ``` +15. **Atomic transition** — hook issues `TransactWriteItems` with two operations: + - Put on `TaskApprovalsTable` (new row with status=PENDING) + - ConditionalUpdate on `TaskTable`: `status = :awaiting, awaiting_approval_request_id = :rid WHERE status = :running` + Both succeed or both fail. On `TransactionCanceledException` (most likely the TaskTable condition fails because another process moved the status), the hook emits `approval_write_failed` and returns DENY. +16. Hook emits `agent_milestone("approval_requested", {...})` to both `ProgressWriter` (DDB audit) and `sse_adapter` (live stream). Best-effort emission — transactional write has already committed; milestone failure is observability degradation, not state degradation. +17. Terminal A stream renders: + ``` + [14:00:00] ★ approval_requested: Bash "git push --force origin feature-x" (medium) + reason: Cedar hard-gate: push_to_protected_branch + bgagent approve 01KPR... [--scope ...] + bgagent deny 01KPR... [--reason "..."] + timeout 300s + ``` + Severity colors the line (respecting `NO_COLOR` env var). +18. Hook enters poll loop with strongly-consistent reads: + ```python + async def _poll_for_decision(task_id, request_id, timeout_s): + start = time.monotonic() + interval = 2 + consecutive_failures = 0 + while True: + elapsed = time.monotonic() - start + if elapsed >= timeout_s: + return TimedOut() + if elapsed > 30: + interval = 5 # backoff + try: + row = await _ddb_get_approval(task_id, request_id, ConsistentRead=True) + consecutive_failures = 0 + if row is None: + # Row disappeared between write and poll — treat as stranded + return TimedOut(reason="approval row missing; fail-closed") + if row["status"] != "PENDING": + return Decided(row) + except Exception as exc: + consecutive_failures += 1 + if consecutive_failures == 3: + log("WARN", f"approval poll degraded for {request_id}: {exc}") + emit_milestone("approval_poll_degraded", {...}) + if consecutive_failures >= 10: + return TimedOut(reason="approval poll consecutive failures") + await asyncio.sleep(interval) + ``` +19. The approval CAP and local-timeout paths ALWAYS attempt to write the row to TIMED_OUT (best-effort conditional update `status = :pending`) before returning. This prevents orphan PENDING rows when the agent bails internally. + +### User responds + +20. User in Terminal B runs `bgagent approve --scope tool_type_session`. +21. CLI validates scope syntax client-side. +22. CLI POSTs `/v1/tasks/{task_id}/approve` with `{request_id, decision: "approve", scope: "tool_type_session"}`. +23. `ApproveTaskFn`: + - Validates Cognito JWT, extracts `sub` as `caller_user_id`. + - Single `UpdateItem` on `TaskApprovalsTable` with compound ConditionExpression: + ``` + #status = :pending AND user_id = :caller AND task_id = :task_id + ``` + If all three conditions hold → atomic flip to APPROVED. Ownership + state + existence check in a single call. No TOCTOU. + - On `ConditionalCheckFailedException` with `ReturnValuesOnConditionCheckFailure=ALL_OLD`: distinguishes between (a) row missing (404 `REQUEST_NOT_FOUND`), (b) wrong user (404 `REQUEST_NOT_FOUND` — don't leak existence), (c) wrong status (409 `REQUEST_ALREADY_DECIDED`). + - Records audit event to TaskEventsTable directly (`approval_decision_recorded`) so the 90-day audit trail is owned by the Lambda, not dependent on agent milestones. + - Returns 202 `{task_id, request_id, status: "APPROVED", scope, decided_at}` or error. +24. Agent's poll reads the `APPROVED` row on next tick (within 2-5s). +25. Hook executes decision in this order: + - a. **Atomic resume transition**: `TransactWriteItems` — TaskTable `status = :running, REMOVE awaiting_approval_request_id WHERE status = :awaiting AND awaiting_approval_request_id = :rid`. If this fails (likely because user cancelled during the poll gap), hook skips allowlist mutation and returns DENY with reason `"task no longer awaiting approval"`. + - b. **Allowlist mutation** (only if `scope != "this_call"`): `PolicyEngine._allowlist.add(scope)`. Synchronously logged. + - c. **Milestone emission** (best-effort): `approval_granted` to both writers. + - d. **Return to SDK**: `{"permissionDecision": "allow"}`. +26. SDK runs the tool. Stream shows: + ``` + [14:00:12] ★ approval_granted: request_id=01KPR... scope=tool_type_session + [14:00:12] ▶ Bash: git push --force origin feature-x + [14:00:14] ◀ Bash: remote: Force pushed. New SHA abc123. + ``` + +### Continuation + +27. Agent continues with its turn, hits another `Bash` call (say `git log --oneline -5`). +28. PreToolUse hook → PolicyEngine.evaluate_tool_use: + - Hard-deny: no match + - Allowlist: `tool_type:Bash` ← matches. Returns ALLOW fast-path. +29. No new approval request. Tool runs immediately. +30. Eventually agent reaches task completion, opens PR, writes memory, task → `COMPLETED`. + +### Denial with steering text + +If instead the user runs `bgagent deny --reason "use --force-with-lease instead"`: +- `DenyTaskFn` runs `output_scanner.scan(reason)` to redact any accidental secrets/PII from the reason **before** writing it to DDB. +- Flips row to DENIED with sanitized reason, atomic ownership check. +- Agent's poll reads DENIED row. +- Hook execution order: + - a. Atomic resume transition to RUNNING (same as approve path). + - b. **Inject denial into agent context** via the Phase 2 `between_turns_hooks` registry. The hook appends a synthetic `sanitized reason` block to the next Stop-seam injection. This reuses the validated nudge mechanism (§5 below) — the agent sees the denial as authoritative context on its next turn boundary. + - c. Milestone emission: `approval_denied`. + - d. Return to SDK: `{"permissionDecision": "deny", "permissionDecisionReason": "User denied — see next-turn context for reason"}`. The SDK's denial text is a terse fallback; the steering text arrives via the robust Stop-hook injection, not via `permissionDecisionReason`. + +Why this dual path: the Claude Agent SDK's `permissionDecisionReason` reaches the model as a tool-call-rejected system hint, which the model treats as a reason-to-retry-differently signal but is NOT the guaranteed steering surface the Phase 2 nudge pattern uses. By layering the denial as a between-turns user message, we guarantee the steering text becomes authoritative context. The `permissionDecisionReason` remains there as belt-and-suspenders for model versions that don't emit a Stop between the rejected tool call and the next turn. + +--- + +## 5. Cedar policy authoring guide + +### 5.1 Policy file layout + +Two physical files, each with exactly one tier: + +- `agent/policies/hard_deny.cedar` — contains ONLY `@tier("hard-deny")` policies +- `agent/policies/hard_gate.cedar` — contains ONLY `@tier("hard-gate")` policies + +Per-repo customization lives in `blueprint.yaml`: + +```yaml +security: + cedarPolicies: + hard_deny: | + @tier("hard-deny") + @rule_id("block_prod_writes") + forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like "prod/**" }; + hard_gate: | + @tier("hard-gate") + @rule_id("deploy_staging") + @approval_timeout_s("900") + @severity("high") + @category("destructive") + forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*terraform apply*" }; + maxPreApprovalScope: "tool_type_session" # optional; caps what --pre-approve can grant +``` + +`PolicyEngine.__init__` concatenates built-in + blueprint rules per tier, validates with a probe `cedarpy.is_authorized()` call. **Any** of the following cause task-start failure (not silent-fallback): +- Malformed policy syntax +- Duplicate `@rule_id` values across tiers +- `@approval_timeout_s` below floor (30s) or non-integer +- `@tier` value mismatches the file section (rules whose @tier disagrees with the file they live in) +- Missing `@rule_id` on a hard-gate rule + +Fail-on-error is the right posture for blueprint misconfiguration — silent-fallback would let broken policies slip into production. + +### 5.2 Annotation reference + +| Annotation | Required? | Values | Purpose | +|---|---|---|---| +| `@rule_id("...")` | **Yes on hard-gate**, recommended on hard-deny | Kebab-case or snake_case identifier, unique across both tiers | Stable ID for `--pre-approve rule:X`, for audit trail, and for the `bgagent policies` discovery endpoint. `PolicyEngine.__init__` raises on duplicates. | +| `@tier("hard-deny"\|"hard-gate")` | **Yes** | Exactly one of "hard-deny" or "hard-gate" | Validates policy is in the correct file/section. Engine rejects mismatch at load time. | +| `@approval_timeout_s("N")` | No | Integer seconds ≥ 30 | Per-rule timeout. If absent, uses task default. Has no effect on hard-deny rules. Values below the floor are rejected at load time. | +| `@severity("low"\|"medium"\|"high")` | No | One of the three | Shown in CLI approval prompt, colored by severity. Default: "medium". | +| `@category("...")` | No | "destructive", "network", "filesystem", "auth", or free-form | UX grouping. CLI could filter approvals by category. Not enforced. | + +### 5.3 Common policy patterns + +**Block absolute dangers** (hard-deny): +```cedar +@tier("hard-deny") +@rule_id("rm_slash") +forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*rm -rf /*" }; + +@tier("hard-deny") +@rule_id("write_git_internals") +forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like ".git/*" }; + +@tier("hard-deny") +@rule_id("write_git_internals_nested") +forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like "*/.git/*" }; + +@tier("hard-deny") +@rule_id("drop_table") +forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*DROP TABLE*" }; +``` + +**Absolute deny on destructive git ops** (hard-deny — part of the built-in starter set): +```cedar +@tier("hard-deny") +@rule_id("force_push_main") +@severity("high") +@category("destructive") +forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*git push --force origin main*" + || context.command like "*git push --force origin prod*" + || context.command like "*git push -f origin main*" + || context.command like "*git push -f origin prod*" }; +``` + +Force-pushing to `main` or `prod` is the canonical "you almost certainly don't want this" action. Absolute deny; not bypassable via `--pre-approve`. A repo that legitimately needs this (release automation) adds an override in its blueprint and removes this rule from the policy set. + +**Gate non-force pushes to protected branches** (hard-gate — part of the built-in starter set): +```cedar +@tier("hard-gate") +@rule_id("push_to_protected_branch") +@approval_timeout_s("300") +@severity("medium") +@category("destructive") +forbid (principal, action == Agent::Action::"execute_bash", resource) + when { context.command like "*git push origin main*" + || context.command like "*git push origin prod*" + || context.command like "*git push origin master*" + || context.command like "*git push origin release/*" }; +``` + +A non-force push to a protected branch gates — catches the case where an agent tries to push directly rather than opening a PR. Low frequency, high impact → worth waiting for a human. + +**Absolute deny on credential writes** (hard-deny — part of the built-in starter set): +```cedar +@tier("hard-deny") +@rule_id("write_credentials") +@severity("high") +@category("auth") +forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like "*credentials*" }; +``` + +Writing a file with "credentials" in the path is a strong signal of accidental secret persistence. Absolute deny. + +**Gate `.env` writes** (hard-gate — part of the built-in starter set): +```cedar +@tier("hard-gate") +@rule_id("write_env_files") +@approval_timeout_s("600") +@severity("high") +@category("filesystem") +forbid (principal, action == Agent::Action::"write_file", resource) + when { context.file_path like "*.env" }; +``` + +`.env` writes are plausibly intentional (template scaffolding, `.env.example` generation) but high-impact enough to warrant a human decision. + +**Optional patterns (not shipped by default — copy into your blueprint if your repo needs them):** +```cedar +// Gate writes under a conventional infrastructure/ directory. Not in the +// built-in set because the "infrastructure/" path is a repo convention, +// not a standard — many repos use cdk/, terraform/, deploy/, etc. Add to +// your blueprint if your repo uses this layout. +// @tier("hard-gate") +// @rule_id("write_infrastructure") +// @approval_timeout_s("900") +// @severity("high") +// @category("filesystem") +// forbid (principal, action == Agent::Action::"write_file", resource) +// when { context.file_path like "infrastructure/*" }; + +// Gate all outbound WebFetch. Not in the built-in set because DNS +// Firewall already restricts egress to an allowlist; gating every +// WebFetch produces high-volume approval requests on doc-heavy tasks. +// Add to your blueprint if your repo wants stricter scrutiny. +// @tier("hard-gate") +// @rule_id("webfetch_any") +// @approval_timeout_s("300") +// @severity("medium") +// @category("network") +// forbid (principal, action == Agent::Action::"invoke_tool", +// resource == Agent::Tool::"WebFetch"); + +// Gate writes to specific CI config. Example — tune paths per repo. +// @tier("hard-gate") +// @rule_id("write_github_workflows") +// @approval_timeout_s("600") +// @severity("high") +// @category("filesystem") +// forbid (principal, action == Agent::Action::"write_file", resource) +// when { context.file_path like ".github/workflows/*" }; +``` + +Per the sentinel trick (see §6.2), `invoke_tool` matches on the real tool-name UID. The other actions (`write_file`, `execute_bash`) use a sentinel UID with the real value in `context`. + +### 5.4 Policy discovery — shared parser + +Because `CreateTaskFn` needs to validate `rule:` pre-approvals against the target repo's actual policy set, we ship a **shared policy-parsing library** used in both places: + +- `cdk/src/handlers/shared/cedar-policy.ts` — thin wrapper around cedarpy's JSON form for TypeScript +- `agent/src/policy.py` — the full engine + +Both consume the blueprint's `security.cedarPolicies` section. `CreateTaskFn` loads the target repo's blueprint (via the existing `RepoTable` store), concatenates with the built-in policies, parses via `cedarpy.policies_to_json_str()`, and extracts `rule_id` + `tier` annotations. `--pre-approve rule:X` is validated: +- `X` exists as some rule's `@rule_id` → ok +- `X` refers to a hard-deny rule → 400 at submit time (hard-deny cannot be bypassed) +- `X` refers to a hard-gate rule → ok; passes through + +Runtime enforcement is still the authoritative layer. Submit-time validation is a UX guard — any drift between submit-time and runtime-loaded policies (possible if blueprint changes between them) causes the task to fail at container start with a clear error, not silently misbehave. + +### 5.5 Gotchas for policy authors + +**`like` is glob, not regex.** Only `*` (zero-or-more) and `?` (exactly-one-char) wildcards. If you need regex, write multiple `forbid` rules. + +**Case sensitivity.** `like` is case-sensitive. `*rm -rf*` won't match `*Rm -Rf*`. If case-insensitivity matters, write both variants. + +**Don't match `resource ==` for user-supplied values.** `Bash` commands and file paths go through the sentinel UID. Always use `context.command` / `context.file_path` in the `when` clause, never `resource == ...`. + +**`@rule_id` must be globally unique.** Including across tiers. `PolicyEngine.__init__` raises on duplicates. + +**Hard-deny rules shouldn't have `@approval_timeout_s`.** It has no effect. Engine logs WARN but doesn't reject (backward compatibility if someone moves a rule between tiers). + +**The default ruleset is shared across all tasks.** Per-task overrides live in the Blueprint and are isolated to tasks on that repo. The engine never allows a task to loosen the default hard-deny set via Blueprint — only add to it. + +**`@approval_timeout_s` values below 30 are rejected at load.** There is no way to configure unusably-short approval windows. + +--- + +## 6. Engine implementation + +### 6.1 Extended `PolicyDecision` shape + +```python +from dataclasses import dataclass +from enum import Enum + +class Outcome(str, Enum): + ALLOW = "allow" + DENY = "deny" # absolute (hard-deny or upstream error or cap-exceeded) + REQUIRE_APPROVAL = "require_approval" # hard-gate hit + +@dataclass(frozen=True) +class PolicyDecision: + outcome: Outcome + reason: str + # Only populated when outcome == REQUIRE_APPROVAL: + timeout_s: int | None = None + severity: str | None = None + matching_rule_ids: tuple[str, ...] = () + duration_ms: float = 0 + + @property + def allowed(self) -> bool: + """Backward-compat shim for Phase 1a/1b callers.""" + return self.outcome == Outcome.ALLOW +``` + +### 6.2 `evaluate_tool_use` skeleton + +```python +def evaluate_tool_use(self, tool_name: str, tool_input: dict) -> PolicyDecision: + start = time.monotonic() + base_context = {"task_type": self._task_type, "repo": self._repo} + input_sha = _sha256(json.dumps(tool_input, sort_keys=True)) + + # STEP 1 — Hard-deny (absolute) + hard = self._eval_tier(self._hard_policies, tool_name, tool_input, base_context) + if hard.decision == "deny": + return PolicyDecision(outcome=Outcome.DENY, + reason=f"Hard-deny: {hard.rule_ids}", + duration_ms=_elapsed(start)) + + # STEP 2 — Allowlist fast-path (covers tool_type, bash_pattern, write_path, all_session) + if self._allowlist.matches(tool_name, tool_input): + return PolicyDecision(outcome=Outcome.ALLOW, + reason="Pre-approved by allowlist", + duration_ms=_elapsed(start)) + + # STEP 2.5 — Recent-decision cache (anti-retry-loop, 60s TTL) + cached = self._recent_decisions.get((tool_name, input_sha)) + if cached is not None: + return PolicyDecision(outcome=Outcome.DENY, + reason=f"Recent decision ({cached.decision}) within 60s: {cached.reason}", + duration_ms=_elapsed(start)) + + # STEP 3 — Hard-gate (require approval) + gate = self._eval_tier(self._hard_gate_policies, tool_name, tool_input, base_context) + if gate.decision == "deny": + # Rule-scope allowlist check happens AFTER hard-gate eval (rule_ids + # aren't known until Cedar tells us which policies matched) + if any(rid in self._allowlist._rule_ids for rid in gate.rule_ids): + return PolicyDecision(outcome=Outcome.ALLOW, + reason=f"Allowlist rule: {gate.rule_ids}", + duration_ms=_elapsed(start)) + + annotations = self._merge_annotations(gate.rule_ids) + return PolicyDecision( + outcome=Outcome.REQUIRE_APPROVAL, + reason=f"Hard-gate: {', '.join(annotations['rule_ids'])}", + timeout_s=annotations["timeout_s"], + severity=annotations["severity"], + matching_rule_ids=tuple(annotations["rule_ids"]), + duration_ms=_elapsed(start), + ) + + # STEP 4 — Default allow + return PolicyDecision(outcome=Outcome.ALLOW, reason="permitted", + duration_ms=_elapsed(start)) +``` + +The recent-decision cache is a simple `dict[(tool_name, input_sha), (decision, reason, inserted_at)]` with a 60-second sliding window. Entries are added by the PreToolUse hook whenever an approval resolves to DENIED or TIMED_OUT — not on APPROVED (we don't want to accidentally auto-deny a tool call the user just approved). Cache is in-process, lost on restart. + +### 6.3 Annotation merging + +When multiple hard-gate rules match a single tool call: + +```python +def _merge_annotations(self, policy_ids: list[str]) -> dict: + rule_ids, timeouts, severities = [], [], [] + for pid in policy_ids: + ann = self._annotations[pid] + rule_ids.append(ann.get("rule_id", pid)) + if "approval_timeout_s" in ann: + try: + t = int(ann["approval_timeout_s"]) + if t >= FLOOR_30S: + timeouts.append(t) + except ValueError: + log("WARN", f"malformed @approval_timeout_s on {ann.get('rule_id', pid)}") + severities.append(ann.get("severity", "medium")) + + # Task default always eligible + timeouts.append(self._task_default_timeout_s) + + raw_min_timeout = min(timeouts) + return { + "rule_ids": rule_ids, + "timeout_s": max(FLOOR_30S, raw_min_timeout), # floor enforcement + "severity": _max_severity(severities), # "high" > "medium" > "low" + } +``` + +**Rationale for min/max choices**: +- **Timeout → min (above floor)**: multiple rules matching means multiple concerns. Users should have *less* time to decide when stakes are higher. Floor prevents unusable 5s windows. +- **Severity → max**: the most severe concern governs the UX coloring. + +### 6.4 Allowlist data structure + +```python +class ApprovalAllowlist: + def __init__(self, initial_scopes: list[str]): + self._all_session = False + self._tool_types: set[str] = set() + self._tool_groups: set[str] = set() # file_write → {Write, Edit} + self._rule_ids: set[str] = set() + self._bash_patterns: list[str] = [] # glob patterns + self._write_path_patterns: list[str] = [] # glob patterns, for Write/Edit file_path + + for scope in initial_scopes: + self.add(scope) + + TOOL_GROUPS = {"file_write": {"Write", "Edit"}} + + def add(self, scope: str) -> None: + if scope == "all_session": + self._all_session = True + elif scope.startswith("tool_type:"): + self._tool_types.add(scope.split(":", 1)[1]) + elif scope.startswith("tool_group:"): + group = scope.split(":", 1)[1] + if group not in self.TOOL_GROUPS: + raise ValueError(f"unknown tool_group: {group!r}") + self._tool_groups.add(group) + elif scope.startswith("rule:"): + self._rule_ids.add(scope.split(":", 1)[1]) + elif scope.startswith("bash_pattern:"): + self._bash_patterns.append(scope.split(":", 1)[1]) + elif scope.startswith("write_path:"): + self._write_path_patterns.append(scope.split(":", 1)[1]) + else: + raise ValueError(f"unknown scope: {scope!r}") + + def matches(self, tool_name: str, tool_input: dict) -> bool: + if self._all_session: + return True + if tool_name in self._tool_types: + return True + for group in self._tool_groups: + if tool_name in self.TOOL_GROUPS[group]: + return True + if tool_name == "Bash": + cmd = tool_input.get("command", "") + if any(fnmatch(cmd, pat) for pat in self._bash_patterns): + return True + if tool_name in ("Write", "Edit"): + path = tool_input.get("file_path", "") + if any(fnmatch(path, pat) for pat in self._write_path_patterns): + return True + # rule_ids matched after hard-gate eval — see evaluate_tool_use + return False +``` + +### 6.5 PreToolUse hook changes + +Phase 3 PreToolUse hook (compressed for doc; implementation will be richer): + +```python +async def pre_tool_use_hook(hook_input, tool_use_id, ctx, *, + engine, task_id, user_id, progress, sse_adapter, + task_default_timeout_s): + tool_name, tool_input = _extract(hook_input) + decision = engine.evaluate_tool_use(tool_name, tool_input) + + if decision.outcome == Outcome.ALLOW: + return _allow() + if decision.outcome == Outcome.DENY: + return _deny(decision.reason) + + # REQUIRE_APPROVAL path. + # Cap + rate-limit check. + if engine.approval_gate_count >= APPROVAL_GATE_CAP_PER_TASK: + return _deny("approval-gate cap exceeded (50/task)") + if engine.approvals_in_last_minute >= APPROVAL_RATE_LIMIT: + return _deny("approval-gate rate limit exceeded (20/min)") + + # Compute effective timeout with floor/ceiling. + remaining = _remaining_maxlifetime_s() + effective_timeout = max( + FLOOR_30S, + min(decision.timeout_s or task_default_timeout_s, + task_default_timeout_s, + remaining - CLEANUP_MARGIN_120S), + ) + if remaining - CLEANUP_MARGIN_120S < FLOOR_30S: + return _deny(f"insufficient maxLifetime remaining ({remaining}s) for approval") + + request_id = _ulid() + engine.approval_gate_count += 1 + + row = { + "task_id": task_id, "request_id": request_id, + "tool_name": tool_name, + "tool_input_preview": _strip_ansi(_preview(tool_input))[:256], + "tool_input_sha256": _sha256(_serialize(tool_input)), + "reason": decision.reason, "severity": decision.severity, + "matching_rule_ids": list(decision.matching_rule_ids), + "status": "PENDING", + "created_at": _iso_now(), + "timeout_s": effective_timeout, + "ttl": int(time.time()) + effective_timeout + CLEANUP_MARGIN_120S, + "user_id": user_id, "repo": engine.repo, + } + + # ATOMIC: put approval row + transition TaskTable status in one transaction. + try: + await _transact_write_approval_request(task_id, request_id, row) + except TransactionCanceledException as exc: + # Either the task was concurrently cancelled, or status wasn't RUNNING. + _emit("approval_write_failed", {"request_id": request_id, "reason": str(exc)}) + return _deny("approval system unavailable") + + _emit("approval_requested", { + "request_id": request_id, "tool_name": tool_name, + "input_preview": row["tool_input_preview"], + "reason": decision.reason, "severity": decision.severity, + "timeout_s": effective_timeout, + "matching_rule_ids": list(decision.matching_rule_ids), + }) + + outcome = await _poll_for_decision(task_id, request_id, effective_timeout) + + # On TIMED_OUT, attempt to write the row to TIMED_OUT so future reads see + # a terminal state (not orphaned PENDING). + if outcome.status == "TIMED_OUT": + await _best_effort_update_status(task_id, request_id, "TIMED_OUT", + reason=outcome.reason) + + # ATOMIC: resume TaskTable status RUNNING, conditional on awaiting_approval_request_id matching. + try: + await _transact_resume(task_id, request_id) + except TransactionCanceledException: + # User cancelled (or some other path) during poll; abandon gracefully. + _emit("approval_resume_failed", {"request_id": request_id}) + return _deny("task no longer awaiting approval") + + if outcome.status == "APPROVED": + if outcome.scope and outcome.scope != "this_call": + engine._allowlist.add(outcome.scope) + _emit("approval_granted", {"request_id": request_id, + "scope": outcome.scope or "this_call", + "decided_at": outcome.decided_at}) + return _allow() + + # DENIED or TIMED_OUT — cache for 60s + inject denial via Stop hook path. + engine._recent_decisions.record( + tool_name, _sha256(_serialize(tool_input)), + decision="DENIED" if outcome.status == "DENIED" else "TIMED_OUT", + reason=outcome.reason, + ) + if outcome.status == "DENIED": + # Queue steering injection via Stop hook's between_turns_hooks. + engine._queue_denial_injection( + request_id=request_id, + reason=outcome.reason, # already sanitized by DenyTaskFn + decided_at=outcome.decided_at, + ) + _emit("approval_denied" if outcome.status == "DENIED" else "approval_timed_out", + {"request_id": request_id, "reason": outcome.reason}) + return _deny(f"User {outcome.status.lower()}: see next turn context for details") +``` + +`engine._queue_denial_injection` appends to a list consumed by a new `_denial_between_turns_hook` — registered alongside `_nudge_between_turns_hook` in the Phase 2 `between_turns_hooks` list. At the next Stop hook fire, the denial is emitted as `…` XML (sanitized via `_xml_escape` from the shared utility introduced with Phase 2). + +--- + +## 7. REST API contract + +### 7.1 `POST /v1/tasks/{task_id}/approve` + +**Request** (CLI → API Gateway → `ApproveTaskFn`): +```http +POST /v1/tasks/01KPW.../approve +Authorization: Bearer +Content-Type: application/json + +{ + "request_id": "01KPR...", + "decision": "approve", + "scope": "tool_type_session" +} +``` + +**Responses**: + +| Status | Code | When | Body | +|---|---|---|---| +| 202 | — | Success | `{task_id, request_id, status: "APPROVED", scope, decided_at}` | +| 400 | `VALIDATION_ERROR` | Bad scope format, missing fields | `{error, message, field}` | +| 401 | `UNAUTHORIZED` | Missing/invalid JWT | — | +| 404 | `REQUEST_NOT_FOUND` | Row missing OR wrong user (both surfaces 404 to prevent enumeration) | — | +| 409 | `REQUEST_ALREADY_DECIDED` | Status != PENDING | `{error, message, current_status}` | +| 409 | `TASK_NOT_AWAITING_APPROVAL` | Task's current status is not AWAITING_APPROVAL | `{error, message, current_status}` | +| 429 | `RATE_LIMIT_EXCEEDED` | Per-user > 30 approve/min | — | +| 503 | `SERVICE_UNAVAILABLE` | DDB throttled or upstream failure | — | + +**Authorization + state + existence check is a single DDB operation**: +```python +response = ddb.update_item( + TableName=TASK_APPROVALS_TABLE, + Key={"task_id": task_id, "request_id": request_id}, + UpdateExpression="SET #s = :approved, decided_at = :now, #sc = :scope", + ConditionExpression="#s = :pending AND user_id = :caller", + ExpressionAttributeNames={"#s": "status", "#sc": "scope"}, + ExpressionAttributeValues={ + ":approved": "APPROVED", ":pending": "PENDING", + ":now": now_iso, ":scope": scope, ":caller": cognito_sub, + }, + ReturnValuesOnConditionCheckFailure="ALL_OLD", +) +``` + +On `ConditionalCheckFailedException`: +- If `OldImage` is absent → row never existed → 404 `REQUEST_NOT_FOUND` +- If `OldImage.user_id != caller` → 404 (same code, prevent existence oracle) +- If `OldImage.status != "PENDING"` → 409 `REQUEST_ALREADY_DECIDED` + +In addition, the Lambda does a separate GetItem on `TaskTable` to check `status == "AWAITING_APPROVAL"` — if the task has already moved (e.g., was cancelled), return 409 `TASK_NOT_AWAITING_APPROVAL` before even attempting the update. This check is belt-and-suspenders; the atomic UpdateItem handles the rest. + +After successful update, `ApproveTaskFn` writes an audit event to `TaskEventsTable` (`approval_decision_recorded` event_type), ensuring the 90-day audit trail is owned by the Lambda path — not dependent on the agent's milestone emission. + +### 7.2 `POST /v1/tasks/{task_id}/deny` + +Identical shape with `decision: "deny"` and optional `reason`: + +```json +{ + "request_id": "01KPR...", + "reason": "use force-with-lease instead; force is too risky" +} +``` + +`DenyTaskFn`: +1. Auth check (Cognito JWT) +2. Run `output_scanner.scan(reason)` — redacts AWS keys, GitHub PATs, API tokens, etc. from the reason text before persisting +3. Truncate sanitized reason to 2000 chars (matches Phase 2 nudge limit for consistency) +4. Atomic conditional update (same shape as approve) +5. Write audit event to TaskEventsTable + +The agent reads the sanitized reason from DDB. It never sees unscanned user text. + +### 7.3 `POST /v1/tasks` — new optional fields + +Extended request shape: + +```json +{ + "repo": "my-org/my-app", + "task": "...", + "task_type": "new_task", + "approval_timeout_s": 600, + "initial_approvals": [ + "tool_type:Read", + "bash_pattern:git status*", + "write_path:docs/**", + "rule:safe_read_config", + "tool_group:file_write" + ] +} +``` + +`CreateTaskFn` validations: +1. Length cap: ≤20 entries +2. Per-entry length cap: ≤128 chars +3. Scope format parsing: normalized to known shape; leading/trailing whitespace trimmed +4. Scope value validation: + - `tool_type:X` — X must be in known tool set (Read, Bash, Write, Edit, Glob, Grep, WebFetch, ...) + - `tool_group:X` — X must be in known group set (currently `file_write`) + - `bash_pattern:X` — X ≤128 chars; reject if X is degenerate (`*`, `**`, `?*`, or patterns where wildcard-char ratio exceeds 50%) — see §7.4 + - `write_path:X` — same rules as bash_pattern + - `rule:X` — X must exist in the (built-in + target repo's blueprint) hard-gate policy set per the shared policy-parsing library; hard-deny rule IDs rejected + - `all_session` — rejected if `Blueprint.security.maxPreApprovalScope` forbids +5. `approval_timeout_s` within `[30, min(3600, maxLifetime - 300)]` — cap at 1 hour OR (maxLifetime - 5min), whichever is smaller. Prevents multi-hour slot-exhaustion attacks and keeps approval windows within the TTL budget. + +### 7.4 Degenerate-pattern detection + +A pattern is considered degenerate if: +- Length ≤ 2, OR +- Consists only of `*`, `?`, and whitespace, OR +- Ratio of wildcard chars (`*` + `?`) to literal chars exceeds 50% + +Degenerate `bash_pattern:` and `write_path:` scopes are rejected at submit with 400 `VALIDATION_ERROR`. Users wanting broad permission must use the explicit `all_session` scope (which is subject to `maxPreApprovalScope` blueprint cap). + +### 7.5 `maxPreApprovalScope` ordering + +Blueprint's `maxPreApprovalScope` is a partial order: + +``` +this_call < { tool_type_session, tool_group, bash_pattern, write_path, rule } < all_session +``` + +If `maxPreApprovalScope: "tool_type_session"`, `all_session` is rejected. All other scopes pass. Setting it to `"this_call"` (meaningless) is rejected at blueprint load. Blueprint absence defaults to unbounded (except `all_session` requires explicit `--yes` on CLI). + +### 7.6 `GET /v1/repos/{repo_id}/policies` + +New read-only endpoint for rule discovery and `bgagent policies list`: + +**Response** (200): +```json +{ + "repo_id": "my-org/my-app", + "policies": { + "hard_deny": [ + {"rule_id": "rm_slash", "category": "destructive", + "summary": "Reject rm -rf / and similar"}, + {"rule_id": "force_push_main", "category": "destructive", + "summary": "Reject force-push to main/prod"}, + {"rule_id": "write_credentials", "category": "auth", + "summary": "Reject writes to paths containing 'credentials'"}, + ... + ], + "hard_gate": [ + {"rule_id": "push_to_protected_branch", "severity": "medium", + "category": "destructive", "approval_timeout_s": 300, + "summary": "Non-force push to a protected branch"}, + {"rule_id": "write_env_files", "severity": "high", + "category": "filesystem", "approval_timeout_s": 600, + "summary": "Write to *.env files"}, + ... + ] + } +} +``` + +Loaded by the Lambda on demand from the target repo's blueprint + built-in policies. `summary` is a human-readable annotation `@summary("...")` if present, else falls back to the first line of the `when` clause rendered as text. + +Rate-limited 30/min/user; cached 5min per repo in-Lambda. + +--- + +## 8. CLI UX + +### 8.1 New commands + +```bash +# Approve a specific pending request +bgagent approve [--scope ] [--output text|json] + +# Deny a specific pending request, optionally with a reason the agent sees (sanitized server-side) +bgagent deny [--reason "..."|--reason-file ] [--output text|json] + +# List all pending approvals across the user's active tasks (solves request-id lookup) +bgagent pending [--output text|json] + +# Discover policies for a repo (solves rule-id lookup) +bgagent policies list --repo [--tier hard-deny|hard-gate] [--output text|json] +bgagent policies show --repo --rule [--output text|json] +``` + +### 8.2 Extended `submit` / `run` flags + +```bash +bgagent submit \ + --repo my-org/my-app \ + --task "..." \ + --approval-timeout 600 \ + --pre-approve tool_type:Read \ + --pre-approve write_path:"docs/**" \ + --pre-approve tool_group:file_write \ + --pre-approve rule:safe_file_read \ + --pre-approve-file ./approvals.yaml + +# Shorthand for no approval gates (requires --yes): +bgagent submit --task "..." --pre-approve all_session --yes +``` + +`--pre-approve-file` reads a YAML/JSON array of scope strings — supports the 20-entry cap without command-line bloat. + +### 8.3 Notification UX + +Approval requests surface through the fan-out plane (see [`INTERACTIVE_AGENTS.md`](/architecture/interactive-agents) §6) — not through a CLI stream. When the agent emits an `approval_required` event to `TaskEventsTable`, `FanOutConsumer` routes it per the user's notification config: + +- **Slack**: posts a message to the configured channel with `Approve` / `Deny` action buttons. Button click invokes an interaction-callback Lambda that writes to `TaskApprovalsTable` via the same path `bgagent approve` uses. +- **Email**: sends a one-line summary with a link that deep-links to the approve/deny REST endpoint (optional authenticated click-through). +- **GitHub issue comment**: appends to the in-place comment that the task is waiting for approval (visible to anyone watching the issue). +- **CLI via `bgagent watch`**: the event shows up in the polling stream as any other event: + +```text +[14:00:00] ★ approval_requested: Bash "git push origin main" (severity=medium) + reason: Cedar hard-gate: push_to_protected_branch + respond: bgagent approve 01KPR... [--scope tool_type_session] + bgagent deny 01KPR... [--reason "..."] + timeout: 300s (or "bgagent pending" to list all) +``` + +`bgagent watch` formats the line with severity color (respecting `NO_COLOR`; emits `[HIGH]` prefix when set). No interactive prompt in the watch stream — approval responses are always explicit commands. + +**Discovery path.** A user who wasn't watching at all finds pending approvals via: + +- `bgagent pending` — lists every open approval across the user's tasks. +- Slack button click — zero commands, one-tap response. +- Inbound from email link → REST API. + +### 8.4 Safety UX + +When `--pre-approve all_session` is passed without `--yes`: + +```bash +$ bgagent submit --task "apply terraform plan" --pre-approve all_session +WARNING: --pre-approve all_session disables Cedar hard-gate approval gates + for this task. Hard-deny policies (rm -rf /, write to .git/, DROP + TABLE, etc.) still apply. + Add --yes to skip this prompt. +Continue? [y/N] +``` + +Hard-deny enforcement is clearly called out so users don't mistake `all_session` for root. + +### 8.5 `bgagent pending` output + +```text +Pending approvals (3): + + 01KPW0...(task) / 01KPR0...(request) + ├─ Bash: git push --force origin feature-x + ├─ severity: high + ├─ reason: Cedar hard-gate: push_to_protected_branch + ├─ timeout: 4m 32s remaining + └─ approve|deny + + 01KPW1.../01KPR1... + ├─ Write: /workspace/.../src/.env + ├─ severity: high + ├─ timeout: 9m 12s remaining + ... +``` + +Picking one (`bgagent approve` or `bgagent deny` with the listed IDs) is straightforward. Shell completion (tab-complete task_id + request_id from `bgagent pending` output) is a Phase 3b enhancement. + +--- + +## 9. State machine + concurrency + +### 9.1 New state: AWAITING_APPROVAL + +Transitions added (extending §7 of INTERACTIVE_AGENTS.md): + +``` +RUNNING → AWAITING_APPROVAL (on REQUIRE_APPROVAL; via TransactWriteItems) +AWAITING_APPROVAL → RUNNING (on approve OR deny OR timeout; via TransactWriteItems) +AWAITING_APPROVAL → CANCELLED (on explicit `bgagent cancel`) +AWAITING_APPROVAL → FAILED (on reconciler detecting stranded approval; new edge) +HYDRATING → AWAITING_APPROVAL (if a hard-gate gate fires during hydration; rare but possible) +``` + +No direct `AWAITING_APPROVAL → COMPLETED/FINALIZING` without RUNNING in between. + +### 9.2 Orchestrator impact + +- `waitStrategy` adds `AWAITING_APPROVAL` as non-terminal. +- `finalizeTask` recognizes `AWAITING_APPROVAL`. +- `ACTIVE_STATUSES` (used by `GET /tasks?status=active` and `reconcile-concurrency.ts`) gains `AWAITING_APPROVAL`. +- `task_state.py::write_terminal` condition expression accepts `AWAITING_APPROVAL` as a valid source state. + +### 9.3 Concurrency slot semantics + +**AWAITING_APPROVAL holds the user's concurrency slot.** + +Rationale: the Docker container is alive. Memory allocated. The AgentCore microVM pool is committed. Releasing the slot while the resource is still held lies to accounting and opens a resource-exhaustion vector. + +Concrete behavior: + +```text +Bob's per-user cap: 10. +t=0: Bob submits 10 tasks. count=10. 11th submit → 429. +t=2m: Task #1 → AWAITING_APPROVAL. count still 10. + Bob's 12th submit → 429. He must approve, cancel, or wait. +t=30m: Bob approves task #1. task → RUNNING. count still 10. +t=45m: Task #1 completes. count → 9. Bob can submit task #11. +``` + +### 9.4 `maxLifetime` clock does not pause + +AgentCore Runtime's `maxLifetime = 28800s` (8h) is an absolute timer from session start. It does NOT pause during `AWAITING_APPROVAL`. + +This has a concrete implication: the hook computes an `effective_timeout` bounded by `maxLifetime - remaining - CLEANUP_MARGIN_120S`. If the task has been running 7h55m and hits a hard-gate gate, the effective timeout might be clamped to a much shorter value than the task default. Below the 30s floor → immediate DENY with reason `"insufficient lifetime"`. + +### 9.5 Stranded-approval reconciliation + +`reconcile-stranded-tasks.ts` gains an AWAITING_APPROVAL-aware branch: + +- Detects tasks in AWAITING_APPROVAL with `age > 2 * timeout_s` +- Best-effort conditional-updates TaskApprovalsTable row → `STRANDED` status +- Transitions TaskTable → `FAILED` with reason `"approval stranded (container eviction)"` +- Emits `approval_stranded` event to TaskEventsTable + +This closes the Phase 3a container-eviction gap. Without this, a container restart mid-approval would leave the task hanging until the user manually cancelled. + +`reconcile-concurrency.ts` (scheduled every 5 min) already scans for orphaned concurrency counters; with `AWAITING_APPROVAL` added to `ACTIVE_STATUSES` it correctly counts awaiting tasks as active. + +### 9.6 Attended vs unattended mode + +The design assumes a human is watching. For truly unattended tasks (scheduled automation, cron-driven runs) the `--pre-approve all_session` path skips hard-gate entirely. No additional mode flag needed — the set of scopes in `initial_approvals` dictates the attendance expectation. + +--- + +## 10. Data model + +### 10.1 New DynamoDB table: `TaskApprovalsTable` + +```typescript +new dynamodb.Table(this, 'Table', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + sortKey: { name: 'request_id', type: dynamodb.AttributeType.STRING }, // ULID + billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, + pointInTimeRecovery: true, + timeToLiveAttribute: 'ttl', + stream: dynamodb.StreamViewType.NEW_AND_OLD_IMAGES, // (evaluated — may drop; see §11) + removalPolicy: RemovalPolicy.RETAIN, +}); +``` + +Attributes: + +| Name | Type | Required | Description | +|---|---|---|---| +| `task_id` | S | Yes | PK; ULID matching TaskTable | +| `request_id` | S | Yes | SK; ULID minted by agent | +| `tool_name` | S | Yes | "Bash", "Write", etc. | +| `tool_input_preview` | S | Yes | First 256 chars of serialized tool input, ANSI/control-stripped | +| `tool_input_sha256` | S | Yes | Full-input hash for audit + recent-decision cache | +| `reason` | S | Yes | Cedar matching rule description | +| `severity` | S | Yes | "low" \| "medium" \| "high" | +| `matching_rule_ids` | L | Yes | List (not Set — can be empty) of hard-gate rule IDs | +| `status` | S | Yes | PENDING \| APPROVED \| DENIED \| TIMED_OUT \| STRANDED | +| `created_at` | S | Yes | ISO8601 | +| `decided_at` | S | No | Set when status != PENDING | +| `scope` | S | No | Set on APPROVED | +| `deny_reason` | S | No | Set on DENIED; sanitized user text | +| `timeout_s` | N | Yes | Resolved timeout for audit | +| `ttl` | N | Yes | `created_at_epoch + timeout_s + CLEANUP_MARGIN_120S` — always covers the decision window | +| `user_id` | S | Yes | Used in ownership check `ConditionExpression` | +| `repo` | S | Yes | Denormalized for fan-out | + +**TTL sizing**: the TTL is always `timeout_s + 120s`, so a 300s approval window has a 420s TTL, a 3600s window has a 3720s TTL. The row never expires during the decision window. After the decision + a short grace period, DDB's eventual-consistency TTL reaper cleans up. + +**Why a list, not a StringSet, for `matching_rule_ids`**: DDB string sets cannot be empty. Pathological no-match hard-gate hits would fail to persist. Lists handle empty gracefully. + +**Why no GSI in v1**: query pattern is always `(task_id, request_id)` for agent polls; the `bgagent pending` listing is implemented as a Scan with FilterExpression `user_id = :caller AND status = :pending` — acceptable at current scale. When pending-approval volume grows, add a GSI on `user_id`. + +### 10.2 `TaskTable` additions + +Four new attributes on the existing task row: + +| Name | Type | Required | Description | +|---|---|---|---| +| `approval_timeout_s` | N | No | Default timeout for hard-gate gates. Default 300. | +| `initial_approvals` | L | No | List of scope strings from submit time | +| `awaiting_approval_request_id` | S | No | Set when status = AWAITING_APPROVAL; cleared on transition back (via joint `UpdateExpression`) | +| `approval_gate_count` | N | No | Running counter of approval gates fired on this task; used to enforce the 50-gate cap | + +Joint updates on AWAITING_APPROVAL transitions always set/clear `awaiting_approval_request_id` in the same `UpdateExpression` as the status change — either within the TransactWriteItems Put+Update, or in the single UpdateItem on resume. + +### 10.3 TaskTable status enum update + +```typescript +export const TASK_STATUSES = [ + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', + 'FINALIZING', 'COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT', +] as const; + +export const ACTIVE_STATUSES = new Set([ + 'SUBMITTED', 'HYDRATING', 'RUNNING', 'AWAITING_APPROVAL', 'FINALIZING', +]); + +export const VALID_TRANSITIONS = { + // ...existing... + RUNNING: ['FINALIZING', 'CANCELLED', 'TIMED_OUT', 'FAILED', 'AWAITING_APPROVAL'], + AWAITING_APPROVAL: ['RUNNING', 'CANCELLED', 'FAILED'], // FAILED via reconciler only + HYDRATING: ['RUNNING', 'FAILED', 'CANCELLED', 'AWAITING_APPROVAL'], // rare but possible + // ... +}; +``` + +--- + +## 11. Observability + +### 11.1 New `agent_milestone` event types + +Emitted to both `ProgressWriter` (DDB, 90d) and `sse_adapter` (live stream). Plus audit events emitted by the REST Lambdas directly to TaskEventsTable. + +| Event | Source | Metadata | +|---|---|---| +| `pre_approvals_loaded` | Agent | `{count, scopes[]}` | +| `approval_requested` | Agent | `{request_id, tool_name, input_preview, reason, severity, timeout_s, matching_rule_ids[]}` | +| `approval_granted` | Agent | `{request_id, scope, decided_at}` | +| `approval_denied` | Agent | `{request_id, reason, decided_at}` | +| `approval_timed_out` | Agent | `{request_id, timeout_s}` | +| `approval_stranded` | Reconciler | `{request_id, age_s, reason}` | +| `approval_write_failed` | Agent | `{request_id?, error}` | +| `approval_resume_failed` | Agent | `{request_id, error}` | +| `approval_poll_degraded` | Agent | `{request_id, consecutive_failures}` | +| `approval_timeout_capped` | Agent | `{requested: N, effective: M, reason}` — surfaces when min-wins clips user's requested timeout | +| `approval_cap_exceeded` | Agent | `{request_id, count, cap}` — when 50-gate cap fires | +| `approval_rate_limit_exceeded` | Agent | `{request_id, rate, limit}` | +| `approval_decision_recorded` | ApproveTaskFn / DenyTaskFn | `{request_id, status, scope?, reason?, decided_at, caller_user_id}` — authoritative audit record | + +### 11.2 Fan-out plane — primary UX channel for approvals + +Approval events flow to the `FanOutConsumer` router via `TaskEventsTable` DDB Streams (see [`INTERACTIVE_AGENTS.md`](/architecture/interactive-agents) §6). The router invokes per-channel dispatcher Lambdas (`SlackDispatchFn`, `EmailDispatchFn`, `GitHubDispatchFn`) according to the user's notification config. + +**`TaskApprovalsTable` Streams are NOT consumed by the fan-out router.** The approval row is working state; the audit trail is in `TaskEventsTable`. Enabling Streams on `TaskApprovalsTable` would duplicate events for no benefit. Final design: `TaskApprovalsTable` does not have Streams enabled. + +**Per-channel event routing for approval events:** + +| Channel | Events subscribed by default | Payload notes | +|---|---|---| +| **Slack** | `approval_required`, `approval_decided` (granted/denied), `approval_timed_out` | Messages include `Approve` / `Deny` action buttons on `approval_required`. Button click → Slack interaction-callback Lambda → POSTs to `/v1/tasks/{id}/approvals/{request_id}` via the user's Cognito-mapped identity. | +| **Email (SES)** | `approval_required` with `severity: high` | Deep-link URL to the REST endpoint; user signs in once, decision routed. | +| **GitHub issue comment** | `approval_required` appended to the in-place comment | Visible to anyone watching the originating issue. | + +**Rate-limited per-user**: 10 approval-related fan-out messages per user per minute. Prevents notification spam. Rate-limit counter shared with other approval-related events (requested, stranded, decided); enforced in the router before dispatcher invocation. + +**Slack button security**: `approve` / `deny` button payloads are signed by Slack; the interaction-callback Lambda validates the signing secret before writing. User mapping from Slack user ID → Cognito user ID is configured per workspace via `bgagent notifications configure --workspace `. + +### 11.3 Dashboard additions + +Extend `TaskDashboard` (`cdk/src/constructs/task-dashboard.ts`). These are read-only CloudWatch widgets that surface approval behavior to operators; no notification channel or on-call action required: + +- **Approval request rate** (line, 7d): count of `approval_requested` per hour, across all tasks. +- **Approval response time** (line + p50/p99): `decided_at - created_at`, per decision; plotted for the three outcome types. +- **Outcome distribution** (stacked bar, per hour): granted / denied / timed_out / stranded. Inverts quickly if notifications break. +- **Active AWAITING_APPROVAL tasks** (gauge): current count across the fleet. +- **Per-task approval-gate count distribution** (histogram): spot tasks approaching the 50-gate cap. +- **Top hard-gate rules by match frequency** (table): which rules are firing; informs rule tuning over time. + +### 11.4 OTEL trace integration + +Every `agent_milestone("approval_*")` event carries `trace_id` / `span_id`. A span `hitl.approval_wait` brackets the PreToolUse poll loop: `span.duration = decided_at - created_at`. `hitl.approval_race_loss` emitted when the agent's local timeout fired <5s before a late user decision (useful for tuning). + +### 11.5 CloudWatch alarms — deferred + +Operator-facing CloudWatch alarms that would page on: +- High approval-timeout rate (users not responding, notifications broken) +- Tasks stuck in AWAITING_APPROVAL beyond `timeout_s + 60s` (reconciler failure) +- High approval-write failure rate (DDB throttled or IAM drift) +- Approval-gate cap hit (suspicious retry loop) + +…are **out of scope for Phase 3a** because the project does not yet have a notification channel (Slack / PagerDuty / SNS topic / email distribution list) configured for operational alerts. Adding alarms without a notification channel produces CloudWatch widgets that nobody sees — no safety benefit. + +If / when an operational channel is added to the stack, these alarms become a small follow-up: wire CloudWatch metric filters on the milestone event types already emitted (§11.1), then an alarm + SNS action per threshold. The supporting metric data already flows (decisions 3-15 guarantee it); only the plumbing is deferred. + +--- + +## 12. Security model + +### 12.1 Trust boundaries + +- **Agent container ↔ TaskApprovalsTable**: IAM role on the runtime has `GetItem` / `PutItem` / conditional `UpdateItem` on the table. Agent writes pending, reads decisions, writes TIMED_OUT on internal timeout. +- **User CLI ↔ API Gateway**: Cognito JWT (same authorizer as `/tasks/*`). +- **ApproveTaskFn/DenyTaskFn ↔ TaskApprovalsTable**: Lambda IAM policy allows `UpdateItem` with authorization condition (`user_id = :caller`) built into the ConditionExpression. +- **Blueprint origin**: blueprints are CDK-deployed constructs (see `cdk/src/constructs/blueprint.ts`). Platform operators deploy them. Users cannot upload arbitrary blueprint.yaml from the target repo. This property is load-bearing for the security model — if blueprint origin ever becomes user-uploaded, the blueprint-injection section (§12.4) must be re-evaluated. + +### 12.2 Ownership encoded in ConditionExpression + +No TOCTOU window. The single `UpdateItem` on `TaskApprovalsTable` encodes: + +``` +#status = :pending AND user_id = :caller +``` + +Authorization and state transition are atomic. A compromised internal caller (Lambda with raw DDB access) or a logic bug in a future refactor that forgets the ownership check still can't flip rows without matching the `user_id`. + +### 12.3 Race prevention + +**Race 1 — user approves at T, agent times out at T+ε**: +- Agent's poll loop times out → best-effort conditional update `status = TIMED_OUT WHERE status = :pending` +- User's CLI writes `APPROVED WHERE status = :pending` +- One wins atomically +- The loser: + - If TIMED_OUT wins: user gets 409 `REQUEST_ALREADY_DECIDED`. User sees "approval expired". + - If APPROVED wins: agent's poll reads APPROVED on next tick. Agent proceeds. + +**Race 2 — double-approve**: +- Two concurrent CLI invocations. Second gets 409 `REQUEST_ALREADY_DECIDED`. Idempotent. + +**Race 3 — cancel during AWAITING_APPROVAL**: +- Agent writes `RUNNING WHERE status = :awaiting AND awaiting_approval_request_id = :rid` +- User writes `CANCELLED WHERE status = :awaiting` (via `bgagent cancel`) +- If CANCELLED wins: agent's resume fails with TransactionCanceledException. Hook emits `approval_resume_failed` and returns DENY. Task is already CANCELLED; agent's turn is aborted. +- If RUNNING wins: `bgagent cancel` gets 409 `TASK_ALREADY_RUNNING` (or similar) — user sees "task resumed before cancel landed". + +### 12.4 Blueprint content safety + +The blueprint trust model (§12.1) means blueprint Cedar policies are trusted by construction. Nonetheless the engine enforces: + +- Cedar syntax validation at load → fail-on-error +- Duplicate `@rule_id` → fail-on-error +- `@tier` mismatch with physical file/section → fail-on-error +- `@approval_timeout_s < 30` → fail-on-error +- Missing `@rule_id` on hard-gate rule → fail-on-error + +These guard against blueprint misconfiguration, not malicious intent. If the blueprint model ever changes to user-uploadable, additional safeguards needed: per-blueprint policy count cap (50), total policy text size cap (64KB), per-eval timeout on `is_authorized` (100ms). + +### 12.5 `all_session` does not override hard-deny + +Hard-deny is evaluated FIRST, before the allowlist fast-path (§6.2). No `initial_approvals` scope can bypass it. `CreateTaskFn` rejects `rule:` at submit. + +### 12.6 Denial reason sanitization in the Lambda + +`DenyTaskFn` runs `output_scanner.scan(reason)` — the existing agent-side scanner that redacts AWS keys, GitHub PATs, OAuth tokens, and common secrets — **before** persisting to DDB. + +Sanitization at the Lambda layer means: +- TaskApprovalsTable stores only sanitized text (visible to operators with DDB read) +- TaskEventsTable audit record stores only sanitized text (90d retention) +- Fan-out Slack/email notifications only see sanitized text +- Agent reads sanitized text verbatim; no secondary scanning needed + +Additionally, both CLI and Lambda log `message_length` not `reason` in CloudWatch logs (matching Phase 2 nudge logging discipline). + +### 12.7 `tool_input_preview` terminal-escape sanitization + +`_strip_ansi` removes: +- ANSI CSI sequences (`\x1b[...m`, etc.) +- OSC sequences (`\x1b]...\x07`) +- Control characters below 0x20 except `\t\n` +- DEL (0x7F) + +Applied at two layers: +- **Agent-side at write**: `tool_input_preview` is sanitized before DDB Put +- **CLI-side at render**: `bgagent pending`, `bgagent approve` output, and the live stream renderer all pass preview text through `_strip_ansi` before display + +Defense in depth: rows written before the agent-side sanitization landed (if any) are still rendered safely. + +### 12.8 Recent-decision cache prevents approval-gate storms + +After a DENIED or TIMED_OUT outcome, the engine caches `(tool_name, tool_input_sha256)` for 60s. The agent's next identical tool call auto-denies without a new approval request. A prompt-injected agent cannot burn through approval gates with the same destructive action. + +Cache is NOT populated on APPROVED (don't want to cache-block a just-approved call). + +### 12.9 Per-task + per-rate caps + +- Per-task hard cap: 50 approval gates. Exceeded → task → FAILED with reason `"approval-gate cap exceeded"`. +- Per-minute rate limit: 20 approval-row writes. Exceeded → fail-closed deny on the gate that tripped it. +- Fan-out notification cap: 10 approval-related messages per user per minute. Exceeded → messages dropped (logged). + +These caps bound the worst-case behavior of a compromised account or prompt-injected agent. + +### 12.10 JWT replay + +Cognito JWT with signature + expiry validation on API Gateway. Approval row conditional-update prevents replay from mutating state. + +--- + +## 13. Failure modes + fail-closed posture + +### 13.1 DDB write failure at approval creation + +TransactWriteItems fails → hook emits `approval_write_failed` and returns DENY. No partial-state leakage. + +### 13.2 Poll read failures + +- Single failed GetItem: log WARN, continue polling +- After 3 consecutive failures: emit `approval_poll_degraded` event +- After 10 consecutive failures: treat as TIMED_OUT, best-effort UpdateItem to TIMED_OUT, fail-closed deny to SDK + +### 13.3 Ownership mismatch + +ApproveTaskFn sees JWT whose sub doesn't match row's user_id: atomic conditional-update fails → returns 404 `REQUEST_NOT_FOUND` (no existence oracle). + +### 13.4 Cedar engine crash mid-evaluation + +`evaluate_tool_use` catches all exceptions from `cedarpy.is_authorized` and returns `Outcome.DENY` with reason `"fail-closed: "`. Matches existing behavior. + +### 13.5 Multiple matching rules with conflicting annotations + +Covered in §6.3 (min timeout clamped by floor; max severity). + +### 13.6 Container restart mid-approval + +Detected by `reconcile-stranded-tasks.ts` (§9.5). Transitions task to FAILED with reason `"approval stranded (container eviction)"`. User sees clear failure, can resubmit. No silent hang. + +### 13.7 Insufficient lifetime remaining for approval + +If `remaining_maxLifetime - CLEANUP_MARGIN_120S < FLOOR_30S`, hook immediately returns DENY with reason `"insufficient maxLifetime for approval"`. Task continues without a gate — or, if the gate was load-bearing, fails gracefully in RUNNING state. + +### 13.8 PreToolUse hook itself crashes + +Existing behavior: hook's outer try/except returns fail-closed deny. Extended in Phase 3 to log hook crash with context (request_id if available) for triage. + +### 13.9 Resume transition fails (user cancelled during poll) + +Hook emits `approval_resume_failed` and returns DENY. Task is already in its new state (CANCELLED); hook doesn't attempt to resume. + +--- + +## 14. Sample scenarios + +### 14.1 Scenario A: force-push with per-rule timeout + +Setup: repo `my-org/my-app` blueprint extends hard-gate with `force_push_main` (@approval_timeout_s=600). Task default is 300s. + +```bash +$ bgagent submit --repo my-org/my-app \ + --task "merge feature-x into main and push" \ + --approval-timeout 300 +``` + +Agent runs `git push origin main`. `push_to_protected_branch` matches (non-force push to a protected branch). Annotations: `timeout_s=300`, `severity=medium`. + +``` +[14:00:00] ★ approval_requested: Bash "git push origin main" (severity=medium) + reason: Cedar hard-gate: push_to_protected_branch + respond: bgagent approve 01KPR... [--scope tool_type_session] + timeout: 300s +``` + +User approves with `tool_type_session` (either via `bgagent approve` or a Slack button). Events: + +``` +[14:00:08] ★ approval_granted: request_id=01KPR... scope=tool_type_session +[14:00:08] ▶ Bash: git push origin main +[14:00:10] ◀ Bash: Everything up-to-date +``` + +Later `git status` call → allowlist fast-path → no new approval. + +### 14.2 Scenario B: Force-push to main hits hard-deny + +Agent proposes `Bash: git push --force origin main`. Hard-deny rule `force_push_main` matches → immediate DENY with reason `"Hard-deny: force_push_main"`. No approval request. Task stays in RUNNING. + +Recent-decision cache now has `(Bash, sha256("git push --force origin main"))` for 60s — a retry would auto-deny without re-running Cedar. + +Agent adapts, opens a PR via `gh pr create` instead. No rule matches. Tool runs. + +### 14.3 Scenario C: Trusted automation with `all_session` + +```bash +$ bgagent submit --repo my-org/infra \ + --task "apply approved terraform plan for staging-v2" \ + --pre-approve all_session --yes +``` + +Blueprint on `my-org/infra` allows `maxPreApprovalScope: "all_session"`. Task runs fully autonomously. Zero approval gates. Hard-deny still enforces. + +Stream shows `[14:20:00] ★ pre_approvals_loaded: count=1 scopes=[all_session]` at startup so operators see the starting posture. + +### 14.4 Scenario D: Denying with steering reason + +```bash +$ bgagent submit --repo my-org/my-app \ + --task "Update the deployment scripts to use the new release branch" \ + --approval-timeout 600 +``` + +Agent tries `Bash: git push origin release/v2`. Hard-gate rule `push_to_protected_branch` hits. `approval_requested` → user: + +```bash +$ bgagent deny 01KPW... 01KPR... \ + --reason "move it to src/dashboard/v1.deprecated instead of deleting; we may need to reference it in migrations" +``` + +`DenyTaskFn` sanitizes (no secrets in this reason, passes through unchanged), writes to DDB. Agent's poll reads DENIED. + +Hook executes: atomic resume to RUNNING → queue denial injection via `between_turns_hooks` → return to SDK with fallback deny reason. + +Next Stop seam fires. The between-turns injector emits: + +```xml + +move it to src/dashboard/v1.deprecated instead of deleting; we may need to reference it in migrations + +``` + +Agent reads the denial on its next turn, adapts: + +``` +[14:30:12] ▶ Bash: git mv src/dashboard/v1 src/dashboard/v1.deprecated +[14:30:13] ◀ Bash: (success) +``` + +Task proceeds. Denial-as-steering worked via the same robust path Phase 2 nudges use. + +### 14.5 Scenario E: AI-DLC phased pre-approvals + +Three-phase workflow with escalating trust: + +```bash +# Phase 1 — analysis only +$ bgagent submit --repo my-org/new-feature \ + --task "analyze the existing auth module and produce a design doc" \ + --pre-approve tool_type:Read \ + --pre-approve tool_type:Glob \ + --pre-approve tool_type:Grep \ + --pre-approve bash_pattern:"ls *" \ + --pre-approve bash_pattern:"find *" + +# Phase 2 — documentation writes +$ bgagent submit --repo my-org/new-feature \ + --task "update docs/auth.md per the approved design doc" \ + --pre-approve tool_type:Read \ + --pre-approve write_path:"docs/**" \ + --pre-approve tool_group:file_write \ + --pre-approve bash_pattern:"git add docs/**" \ + --pre-approve bash_pattern:"git commit *" + +# Phase 3 — full implementation +$ bgagent submit --repo my-org/new-feature \ + --task "implement the auth module per approved design + docs" \ + --pre-approve all_session --yes +``` + +Each phase has explicit scope. Matches real-world review workflows. Visible in audit via `pre_approvals_loaded` event. + +--- + +## 15. Implementation plan + +### 15.1 Milestone structure + +**Phase 3a** — core feature (3-4 weeks of work): +- Day 1: commit the cedarpy annotation round-trip test (agent side, `agent/tests/test_cedarpy_annotations_contract.py`) + the `@cedar-policy/cedar-wasm` parse test (Lambda side, `cdk/test/handlers/shared/cedar-policy.test.ts`). Both packages already spiked 2026-04-24: `cedarpy.policies_to_json_str()` returns annotations verbatim under `staticPolicies..annotations`; `@cedar-policy/cedar-wasm/nodejs` exports `policySetTextToParts` + `policyToJson(text)` which together expose the same data (see §15.6). +- Engine refactor (hard-deny + hard-gate + annotations + allowlist + recent-decisions) +- New DDB table, new Lambdas, new CLI commands +- PreToolUse hook extension (atomic transitions) +- `bgagent policies list` + `bgagent pending` (support UX that unblocks real usage) +- Happy path + fail-closed tests +- E2E on `backgroundagent-dev` + +**Phase 3b** — polish (1-2 weeks): +- CLI inline streaming prompt (UX research first) +- `approve --defer` / allowlist revocation (`bgagent revoke-approval`) +- CloudWatch alarm plumbing (§11.5) — deferred until an operational notification channel is available +- More hard-gate policies in the default set based on real usage + +### 15.2 Phase 3a task list + +~35 focused items. Ordered by dependency. + +| # | Package | File | Change | +|---|---|---|---| +| 1 | agent | Spike | Validate cedarpy.policies_to_json_str() returns annotations. Confirm `diagnostics.reasons` shape for multi-match. If API diverges, update §6 before proceeding. | +| 2 | agent | `src/policy.py` | Extend `PolicyDecision` (outcome/timeout_s/severity/matching_rule_ids/allowed-property). Split `_DEFAULT_POLICIES` into hard-deny + hard-gate. Add annotation parsing. Implement `ApprovalAllowlist` + `RecentDecisionCache`. Load-time validation (rule_id uniqueness, tier mismatch, annotation floor). | +| 3 | agent | `policies/hard_deny.cedar` (new) | Migrate current hard-deny rules + add DROP TABLE. Annotations. | +| 4 | agent | `policies/hard_gate.cedar` (new) | force-push, *.env, infrastructure/**, credentials. Annotations. | +| 5 | agent | `tests/test_policy.py` | Three-outcome, annotation merging, allowlist (incl. write_path, tool_group), recent-decision cache, pre-approval seeding, annotation round-trip. | +| 6 | cdk | `src/constructs/task-approvals-table.ts` (new) | Table + TTL + PITR (no Streams). | +| 7 | cdk | `src/handlers/shared/cedar-policy.ts` (new) | Shared policy-parsing library for Lambda-side rule-id validation. | +| 8 | cdk | `src/handlers/approve-task.ts` (new) | POST /approve with ownership-in-condition + audit event. | +| 9 | cdk | `src/handlers/deny-task.ts` (new) | POST /deny with output_scanner sanitization + audit event. | +| 10 | cdk | `src/handlers/get-policies.ts` (new) | GET /v1/repos/{repo}/policies. | +| 11 | cdk | `src/handlers/shared/types.ts` | ApprovalRequest/Response/DenyRequest + Scope union + extended CreateTaskRequest. | +| 12 | cdk | `src/handlers/shared/response.ts` | New error codes (REQUEST_NOT_FOUND, REQUEST_ALREADY_DECIDED, TASK_NOT_AWAITING_APPROVAL). | +| 13 | cdk | `src/constructs/task-api.ts` | Wire /approve, /deny, /repos/{}/policies routes. Grants. | +| 14 | cdk | `src/stacks/agent.ts` | Instantiate TaskApprovalsTable. Env var on runtimes. | +| 15 | cdk | `src/constructs/task-status.ts` | AWAITING_APPROVAL enum + transitions. | +| 16 | cdk | `src/handlers/create-task.ts` | Validate initial_approvals + approval_timeout_s with all safeguards (degenerate patterns, hard-deny rule rejection, maxPreApprovalScope ceiling, blueprint-resolved rule lookup). | +| 17 | cdk | `src/handlers/orchestrate-task.ts` | waitStrategy + finalizeTask handle AWAITING_APPROVAL. | +| 18 | cdk | `src/constructs/stranded-task-reconciler.ts` | Detect + transition stranded AWAITING_APPROVAL tasks. | +| 19 | cdk | `src/handlers/fanout-task-events.ts` | Dispatch rules for approval_* events + per-user notification rate limit. | +| 20 | agent | `src/hooks.py` | PreToolUse REQUIRE_APPROVAL path: atomic transitions, caps, poll, resume, denial-injection queue. | +| 21 | agent | `src/hooks.py` | `_denial_between_turns_hook` registered alongside `_nudge_between_turns_hook`. Shared `_xml_escape`. | +| 22 | agent | `src/task_state.py` | AWAITING_APPROVAL in transition helpers (TransactWriteItems primitive). | +| 23 | agent | `src/progress_writer.py` | `write_approval_*` convenience methods over `write_agent_milestone`. | +| 24 | cli | `src/commands/approve.ts` (new) | + 429 handling, `NO_COLOR` check. | +| 25 | cli | `src/commands/deny.ts` (new) | + `--reason-file` support. | +| 26 | cli | `src/commands/pending.ts` (new) | `bgagent pending` listing across active tasks. | +| 27 | cli | `src/commands/policies.ts` (new) | `bgagent policies list` + `policies show`. | +| 28 | cli | `src/commands/submit.ts` + `run.ts` | --approval-timeout, --pre-approve (repeatable), --pre-approve-file, all_session confirmation with --yes bypass. | +| 29 | cli | `src/api-client.ts` | approveTask, denyTask, listPending, listPolicies, extended createTask. | +| 30 | cli | `src/types.ts` | Mirror CDK types. Scope union + validator. | +| 31 | cdk | `test/handlers/approve-task.test.ts` (new) | Happy path, race, ownership-in-condition, scope validation, 409/404 distinction. | +| 32 | cdk | `test/handlers/deny-task.test.ts` (new) | Same shape + output_scanner integration. | +| 33 | cdk | `test/handlers/get-policies.test.ts` (new) | Discovery endpoint tests. | +| 34 | cdk | `test/handlers/create-task.test.ts` | initial_approvals validation (degenerate patterns, hard-deny rule rejection, blueprint resolution). | +| 35 | cli | `test/commands/*.test.ts` | CLI command tests. | +| 36 | agent | `tests/test_hooks.py` | REQUIRE_APPROVAL path, atomic transitions, caps, recent-decision cache, denial injection. | +| 37 | docs | `docs/design/INTERACTIVE_AGENTS.md` | Confirm §5.6 (approval CLI commands) and §8.2 (state machine) reflect Phase 3 wiring. | + +### 15.3 Testing strategy + +- **Unit**: ~80% coverage target, matching Phase 2. +- **Integration**: + - Cedar annotation round-trip test (write, parse, recover all 5 annotations) + - Full PreToolUse → PolicyDecision → DDB pipeline + - Allowlist seeding from initial_approvals + - Shared policy-parsing library consistency (Lambda side == agent side) +- **E2E** on `backgroundagent-dev`: 5 scenarios (A-E from §14). Both RuntimeJwt and Runtime-IAM paths. +- **Race tests**: + - Approve vs. timeout concurrent + - Deny vs. timeout concurrent + - Double-approve + - Cancel during AWAITING_APPROVAL + - Late approval after TIMED_OUT (expect 409) +- **Chaos tests**: + - Container restart mid-approval (simulated via kill + reconciler) + - DDB throttle during poll (simulated via mock) + - Bash retry loop after DENIED (expect recent-decision cache auto-deny) +- **Security tests**: + - Wrong user JWT → 404 (not 403) + - ANSI-injected tool_input_preview → stripped at both layers + - Malformed Cedar annotations → task fails to start + - Degenerate bash_pattern → 400 at submit + - Sanitizer-removing-secret test (OUTPUT_SCANNER integration) + +### 15.4 Rollout — no feature flag + +Cedar-HITL is shipped as standard functionality — no per-repo enable/disable flag. The safety posture of a given task is determined entirely by the content of the loaded policy set (built-in + blueprint) and the user's `--pre-approve` scopes at submit time. + +Built-in policies shipped with the agent: + +**Hard-deny (absolute, no scope bypasses them)**: +- `rm_slash` — `rm -rf /` +- `write_git_internals`, `write_git_internals_nested` — writes under `.git/` +- `drop_table` — SQL destructive DDL +- `force_push_main` — `git push --force` (or `-f`) to `main`/`prod` +- `write_credentials` — writes to files with `credentials` in the path + +**Hard-gate starter set (require approval by default)**: +- `push_to_protected_branch` — non-force push to `main`/`master`/`prod`/`release/*` — medium, 300s +- `write_env_files` — `like "*.env"` — high, 600s + +Users who want fully autonomous execution (no approval gates) pass `--pre-approve all_session --yes` at submit. Repos that want additional gates add them via `Blueprint.security.cedarPolicies.hard_gate`. Repos that want a different policy set can override specific built-in rules by `@rule_id` via the blueprint's `security.cedarPolicies.disable` list (see §17 for the disable-by-id mechanism, implemented as part of 3a). + +Rollout steps: + +1. **Implement + merge to main.** Built-in policies ship with the hard-deny + hard-gate sets above. No flag, no global kill switch. Any task on any repo instantly has the gate behavior for rules in the starter set; any task with `--pre-approve all_session` bypasses hard-gate rules (hard-deny rules remain enforced regardless). +2. **`backgroundagent-dev` validation.** Deploy merged code. Run E2E scenarios A–E: + - A: force-push gated + approved via CLI + - B: hard-deny path (DROP TABLE blocked, not gated) + - C: `--pre-approve all_session` bypasses hard-gate + - D: deny-with-reason steers agent via `` injection + - E: AI-DLC-style phased pre-approvals + Confirm Phase 1a/1b/2 regressions still pass. Confirm dashboards render. +3. **Pilot period (2 weeks).** Designate `scoropeza/agent-plugins` as the pilot repo (non-critical, active usage). Monitor: + - Any stranded tasks → indicates reconciler gap + - Timeout rate on approval_requested + - Per-task approval-gate count distribution — spot anomalous retry loops + - User-reported friction: "is the gate firing on things it shouldn't?" + If the starter set is too noisy, tune. If reliability is solid, proceed. +4. **Default for all repos.** Once the pilot is stable, the starter set is already live for everyone — no "flip the switch" step because there was no flag. Ongoing tuning happens by modifying built-in policies in code or via repo blueprints. + +**Rollback mechanism.** If the pilot surfaces a bug: remove the problem rule from `hard_gate.cedar` and redeploy (~5 min). No flag to flip. If the bug is more fundamental (engine regression), `git revert` the Phase 3 merge and redeploy — Phase 2 tests continue to pass because the backward-compat shim on `PolicyDecision.allowed` preserves the hook contract. + +**Success criteria for "pilot done":** +- Zero stranded tasks in 2 weeks +- <10% timeout rate on `approval_requested` +- Zero `approval_cap_exceeded` events (if any fire, either the cap is wrong or adversarial traffic to investigate) +- No regressions in Phase 1a/1b/2 tests (CI enforced on every commit) +- User-initiated gates that work: every hard-gate match produces a visible `★ approval_requested` in the stream and a responsive `bgagent approve/deny` cycle + +### 15.5 Backward compatibility + +- Existing tasks without `initial_approvals` → empty list → no pre-approvals, default `approval_timeout_s = 300` +- Existing policies without `@rule_id` / `@tier` → engine fails to start (fail-closed). Blueprint authors must add annotations explicitly during migration. +- `PolicyDecision.allowed` property provides backward compat for existing `if not decision.allowed` callers +- Hook return shape unchanged — Phase 1a/1b tests continue to pass + +### 15.6 Shared Cedar parsing — `@cedar-policy/cedar-wasm` API quickref + +The Lambda side (`CreateTaskFn`, `ApproveTaskFn`, `GetPoliciesFn`) uses [`@cedar-policy/cedar-wasm`](https://www.npmjs.com/package/@cedar-policy/cedar-wasm) — AWS's official WASM-compiled Cedar engine. Same Rust core as the Python `cedarpy` binding we already use in the agent. Spiked + verified 2026-04-24. + +**Package:** `@cedar-policy/cedar-wasm@4.10.0` (or latest major 4.x). +**Size:** 4.1 MB unzipped / ~1.5 MB zipped — well under Lambda limits. +**Import:** `const cedar = require('@cedar-policy/cedar-wasm/nodejs');` — use the CJS nodejs sub-export, NOT the default ESM export (ESM fails with `ERR_UNKNOWN_FILE_EXTENSION` on the `.wasm` file in Node 22). + +**Core functions used by the design:** + +| Function | Purpose | +|---|---| +| `policySetTextToParts(text: string)` | Split a multi-policy Cedar text into an array of individual policy texts. Returns `{type: "success", policies: string[]}` or `{type: "failure", errors: [...]}` | +| `policyToJson(text: string)` | Parse a single policy text into structured JSON. Returns `{type: "success", json: {annotations, effect, principal, action, resource, conditions}}` — annotations preserved verbatim under `json.annotations` as a `Record` | +| `isAuthorized({principal, action, resource, context, policies: {staticPolicies: string}, entities: []})` | Main authorization call. Entity references are `{type, id}` objects, **not** string literals. Returns `{type, response: {decision, diagnostics: {reason: string[]}}}` — `diagnostics.reason` is the list of matching policy IDs (e.g. `["policy1", "policy2"]`) for multi-match | + +**Minimal annotation-extraction pattern (the only thing `CreateTaskFn` needs for rule validation):** + +```typescript +// cdk/src/handlers/shared/cedar-policy.ts (sketch) +import * as cedar from '@cedar-policy/cedar-wasm/nodejs'; + +export interface ParsedRule { + ruleId: string; + tier: 'hard-deny' | 'hard-gate'; + severity?: 'low' | 'medium' | 'high'; + category?: string; + approvalTimeoutS?: number; +} + +export function parseRules(policiesText: string): ParsedRule[] { + const splitResult = cedar.policySetTextToParts(policiesText); + if (splitResult.type !== 'success') { + throw new Error(`Cedar policy parse failed: ${JSON.stringify(splitResult.errors)}`); + } + const rules: ParsedRule[] = []; + for (const policyText of splitResult.policies ?? []) { + const jsonResult = cedar.policyToJson(policyText); + if (jsonResult.type !== 'success') continue; + const annotations = jsonResult.json.annotations ?? {}; + const tier = annotations.tier; + const ruleId = annotations.rule_id; + if (tier !== 'hard-deny' && tier !== 'hard-gate') { + throw new Error(`Missing or invalid @tier annotation on policy (rule_id=${ruleId})`); + } + if (!ruleId) { + throw new Error(`Missing @rule_id annotation on ${tier}-deny policy`); + } + rules.push({ + ruleId, + tier, + severity: annotations.severity as ParsedRule['severity'], + category: annotations.category, + approvalTimeoutS: annotations.approval_timeout_s ? parseInt(annotations.approval_timeout_s, 10) : undefined, + }); + } + return rules; +} + +export function isHardDenyRule(rules: ParsedRule[], ruleId: string): boolean { + return rules.some(r => r.ruleId === ruleId && r.tier === 'hard'); +} +``` + +**API differences from Python cedarpy to be aware of during implementation:** + +1. Results are always wrapped in `{type: "success" | "failure", ...}`. Always check `.type` before accessing payload. +2. `isAuthorized` takes a single call object (not 3 positional args). Entities are `{type, id}` objects. +3. The Lambda cold-start penalty is ~30ms for the first `require()` (WASM module instantiation). Keep the import at module scope — not inside the handler — so subsequent invocations reuse the already-instantiated module. +4. The Node binding is CJS; the Lambda bundler (esbuild) treats the `.wasm` file as an external asset and Lambda's layer mechanism handles it automatically. No custom esbuild loader needed. + +--- + +## 16. Implementation notes (carry-forward tasks) + +Items from the 2026-04-24 design review not captured above as design changes — to be addressed during implementation and removed from this list once completed. These are P1-P2 findings; P0s have been integrated into the main design body. + +**IMPL-1** (data-flow P1-5): Scope string normalization. CLI + Lambda must agree. Document: trim whitespace, preserve case on `tool_type:` (Bash/Read/Write are canonical; reject case-shifted variants). + +**IMPL-2** (data-flow P1-7): Dual-write ordering between `progress_writer` and `sse_adapter` is best-effort; canonical source is TaskEventsTable. Document this in the implementation guide alongside Phase 2. + +**IMPL-3** (data-flow P2-1): Catch `ValueError` in `_merge_annotations` on malformed `@approval_timeout_s`; skip the annotation, log WARN. Engine already fails the task at load time if below floor, so this is a belt-and-suspenders. + +**IMPL-4** (data-flow P2-4): Test constraint — tests MUST NOT assert specific positional Cedar policy IDs. Use `@rule_id` annotations exclusively. + +**IMPL-5** (security SA-11 residual): Both the Lambda (audit event) and the agent (milestone) write approval decisions. The Lambda's write is canonical; the agent's is observational. Tests should verify the Lambda write completes even if agent milestone fails. + +**IMPL-6** (security P1-8): Audit trail ownership. `ApproveTaskFn` / `DenyTaskFn` write `approval_decision_recorded` to TaskEventsTable directly (not via agent milestone). Implement as part of the Lambda request flow. + +**IMPL-7** (security blind-spot #5): PolicyEngine MUST be instantiated per task, NOT per container. Verify in server.py bootstrap that a new instance is created on each task invocation (even when attach-don't-spawn logic reuses the container). + +**IMPL-8** (security blind-spot #6): TaskApprovalsTable Streams — confirmed off (§11.2). Do not subscribe any consumer. + +**IMPL-9** (functional P1-3): Runtime allowlist revocation. Not shipped in 3a. Placeholder: `bgagent revoke-approval ` noted in §17. + +**IMPL-10** (functional P1-12): `approval_timeout_s` default 300 documented consistently in §3 #6, §7.3 table, §10.2 attribute description. + +**IMPL-11** (functional P2-8): CLI `submit.ts` gains `--pre-approve` / `--approval-timeout` flags. + +**IMPL-12** (functional P2-9): Poll cadence in §3 #3 reconciled — describe as "initial 2s for 30s, then 5s" without specific call count math (it varies with timeout_s). + +**IMPL-13** (functional FC-5): `bgagent status --allowlist` — inspects current in-process allowlist state. Useful for debugging "why is this tool being gated again?". Low priority; add to `bgagent status` if cheap. + +**IMPL-14** (functional FC-6): Tool_use_id correlation. SDK handles internally. No hook-side changes needed; tests should verify the hook does not echo tool_use_id in its response. + +**IMPL-15** (functional FC-9): Recent-decision cache 60s window — tune after observation. Default 60s is a reasonable starting point. + +**IMPL-16** (CLI UX): ULID length is 26 chars, not 33. Update all CLI help text and error messages. + +**IMPL-17** (CLI UX): Shell completion (tab-complete task_id + request_id from `bgagent pending`). Deferred to 3b; document in §17. + +**IMPL-18** (FC-7): PolicyEngine freezing is implicit (single `__init__` call, no reload path). Add a test: assert that no code path calls `load_policies` after `__init__` completes. + +--- + +## 17. Deferred / out of scope + +### 17.1 Multi-user approval + +Future: multi-user approval (e.g., two of three reviewers must approve for `rule:deploy_prod`). Scope: §9.8 INTERACTIVE_AGENTS.md, Iteration 5. + +### 17.2 Per-rule auto-approve on timeout + +`@on_timeout("allow")` annotation sketched. Safety footgun. Revisit in 3b if demand. + +### 17.3 `@tier("advise")` — non-blocking advisory rules + +A third policy tier for rules that should surface but not block. Semantics sketch: + +- Cedar matches → emit `agent_milestone("advise_matched", {rule_ids, severity, tool_name, input_preview})` via `ProgressWriter` + fan-out. +- **No block.** Tool call proceeds immediately as if ALLOWED. +- **No timeout, no approval row, no state transition.** The engine never pauses. +- `PolicyDecision` gains `Outcome.ADVISE` but `evaluate_tool_use` returns ALLOW to the hook (internal tier, not a new SDK `permissionDecision`). +- Event framing: past-tense ("agent did X, matched rule Y"). Fan-out to Slack/email is FYI — no action buttons, audit-only. +- File layout: `agent/policies/advise.cedar`. Third file alongside `hard_deny.cedar` + `hard_gate.cedar`. + +Deferred because (a) shipping with gate-or-not is the simpler mental model for v1 users, (b) we want to observe whether hard-gates alone produce acceptable UX before introducing a third outcome, and (c) a concrete "I want to know but not be blocked" use case hasn't surfaced yet. First candidate rule if we ship it: `push_to_protected_branch` (force-push to any branch — informational for feature-branch workflows where force-pushing is routine). + +### 17.4 Interactive streaming prompts + +UX research first. Unlikely to ship — the async-only direction for the platform suggests notification-plane delivery is the right shape. + +### 17.4 Persistent allowlist across container restarts + +Today: in-process; reconciler fails stranded tasks. Phase 3b could persist to TaskTable + hydrate on restart. Not critical given rare restarts. + +### 17.5 `bgagent approve --defer` + +Escape hatch: "cancel + release slot". Clearer than silent timeout. Phase 3b. + +### 17.6 Policy hot-reload + +Today: policies frozen at task start. A long-running task can't benefit from a fresh hard-gate rule added mid-task. Probably fine; submission is the authoritative moment. Not a Phase 3 goal. + +### 17.7 Severity-based routing + +CLI: `bgagent approve --severity high` auto-approves high only, leaves medium/low. Phase 3b. + +### 17.8 Runtime allowlist revocation + +`bgagent revoke-approval `. User realization "oh wait, I didn't mean to approve ALL Bash". Phase 3b — implementation is straightforward (remove from in-process allowlist + emit `approval_revoked` milestone). + +### 17.9 Bulk approve + +`bgagent approve --all-pending` to approve everything pending. Power-user. Low priority; users WILL ask. + +### 17.10 Shell completion for task_id / request_id + +Tab-complete from `bgagent pending`. Deferred to 3b. + +### 17.11 Policy linting + +`bgagent lint-policies --repo ` to validate blueprint Cedar before submission. Catches annotation errors in development rather than at container start. Phase 3b. + +### 17.12 Richer approval annotations + +`@approval_requires_mfa("true")`, `@approval_channel("slack")` for enterprise workflows (step-up auth, audit channel). Good ideas; deferred. + +### 17.13 Cross-task scope inheritance + +"Apply the same pre-approvals I used on my last task." Convenience. Phase 3b. + +--- + +## Appendix A — Key file change map + +See §15.2. Net new files: ~13. Net modified files: ~15. Total LOC estimate: ~3500 production + ~2000 test = ~5500 lines. Larger than Phase 2 (+2950 / -34) because of the new Lambda × 3 + discovery endpoint + shared parser + state machine + reconciler updates. + +## Appendix B — Review checklist (pre-merge) + +- [ ] Day-1 cedarpy spike run; annotation round-trip confirmed +- [ ] All 5 Cedar annotations parse + recover via `policies_to_json_str()` round-trip test +- [ ] Every hard-deny rule has `@tier("hard-deny")` + `@rule_id` +- [ ] Every hard-gate rule has `@tier("hard-gate")` + `@rule_id` + `@severity` (default medium if missing) +- [ ] `@rule_id` uniqueness enforced at engine load (fail-on-error, not fall-back) +- [ ] `@approval_timeout_s < 30` rejected at load +- [ ] Atomic TransactWriteItems for approval-request creation and resume transitions +- [ ] Ownership encoded in ConditionExpression on ApproveTaskFn / DenyTaskFn +- [ ] Scope validation: rejects `rule:`, degenerate patterns, blueprint-maxPreApprovalScope violations +- [ ] ANSI/control-char stripping in `tool_input_preview` (both layers) +- [ ] `output_scanner.scan` runs in DenyTaskFn before persisting `reason` +- [ ] Recent-decision cache blocks 60s retries +- [ ] Per-task cap (50) + per-minute rate limit (20) + per-user notification cap (10/min) +- [ ] Denial injection via Stop hook `between_turns_hooks` (not `permissionDecisionReason` alone) +- [ ] Stranded-task reconciler transitions AWAITING_APPROVAL > 2×timeout_s to FAILED +- [ ] Race tests pass: approve+timeout, deny+timeout, double-approve, cancel-during-awaiting, late-approval-after-TIMED_OUT +- [ ] E2E on `backgroundagent-dev`: Scenarios A-E, both runtime paths +- [ ] `bgagent pending` + `bgagent policies list` functional +- [ ] Dashboard widgets emitting all approval-* metrics +- [ ] `bgagent status --allowlist` (if IMPL-13 shipped) +- [ ] Built-in starter set loaded: hard-deny = {rm_slash, write_git_internals, write_git_internals_nested, drop_table, force_push_main, write_credentials}; hard-gate = {push_to_protected_branch, write_env_files} +- [ ] No feature flag — Cedar-HITL is standard functionality; `--pre-approve all_session --yes` is the opt-out +- [ ] Backward compat: Phase 1a/1b tests pass without modification +- [ ] ULID length references are 26 chars throughout CLI + docs + +--- + +*End of Phase 3 design doc, rev 2.* diff --git a/docs/src/content/docs/architecture/Security.md b/docs/src/content/docs/architecture/Security.md index 7777f16..bf4a9f2 100644 --- a/docs/src/content/docs/architecture/Security.md +++ b/docs/src/content/docs/architecture/Security.md @@ -75,6 +75,8 @@ The blueprint framework ([REPO_ONBOARDING.md](/architecture/repo-onboarding)) al **Deployment control** - Custom steps are defined in the `Blueprint` CDK construct and deployed via `cdk deploy`. Only principals with CDK deployment permissions can add or modify them. There is no runtime API for custom step CRUD. +The **same deploy-only property extends to `Blueprint.security.cedarPolicies`** — user-authored Cedar policies live in the CDK source, are typed as `readonly string[]` on the construct, and reach `RepoTable` only through a CloudFormation custom resource invoked at deploy time. Phase 3 (Cedar-driven HITL approval gates — see [`PHASE3_CEDAR_HITL.md`](/architecture/phase3-cedar-hitl)) is load-bearing on this property: the engine treats Cedar policies loaded at task start as trusted content. If the blueprint model ever changes to accept user-uploaded policy text via an API path, Phase 3's §12 trust model must be re-evaluated (add per-blueprint policy count cap, per-eval timeout, size cap). + **Input filtering** - The framework strips credential ARNs (`github_token_secret_arn`) and networking configuration (`egress_allowlist`) from the config before passing it to custom Lambda steps. If a custom step needs secrets, it must declare them explicitly and the operator must grant IAM permissions. **What a custom step can do:** diff --git a/docs/src/content/docs/developer-guide/Installation.md b/docs/src/content/docs/developer-guide/Installation.md index 97e2414..b439237 100644 --- a/docs/src/content/docs/developer-guide/Installation.md +++ b/docs/src/content/docs/developer-guide/Installation.md @@ -108,6 +108,27 @@ docker stats bgagent-run # CPU, memory usage docker exec -it bgagent-run bash # shell into the container ``` +#### Testing with progress events (DynamoDB Local) + +By default, progress events and task state writes are silently skipped during local runs (the `TASK_EVENTS_TABLE_NAME` and `TASK_TABLE_NAME` env vars are not set). To enable them locally using DynamoDB Local: + +```bash +# 1. Start DynamoDB Local and create tables +cd agent && mise run local:up + +# 2. Run the agent with --local-events +./agent/run.sh --local-events "owner/repo" 42 + +# 4. In another terminal — query progress events +mise run local:events # table format +mise run local:events:json # JSON format + +# 5. When done — tear down DynamoDB Local +mise run local:down +``` + +The `--local-events` flag connects the agent container to DynamoDB Local on the `agent-local` Docker network and sets the appropriate env vars. The agent code writes to DDB Local using the same code path as production — no mocks or alternate implementations. + #### Environment variables | Variable | Default | Description | diff --git a/docs/src/content/docs/using/Using-the-cli.md b/docs/src/content/docs/using/Using-the-cli.md index a1ec709..193f7cb 100644 --- a/docs/src/content/docs/using/Using-the-cli.md +++ b/docs/src/content/docs/using/Using-the-cli.md @@ -132,6 +132,20 @@ node lib/bin/bgagent.js events --output json Use **`--output json`** to see the full payload for **`preflight_failed`** (`reason`, `detail`, and per-check metadata). See **Task events** under **Task lifecycle** for how to interpret common `reason` values. +### Watching a task in real time + +Stream progress events (turns, tool calls, tool results, milestones, cost updates) from a running task and exit automatically when it reaches a terminal state. + +```bash +node lib/bin/bgagent.js watch + +# JSON output (one event per line) — useful for scripting +node lib/bin/bgagent.js watch --output json +``` + +Exit codes: `0` on `COMPLETED`, `1` on `FAILED` / `CANCELLED` / `TIMED_OUT`. Press Ctrl+C to exit early without affecting the task. + + ### Cancelling a task ```bash diff --git a/yarn.lock b/yarn.lock index 7fb806e..7f84385 100644 --- a/yarn.lock +++ b/yarn.lock @@ -221,6 +221,27 @@ "@aws-sdk/types" "^3.222.0" tslib "^2.6.2" +"@aws-crypto/crc32c@5.2.0": + version "5.2.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz#4e34aab7f419307821509a98b9b08e84e0c1917e" + integrity sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag== + dependencies: + "@aws-crypto/util" "^5.2.0" + "@aws-sdk/types" "^3.222.0" + tslib "^2.6.2" + +"@aws-crypto/sha1-browser@5.2.0": + version "5.2.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz#b0ee2d2821d3861f017e965ef3b4cb38e3b6a0f4" + integrity sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg== + dependencies: + "@aws-crypto/supports-web-crypto" "^5.2.0" + "@aws-crypto/util" "^5.2.0" + "@aws-sdk/types" "^3.222.0" + "@aws-sdk/util-locate-window" "^3.0.0" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.6.2" + "@aws-crypto/sha256-browser@5.2.0": version "5.2.0" resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz#153895ef1dba6f9fce38af550e0ef58988eb649e" @@ -250,7 +271,7 @@ dependencies: tslib "^2.6.2" -"@aws-crypto/util@^5.2.0": +"@aws-crypto/util@5.2.0", "@aws-crypto/util@^5.2.0": version "5.2.0" resolved "https://registry.yarnpkg.com/@aws-crypto/util/-/util-5.2.0.tgz#71284c9cffe7927ddadac793c14f14886d3876da" integrity sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ== @@ -550,6 +571,67 @@ "@smithy/util-waiter" "^4.2.14" tslib "^2.6.2" +"@aws-sdk/client-s3@^3.1021.0": + version "3.1040.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.1040.0.tgz#96fa3975b815e6cd6d0855a7bd4a72adf7dc1016" + integrity sha512-Ldfby1xDrlZwNY2NxP9pwdVrf8sqHbGBKP1UkoG/oWcePGlGhjY8iVwy8hRy9f1EQfHVFWIFunwHaPQxhYTnWQ== + dependencies: + "@aws-crypto/sha1-browser" "5.2.0" + "@aws-crypto/sha256-browser" "5.2.0" + "@aws-crypto/sha256-js" "5.2.0" + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/credential-provider-node" "^3.972.38" + "@aws-sdk/middleware-bucket-endpoint" "^3.972.10" + "@aws-sdk/middleware-expect-continue" "^3.972.10" + "@aws-sdk/middleware-flexible-checksums" "^3.974.15" + "@aws-sdk/middleware-host-header" "^3.972.10" + "@aws-sdk/middleware-location-constraint" "^3.972.10" + "@aws-sdk/middleware-logger" "^3.972.10" + "@aws-sdk/middleware-recursion-detection" "^3.972.11" + "@aws-sdk/middleware-sdk-s3" "^3.972.36" + "@aws-sdk/middleware-ssec" "^3.972.10" + "@aws-sdk/middleware-user-agent" "^3.972.37" + "@aws-sdk/region-config-resolver" "^3.972.13" + "@aws-sdk/signature-v4-multi-region" "^3.996.24" + "@aws-sdk/types" "^3.973.8" + "@aws-sdk/util-endpoints" "^3.996.8" + "@aws-sdk/util-user-agent-browser" "^3.972.10" + "@aws-sdk/util-user-agent-node" "^3.973.23" + "@smithy/config-resolver" "^4.4.17" + "@smithy/core" "^3.23.17" + "@smithy/eventstream-serde-browser" "^4.2.14" + "@smithy/eventstream-serde-config-resolver" "^4.3.14" + "@smithy/eventstream-serde-node" "^4.2.14" + "@smithy/fetch-http-handler" "^5.3.17" + "@smithy/hash-blob-browser" "^4.2.15" + "@smithy/hash-node" "^4.2.14" + "@smithy/hash-stream-node" "^4.2.14" + "@smithy/invalid-dependency" "^4.2.14" + "@smithy/md5-js" "^4.2.14" + "@smithy/middleware-content-length" "^4.2.14" + "@smithy/middleware-endpoint" "^4.4.32" + "@smithy/middleware-retry" "^4.5.7" + "@smithy/middleware-serde" "^4.2.20" + "@smithy/middleware-stack" "^4.2.14" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/node-http-handler" "^4.6.1" + "@smithy/protocol-http" "^5.3.14" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + "@smithy/url-parser" "^4.2.14" + "@smithy/util-base64" "^4.3.2" + "@smithy/util-body-length-browser" "^4.2.2" + "@smithy/util-body-length-node" "^4.2.3" + "@smithy/util-defaults-mode-browser" "^4.3.49" + "@smithy/util-defaults-mode-node" "^4.2.54" + "@smithy/util-endpoints" "^3.4.2" + "@smithy/util-middleware" "^4.2.14" + "@smithy/util-retry" "^4.3.6" + "@smithy/util-stream" "^4.5.25" + "@smithy/util-utf8" "^4.2.2" + "@smithy/util-waiter" "^4.3.0" + tslib "^2.6.2" + "@aws-sdk/client-secrets-manager@^3.1021.0": version "3.1021.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.1021.0.tgz#57c6348c63146642132ffa7e885a2abba08c6ff4" @@ -633,6 +715,34 @@ "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" +"@aws-sdk/core@^3.974.7": + version "3.974.7" + resolved "https://registry.yarnpkg.com/@aws-sdk/core/-/core-3.974.7.tgz#1b78801c86f54947971ead2d4b9913a2b5b7d860" + integrity sha512-YhRC90ofz5oolTJZlA8voU/oUrCB2azi8Usx51k8hhB5LpWbYQMMXKUqSqkoL0Cru+RQJgWTHpAfEDDIwfUhJw== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@aws-sdk/xml-builder" "^3.972.22" + "@smithy/core" "^3.23.17" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/property-provider" "^4.2.14" + "@smithy/protocol-http" "^5.3.14" + "@smithy/signature-v4" "^5.3.14" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + "@smithy/util-base64" "^4.3.2" + "@smithy/util-middleware" "^4.2.14" + "@smithy/util-retry" "^4.3.6" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + +"@aws-sdk/crc64-nvme@^3.972.7": + version "3.972.7" + resolved "https://registry.yarnpkg.com/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.7.tgz#0e56fb3ccc0242ed05ffd0bc993d724ce8b3dde2" + integrity sha512-QUagVVBbC8gODCF6e1aV0mE2TXWB9Opz4k8EJFdNrujUVQm5R4AjJa1mpOqzwOuROBzqJU9zawzig7M96L8Ejg== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-env@^3.972.24": version "3.972.24" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.24.tgz#bc33a34f15704d02552aa8b3994d17008b991f86" @@ -655,6 +765,17 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-env@^3.972.33": + version "3.972.33" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.33.tgz#8a3703571871e85e064f3cabda4b7b37f2344aea" + integrity sha512-bJV7eViSJV6GSuuN+VIdNVPdwPsNSf75BiC2v5alPrjR/OCcqgKwSZInKbDFz9mNeizldsyf67jt6YSIiv53Cw== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/types" "^3.973.8" + "@smithy/property-provider" "^4.2.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-http@^3.972.26": version "3.972.26" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.26.tgz#6524c3681dbb62d3c4de82262631ab94b800f00e" @@ -687,6 +808,22 @@ "@smithy/util-stream" "^4.5.22" tslib "^2.6.2" +"@aws-sdk/credential-provider-http@^3.972.35": + version "3.972.35" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.35.tgz#913eaf3a66484cb1d678ab0691943cb4f57e230d" + integrity sha512-x/BQGEIdq0oI+4WxLjKmnQvT7CnF9r8ezdGt7wXwxb7ckHXQz0Zmgxt8v3Ne0JaT3R5YefmuybHX6E8EnsDXyA== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/types" "^3.973.8" + "@smithy/fetch-http-handler" "^5.3.17" + "@smithy/node-http-handler" "^4.6.1" + "@smithy/property-provider" "^4.2.14" + "@smithy/protocol-http" "^5.3.14" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + "@smithy/util-stream" "^4.5.25" + tslib "^2.6.2" + "@aws-sdk/credential-provider-ini@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.28.tgz#6bc0d684c245914dca7a1a4dd3c2d84212833320" @@ -727,6 +864,26 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-ini@^3.972.37": + version "3.972.37" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.37.tgz#86e0d92abd993ee8866ff72865c47448a4ac1d16" + integrity sha512-eUTpmWfd/BKsq9medhCRcu+GRAhFP2Zrn7/2jKDHHOOjCkhrMoTp/t4cEthqFoG7gE0VGp5wUxrXTdvBCmSmJg== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/credential-provider-env" "^3.972.33" + "@aws-sdk/credential-provider-http" "^3.972.35" + "@aws-sdk/credential-provider-login" "^3.972.37" + "@aws-sdk/credential-provider-process" "^3.972.33" + "@aws-sdk/credential-provider-sso" "^3.972.37" + "@aws-sdk/credential-provider-web-identity" "^3.972.37" + "@aws-sdk/nested-clients" "^3.997.5" + "@aws-sdk/types" "^3.973.8" + "@smithy/credential-provider-imds" "^4.2.14" + "@smithy/property-provider" "^4.2.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-login@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.28.tgz#b2d47d4d43690d2d824edc94ce955d86dd3877f1" @@ -755,6 +912,20 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-login@^3.972.37": + version "3.972.37" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.37.tgz#43fd32e4140b4fe3a3c243ab21d0ac306e772e28" + integrity sha512-Ty68y8ISSC+g5Q3D0K8uAaoINwvfaOslnNpsF/LgVUxyosYXHawcK2yV4HLXDVugiTTYLQfJfcw0ce5meAGkKw== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/nested-clients" "^3.997.5" + "@aws-sdk/types" "^3.973.8" + "@smithy/property-provider" "^4.2.14" + "@smithy/protocol-http" "^5.3.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-node@^3.972.29": version "3.972.29" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.29.tgz#4bcc991fcbf245f75494a119b3446a678a51e019" @@ -791,6 +962,24 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-node@^3.972.38": + version "3.972.38" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.38.tgz#2fc1742b626c6b80534e42ae91359a25262e9e91" + integrity sha512-BQ9XYnBDVxR2HuV5huXYQYF/PZMTsY+EnwfGnCU2cA8Zw63XpkOtPY8WqiMIZMQCrKPQQEiFURS/o9CIolRLqg== + dependencies: + "@aws-sdk/credential-provider-env" "^3.972.33" + "@aws-sdk/credential-provider-http" "^3.972.35" + "@aws-sdk/credential-provider-ini" "^3.972.37" + "@aws-sdk/credential-provider-process" "^3.972.33" + "@aws-sdk/credential-provider-sso" "^3.972.37" + "@aws-sdk/credential-provider-web-identity" "^3.972.37" + "@aws-sdk/types" "^3.973.8" + "@smithy/credential-provider-imds" "^4.2.14" + "@smithy/property-provider" "^4.2.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-process@^3.972.24": version "3.972.24" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.24.tgz#940c76a2db0aece23879dcf75ac5b6ee8f8fa135" @@ -815,6 +1004,18 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-process@^3.972.33": + version "3.972.33" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.33.tgz#9f5ae9a63e76fcdaf684c03e5479c2e53305ce1c" + integrity sha512-yfjGksI9WQbdMObb0VeLXqzTLI+a0qXLJT9gCDiv0+X/xjPpI3mTz6a5FibrhpuEKIe0gSgvs3MaoFZy5cx4WA== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/types" "^3.973.8" + "@smithy/property-provider" "^4.2.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-sso@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.28.tgz#bf150bfb7e708d58f35bb2b5786b902df19fd92d" @@ -843,6 +1044,20 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-sso@^3.972.37": + version "3.972.37" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.37.tgz#91cdba4a3aef35c1b4178610db782dfc8a6bc490" + integrity sha512-fpwE+20ntpp3i9Xb9vUuQfXLDKYHH+5I2V+ZG96SX1nBzrruhy10RXDgmN7t1etOz3c55stlA3TeQASUA451NQ== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/nested-clients" "^3.997.5" + "@aws-sdk/token-providers" "3.1039.0" + "@aws-sdk/types" "^3.973.8" + "@smithy/property-provider" "^4.2.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-web-identity@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.28.tgz#27fc2a0fe0d2ff1460171d2a6912898c2235a7df" @@ -869,6 +1084,19 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-web-identity@^3.972.37": + version "3.972.37" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.37.tgz#9b74f250a644f9d6248950e02ed311b0b4959d3e" + integrity sha512-aryawqyebf+3WhAFNHfF62rekFpYtVcVN7dQ89qnAWsa4n5hJst8qBG6gXC24WHtW7Nnhkf9ScYnjwo0Brn3bw== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/nested-clients" "^3.997.5" + "@aws-sdk/types" "^3.973.8" + "@smithy/property-provider" "^4.2.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/dynamodb-codec@^3.972.27": version "3.972.27" resolved "https://registry.yarnpkg.com/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.972.27.tgz#3d29a2f00bbc145260419878a5f3640af81d36b3" @@ -911,6 +1139,19 @@ "@smithy/types" "^4.13.1" tslib "^2.6.2" +"@aws-sdk/middleware-bucket-endpoint@^3.972.10": + version "3.972.10" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.10.tgz#d26aa88b441d6d1b6e9275ffdc61e0fbfb55a513" + integrity sha512-Vbc2frZH7wXlMNd+ZZSXUEs/l1Sv8Jj4zUnIfwrYF5lwaLdXHZ9xx4U3rjUcaye3HRhFVc+E5DbBxpRAbB16BA== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@aws-sdk/util-arn-parser" "^3.972.3" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + "@smithy/util-config-provider" "^4.2.2" + tslib "^2.6.2" + "@aws-sdk/middleware-endpoint-discovery@^3.972.9": version "3.972.9" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.9.tgz#664f9074b0017255680c200bd9b8b23a864c0ad5" @@ -933,6 +1174,46 @@ "@smithy/types" "^4.13.1" tslib "^2.6.2" +"@aws-sdk/middleware-expect-continue@^3.972.10": + version "3.972.10" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.10.tgz#b685287951156a5d093cfdd37364894c6a8c966c" + integrity sha512-2Yn0f1Qiq/DjxYR3wfI3LokXnjOhFM7Ssn4LTdFDIxRMCE6I32MAsVnhPX1cUZsuVA9tiZtwwhlSLAtFGxAZlQ== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + +"@aws-sdk/middleware-flexible-checksums@^3.974.15": + version "3.974.15" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.15.tgz#11e688424dd08fae175d08597dd2a7edeaa4773a" + integrity sha512-j4Zp7rA1HfhDTteICnx/tPax4N/v5wmytgguXExUGyEwQ8Ug4EBA4kjp9puFAN1UZoBVpxoiXMiuTFvjaHjeEw== + dependencies: + "@aws-crypto/crc32" "5.2.0" + "@aws-crypto/crc32c" "5.2.0" + "@aws-crypto/util" "5.2.0" + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/crc64-nvme" "^3.972.7" + "@aws-sdk/types" "^3.973.8" + "@smithy/is-array-buffer" "^4.2.2" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + "@smithy/util-middleware" "^4.2.14" + "@smithy/util-stream" "^4.5.25" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + +"@aws-sdk/middleware-host-header@^3.972.10": + version "3.972.10" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.10.tgz#e63b91959ce46948d789582351b2a44c4876e924" + integrity sha512-IJSsIMeVQ8MMCPbuh1AbltkFhLBLXn7aejzfX5YKT/VLDHn++Dcz8886tXckE+wQssyPUhaXrJhdakO2VilRhg== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/middleware-host-header@^3.972.8": version "3.972.8" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.8.tgz#72186e96500b49b38fb5482d6b7bf95e5b985281" @@ -953,6 +1234,24 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/middleware-location-constraint@^3.972.10": + version "3.972.10" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.10.tgz#5265ea472f735c50b016bb5d1b46c7a616653733" + integrity sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + +"@aws-sdk/middleware-logger@^3.972.10": + version "3.972.10" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-logger/-/middleware-logger-3.972.10.tgz#d92b3374dcaddd523930bdff441207946343c270" + integrity sha512-OOuGvvz1Dm20SjZo5oEBePFqxt5nf8AwkNDSyUHvD9/bfNASmstcYxFAHUowy4n6Io7mWUZ04JURZwSBvyQanQ== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/middleware-logger@^3.972.8": version "3.972.8" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-logger/-/middleware-logger-3.972.8.tgz#7fee4223afcb6f7828dbdf4ea745ce15027cf384" @@ -982,6 +1281,17 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/middleware-recursion-detection@^3.972.11": + version "3.972.11" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.11.tgz#5659982a34fa58c69cbd358c2987c32aefd2bd91" + integrity sha512-+zz6f79Kj9V5qFK2P+D8Ehjnw4AhphAlCAsPjUqEcInA9umtSSKMrHbSagEeOIsDNuvVrH98bjRHcyQukTrhaQ== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@aws/lambda-invoke-store" "^0.2.2" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/middleware-recursion-detection@^3.972.9": version "3.972.9" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.9.tgz#53a2cc0cf827863163b2351209212f642015c2e2" @@ -993,6 +1303,35 @@ "@smithy/types" "^4.13.1" tslib "^2.6.2" +"@aws-sdk/middleware-sdk-s3@^3.972.36": + version "3.972.36" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.36.tgz#d67c778ca2c385a35ef48986d547dd4693fb6a0a" + integrity sha512-YhPix+0x/MdQrb1Ug1GDKeS5fqylIy+naz800asX8II4jqfTk2KY2KhmmYCwZcky8YWtRQQwWCGdoqeAnip8Uw== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/types" "^3.973.8" + "@aws-sdk/util-arn-parser" "^3.972.3" + "@smithy/core" "^3.23.17" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/protocol-http" "^5.3.14" + "@smithy/signature-v4" "^5.3.14" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + "@smithy/util-config-provider" "^4.2.2" + "@smithy/util-middleware" "^4.2.14" + "@smithy/util-stream" "^4.5.25" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + +"@aws-sdk/middleware-ssec@^3.972.10": + version "3.972.10" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.10.tgz#46b5c030c0116f51110e18042ad3cf863ab5c81c" + integrity sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/middleware-user-agent@^3.972.28": version "3.972.28" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.28.tgz#7f81d96d2fed0334ff601af62d77e14f67fb9d22" @@ -1021,6 +1360,20 @@ "@smithy/util-retry" "^4.3.0" tslib "^2.6.2" +"@aws-sdk/middleware-user-agent@^3.972.37": + version "3.972.37" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.37.tgz#61ee85d964a3f36091a5b36dea630fb30e8e6913" + integrity sha512-N1oNpdiLoVAWYD3WFBnUi3LlfoDA06ZHo4ozyjbsJNLvILzvt//0CnR8N+CZ0NWeYgVB/5V59ivixHCWCx2ALw== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/types" "^3.973.8" + "@aws-sdk/util-endpoints" "^3.996.8" + "@smithy/core" "^3.23.17" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + "@smithy/util-retry" "^4.3.6" + tslib "^2.6.2" + "@aws-sdk/middleware-websocket@^3.972.14": version "3.972.14" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.14.tgz#52ea3b4fddb4320bd23891a4ce103f193b94cadf" @@ -1127,6 +1480,51 @@ "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" +"@aws-sdk/nested-clients@^3.997.5": + version "3.997.5" + resolved "https://registry.yarnpkg.com/@aws-sdk/nested-clients/-/nested-clients-3.997.5.tgz#0b66825b14b1a06b43b71e95354f22cb6b4926df" + integrity sha512-jGFr6DxtcMTmzOkG/a0jCZYv4BBDmeNYVeO+/memSoDkYCJu4Y58xviYmzwJfYyIVSts+X/BVjJm1uGBnwHEMg== + dependencies: + "@aws-crypto/sha256-browser" "5.2.0" + "@aws-crypto/sha256-js" "5.2.0" + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/middleware-host-header" "^3.972.10" + "@aws-sdk/middleware-logger" "^3.972.10" + "@aws-sdk/middleware-recursion-detection" "^3.972.11" + "@aws-sdk/middleware-user-agent" "^3.972.37" + "@aws-sdk/region-config-resolver" "^3.972.13" + "@aws-sdk/signature-v4-multi-region" "^3.996.24" + "@aws-sdk/types" "^3.973.8" + "@aws-sdk/util-endpoints" "^3.996.8" + "@aws-sdk/util-user-agent-browser" "^3.972.10" + "@aws-sdk/util-user-agent-node" "^3.973.23" + "@smithy/config-resolver" "^4.4.17" + "@smithy/core" "^3.23.17" + "@smithy/fetch-http-handler" "^5.3.17" + "@smithy/hash-node" "^4.2.14" + "@smithy/invalid-dependency" "^4.2.14" + "@smithy/middleware-content-length" "^4.2.14" + "@smithy/middleware-endpoint" "^4.4.32" + "@smithy/middleware-retry" "^4.5.7" + "@smithy/middleware-serde" "^4.2.20" + "@smithy/middleware-stack" "^4.2.14" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/node-http-handler" "^4.6.1" + "@smithy/protocol-http" "^5.3.14" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + "@smithy/url-parser" "^4.2.14" + "@smithy/util-base64" "^4.3.2" + "@smithy/util-body-length-browser" "^4.2.2" + "@smithy/util-body-length-node" "^4.2.3" + "@smithy/util-defaults-mode-browser" "^4.3.49" + "@smithy/util-defaults-mode-node" "^4.2.54" + "@smithy/util-endpoints" "^3.4.2" + "@smithy/util-middleware" "^4.2.14" + "@smithy/util-retry" "^4.3.6" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + "@aws-sdk/region-config-resolver@^3.972.10": version "3.972.10" resolved "https://registry.yarnpkg.com/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.10.tgz#cbabd969a2d4fedb652273403e64d98b79d0144c" @@ -1149,6 +1547,43 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/region-config-resolver@^3.972.13": + version "3.972.13" + resolved "https://registry.yarnpkg.com/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.13.tgz#bd32748c2d41b62be838fec76c4b87d4370939c6" + integrity sha512-CvJ2ZIjK/jVD/lbOpowBVElJyC1YxLTIJ13yM0AEo0t2v7swOzGjSA6lJGH+DwZXQhcjUjoYwc8bVYCX5MDr1A== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/config-resolver" "^4.4.17" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + +"@aws-sdk/s3-request-presigner@^3.1021.0": + version "3.1040.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.1040.0.tgz#c81d96024325436dd4ff0f412ca5c22d2674a6e6" + integrity sha512-AmesZGG/B5sDIiWahyY11fOkXSsuHc7LciE88YFURehMVSdEORo2Vzz1d2kBgmJG9oar5Vmmwf9X/w7mqb7ytg== + dependencies: + "@aws-sdk/signature-v4-multi-region" "^3.996.24" + "@aws-sdk/types" "^3.973.8" + "@aws-sdk/util-format-url" "^3.972.10" + "@smithy/middleware-endpoint" "^4.4.32" + "@smithy/protocol-http" "^5.3.14" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + +"@aws-sdk/signature-v4-multi-region@^3.996.24": + version "3.996.24" + resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.24.tgz#efe204595832e418aad404163f55d7ffc7d21dad" + integrity sha512-amP7tLikppN940wbBFISYqiuzVmpzMS9U3mcgtmVLjX4fdWI/SNCvrXv6ZxfVzTT4cT0rPKOLhFah2xLwzREWw== + dependencies: + "@aws-sdk/middleware-sdk-s3" "^3.972.36" + "@aws-sdk/types" "^3.973.8" + "@smithy/protocol-http" "^5.3.14" + "@smithy/signature-v4" "^5.3.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/token-providers@3.1021.0": version "3.1021.0" resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1021.0.tgz#90905a8def49f90e54a73849e25ad4bcc4dbea2a" @@ -1175,6 +1610,19 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/token-providers@3.1039.0": + version "3.1039.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1039.0.tgz#98fac2aa3c22d2ba8b2375d35dcd67f96ea3e990" + integrity sha512-NMSFL2HwkAOoCeLCQiqoOq5pT3vVbSjww2QZTuYgYknVwhhv125PSDzZIcL5EYnlxuPWjEOdauZK+FspkZDVdw== + dependencies: + "@aws-sdk/core" "^3.974.7" + "@aws-sdk/nested-clients" "^3.997.5" + "@aws-sdk/types" "^3.973.8" + "@smithy/property-provider" "^4.2.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/types@^3.222.0", "@aws-sdk/types@^3.973.6": version "3.973.6" resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.973.6.tgz#1964a7c01b5cb18befa445998ad1d02f86c5432d" @@ -1191,6 +1639,21 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/types@^3.973.8": + version "3.973.8" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.973.8.tgz#7352cb74a5f8bae1218eee63e714cf94302911c5" + integrity sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + +"@aws-sdk/util-arn-parser@^3.972.3": + version "3.972.3" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.972.3.tgz#ed989862bbb172ce16d9e1cd5790e5fe367219c2" + integrity sha512-HzSD8PMFrvgi2Kserxuff5VitNq2sgf3w9qxmskKDiDTThWfVteJxuCS9JXiPIPtmCrp+7N9asfIaVhBFORllA== + dependencies: + tslib "^2.6.2" + "@aws-sdk/util-dynamodb@^3.996.2": version "3.996.2" resolved "https://registry.yarnpkg.com/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.2.tgz#9521dfe84c031809f8cf2e32f03c58fd8a4bb84f" @@ -1220,6 +1683,27 @@ "@smithy/util-endpoints" "^3.3.4" tslib "^2.6.2" +"@aws-sdk/util-endpoints@^3.996.8": + version "3.996.8" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-endpoints/-/util-endpoints-3.996.8.tgz#ad5c4f09b93482c0861d49d8a025edc2b0d2f5ec" + integrity sha512-oOZHcRDihk5iEe5V25NVWg45b3qEA8OpHWVdU/XQh8Zj4heVPAJqWvMphQnU7LkufmUo10EpvFPZuQMiFLJK3g== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/types" "^4.14.1" + "@smithy/url-parser" "^4.2.14" + "@smithy/util-endpoints" "^3.4.2" + tslib "^2.6.2" + +"@aws-sdk/util-format-url@^3.972.10": + version "3.972.10" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-format-url/-/util-format-url-3.972.10.tgz#63184b56627b50842cf37cc0e63251944fc234ed" + integrity sha512-DEKiHNJVtNxdyTeQspzY+15Po/kHm6sF0Cs4HV9Q2+lplB63+DrvdeiSoOSdWEWAoO2RcY1veoXVDz2tWxWCgQ== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/querystring-builder" "^4.2.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@aws-sdk/util-format-url@^3.972.8": version "3.972.8" resolved "https://registry.yarnpkg.com/@aws-sdk/util-format-url/-/util-format-url-3.972.8.tgz#803273f72617edb16b4087bcff2e52d740a26250" @@ -1237,6 +1721,16 @@ dependencies: tslib "^2.6.2" +"@aws-sdk/util-user-agent-browser@^3.972.10": + version "3.972.10" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.10.tgz#e29be10389db9db12b2d8246ad247a89038f4c60" + integrity sha512-FAzqXvfEssGdSIz8ejatan0bOdx1qefBWKF/gWmVBXIP1HkS7v/wjjaqrAGGKvyihrXTXW00/2/1nTJtxpXz7g== + dependencies: + "@aws-sdk/types" "^3.973.8" + "@smithy/types" "^4.14.1" + bowser "^2.11.0" + tslib "^2.6.2" + "@aws-sdk/util-user-agent-browser@^3.972.8": version "3.972.8" resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.8.tgz#1044845c97c898cd68fc3f9c773494a6a98cdf80" @@ -1281,6 +1775,18 @@ "@smithy/util-config-provider" "^4.2.2" tslib "^2.6.2" +"@aws-sdk/util-user-agent-node@^3.973.23": + version "3.973.23" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.23.tgz#3e29535e887ad72deaecdfd4667ec710e4086f90" + integrity sha512-gGwq8L2Euw0aNG6Ey4EktiAo3fSCVoDy1CaBIthd+oeaKHPXUrNaApMewQ6La5Hv0lcznOtECZaNvYyc5LXXfA== + dependencies: + "@aws-sdk/middleware-user-agent" "^3.972.37" + "@aws-sdk/types" "^3.973.8" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/types" "^4.14.1" + "@smithy/util-config-provider" "^4.2.2" + tslib "^2.6.2" + "@aws-sdk/xml-builder@^3.972.16": version "3.972.16" resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.972.16.tgz#ea22fe022cf12d12b07f6faf75c4fa214dea00bc" @@ -1299,6 +1805,16 @@ fast-xml-parser "5.5.8" tslib "^2.6.2" +"@aws-sdk/xml-builder@^3.972.22": + version "3.972.22" + resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.972.22.tgz#1e44ca9fd9c3fdc3d9af9540ced024f34cfc60b2" + integrity sha512-PMYKKtJd70IsSG0yHrdAbxBr+ZWBKLvzFZfD3/urxgf6hXVMzuU5M+3MJ5G67RpOmLBu1fAUN65SbWuKUCOlAA== + dependencies: + "@nodable/entities" "2.1.0" + "@smithy/types" "^4.14.1" + fast-xml-parser "5.7.2" + tslib "^2.6.2" + "@aws/durable-execution-sdk-js@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@aws/durable-execution-sdk-js/-/durable-execution-sdk-js-1.1.0.tgz#c32a4a358cc5940414accc13cd9825766299898d" @@ -2457,7 +2973,7 @@ "@emnapi/runtime" "^1.4.3" "@tybys/wasm-util" "^0.10.0" -"@nodable/entities@^2.1.0": +"@nodable/entities@2.1.0", "@nodable/entities@^2.1.0": version "2.1.0" resolved "https://registry.yarnpkg.com/@nodable/entities/-/entities-2.1.0.tgz#f543e5c6446720d4cf9e498a83019dd159973bc2" integrity sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA== @@ -2788,6 +3304,21 @@ dependencies: "@sinonjs/commons" "^3.0.1" +"@smithy/chunked-blob-reader-native@^4.2.3": + version "4.2.3" + resolved "https://registry.yarnpkg.com/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-4.2.3.tgz#9e79a80d8d44798e7ce7a8f968cbbbaf5a40d950" + integrity sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw== + dependencies: + "@smithy/util-base64" "^4.3.2" + tslib "^2.6.2" + +"@smithy/chunked-blob-reader@^5.2.2": + version "5.2.2" + resolved "https://registry.yarnpkg.com/@smithy/chunked-blob-reader/-/chunked-blob-reader-5.2.2.tgz#3af48e37b10e5afed478bb31d2b7bc03c81d196c" + integrity sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw== + dependencies: + tslib "^2.6.2" + "@smithy/config-resolver@^4.4.13": version "4.4.13" resolved "https://registry.yarnpkg.com/@smithy/config-resolver/-/config-resolver-4.4.13.tgz#8bffd41de647ec349b4a74bf02bdd1b32452bacd" @@ -2812,6 +3343,18 @@ "@smithy/util-middleware" "^4.2.13" tslib "^2.6.2" +"@smithy/config-resolver@^4.4.17": + version "4.4.17" + resolved "https://registry.yarnpkg.com/@smithy/config-resolver/-/config-resolver-4.4.17.tgz#5bd7ccf461e126c79072ce84c6b0f3d00b3409bc" + integrity sha512-TzDZcAnhTyAHbXVxWZo7/tEcrIeFq20IBk8So3OLOetWpR8EwY/yEqBMBFaJMeyEiREDq4NfEl+qO3OAUD+vbQ== + dependencies: + "@smithy/node-config-provider" "^4.3.14" + "@smithy/types" "^4.14.1" + "@smithy/util-config-provider" "^4.2.2" + "@smithy/util-endpoints" "^3.4.2" + "@smithy/util-middleware" "^4.2.14" + tslib "^2.6.2" + "@smithy/core@^3.23.13": version "3.23.13" resolved "https://registry.yarnpkg.com/@smithy/core/-/core-3.23.13.tgz#343e0d78b907f463b560d9e50d8ae16456281830" @@ -2844,6 +3387,22 @@ "@smithy/uuid" "^1.1.2" tslib "^2.6.2" +"@smithy/core@^3.23.17": + version "3.23.17" + resolved "https://registry.yarnpkg.com/@smithy/core/-/core-3.23.17.tgz#23d02277c8d6d30a1605afd756696265e48ed67e" + integrity sha512-x7BlLbUFL8NWCGjMF9C+1N5cVCxcPa7g6Tv9B4A2luWx3be3oU8hQ96wIwxe/s7OhIzvoJH73HAUSg5JXVlEtQ== + dependencies: + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + "@smithy/url-parser" "^4.2.14" + "@smithy/util-base64" "^4.3.2" + "@smithy/util-body-length-browser" "^4.2.2" + "@smithy/util-middleware" "^4.2.14" + "@smithy/util-stream" "^4.5.25" + "@smithy/util-utf8" "^4.2.2" + "@smithy/uuid" "^1.1.2" + tslib "^2.6.2" + "@smithy/credential-provider-imds@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.12.tgz#fa2e52116cac7eaf5625e0bfd399a4927b598f66" @@ -2866,6 +3425,17 @@ "@smithy/url-parser" "^4.2.13" tslib "^2.6.2" +"@smithy/credential-provider-imds@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.14.tgz#b5dcc198ee240eaf68069e7449bcec29ce279827" + integrity sha512-Au28zBN48ZAoXdooGUHemuVBrkE+Ie6RPmGNIAJsFqj33Vhb6xAgRifUydZ2aY+M+KaMAETAlKk5NC5h1G7wpg== + dependencies: + "@smithy/node-config-provider" "^4.3.14" + "@smithy/property-provider" "^4.2.14" + "@smithy/types" "^4.14.1" + "@smithy/url-parser" "^4.2.14" + tslib "^2.6.2" + "@smithy/eventstream-codec@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/eventstream-codec/-/eventstream-codec-4.2.12.tgz#8cd62d08709344fb8b35fd17870fdf1435de61a3" @@ -2876,6 +3446,16 @@ "@smithy/util-hex-encoding" "^4.2.2" tslib "^2.6.2" +"@smithy/eventstream-codec@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-codec/-/eventstream-codec-4.2.14.tgz#4963ca27242b80c5b1d11dcd3ea1bee2a3c5f96d" + integrity sha512-erZq0nOIpzfeZdCyzZjdJb4nVSKLUmSkaQUVkRGQTXs30gyUGeKnrYEg+Xe1W5gE3aReS7IgsvANwVPxSzY6Pw== + dependencies: + "@aws-crypto/crc32" "5.2.0" + "@smithy/types" "^4.14.1" + "@smithy/util-hex-encoding" "^4.2.2" + tslib "^2.6.2" + "@smithy/eventstream-serde-browser@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.12.tgz#3ceb8743750edaf5d6e42cd1a2327e048f85ba4e" @@ -2885,6 +3465,15 @@ "@smithy/types" "^4.13.1" tslib "^2.6.2" +"@smithy/eventstream-serde-browser@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.14.tgz#b483667ea358975afb2170cd2618b9aa53a0fb29" + integrity sha512-8IelTCtTctWRbb+0Dcy+C0aICh1qa0qWXqgjcXDmMuCvPJRnv26hiDZoAau2ILOniki65mCPKqOQs/BaWvO4CQ== + dependencies: + "@smithy/eventstream-serde-universal" "^4.2.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/eventstream-serde-config-resolver@^4.3.12": version "4.3.12" resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.12.tgz#a29164bc5480d935ece9dbdca0f79924259e519a" @@ -2893,6 +3482,14 @@ "@smithy/types" "^4.13.1" tslib "^2.6.2" +"@smithy/eventstream-serde-config-resolver@^4.3.14": + version "4.3.14" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.14.tgz#2eb23acad43414b9bc0b43f34ae9afbd5464e484" + integrity sha512-sqHiHpYRYo3FJlaIxD1J8PhbcmJAm7IuM16mVnwSkCToD7g00IBZzKuiLNMGmftULmEUX6/UAz8/NN5uMP8bVA== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/eventstream-serde-node@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.12.tgz#2cc06a1ea1108f679d376aab81e95a6f69877b4a" @@ -2902,6 +3499,15 @@ "@smithy/types" "^4.13.1" tslib "^2.6.2" +"@smithy/eventstream-serde-node@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.14.tgz#402c2a3b0437b7ac9747090a38a60d3642813490" + integrity sha512-Ht/8BuGlKfFTy0H3+8eEu0vdpwGztCnaLLXtpXNdQqiR7Hj4vFScU3T436vRAjATglOIPjJXronY+1WxxNLSiw== + dependencies: + "@smithy/eventstream-serde-universal" "^4.2.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/eventstream-serde-universal@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.12.tgz#a3640d1e7c3e348168360035661db8d21b51e078" @@ -2911,6 +3517,15 @@ "@smithy/types" "^4.13.1" tslib "^2.6.2" +"@smithy/eventstream-serde-universal@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.14.tgz#1e1d29c111e580a93f3c197139c5ca8c976ec205" + integrity sha512-lWyt4T2XQZUZgK3tQ3Wn0w3XBvZsK/vjTuJl6bXbnGZBHH0ZUSONTYiK9TgjTTzU54xQr3DRFwpjmhp0oLm3gg== + dependencies: + "@smithy/eventstream-codec" "^4.2.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/fetch-http-handler@^5.3.15": version "5.3.15" resolved "https://registry.yarnpkg.com/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.15.tgz#acf69a8b3bab0396d2782fc901bad0b957c8c6a2" @@ -2933,6 +3548,27 @@ "@smithy/util-base64" "^4.3.2" tslib "^2.6.2" +"@smithy/fetch-http-handler@^5.3.17": + version "5.3.17" + resolved "https://registry.yarnpkg.com/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.17.tgz#bf13a4b03eb8afe101775fef59a1758f8fb5cd4b" + integrity sha512-bXOvQzaSm6MnmLaWA1elgfQcAtN4UP3vXqV97bHuoOrHQOJiLT3ds6o9eo5bqd0TJfRFpzdGnDQdW3FACiAVdw== + dependencies: + "@smithy/protocol-http" "^5.3.14" + "@smithy/querystring-builder" "^4.2.14" + "@smithy/types" "^4.14.1" + "@smithy/util-base64" "^4.3.2" + tslib "^2.6.2" + +"@smithy/hash-blob-browser@^4.2.15": + version "4.2.15" + resolved "https://registry.yarnpkg.com/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.15.tgz#1323f9717cad352b3e18065b738387bb9684f993" + integrity sha512-0PJ4Al3fg2nM4qKrAIxyNcApgqHAXcBkN8FeizOz69z0rb26uZ6lMESYtxegaTlXB5Hj84JfwMPavMrwDMjucA== + dependencies: + "@smithy/chunked-blob-reader" "^5.2.2" + "@smithy/chunked-blob-reader-native" "^4.2.3" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/hash-node@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/hash-node/-/hash-node-4.2.12.tgz#0ee7f6a1d2958c313ee24b07159dcb9547792441" @@ -2953,6 +3589,25 @@ "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" +"@smithy/hash-node@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/hash-node/-/hash-node-4.2.14.tgz#e3ed33dc614e26fff5f043e097750c6931b48592" + integrity sha512-8ZBDY2DD4wr+GGjTpPtiglEsqr0lUP+KHqgZcWczFf6qeZ/YRjMIOoQWVQlmwu7EtxKTd8YXD8lblmYcpBIA1g== + dependencies: + "@smithy/types" "^4.14.1" + "@smithy/util-buffer-from" "^4.2.2" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + +"@smithy/hash-stream-node@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/hash-stream-node/-/hash-stream-node-4.2.14.tgz#98bc14e79e2be852d04ff6cbfe4b0babe48fb10d" + integrity sha512-tw4GANWkZPb6+BdD4Fgucqzey2+r73Z/GRo9zklsCdwrnxxumUV83ZIaBDdudV4Ylazw3EPTiJZhpX42105ruQ== + dependencies: + "@smithy/types" "^4.14.1" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + "@smithy/invalid-dependency@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/invalid-dependency/-/invalid-dependency-4.2.12.tgz#1a28c13fb33684b91848d4d6ec5104a1c1413e7f" @@ -2969,6 +3624,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/invalid-dependency@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/invalid-dependency/-/invalid-dependency-4.2.14.tgz#a52766f9d4299abcd9d6cd23b5a76f34fc59c7a0" + integrity sha512-c21qJiTSb25xvvOp+H2TNZzPCngrvl5vIPqPB8zQ/DmJF4QWXO19x1dWfMJZ6wZuuWUPPm0gV8C0cU3+ifcWuw== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/is-array-buffer@^2.2.0": version "2.2.0" resolved "https://registry.yarnpkg.com/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz#f84f0d9f9a36601a9ca9381688bd1b726fd39111" @@ -2983,6 +3646,15 @@ dependencies: tslib "^2.6.2" +"@smithy/md5-js@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/md5-js/-/md5-js-4.2.14.tgz#c066572ec84def147af24e55a402c44d0d7dcd7b" + integrity sha512-V2v0vx+h0iUSNG1Alt+GNBMSLGCrl9iVsdd+Ap67HPM9PN479x12V8LkuMoKImNZxn3MXeuyUjls+/7ZACZghA== + dependencies: + "@smithy/types" "^4.14.1" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + "@smithy/middleware-content-length@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/middleware-content-length/-/middleware-content-length-4.2.12.tgz#dec97ea1444b12e734156b764e9953b2b37c70fd" @@ -3001,6 +3673,15 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/middleware-content-length@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/middleware-content-length/-/middleware-content-length-4.2.14.tgz#d8b17f94c4d8f9c3b7992f1db84d3299c83efe78" + integrity sha512-xhHq7fX4/3lv5NHxLUk3OeEvl0xZ+Ek3qIbWaCL4f9JwgDZEclPBElljaZCAItdGPQl/kSM4LPMOpy1MYgprpw== + dependencies: + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/middleware-endpoint@^4.4.28": version "4.4.28" resolved "https://registry.yarnpkg.com/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.28.tgz#201b568f3669bd816f60a6043d914c134d80f46c" @@ -3029,6 +3710,20 @@ "@smithy/util-middleware" "^4.2.13" tslib "^2.6.2" +"@smithy/middleware-endpoint@^4.4.32": + version "4.4.32" + resolved "https://registry.yarnpkg.com/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.32.tgz#4c7dcf06b637b40dfcc53d3b18d1a784a747c530" + integrity sha512-ZZkgyjnJppiZbIm6Qbx92pbXYi1uzenIvGhBSCDlc7NwuAkiqSgS75j1czAD25ZLs2FjMjYy1q7gyRVWG6JA0Q== + dependencies: + "@smithy/core" "^3.23.17" + "@smithy/middleware-serde" "^4.2.20" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + "@smithy/url-parser" "^4.2.14" + "@smithy/util-middleware" "^4.2.14" + tslib "^2.6.2" + "@smithy/middleware-retry@^4.4.46": version "4.4.46" resolved "https://registry.yarnpkg.com/@smithy/middleware-retry/-/middleware-retry-4.4.46.tgz#dbbf0af08c1bd03fe2afa09a6cfb7a9056387ce6" @@ -3060,6 +3755,22 @@ "@smithy/uuid" "^1.1.2" tslib "^2.6.2" +"@smithy/middleware-retry@^4.5.7": + version "4.5.7" + resolved "https://registry.yarnpkg.com/@smithy/middleware-retry/-/middleware-retry-4.5.7.tgz#a2da0c472d631ee408ff566186c99571b3efb70b" + integrity sha512-bRt6ZImqVSeTk39Nm81K20ObIiAZ3WefY7G6+iz/0tZjs4dgRRjvRX2sgsH+zi6iDCRR/aQvQofLKxxz4rPBZg== + dependencies: + "@smithy/core" "^3.23.17" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/protocol-http" "^5.3.14" + "@smithy/service-error-classification" "^4.3.1" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + "@smithy/util-middleware" "^4.2.14" + "@smithy/util-retry" "^4.3.6" + "@smithy/uuid" "^1.1.2" + tslib "^2.6.2" + "@smithy/middleware-serde@^4.2.16": version "4.2.16" resolved "https://registry.yarnpkg.com/@smithy/middleware-serde/-/middleware-serde-4.2.16.tgz#7f259e1e4e43332ad29b53cf3b4d9f14fde690ce" @@ -3080,6 +3791,16 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/middleware-serde@^4.2.20": + version "4.2.20" + resolved "https://registry.yarnpkg.com/@smithy/middleware-serde/-/middleware-serde-4.2.20.tgz#76862c8f9b39b08501539440a2e6bca7a77de508" + integrity sha512-Lx9JMO9vArPtiChE3wbEZ5akMIDQpWQtlu90lhACQmNOXcGXRbaDywMHDzuDZ2OkZzP+9wQfZi3YJT9F67zTQQ== + dependencies: + "@smithy/core" "^3.23.17" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/middleware-stack@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/middleware-stack/-/middleware-stack-4.2.12.tgz#96b43b2fab0d4a6723f813f76b72418b0fdb6ba0" @@ -3096,6 +3817,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/middleware-stack@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/middleware-stack/-/middleware-stack-4.2.14.tgz#23a4cf643ccdbde52c8780fe5cc080611efef1c7" + integrity sha512-2dvkUKLuFdKsCRmOE4Mn63co0Djtsm+JMh0bYZQupN1pJwMeE8FmQmRLLzzEMN0dnNi7CDCYYH8F0EVwWiPBeA== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/node-config-provider@^4.3.12": version "4.3.12" resolved "https://registry.yarnpkg.com/@smithy/node-config-provider/-/node-config-provider-4.3.12.tgz#bb722da6e2a130ae585754fa7bc8d909f9f5d702" @@ -3116,6 +3845,16 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/node-config-provider@^4.3.14": + version "4.3.14" + resolved "https://registry.yarnpkg.com/@smithy/node-config-provider/-/node-config-provider-4.3.14.tgz#8ca13b86b6123cbb0425d669bd847fcd333ca4bd" + integrity sha512-S+gFjyo/weSVL0P1b9Ts8C/CwIfNCgUPikk3sl6QVsfE/uUuO+QsF+NsE/JkpvWqqyz1wg7HFdiaZuj5CoBMRg== + dependencies: + "@smithy/property-provider" "^4.2.14" + "@smithy/shared-ini-file-loader" "^4.4.9" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/node-http-handler@^4.5.1": version "4.5.1" resolved "https://registry.yarnpkg.com/@smithy/node-http-handler/-/node-http-handler-4.5.1.tgz#9f05b4478ccfc6db82af37579a36fa48ee8f6067" @@ -3136,6 +3875,16 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/node-http-handler@^4.6.1": + version "4.6.1" + resolved "https://registry.yarnpkg.com/@smithy/node-http-handler/-/node-http-handler-4.6.1.tgz#cb25b9445e46294a6f0dfb1566dbf2a1a19510af" + integrity sha512-iB+orM4x3xrr57X3YaXazfKnntl0LHlZB1kcXSGzMV1Tt0+YwEjGlbjk/44qEGtBzXAz6yFDzkYTKSV6Pj2HUg== + dependencies: + "@smithy/protocol-http" "^5.3.14" + "@smithy/querystring-builder" "^4.2.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/property-provider@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/property-provider/-/property-provider-4.2.12.tgz#e9f8e5ce125413973b16e39c87cf4acd41324e21" @@ -3152,6 +3901,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/property-provider@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/property-provider/-/property-provider-4.2.14.tgz#8072418672d8c29d3f9ef35e452437ba2c59100a" + integrity sha512-WuM31CgfsnQ/10i7NYr0PyxqknD72Y5uMfUMVSniPjbEPceiTErb4eIqJQ+pdxNEAUEWrewrGjIRjVbVHsxZiQ== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/protocol-http@^5.3.12": version "5.3.12" resolved "https://registry.yarnpkg.com/@smithy/protocol-http/-/protocol-http-5.3.12.tgz#c913053e7dfbac6cdd7f374f0b4f5aa7c518d0e1" @@ -3168,6 +3925,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/protocol-http@^5.3.14": + version "5.3.14" + resolved "https://registry.yarnpkg.com/@smithy/protocol-http/-/protocol-http-5.3.14.tgz#ed1e65cdb0fffb7fd00dce997c04baa236f180cc" + integrity sha512-dN5F8kHx8RNU0r+pCwNmFZyz6ChjMkzShy/zup6MtkRmmix4vZzJdW+di7x//b1LiynIev88FM18ie+wwPcQtQ== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/querystring-builder@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/querystring-builder/-/querystring-builder-4.2.12.tgz#20a0266b151a4b58409f901e1463257a72835c16" @@ -3186,6 +3951,15 @@ "@smithy/util-uri-escape" "^4.2.2" tslib "^2.6.2" +"@smithy/querystring-builder@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/querystring-builder/-/querystring-builder-4.2.14.tgz#102429e0fb004108babf219edfcf6f111e66d782" + integrity sha512-XYA5Z0IqTeF+5XDdh4BBmSA0HvbgVZIyv4cmOoUheDNR57K1HgBp9ukUMx3Cr3XpDHHpLBnexPE3LAtDsZkj2A== + dependencies: + "@smithy/types" "^4.14.1" + "@smithy/util-uri-escape" "^4.2.2" + tslib "^2.6.2" + "@smithy/querystring-parser@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/querystring-parser/-/querystring-parser-4.2.12.tgz#918cb609b2d606ab81f2727bfde0265d2ebb2758" @@ -3202,6 +3976,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/querystring-parser@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/querystring-parser/-/querystring-parser-4.2.14.tgz#c479ba1f346656b9f8ce46d9a91c229e4e50420f" + integrity sha512-hr+YyqBD23GVvRxGGrcc/oOeNlK3PzT5Fu4dzrDXxzS1LpFiuL2PQQqKPs87M79aW7ziMs+nvB3qdw77SqE7Lw== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/service-error-classification@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/service-error-classification/-/service-error-classification-4.2.12.tgz#795e9484207acf63817a9e9cf67e90b42e720840" @@ -3216,6 +3998,13 @@ dependencies: "@smithy/types" "^4.14.0" +"@smithy/service-error-classification@^4.3.1": + version "4.3.1" + resolved "https://registry.yarnpkg.com/@smithy/service-error-classification/-/service-error-classification-4.3.1.tgz#5303d4fc3c3eea0f79c3b88cb4436498a31e9f12" + integrity sha512-aUQuDGh760ts/8MU+APjIZhlLPKhIIfqyzZaJikLEIMrdxFvxuLYD0WxWzaYWpmLbQlXDe9p7EWM3HsBe0K6Gw== + dependencies: + "@smithy/types" "^4.14.1" + "@smithy/shared-ini-file-loader@^4.4.7": version "4.4.7" resolved "https://registry.yarnpkg.com/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.7.tgz#18cc5a21f871509fafbe535a7bf44bde5a500727" @@ -3232,6 +4021,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/shared-ini-file-loader@^4.4.9": + version "4.4.9" + resolved "https://registry.yarnpkg.com/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.9.tgz#fb3719b401d101a65a682380b40efd3a116162f0" + integrity sha512-495/V2I15SHgedSJoDPD23JuSfKAp726ZI1V0wtjB07Wh7q/0tri/0e0DLefZCHgxZonrGKt/OCTpAtP1wE1kQ== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/signature-v4@^5.3.12": version "5.3.12" resolved "https://registry.yarnpkg.com/@smithy/signature-v4/-/signature-v4-5.3.12.tgz#b61ce40a94bdd91dfdd8f5f2136631c8eb67f253" @@ -3260,6 +4057,33 @@ "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" +"@smithy/signature-v4@^5.3.14": + version "5.3.14" + resolved "https://registry.yarnpkg.com/@smithy/signature-v4/-/signature-v4-5.3.14.tgz#2b28c7d190301a67a520227a2343d1e7bb1c6d22" + integrity sha512-1D9Y/nmlVjCeSivCbhZ7hgEpmHyY1h0GvpSZt3l0xcD9JjmjVC1CHOozS6+Gh+/ldMH8JuJ6cujObQqfayAVFA== + dependencies: + "@smithy/is-array-buffer" "^4.2.2" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + "@smithy/util-hex-encoding" "^4.2.2" + "@smithy/util-middleware" "^4.2.14" + "@smithy/util-uri-escape" "^4.2.2" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + +"@smithy/smithy-client@^4.12.13": + version "4.12.13" + resolved "https://registry.yarnpkg.com/@smithy/smithy-client/-/smithy-client-4.12.13.tgz#dec184a1d2d5027370ae1582bddbdbc068c97da5" + integrity sha512-y/Pcj1V9+qG98gyu1gvftHB7rDpdh+7kIBIggs55yGm3JdtBV8GT8IFF3a1qxZ79QnaJHX9GXzvBG6tAd+czJA== + dependencies: + "@smithy/core" "^3.23.17" + "@smithy/middleware-endpoint" "^4.4.32" + "@smithy/middleware-stack" "^4.2.14" + "@smithy/protocol-http" "^5.3.14" + "@smithy/types" "^4.14.1" + "@smithy/util-stream" "^4.5.25" + tslib "^2.6.2" + "@smithy/smithy-client@^4.12.8": version "4.12.8" resolved "https://registry.yarnpkg.com/@smithy/smithy-client/-/smithy-client-4.12.8.tgz#b2982fe8b72e44621c139045d991555c07df0e1a" @@ -3300,6 +4124,13 @@ dependencies: tslib "^2.6.2" +"@smithy/types@^4.14.1": + version "4.14.1" + resolved "https://registry.yarnpkg.com/@smithy/types/-/types-4.14.1.tgz#aba92b4cdb406f2a2b062e82f1e3728d809a7c23" + integrity sha512-59b5HtSVrVR/eYNei3BUj3DCPKD/G7EtDDe7OEJE7i7FtQFugYo6MxbotS8mVJkLNVf8gYaAlEBwwtJ9HzhWSg== + dependencies: + tslib "^2.6.2" + "@smithy/url-parser@^4.2.12": version "4.2.12" resolved "https://registry.yarnpkg.com/@smithy/url-parser/-/url-parser-4.2.12.tgz#e940557bf0b8e9a25538a421970f64bd827f456f" @@ -3318,6 +4149,15 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/url-parser@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/url-parser/-/url-parser-4.2.14.tgz#349a442a62eb5907533f204b73a010618198b073" + integrity sha512-p06BiBigJ8bTA3MgnOfCtDUWnAMY0YfedO/GRpmc7p+wg3KW8vbXy1xwSu5ASy0wV7rRYtlfZOIKH4XqfhjSQQ== + dependencies: + "@smithy/querystring-parser" "^4.2.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/util-base64@^4.3.2": version "4.3.2" resolved "https://registry.yarnpkg.com/@smithy/util-base64/-/util-base64-4.3.2.tgz#be02bcb29a87be744356467ea25ffa413e695cea" @@ -3384,6 +4224,16 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/util-defaults-mode-browser@^4.3.49": + version "4.3.49" + resolved "https://registry.yarnpkg.com/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.49.tgz#926ce84bf65e56307f25cce7a13b427d33442939" + integrity sha512-a5bNrdiONYB/qE2BuKegvUMd/+ZDwdg4vsNuuSzYE8qs2EYAdK9CynL+Rzn29PbPiUqoz/cbpRbcLzD5lEevHw== + dependencies: + "@smithy/property-provider" "^4.2.14" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/util-defaults-mode-node@^4.2.48": version "4.2.48" resolved "https://registry.yarnpkg.com/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.48.tgz#8ee63e2ea706bd111104e8f3796d858cc186625f" @@ -3410,6 +4260,19 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/util-defaults-mode-node@^4.2.54": + version "4.2.54" + resolved "https://registry.yarnpkg.com/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.54.tgz#32c4ea9f8a8c74ef9fe0ca5e3d6a10df0327f87e" + integrity sha512-g1cvrJvOnzeJgEdf7AE4luI7gp6L8weE0y9a9wQUSGtjb8QRHDbCJYuE4Sy0SD9N8RrnNPFsPltAz/OSoBR9Zw== + dependencies: + "@smithy/config-resolver" "^4.4.17" + "@smithy/credential-provider-imds" "^4.2.14" + "@smithy/node-config-provider" "^4.3.14" + "@smithy/property-provider" "^4.2.14" + "@smithy/smithy-client" "^4.12.13" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/util-endpoints@^3.3.3": version "3.3.3" resolved "https://registry.yarnpkg.com/@smithy/util-endpoints/-/util-endpoints-3.3.3.tgz#0119f15bcac30b3b9af1d3cc0a8477e7199d0185" @@ -3428,6 +4291,15 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/util-endpoints@^3.4.2": + version "3.4.2" + resolved "https://registry.yarnpkg.com/@smithy/util-endpoints/-/util-endpoints-3.4.2.tgz#ee59c42d039a642b6c6eb2d38e0ae3db6fc48e97" + integrity sha512-a55Tr+3OKld4TTtnT+RhKOQHyPxm3j/xL4OR83WBUhLJaKDS9dnJ7arRMOp3t31dcLhApwG9bgvrRXBHlLdIkg== + dependencies: + "@smithy/node-config-provider" "^4.3.14" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/util-hex-encoding@^4.2.2": version "4.2.2" resolved "https://registry.yarnpkg.com/@smithy/util-hex-encoding/-/util-hex-encoding-4.2.2.tgz#4abf3335dd1eb884041d8589ca7628d81a6fd1d3" @@ -3451,6 +4323,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/util-middleware@^4.2.14": + version "4.2.14" + resolved "https://registry.yarnpkg.com/@smithy/util-middleware/-/util-middleware-4.2.14.tgz#9985dd82b4036db2d03835229b9b0c63d2bb85fa" + integrity sha512-1Su2vj9RYNDEv/V+2E+jXkkwGsgR7dc4sfHn9Z7ruzQHJIEni9zzw5CauvRXlFJfmgcqYP8fWa0dkh2Q2YaQyw== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/util-retry@^4.2.13": version "4.2.13" resolved "https://registry.yarnpkg.com/@smithy/util-retry/-/util-retry-4.2.13.tgz#ad816d6ddf197095d188e9ef56664fbd392a39c9" @@ -3469,6 +4349,15 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/util-retry@^4.3.6": + version "4.3.6" + resolved "https://registry.yarnpkg.com/@smithy/util-retry/-/util-retry-4.3.6.tgz#8d242d7e736593ca3f1c0f056279909b881d6e2a" + integrity sha512-p6/FO1n2KxMeQyna067i0uJ6TSbb165ZhnRtCpWh4Foxqbfc6oW+XITaL8QkFJj3KFnDe2URt4gOhgU06EP9ew== + dependencies: + "@smithy/service-error-classification" "^4.3.1" + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/util-stream@^4.5.21": version "4.5.21" resolved "https://registry.yarnpkg.com/@smithy/util-stream/-/util-stream-4.5.21.tgz#a9ea13d0299d030c72ab4b4e394db111cd581629" @@ -3497,6 +4386,20 @@ "@smithy/util-utf8" "^4.2.2" tslib "^2.6.2" +"@smithy/util-stream@^4.5.25": + version "4.5.25" + resolved "https://registry.yarnpkg.com/@smithy/util-stream/-/util-stream-4.5.25.tgz#f48385a284151c7e099395af4e5fb0978fffe4ff" + integrity sha512-/PFpG4k8Ze8Ei+mMKj3oiPICYekthuzePZMgZbCqMiXIHHf4n2aZ4Ps0aSRShycFTGuj/J6XldmC0x0DwednIA== + dependencies: + "@smithy/fetch-http-handler" "^5.3.17" + "@smithy/node-http-handler" "^4.6.1" + "@smithy/types" "^4.14.1" + "@smithy/util-base64" "^4.3.2" + "@smithy/util-buffer-from" "^4.2.2" + "@smithy/util-hex-encoding" "^4.2.2" + "@smithy/util-utf8" "^4.2.2" + tslib "^2.6.2" + "@smithy/util-uri-escape@^4.2.2": version "4.2.2" resolved "https://registry.yarnpkg.com/@smithy/util-uri-escape/-/util-uri-escape-4.2.2.tgz#48e40206e7fe9daefc8d44bb43a1ab17e76abf4a" @@ -3536,6 +4439,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@smithy/util-waiter@^4.3.0": + version "4.3.0" + resolved "https://registry.yarnpkg.com/@smithy/util-waiter/-/util-waiter-4.3.0.tgz#6122ce27939edb5550d1d6c7c8d506323f3a17f7" + integrity sha512-JyjYmLAfS+pdxF92o4yLgEoy0zhayKTw73FU1aofLWwLcJw7iSqIY2exGmMTrl/lmZugP5p/zxdFSippJDfKWA== + dependencies: + "@smithy/types" "^4.14.1" + tslib "^2.6.2" + "@smithy/uuid@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@smithy/uuid/-/uuid-1.1.2.tgz#b6e97c7158615e4a3c775e809c00d8c269b5a12e" @@ -5636,7 +6547,7 @@ fast-xml-builder@^1.1.5: dependencies: path-expression-matcher "^1.1.3" -fast-xml-parser@5.5.8, fast-xml-parser@^5.7.0: +fast-xml-parser@5.5.8, fast-xml-parser@5.7.2, fast-xml-parser@^5.7.0: version "5.7.2" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-5.7.2.tgz#fecd0b054c6c132fc03dab994a413da781e0eb9f" integrity sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w== @@ -9192,16 +10103,7 @@ string-length@^4.0.2: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -9259,14 +10161,7 @@ stringify-entities@^4.0.0: character-entities-html4 "^2.0.0" character-entities-legacy "^3.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -10073,16 +10968,7 @@ wordwrap@^1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==