fix(security): sanitize the remaining log-injection sinks, guard the nightly allowlist

.claude/skills/cutting-a-release/SKILL.md

@@ -225,12 +225,27 @@ required" — that is the note that sends someone digging through `git log`:
 > for tool rows written before it existed. Idempotent; dry-run by default.
 >
 > ```bash
-> AWS_PROFILE=<env> python backend/scripts/backfill_tool_catalog_index.py \
+> AWS_PROFILE=<env> backend/.venv/bin/python backend/scripts/backfill_tool_catalog_index.py \
 >     --table <prefix>-app-roles --region us-west-2 --apply
 > ```
 >
-> Verify `skipped=0 failed=0` and that the index item count matches the tool
-> count before considering the deploy complete.
+> Verify `skipped=0 failed=0`, then confirm with a live Query on the index.
+
+Two details that cost real time on the v1.21.0 prod run, both worth carrying
+into whatever backfill note you write:
+
+- **Give the backend venv's interpreter, not a bare `python`.** These scripts
+  need `boto3`, which is not in the system Python — a bare `python` fails with
+  `ModuleNotFoundError: No module named 'boto3'` before it does anything. All
+  six `backfill_*.py` scripts are runnable as
+  `backend/.venv/bin/python backend/scripts/<script>.py …` from the repo root
+  (`uv run --project backend python …` works too).
+- **Never tell an operator to verify with `describe-table` `ItemCount`.**
+  DynamoDB refreshes table and index item counts roughly every **six hours**,
+  so a correct backfill still reports `0` immediately afterwards and reads as a
+  failure. Name a live `query … --select COUNT` instead, and pair it with a
+  scan for rows the backfill missed — a *partial* backfill is silent, and is
+  exactly what a sparse-index read path's fallback does not cover.
 
 ### Ordering, when the release also switches a read onto the backfilled data
 

RELEASE_NOTES.md

@@ -363,18 +363,43 @@ aws dynamodb describe-table --table-name <prefix>-app-roles \
 Populates `GSI5PK`/`GSI5SK` on tool rows written before the keys existed. **Dry-run by default; idempotent; guarded by `attribute_not_exists(GSI5PK)` and `attribute_exists(SK)`, so it never resurrects a deleted row and never overwrites one the writer has since stamped.** It touches only tool metadata rows — `CAPABILITIES` snapshots, skills, roles and user preferences are left alone.
 
 ```bash
-AWS_PROFILE=<env> python backend/scripts/backfill_tool_catalog_index.py \
+AWS_PROFILE=<env> backend/.venv/bin/python backend/scripts/backfill_tool_catalog_index.py \
     --table <prefix>-app-roles --region us-west-2
 ```
 
+> Use the backend venv's interpreter, not the system `python` — the script needs `boto3`
+> and a bare `python` fails with `ModuleNotFoundError: No module named 'boto3'`.
+> `uv run --project backend python …` works equally well.
+
 Then, once the dry run looks right:
 
 ```bash
-AWS_PROFILE=<env> python backend/scripts/backfill_tool_catalog_index.py \
+AWS_PROFILE=<env> backend/.venv/bin/python backend/scripts/backfill_tool_catalog_index.py \
     --table <prefix>-app-roles --region us-west-2 --apply
 ```
 
-**Verify `skipped=0 failed=0` and that the index item count matches the tool count before considering the deploy complete.** Run against dev first, then prod. If the backfill is skipped, the Query returns zero rows, the zero-result fallback re-reads via Scan and logs an ERROR naming this script — so the catalog still serves, but at Scan cost and with a standing error in the logs.
+**Verify `skipped=0 failed=0`, then confirm the index is populated with a live Query.** Run against dev first, then prod.
+
+```bash
+aws dynamodb query --table-name <prefix>-app-roles --index-name EntityTypeIndex \
+  --key-condition-expression "GSI5PK = :pk" \
+  --expression-attribute-values '{":pk":{"S":"ENTITY#TOOL"}}' --select COUNT
+```
+
+> ⚠️ **Do not verify with `describe-table` `ItemCount`.** DynamoDB refreshes table and index
+> item counts roughly every **six hours**, so immediately after a successful backfill it still
+> reads `0` and looks like a failure. A Query is the only live check.
+
+Pair it with a scan for rows the backfill missed — a **partial** backfill is the one case the
+read path's zero-result fallback deliberately does not cover, so it is silent:
+
+```bash
+aws dynamodb scan --table-name <prefix>-app-roles \
+  --filter-expression "begins_with(PK, :p) AND SK = :s AND attribute_not_exists(GSI5PK)" \
+  --expression-attribute-values '{":p":{"S":"TOOL#"},":s":{"S":"METADATA"}}' --select COUNT
+```
+
+That must return `Count: 0`. If the backfill is skipped, the Query returns zero rows, the zero-result fallback re-reads via Scan and logs an ERROR naming this script — so the catalog still serves, but at Scan cost and with a standing error in the logs.
 
 ### 4. Decide on the Conversation Mode regression
 

backend/scripts/backfill_artifact_tool_merge.py

@@ -48,10 +48,10 @@
 
 Run against dev first, then prod::
 
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_artifact_tool_merge.py \\
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_artifact_tool_merge.py \\
         --table dev-boisestateai-v2-app-roles \\
         --assistants-table dev-boisestateai-v2-assistants          # dry-run
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_artifact_tool_merge.py \\
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_artifact_tool_merge.py \\
         --table dev-boisestateai-v2-app-roles \\
         --assistants-table dev-boisestateai-v2-assistants --apply
 """

backend/scripts/backfill_artifact_user_index_keys.py

@@ -52,9 +52,9 @@
 
 Run against dev first, then prod::
 
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_artifact_user_index_keys.py \\
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_artifact_user_index_keys.py \\
         --table dev-boisestateai-v2-user-artifacts --region us-west-2
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_artifact_user_index_keys.py \\
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_artifact_user_index_keys.py \\
         --table dev-boisestateai-v2-user-artifacts --region us-west-2 --apply
 """
 

backend/scripts/backfill_false_interrupted_markers.py

@@ -42,10 +42,10 @@
 * Idempotent: a second run finds nothing left to do.
 
 USAGE
-    python scripts/backfill_false_interrupted_markers.py \
+    backend/.venv/bin/python backend/scripts/backfill_false_interrupted_markers.py \
         --table boisestateai-v2-sessions-metadata --profile prod-ai
     # …review the dry-run summary, then:
-    python scripts/backfill_false_interrupted_markers.py \
+    backend/.venv/bin/python backend/scripts/backfill_false_interrupted_markers.py \
         --table boisestateai-v2-sessions-metadata --profile prod-ai --apply
 """
 

backend/scripts/backfill_session_static_sk.py

@@ -21,9 +21,9 @@
 
 Run against dev first, then prod::
 
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_session_static_sk.py \
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_session_static_sk.py \
         --table dev-boisestateai-v2-sessions-metadata            # dry-run
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_session_static_sk.py \
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_session_static_sk.py \
         --table dev-boisestateai-v2-sessions-metadata --apply --set-marker
 """
 

backend/scripts/backfill_skill_bundles.py

@@ -31,10 +31,10 @@
 
 Run against dev first, then prod::
 
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_skill_bundles.py \\
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_skill_bundles.py \\
         --table dev-boisestateai-v2-app-roles \\
         --bucket dev-boisestateai-v2-skill-resources            # dry-run
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_skill_bundles.py \\
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_skill_bundles.py \\
         --table dev-boisestateai-v2-app-roles \\
         --bucket dev-boisestateai-v2-skill-resources --apply
 """

backend/scripts/backfill_tool_catalog_index.py

@@ -49,11 +49,16 @@
 * **Invents nothing.** Both key values derive from the row's own PK, so a
   stamped row is byte-identical to what the writer would have written.
 
-Run against dev first, then prod::
-
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_tool_catalog_index.py \\
+Run from the repo root, against dev first, then prod. The interpreter is the
+backend venv's, not the system ``python`` — this script needs ``boto3``, and a
+bare ``python`` fails with ``ModuleNotFoundError: No module named 'boto3'``
+(hit for real during the v1.21.0 prod run). Nothing here imports ``apis.*``, so
+the venv's interpreter is the only requirement; ``uv run --project backend
+python …`` works too::
+
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_tool_catalog_index.py \\
         --table dev-boisestateai-v2-app-roles --region us-west-2
-    AWS_PROFILE=dev-ai python backend/scripts/backfill_tool_catalog_index.py \\
+    AWS_PROFILE=dev-ai backend/.venv/bin/python backend/scripts/backfill_tool_catalog_index.py \\
         --table dev-boisestateai-v2-app-roles --region us-west-2 --apply
 """
 

backend/src/apis/app_api/admin/roles/agent_pins.py

@@ -124,7 +124,7 @@ async def put_role_agent_pins(
             f"Admin {scrub_log(admin.email)} set {len(request.pins)} default pin(s) on role {scrub_log(role_id)}",
             extra={
                 "event": "role_agent_pins_updated",
-                "role_id": role_id,
+                "role_id": scrub_log(role_id),
                 "pin_count": len(request.pins),
                 "admin_user_id": admin.user_id,
             },

backend/src/apis/app_api/admin/services/model_icons.py

@@ -24,6 +24,7 @@
     model_icon_version,
     normalize_icon,
 )
+from apis.shared.security.log_sanitize import scrub_log
 
 logger = logging.getLogger(__name__)
 
@@ -68,15 +69,17 @@ async def upload_model_icon(model_id: str, content: bytes) -> Tuple[Optional[str
     try:
         key = store.put(model_id=model_id, content=data, ext=ext, content_type=content_type)
     except IconStoreError as e:
-        logger.error(f"Icon storage unavailable for model {model_id}: {e}")
+        logger.error(
+            f"Icon storage unavailable for model {scrub_log(model_id)}: {scrub_log(e)}"
+        )
         raise ModelIconError("Icon storage is unavailable.", status_code=503) from e
 
     previous = model.icon_key
     await write_model_icon_key(model_id, key)
     if previous and previous != key:
         store.delete(previous)
 
-    logger.info(f"🖼️ model-icons: uploaded icon for model {model_id}")
+    logger.info(f"🖼️ model-icons: uploaded icon for model {scrub_log(model_id)}")
     return key, model_icon_url(model_id, key)
 
 
@@ -90,7 +93,7 @@ async def remove_model_icon(model_id: str) -> Tuple[Optional[str], Optional[str]
     if previous:
         get_model_icon_store().delete(previous)
 
-    logger.info(f"🖼️ model-icons: removed icon for model {model_id}")
+    logger.info(f"🖼️ model-icons: removed icon for model {scrub_log(model_id)}")
     return None, None
 
 

backend/src/apis/app_api/fine_tuning/routes.py

@@ -42,6 +42,7 @@
 from .inference_repository import InferenceRepository, get_inference_repository
 from .script_packaging_service import ScriptPackagingService, get_script_packaging_service
 from .dependencies import require_fine_tuning_access
+from apis.shared.security.log_sanitize import scrub_log
 
 logger = logging.getLogger(__name__)
 
@@ -285,7 +286,9 @@ async def preflight_huggingface_model(hf_id: str, spec) -> None:
         async with httpx.AsyncClient(timeout=10.0) as client:
             response = await client.get(f"https://huggingface.co/api/models/{hf_id}")
     except httpx.HTTPError as e:
-        logger.warning(f"HuggingFace pre-flight unavailable for {hf_id}: {e}")
+        logger.warning(
+            f"HuggingFace pre-flight unavailable for {scrub_log(hf_id)}: {scrub_log(e)}"
+        )
         return
 
     if response.status_code == 404:
@@ -298,7 +301,7 @@ async def preflight_huggingface_model(hf_id: str, spec) -> None:
         )
     if response.status_code >= 400:
         logger.warning(
-            f"HuggingFace pre-flight returned {response.status_code} for {hf_id}"
+            f"HuggingFace pre-flight returned {response.status_code} for {scrub_log(hf_id)}"
         )
         return
 
@@ -425,7 +428,7 @@ def _budgeted_runtime(
     if effective < requested_seconds:
         logger.info(
             f"Clamped max runtime from {requested_seconds}s to {effective}s "
-            f"to fit ${remaining_usd:.2f} remaining on {instance_type}"
+            f"to fit ${remaining_usd:.2f} remaining on {scrub_log(instance_type)}"
         )
     return effective
 

backend/src/apis/app_api/sessions/routes.py

@@ -724,7 +724,7 @@ async def signal_turn_interrupted_endpoint(
             if not await is_session_lease_held(session_id, user_id):
                 logger.info(
                     "Ignoring navigated_away for session %s — no turn in flight",
-                    session_id,
+                    scrub_log(session_id),
                 )
                 return Response(status_code=204)
 

backend/src/apis/app_api/skills/routes.py

@@ -62,6 +62,7 @@
     UserSkillNotFoundError,
     get_user_skill_service,
 )
+from apis.shared.security.log_sanitize import scrub_log
 
 logger = logging.getLogger(__name__)
 
@@ -548,7 +549,9 @@ async def get_accessible_skill(
     user: User = Depends(get_current_user_from_session),
 ) -> SkillDetailResponse:
     """Read one skill the current user can reach, catalog or self-authored."""
-    logger.info(f"User {user.name} reading skill '{skill_id}'")
+    logger.info(
+        f"User {scrub_log(user.name)} reading skill '{scrub_log(skill_id)}'"
+    )
 
     skill = await _require_accessible_skill(skill_id, user)
 

backend/src/apis/app_api/skills/service.py

@@ -144,7 +144,7 @@ async def get_catalog_skill(self, skill_id: str) -> Optional[SkillDefinition]:
                 "as not found",
                 extra={
                     "event": "admin_skill_cross_tier_denied",
-                    "skill_id": skill_id,
+                    "skill_id": scrub_log(skill_id),
                 },
             )
             return None
@@ -237,7 +237,7 @@ async def update_skill(
                 f"Admin {scrub_log(admin.email)} updated skill: {scrub_log(skill_id)}",
                 extra={
                     "event": "skill_updated",
-                    "skill_id": skill_id,
+                    "skill_id": scrub_log(skill_id),
                     "admin_user_id": admin.user_id,
                     "admin_email": admin.email,
                     "changes": list(updates.keys()),
@@ -279,7 +279,7 @@ async def delete_skill(
                 f"Admin {scrub_log(admin.email)} deleted skill: {scrub_log(skill_id)}",
                 extra={
                     "event": "skill_deleted",
-                    "skill_id": skill_id,
+                    "skill_id": scrub_log(skill_id),
                     "admin_user_id": admin.user_id,
                     "admin_email": admin.email,
                     "soft_delete": soft,
@@ -401,10 +401,10 @@ async def add_resource(
             f"Admin {scrub_log(admin.email)} uploaded reference file to skill {scrub_log(skill_id)}",
             extra={
                 "event": "skill_resource_added",
-                "skill_id": skill_id,
+                "skill_id": scrub_log(skill_id),
                 # NB: not "filename" — that key is reserved on LogRecord and
                 # raises KeyError when the record is actually emitted.
-                "resource_filename": filename,
+                "resource_filename": scrub_log(filename),
                 "size": len(content),
                 "admin_user_id": admin.user_id,
             },
@@ -460,9 +460,9 @@ async def delete_resource(
             f"Admin {scrub_log(admin.email)} deleted reference file from skill {scrub_log(skill_id)}",
             extra={
                 "event": "skill_resource_deleted",
-                "skill_id": skill_id,
+                "skill_id": scrub_log(skill_id),
                 # NB: not "filename" — reserved on LogRecord (see add_resource).
-                "resource_filename": filename,
+                "resource_filename": scrub_log(filename),
                 "admin_user_id": admin.user_id,
             },
         )
@@ -632,7 +632,7 @@ async def set_roles_for_skill(
             f"Admin {scrub_log(admin.email)} set roles for skill {scrub_log(skill_id)}",
             extra={
                 "event": "skill_roles_updated",
-                "skill_id": skill_id,
+                "skill_id": scrub_log(skill_id),
                 "admin_user_id": admin.user_id,
                 "roles_added": list(to_add),
                 "roles_removed": list(to_remove),

backend/src/apis/app_api/skills/user_service.py

@@ -183,7 +183,7 @@ async def create_my_skill(
             f"User {scrub_log(user.email)} created skill: {scrub_log(skill_id)}",
             extra={
                 "event": "user_skill_created",
-                "skill_id": skill_id,
+                "skill_id": scrub_log(skill_id),
                 "owner_user_id": user.user_id,
             },
         )
@@ -213,7 +213,7 @@ async def update_my_skill(
             f"User {scrub_log(user.email)} updated skill: {scrub_log(skill_id)}",
             extra={
                 "event": "user_skill_updated",
-                "skill_id": skill_id,
+                "skill_id": scrub_log(skill_id),
                 "owner_user_id": user.user_id,
                 "changes": list(updates.keys()),
             },
@@ -242,7 +242,7 @@ async def delete_my_skill(self, skill_id: str, user: User) -> None:
             f"User {scrub_log(user.email)} deleted skill: {scrub_log(skill_id)}",
             extra={
                 "event": "user_skill_deleted",
-                "skill_id": skill_id,
+                "skill_id": scrub_log(skill_id),
                 "owner_user_id": user.user_id,
             },
         )

backend/src/apis/app_api/tools/discovery.py

@@ -38,6 +38,7 @@
     ToolDefinition,
     _clip,
 )
+from apis.shared.security.log_sanitize import scrub_log
 
 logger = logging.getLogger(__name__)
 
@@ -367,7 +368,10 @@ def _get() -> ResolvedPrompt:
         return await asyncio.to_thread(_get)
     except Exception as exc:  # noqa: BLE001 - surfaced as a 502 by the route
         logger.warning(
-            "prompts/get failed for %s/%s: %s", tool.tool_id, prompt_name, exc
+            "prompts/get failed for %s/%s: %s",
+            scrub_log(tool.tool_id),
+            scrub_log(prompt_name),
+            scrub_log(exc),
         )
         raise RuntimeError(f"The server could not compose that prompt: {exc}") from exc
 

backend/src/apis/inference_api/chat/routes.py

@@ -2001,7 +2001,10 @@ async def invocations(request: InvocationRequest, current_user: User = Depends(g
                 if await bump_last_used_at(input_data.rag_assistant_id):
                     await resume_inactive_policies(input_data.rag_assistant_id)
             except Exception as bump_err:
-                logger.warning(f"lastUsedAt bump failed for assistant {input_data.rag_assistant_id}: {bump_err}")
+                logger.warning(
+                    f"lastUsedAt bump failed for assistant "
+                    f"{scrub_log(input_data.rag_assistant_id)}: {scrub_log(bump_err)}"
+                )
 
         # 2b. Agent Designer Phase 3 — resolve the Agent's governed capabilities
         # for the INVOKING user (D5), before the expensive KB search. v1 blocks