Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions backend/src/apis/app_api/admin/costs/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,16 @@ class FeedbackByTurnClass(BaseModel):
none: FeedbackCounts = Field(default_factory=FeedbackCounts)


class ImplicitSignalCounts(BaseModel):
"""Implicit signals (spec §10) as *messages touched* per kind — a message
copied three times counts once here. Kept apart from the thumbs; the two
have different base rates and are never summed."""
model_config = ConfigDict(populate_by_name=True)

copied: int = 0
continued: int = 0


class FeedbackProfile(BaseModel):
"""The outcome signal joined to the session's cost rows. ``byTurnClass``
is ``None`` when no cost row carries the turn-class fields (they arrive
Expand All @@ -505,6 +515,8 @@ class FeedbackProfile(BaseModel):
# a cost row to price.
retried: int = 0
rework_usd: Optional[float] = Field(None, alias="reworkUsd")
# Implicit signals, or None when the session has none.
implicit: Optional[ImplicitSignalCounts] = None


class DataCoverage(BaseModel):
Expand Down
15 changes: 13 additions & 2 deletions backend/src/apis/app_api/admin/costs/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
FeedbackByTurnClass,
FeedbackCounts,
FeedbackProfile,
ImplicitSignalCounts,
FingerprintChanges,
SessionDiagnosis,
SessionProfile,
Expand Down Expand Up @@ -126,6 +127,7 @@ def _join_feedback(
any_turn_class = any(_turn_class(r) is not None for r in records)
buckets = FeedbackByTurnClass() if any_turn_class else None
profile = FeedbackProfile()
implicit_messages: Dict[str, set] = {"copy": set(), "continue": set()}
# Every cost row per assistant message index, for pricing rework.
cost_by_message: Dict[int, float] = {}
for record in records:
Expand All @@ -134,9 +136,13 @@ def _join_feedback(
cost_by_message[message_id] = cost_by_message.get(message_id, 0.0) + (_record_cost(record) or 0.0)
rework_total: Optional[float] = None
for row in feedback_rows:
# Explicit thumbs only — implicit signals (spec §10) share the row
# family but answer a different question and must never be summed in.
# Explicit thumbs only below — implicit signals (spec §10) share the
# row family but answer a different question and are counted apart.
if row.get("signal") not in (None, "explicit"):
if row.get("signal") == "implicit" and row.get("kind") in implicit_messages:
message_id = _as_int(row.get("messageId"))
if message_id is not None:
implicit_messages[row["kind"]].add(message_id)
continue
value = _as_int(row.get("value"))
if value not in (1, -1):
Expand Down Expand Up @@ -170,6 +176,11 @@ def _join_feedback(
bucket.down += 1
profile.by_turn_class = buckets
profile.rework_usd = round(rework_total, 6) if rework_total is not None else None
if any(implicit_messages.values()):
profile.implicit = ImplicitSignalCounts(
copied=len(implicit_messages["copy"]),
continued=len(implicit_messages["continue"]),
)
return profile


Expand Down
18 changes: 18 additions & 0 deletions backend/src/apis/app_api/messages/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,24 @@ class Citation(BaseModel):
FeedbackReason = Literal["wrong", "instructions", "length", "tool_failed", "outdated", "other"]


#: Implicit signals (response-feedback spec §10): denser than thumbs, no UI
#: cost, written to the same ``F#`` family under ``signal: "implicit"`` and
#: never summed with them. ``copy`` = the response was copied out;
#: ``continue`` = a truncated / interrupted response was resumed. Edit-and-
#: resend has no affordance in the SPA yet; abandonment is deferred (its
#: base rate is indistinguishable from a satisfied user going quiet).
IMPLICIT_SIGNAL_KINDS = ("copy", "continue")
ImplicitSignalKind = Literal["copy", "continue"]


class ImplicitSignalRequest(BaseModel):
"""Body of ``POST /sessions/{id}/messages/{messageId}/signals``."""

model_config = ConfigDict(populate_by_name=True)

kind: ImplicitSignalKind = Field(..., description="Which implicit signal fired (closed enum)")


class MessageFeedback(BaseModel):
"""One user's thumb on one assistant message (``F#`` row, see
``apis.shared.sessions.metadata``). ``value`` is +1 (up) or -1 (down);
Expand Down
32 changes: 32 additions & 0 deletions backend/src/apis/app_api/sessions/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@
MessagesListResponse,
MessageFeedback,
MessageFeedbackRequest,
ImplicitSignalRequest,
)
from apis.shared.sessions.feedback import (
SessionNotOwned,
delete_message_feedback,
put_message_feedback,
record_implicit_signal,
)
from apis.shared.sessions.messages import get_messages
from apis.shared.sessions.metadata import (
Expand Down Expand Up @@ -746,6 +748,36 @@ async def delete_message_feedback_endpoint(
return Response(status_code=204)


@router.post("/{session_id}/messages/{message_id}/signals", status_code=204)
async def record_implicit_signal_endpoint(
session_id: str,
message_id: int = Path(..., ge=0, description="0-based message index"),
body: ImplicitSignalRequest = ...,
current_user: User = Depends(get_current_user_from_session),
):
"""Record an implicit signal (``copy`` / ``continue``) on an assistant
message — response-feedback spec §10. Fire-and-forget from the SPA:
always 204 once accepted, never a reason to show the user anything.
Content-free: the body is a closed enum."""
_require_message_feedback()
try:
await record_implicit_signal(
session_id=session_id,
user_id=current_user.user_id,
message_id=message_id,
kind=body.kind,
)
except SessionNotOwned:
raise HTTPException(status_code=404, detail=f"Session not found: {session_id}")
except RuntimeError as e:
logger.warning("Message feedback unavailable: %s", scrub_log(str(e)))
raise HTTPException(status_code=503, detail="Message feedback is not available")
except Exception:
logger.error("Error recording implicit signal", exc_info=True)
raise HTTPException(status_code=500, detail="Failed to record signal")
return Response(status_code=204)


@router.post("/{session_id}/interrupt", status_code=204)
async def signal_turn_interrupted_endpoint(
session_id: str,
Expand Down
2 changes: 2 additions & 0 deletions backend/src/apis/shared/observability/content_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,8 @@ def is_content_bearing(path: str) -> bool:
"value",
"reason",
"signal",
"kind", # implicit rows: copy / continue (closed enum)
"count", # implicit rows: how many times it fired
"retryMessageId", # a message index, the retry-with-correction link
"updatedAt",
)
Expand Down
51 changes: 46 additions & 5 deletions backend/src/apis/shared/sessions/feedback.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,17 @@
GSI_SK: F#{message_id}
sessionId, messageId, userId, value, reason?, signal, retryMessageId?, updatedAt, ttl

``signal`` is ``"explicit"`` for a thumb. ``docs/specs/response-feedback.md``
§10 adds implicit signals (copy, continue, edit-and-resend, abandonment) to
this same row family under ``signal: "implicit"``; every reader here filters
to explicit rows so that phase needs no backfill and the two are never summed.
``signal`` is ``"explicit"`` for a thumb. Implicit signals
(``docs/specs/response-feedback.md`` §10 — today ``copy`` and ``continue``)
live in the same family under their own key so they never collide with the
thumb::

SK: F#{session_id}#{message_id}#{kind}
GSI_SK: F#{message_id}#{kind}
sessionId, messageId, userId, signal="implicit", kind, count (ADD), updatedAt, ttl

Every thumb reader filters to explicit rows, so the two are never summed
(§10's rule); the admin profile counts implicit rows per kind separately.

``retryMessageId`` is the index of the user message the SPA sent as a
*retry with correction* after a down-thumb (response-feedback spec §7 "the
Expand Down Expand Up @@ -53,7 +60,7 @@
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional

from .models import FEEDBACK_REASONS, MessageFeedback
from .models import FEEDBACK_REASONS, IMPLICIT_SIGNAL_KINDS, MessageFeedback
from .preview import is_preview_session

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -235,6 +242,40 @@ async def delete_message_feedback(session_id: str, user_id: str, message_id: int
return True


async def record_implicit_signal(session_id: str, user_id: str, message_id: int, kind: str) -> None:
"""Count one implicit signal on one message (``ADD count :one``). Same
ownership rule as a thumb; a kind outside :data:`IMPLICIT_SIGNAL_KINDS`
is refused so the family stays a closed vocabulary. No session rollup:
the rows are the read model and the profile counts them directly."""
if kind not in IMPLICIT_SIGNAL_KINDS:
raise ValueError("implicit signal kind must be one of the fixed codes")
if is_preview_session(session_id):
return
table = _table()
await _owned_session_sk(session_id, user_id, table)
ttl = int((datetime.now(timezone.utc) + timedelta(days=FEEDBACK_TTL_DAYS)).timestamp())
table.update_item(
Key={"PK": f"USER#{user_id}", "SK": f"{feedback_sk(session_id, message_id)}#{kind}"},
UpdateExpression=(
"SET GSI_PK = :gpk, GSI_SK = :gsk, sessionId = :sid, messageId = :mid, userId = :uid, "
"#signal = :sig, kind = :kind, updatedAt = :now, #ttl = :ttl ADD #count :one"
),
ExpressionAttributeNames={"#signal": "signal", "#ttl": "ttl", "#count": "count"},
ExpressionAttributeValues={
":gpk": f"SESSION#{session_id}",
":gsk": f"F#{message_id}#{kind}",
":sid": session_id,
":mid": int(message_id),
":uid": user_id,
":sig": "implicit",
":kind": kind,
":now": _now(),
":ttl": ttl,
":one": 1,
},
)


def query_session_feedback(table, session_id: str, user_id: Optional[str] = None) -> Dict[str, MessageFeedback]:
"""All ``F#`` rows for a session via ``SessionLookupIndex``, keyed by
message id (as ``str``, matching the metadata index). With ``user_id``
Expand Down
18 changes: 18 additions & 0 deletions backend/src/apis/shared/sessions/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -653,6 +653,24 @@ class Citation(BaseModel):
FeedbackReason = Literal["wrong", "instructions", "length", "tool_failed", "outdated", "other"]


#: Implicit signals (response-feedback spec §10): denser than thumbs, no UI
#: cost, written to the same ``F#`` family under ``signal: "implicit"`` and
#: never summed with them. ``copy`` = the response was copied out;
#: ``continue`` = a truncated / interrupted response was resumed. Edit-and-
#: resend has no affordance in the SPA yet; abandonment is deferred (its
#: base rate is indistinguishable from a satisfied user going quiet).
IMPLICIT_SIGNAL_KINDS = ("copy", "continue")
ImplicitSignalKind = Literal["copy", "continue"]


class ImplicitSignalRequest(BaseModel):
"""Body of ``POST /sessions/{id}/messages/{messageId}/signals``."""

model_config = ConfigDict(populate_by_name=True)

kind: ImplicitSignalKind = Field(..., description="Which implicit signal fired (closed enum)")


class MessageFeedback(BaseModel):
"""One user's thumb on one assistant message (``F#`` row, see
``apis.shared.sessions.metadata``). ``value`` is +1 (up) or -1 (down);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def test_session_profile_returns_200():
"feedback": False,
"documents": False,
}
assert body["feedback"] == {"up": 0, "down": 0, "byTurnClass": None, "unjoined": 0, "retried": 0, "reworkUsd": None}
assert body["feedback"] == {"up": 0, "down": 0, "byTurnClass": None, "unjoined": 0, "retried": 0, "reworkUsd": None, "implicit": None}
service.get_session_profile.assert_awaited_once_with("s1")


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,23 @@ async def test_implicit_signal_rows_are_never_summed_into_the_thumb_counts():
assert (p.feedback.by_turn_class.full.up, p.feedback.by_turn_class.full.down) == (1, 1)


@pytest.mark.asyncio
async def test_implicit_signals_are_counted_as_messages_touched_per_kind():
def implicit(message_id, kind, count):
return {"sessionId": "s1", "messageId": message_id, "signal": "implicit", "kind": kind, "count": count, "updatedAt": "t"}

feedback = [implicit(0, "copy", 3), implicit(2, "copy", 1), implicit(2, "continue", 1), implicit(4, "weird", 1)]
p = await _service_with_feedback(_row(), [_call(0), _call(2)], feedback).get_session_profile("s1")
assert p.feedback.implicit is not None
assert (p.feedback.implicit.copied, p.feedback.implicit.continued) == (2, 1)
assert (p.feedback.up, p.feedback.down) == (0, 0)
assert p.data_coverage.feedback is True
assert p.model_dump(by_alias=True)["feedback"]["implicit"] == {"copied": 2, "continued": 1}

p = await _service_with_feedback(_row(), [_call(0)], [_feedback(0, 1)]).get_session_profile("s1")
assert p.feedback.implicit is None


@pytest.mark.asyncio
async def test_feedback_reader_failure_never_breaks_the_profile():
service = _service(_row(), [_call(0)])
Expand Down
14 changes: 13 additions & 1 deletion backend/tests/apis/app_api/test_message_feedback_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ def _user() -> User:
return User(user_id="user-1", email="u@example.com", name="U", roles=["default"], raw_token="tok")


def _client(monkeypatch, put=None, delete=None) -> TestClient:
def _client(monkeypatch, put=None, delete=None, signal=None) -> TestClient:
monkeypatch.delenv("RESPONSE_FEEDBACK_ENABLED", raising=False)
monkeypatch.setattr(session_routes, "record_implicit_signal", signal or AsyncMock(return_value=None))
monkeypatch.setattr(session_routes, "put_message_feedback", put or AsyncMock(
return_value=MessageFeedback(value=1, updated_at="2026-09-16T00:00:00Z")
))
Expand Down Expand Up @@ -96,6 +97,17 @@ def test_kill_switch_hides_the_surface(monkeypatch):
put.assert_not_awaited()


def test_implicit_signal_is_fire_and_forget_204(monkeypatch):
signal = AsyncMock(return_value=None)
client = _client(monkeypatch, signal=signal)
assert client.post("/sessions/s1/messages/3/signals", json={"kind": "copy"}).status_code == 204
signal.assert_awaited_once_with(session_id="s1", user_id="user-1", message_id=3, kind="copy")
assert client.post("/sessions/s1/messages/3/signals", json={"kind": "abandon"}).status_code == 422
assert client.post("/sessions/s1/messages/3/signals", json={"kind": "I copied it"}).status_code == 422
monkeypatch.setenv("RESPONSE_FEEDBACK_ENABLED", "false")
assert client.post("/sessions/s1/messages/3/signals", json={"kind": "copy"}).status_code == 404


def test_missing_table_is_503_not_500(monkeypatch):
client = _client(monkeypatch, put=AsyncMock(side_effect=RuntimeError("DYNAMODB_SESSIONS_METADATA_TABLE_NAME")))
assert client.put("/sessions/s1/messages/3/feedback", json={"value": 1}).status_code == 503
29 changes: 29 additions & 0 deletions backend/tests/shared/test_message_feedback.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,35 @@ async def test_implicit_signal_rows_share_the_family_but_never_read_as_thumbs(ta
assert fb.is_explicit({"value": 1}) and not fb.is_explicit({"value": 1, "signal": "implicit"})


@pytest.mark.asyncio
async def test_implicit_signals_count_under_their_own_key_beside_the_thumb(table):
"""Spec §10: copy / continue rows share the F# family, keyed per kind so
they never collide with the thumb, ADD a count, and are invisible to
every thumb reader."""
await fb.put_message_feedback(SESSION, OWNER, 3, -1, reason="wrong")
await fb.record_implicit_signal(SESSION, OWNER, 3, "copy")
await fb.record_implicit_signal(SESSION, OWNER, 3, "copy")
await fb.record_implicit_signal(SESSION, OWNER, 3, "continue")

items = {i["SK"]: i for i in _feedback_items()}
assert set(items) == {f"F#{SESSION}#3", f"F#{SESSION}#3#copy", f"F#{SESSION}#3#continue"}
copy = items[f"F#{SESSION}#3#copy"]
assert copy["signal"] == "implicit" and copy["kind"] == "copy" and int(copy["count"]) == 2
assert copy["GSI_SK"] == "F#3#copy" and int(copy["messageId"]) == 3 and "value" not in copy
assert int(items[f"F#{SESSION}#3"]["value"]) == -1, "the thumb is untouched"

# Thumb readers see only the thumb; the session rollups did not move.
assert {k: v.value for k, v in fb.query_session_feedback(table, SESSION, OWNER).items()} == {"3": -1}
assert _session_row()["thumbsDown"] == 1 and _session_row()["thumbsUp"] == 0

with pytest.raises(ValueError):
await fb.record_implicit_signal(SESSION, OWNER, 3, "abandon")
with pytest.raises(fb.SessionNotOwned):
await fb.record_implicit_signal(SESSION, OTHER, 3, "copy")
await fb.record_implicit_signal("preview-x", OWNER, 0, "copy")
assert _feedback_items("preview-x") == []


@pytest.mark.asyncio
async def test_messages_list_metadata_index_carries_feedback(table, monkeypatch):
"""The read path the SPA uses on reload: F# rows merge into the metadata
Expand Down
22 changes: 18 additions & 4 deletions docs/specs/response-feedback.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
**Status:** PARTIALLY BUILT — capture and the read model shipped in PR #1142
(2026-09-16, as document-context-offload PR-7) and the consequence
(retry-with-correction) in the PR stacked on it; §11 PR-1 is therefore
complete. Implicit signals and eval sampling are not built. See §13. Written
complete, and §11 PR-2 (implicit signals) followed for copy and continue.
Eval sampling and the author surfaces are not built. See §13. Written
2026-09-04 from the "how would we benefit?" conversation.
**Refs:** `docs/specs/agentcore-evaluations-spike-findings.md` (the eval
harness this feeds), `docs/specs/mid-turn-steering.md` (the injection path
Expand Down Expand Up @@ -368,7 +369,20 @@ points:
so Phase 6 can find it once the consent decision is made. The profile
reports `feedback.retried` and `reworkUsd` (§7 "rework cost": the thumbed
call rows plus the retry turn's consecutive assistant rows).
- **Not built**: implicit signals (PR-2), eval sampling (PR-4),
author/marketplace surfaces (PR-5), the report-dialog escape hatch in the
reason row.
- **Implicit signals (§11 PR-2), third PR — copy and continue only.**
Same `F#` family under their own key, `F#{session}#{message}#{kind}`, so
they never collide with the thumb; `signal: "implicit"`, `kind`, and an
`ADD`ed `count`. `POST /sessions/{id}/messages/{message_id}/signals`,
fire-and-forget from the SPA (the Copy and Continue clicks in the actions
rail, once per message per kind per page load). The profile reports
`feedback.implicit` as *messages touched* per kind, on its own line — the
§10 rule that explicit and implicit are never summed is enforced in every
reader. Two of §10's four signals are deliberately not here:
**edit-and-resend** has no affordance in the SPA to hook, and
**abandonment** is deferred — a session that goes quiet after a good
answer is indistinguishable from one that goes quiet after a bad one, so
it needs its own design (or the §10 "dissatisfaction in the next message"
offline classifier) before it is worth a row.
- **Not built**: eval sampling (PR-4), author/marketplace surfaces (PR-5),
the report-dialog escape hatch in the reason row, abandonment.

Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,8 @@ export interface FeedbackProfile {
/** Down-thumbs followed by a retry-with-correction, and what the rework cost. */
retried?: number;
reworkUsd?: number | null;
/** Implicit signals as messages touched per kind; null when none. Never summed with thumbs. */
implicit?: { copied: number; continued: number } | null;
}

/** The content-free diagnostic profile of one conversation. */
Expand Down
Loading