From 22709b997c7ca688d2df288b1ccb6a9b1d81e8b4 Mon Sep 17 00:00:00 2001 From: Jinglong Wang Date: Thu, 23 Jul 2026 15:19:15 +0800 Subject: [PATCH 01/19] feat(memory): implement Dreaming consolidation flow --- backend/apps/config_app.py | 4 + backend/apps/memory_dreaming_app.py | 54 +++ backend/database/db_models.py | 37 ++ backend/database/memory_dreaming_db.py | 132 ++++++ backend/database/memory_retrieval_hit_db.py | 43 +- backend/services/memory_dreaming_scheduler.py | 415 +---------------- backend/services/memory_dreaming_service.py | 237 ++++++++++ backend/utils/memory_utils.py | 13 + deploy/sql/init.sql | 27 ++ .../v2.4.0_0723_add_memory_dreaming_audit.sql | 33 ++ sdk/nexent/memory/dreaming/__init__.py | 23 + sdk/nexent/memory/dreaming/models.py | 58 +++ sdk/nexent/memory/dreaming/scoring.py | 146 ++++++ sdk/nexent/memory/dreaming/service.py | 63 +++ sdk/nexent/memory/memory_service.py | 28 ++ test/backend/apps/test_memory_dreaming_app.py | 86 ++++ .../database/test_memory_dreaming_schema.py | 84 ++++ .../test_memory_dreaming_scheduler.py | 425 ------------------ .../services/test_memory_dreaming_service.py | 143 ++++++ test/sdk/memory/test_dreaming.py | 132 ++++++ 20 files changed, 1350 insertions(+), 833 deletions(-) create mode 100644 backend/apps/memory_dreaming_app.py create mode 100644 backend/database/memory_dreaming_db.py create mode 100644 backend/services/memory_dreaming_service.py create mode 100644 backend/utils/memory_utils.py create mode 100644 deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_audit.sql create mode 100644 sdk/nexent/memory/dreaming/__init__.py create mode 100644 sdk/nexent/memory/dreaming/models.py create mode 100644 sdk/nexent/memory/dreaming/scoring.py create mode 100644 sdk/nexent/memory/dreaming/service.py create mode 100644 sdk/nexent/memory/memory_service.py create mode 100644 test/backend/apps/test_memory_dreaming_app.py create mode 100644 test/backend/database/test_memory_dreaming_schema.py delete mode 100644 test/backend/services/test_memory_dreaming_scheduler.py create mode 100644 test/backend/services/test_memory_dreaming_service.py create mode 100644 test/sdk/memory/test_dreaming.py diff --git a/backend/apps/config_app.py b/backend/apps/config_app.py index fe9e9ff5a9..94cb18ae14 100644 --- a/backend/apps/config_app.py +++ b/backend/apps/config_app.py @@ -43,6 +43,8 @@ from apps.memory_config_app import router as memory_config_router from apps.memory_record_app import router as memory_record_router from apps.quota_app import tenant_quota_router, platform_quota_router +from apps.memory_record_app import router as memory_record_router +from apps.memory_dreaming_app import router as memory_dreaming_router from consts.const import IS_SPEED_MODE from services.prompt_template_service import sync_system_default_prompt_template @@ -111,3 +113,5 @@ async def sync_default_prompt_template_on_startup(): app.include_router(memory_record_router) app.include_router(tenant_quota_router) app.include_router(platform_quota_router) +app.include_router(memory_record_router) +app.include_router(memory_dreaming_router) diff --git a/backend/apps/memory_dreaming_app.py b/backend/apps/memory_dreaming_app.py new file mode 100644 index 0000000000..d55a353365 --- /dev/null +++ b/backend/apps/memory_dreaming_app.py @@ -0,0 +1,54 @@ +"""Manual Dreaming run and audit endpoints.""" + +from http import HTTPStatus +from typing import Optional + +from fastapi import APIRouter, Header, HTTPException, Query +from pydantic import BaseModel, Field + +from services.memory_dreaming_service import ( + DreamingRunError, + get_memory_dreaming_service, +) +from utils.auth_utils import get_current_user_id + +router = APIRouter(prefix="/memory/dreaming", tags=["memory-dreaming"]) + + +class DreamingRunRequest(BaseModel): + agent_id: str = Field(..., min_length=1) + + +@router.post("/run") +async def run_dreaming( + payload: DreamingRunRequest, + authorization: Optional[str] = Header(None), +): + user_id, tenant_id = get_current_user_id(authorization) + try: + return get_memory_dreaming_service().run( + tenant_id=tenant_id, + user_id=user_id, + agent_id=payload.agent_id, + ) + except DreamingRunError as exc: + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=str(exc) + ) from exc + + +@router.get("/audit") +async def list_dreaming_audits( + authorization: Optional[str] = Header(None), + agent_id: Optional[str] = Query(default=None), + run_id: Optional[int] = Query(default=None, ge=1), + limit: int = Query(default=100, ge=1, le=500), +): + user_id, tenant_id = get_current_user_id(authorization) + return get_memory_dreaming_service().list_audits( + tenant_id, + user_id, + agent_id=agent_id, + run_id=run_id, + limit=limit, + ) diff --git a/backend/database/db_models.py b/backend/database/db_models.py index aec4524790..f47f9ec50b 100644 --- a/backend/database/db_models.py +++ b/backend/database/db_models.py @@ -937,6 +937,43 @@ class MemoryRetrievalHit(TableBase): doc="Soft delete flag (N = active, Y = deleted).") +class MemoryDreamingAudit(TableBase): + """One durable audit row per manual Dreaming run.""" + + __tablename__ = "memory_dreaming_audit_t" + __table_args__ = ( + Index( + "idx_memory_dreaming_audit_scope", + "tenant_id", + "user_id", + "agent_id", + "started_at", + ), + {"schema": SCHEMA}, + ) + + run_id = Column( + BigInteger, + Sequence("memory_dreaming_audit_t_run_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + ) + tenant_id = Column(String(100), nullable=False) + user_id = Column(String(100), nullable=False) + agent_id = Column(String(100), nullable=False) + trigger_source = Column(String(30), nullable=False, default="manual") + status = Column(String(30), nullable=False, default="running") + current_phase = Column(String(30)) + started_at = Column(TIMESTAMP(timezone=False), nullable=False, server_default=func.now()) + finished_at = Column(TIMESTAMP(timezone=False)) + light_count = Column(Integer, nullable=False, default=0) + rem_count = Column(Integer, nullable=False, default=0) + promoted_count = Column(Integer, nullable=False, default=0) + deferred_count = Column(Integer, nullable=False, default=0) + result_json = Column(JSONB) + error = Column(Text) + + class McpRecord(TableBase): """ MCP (Model Context Protocol) records table diff --git a/backend/database/memory_dreaming_db.py b/backend/database/memory_dreaming_db.py new file mode 100644 index 0000000000..545a929833 --- /dev/null +++ b/backend/database/memory_dreaming_db.py @@ -0,0 +1,132 @@ +"""Persistence and PostgreSQL advisory locking for manual Dreaming runs.""" + +from __future__ import annotations + +import hashlib +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Dict, Iterator, List, Optional + +from sqlalchemy import text + +from .client import get_db_session +from .db_models import MemoryDreamingAudit + + +def advisory_lock_key(tenant_id: str, user_id: str, agent_id: str) -> int: + digest = hashlib.sha256( + f"{tenant_id}:{user_id}:{agent_id}".encode("utf-8") + ).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + +@contextmanager +def try_scope_lock(tenant_id: str, user_id: str, agent_id: str) -> Iterator[bool]: + """Hold a transaction-scoped advisory lock for the context lifetime.""" + with get_db_session() as session: + acquired = bool( + session.execute( + text("SELECT pg_try_advisory_xact_lock(:lock_key)"), + {"lock_key": advisory_lock_key(tenant_id, user_id, agent_id)}, + ).scalar() + ) + try: + yield acquired + session.commit() + except Exception: + session.rollback() + raise + + +def create_audit(tenant_id: str, user_id: str, agent_id: str) -> int: + with get_db_session() as session: + row = MemoryDreamingAudit( + tenant_id=tenant_id, + user_id=user_id, + agent_id=agent_id, + trigger_source="manual", + status="running", + current_phase="light", + ) + session.add(row) + session.commit() + return int(row.run_id) + + +def update_audit(run_id: int, values: Dict[str, Any]) -> bool: + allowed = { + "status", + "current_phase", + "finished_at", + "light_count", + "rem_count", + "promoted_count", + "deferred_count", + "result_json", + "error", + } + with get_db_session() as session: + row = ( + session.query(MemoryDreamingAudit) + .filter(MemoryDreamingAudit.run_id == run_id) + .first() + ) + if row is None: + return False + for key, value in values.items(): + if key in allowed: + setattr(row, key, value) + session.commit() + return True + + +def finish_audit(run_id: int, *, status: str, **values: Any) -> bool: + payload = { + **values, + "status": status, + "finished_at": datetime.utcnow(), + } + if status != "failed": + payload["current_phase"] = None + return update_audit(run_id, payload) + + +def list_audits( + tenant_id: str, + user_id: str, + *, + agent_id: Optional[str] = None, + run_id: Optional[int] = None, + limit: int = 100, +) -> List[Dict[str, Any]]: + with get_db_session() as session: + query = session.query(MemoryDreamingAudit).filter( + MemoryDreamingAudit.tenant_id == tenant_id, + MemoryDreamingAudit.user_id == user_id, + MemoryDreamingAudit.delete_flag == "N", + ) + if agent_id is not None: + query = query.filter(MemoryDreamingAudit.agent_id == agent_id) + if run_id is not None: + query = query.filter(MemoryDreamingAudit.run_id == run_id) + rows = query.order_by(MemoryDreamingAudit.run_id.desc()).limit(limit).all() + return [ + { + "run_id": row.run_id, + "tenant_id": row.tenant_id, + "user_id": row.user_id, + "agent_id": row.agent_id, + "trigger_source": row.trigger_source, + "status": row.status, + "current_phase": row.current_phase, + "started_at": row.started_at.isoformat() if row.started_at else None, + "finished_at": row.finished_at.isoformat() if row.finished_at else None, + "light_count": row.light_count, + "rem_count": row.rem_count, + "promoted_count": row.promoted_count, + "deferred_count": row.deferred_count, + "result": row.result_json, + "error": row.error, + } + for row in rows + ] diff --git a/backend/database/memory_retrieval_hit_db.py b/backend/database/memory_retrieval_hit_db.py index 9028ad41bf..d69b512905 100644 --- a/backend/database/memory_retrieval_hit_db.py +++ b/backend/database/memory_retrieval_hit_db.py @@ -186,6 +186,47 @@ def aggregate_memory_stats( return out +def aggregate_dreaming_stats( + tenant_id: str, + user_id: str, + agent_id: str, + *, + since: datetime, +) -> List[Dict[str, Any]]: + """Return complete Light/Deep evidence for one isolation scope.""" + hits = list_hits_for_user(tenant_id, user_id, since=since, limit=10000) + grouped: Dict[int, Dict[str, Any]] = {} + for hit in hits: + if str(hit.get("agent_id")) != str(agent_id) or hit.get("memory_id") is None: + continue + memory_id = int(hit["memory_id"]) + entry = grouped.setdefault( + memory_id, + { + "memory_id": memory_id, + "hit_count": 0, + "grounded_count": 0, + "days": set(), + "query_hashes": set(), + "total_retrieval_score": 0.0, + "last_recalled_at": None, + }, + ) + entry["hit_count"] += 1 + entry["grounded_count"] += int(bool(hit.get("grounded"))) + if hit.get("day"): + entry["days"].add(str(hit["day"])) + if hit.get("query_hash"): + entry["query_hashes"].add(str(hit["query_hash"])) + entry["total_retrieval_score"] += float(hit.get("retrieval_score") or 0) + occurred_at = hit.get("occurred_at") + if occurred_at and ( + entry["last_recalled_at"] is None or occurred_at > entry["last_recalled_at"] + ): + entry["last_recalled_at"] = occurred_at + return list(grouped.values()) + + def delete_hits_before(cutoff: datetime) -> int: """Delete hit rows older than ``cutoff`` (housekeeping).""" with get_db_session() as session: @@ -223,4 +264,4 @@ def _hit_to_dict(row: MemoryRetrievalHit) -> Dict[str, Any]: "occurred_at": row.occurred_at, "day": row.day, "grounded": bool(row.grounded), - } \ No newline at end of file + } diff --git a/backend/services/memory_dreaming_scheduler.py b/backend/services/memory_dreaming_scheduler.py index 4f56345236..3ec4572986 100644 --- a/backend/services/memory_dreaming_scheduler.py +++ b/backend/services/memory_dreaming_scheduler.py @@ -1,411 +1,12 @@ -"""Dreaming consolidation runner for the Memory Architecture (Phase 2). +"""Compatibility facade for the former Phase 2 Dreaming placeholder.""" -Dreaming promotes agent short-term memories into user long-term memory. It -runs in three phases: +try: + from services.memory_dreaming_service import get_memory_dreaming_service +except (ImportError, ModuleNotFoundError): # package-style unit-test imports + from .memory_dreaming_service import get_memory_dreaming_service -1. **Light Sleep** - aggregate ``memory_retrieval_hits_t`` rows into the - per-memory ``light_hits`` counter and ``recall_count`` / - ``recall_days`` / ``query_hashes`` columns. -2. **REM Sleep** - extract repeating concepts / patterns, write concept - tags to ``memory_records_t``. The phase is implemented as a lightweight - keyword-frequency pass; LLM-driven concept extraction can be wired in - later without changing the public API. -3. **Deep Sleep** - select eligible agent memories and promote them to - ``user`` long-term memory using the documented scoring formula - (frequency / relevance / diversity / recency / consolidation / - concept + phase boost). -Promotion thresholds and weights live in ``consts.const``. The phases are -exposed as standalone ``run_light_sleep`` / ``run_rem_sleep`` / -``run_deep_sleep`` functions plus the aggregate ``run_once`` so callers -can trigger a single pass per tenant on demand (e.g. from a future agent -timer). The module deliberately does **not** ship an internal scheduler, -background thread, or cron expression: agent-driven scheduling will be -added in a later phase, and we want to avoid having to coordinate cron, -lock watchdog and lifecycle here before the agent timer feature lands. -""" - -from __future__ import annotations - -import logging -import math -import time -from collections import Counter -from datetime import datetime, timedelta -from typing import Any, Dict, List, Optional, Sequence, Set - -from consts.const import ( - AGENT_SHORT_TERM_HALF_LIFE_DAYS, - LIGHT_SLEEP_WINDOW_DAYS, - MIN_PROMOTION_SCORE, - MIN_RECALL_COUNT, - MIN_UNIQUE_QUERIES, - RECENCY_HALF_LIFE_DAYS, -) -from database import memory_record_db, memory_retrieval_hit_db -from services.memory_record_service import ( - MemoryRecordError, - get_memory_record_service, -) - - -logger = logging.getLogger("memory_dreaming_scheduler") - - -# --------------------------------------------------------------------------- -# Scoring helpers -# --------------------------------------------------------------------------- - - -def _clamp01(value: float) -> float: - if value < 0.0: - return 0.0 - if value > 1.0: - return 1.0 - return value - - -def _frequency(recall_count: int, daily_count: int, grounded_count: int) -> float: - """Log-scaled accumulation of recall signals.""" - signal = max(0, recall_count) + max(0, daily_count) + max(0, grounded_count) - return _clamp01(math.log1p(signal) / math.log1p(10)) - - -def _relevance(hit_count: int, total_score: float) -> float: - """Average retrieval score across hits, clamped to [0, 1].""" - if hit_count <= 0: - return 0.0 - return _clamp01(total_score / max(1, hit_count)) - - -def _diversity(unique_queries: int) -> float: - """Smoothed saturation in [0, 1].""" - return _clamp01(math.log1p(unique_queries) / math.log1p(5)) - - -def _recency(last_recalled_at: Optional[datetime]) -> float: - """Exponential decay based on ``RECENCY_HALF_LIFE_DAYS``.""" - if last_recalled_at is None: - return 0.0 - delta_days = (datetime.utcnow() - last_recalled_at).total_seconds() / 86400.0 - if delta_days < 0: - delta_days = 0 - half_life = max(1, RECENCY_HALF_LIFE_DAYS) - return _clamp01(math.pow(0.5, delta_days / half_life)) - - -def _consolidation(light_hits: int, rem_hits: int) -> float: - """Boost when both Light and REM phases have seen the memory.""" - combined = max(0, light_hits) + max(0, rem_hits) - return _clamp01(math.log1p(combined) / math.log1p(6)) - - -def _concept(concept_tags: Sequence[str]) -> float: - """Higher when concept tags have been attached.""" - if not concept_tags: - return 0.0 - return _clamp01(math.log1p(len(concept_tags)) / math.log1p(8)) - - -# Weights reflect ``openclaw_dreaming.md`` §深眠阶段评分体系. -_PROMOTION_WEIGHTS: Dict[str, float] = { - "relevance": 0.30, - "frequency": 0.24, - "diversity": 0.15, - "recency": 0.15, - "consolidation": 0.10, - "concept": 0.06, -} - - -def _normalize_weights(weights: Dict[str, float]) -> Dict[str, float]: - total = sum(weights.values()) or 1.0 - return {key: value / total for key, value in weights.items()} - - -def _phase_boost(light_hits: int, rem_hits: int) -> float: - """PhaseBoost from the design doc; kept small to avoid runaway scores.""" - if light_hits <= 0 or rem_hits <= 0: - return 0.0 - return _clamp01(min(0.05, light_hits * 0.01 + rem_hits * 0.01)) - - -def compute_promotion_score(record: Dict[str, Any]) -> float: - """Compute the composite score for a single memory record.""" - recall_count = int(record.get("recall_count") or 0) - daily_count = int(record.get("daily_count") or 0) - grounded_count = int(record.get("grounded_count") or 0) - light_hits = int(record.get("light_hits") or 0) - rem_hits = int(record.get("rem_hits") or 0) - last_recalled_at = record.get("last_recalled_at") - query_hashes = record.get("query_hashes") or [] - - metrics = { - "frequency": _frequency(recall_count, daily_count, grounded_count), - "relevance": _relevance(recall_count, 1.0), - "diversity": _diversity(len(query_hashes)), - "recency": _recency(last_recalled_at), - "consolidation": _consolidation(light_hits, rem_hits), - "concept": _concept(record.get("concept_tags") or []), - } - weights = _normalize_weights(_PROMOTION_WEIGHTS) - score = sum(metrics[key] * weights[key] for key in metrics) - score += _phase_boost(light_hits, rem_hits) - return _clamp01(score) - - -# --------------------------------------------------------------------------- -# Phase runners -# --------------------------------------------------------------------------- - - -def run_light_sleep( - *, - tenant_id: str, - user_id: str, - agent_id: Optional[str] = None, - window_days: int = LIGHT_SLEEP_WINDOW_DAYS, -) -> int: - """Aggregate recent hits into memory row counters. - - Returns the number of memory rows touched. - """ - since = datetime.utcnow() - timedelta(days=max(1, window_days)) - stats = memory_retrieval_hit_db.aggregate_memory_stats( - tenant_id, - user_id=user_id, - agent_id=agent_id, - since=since, - ) - touched = 0 - for entry in stats: - memory_id = entry["memory_id"] - # Last hit day is the most recent value in the per-memory hit set. - last_day = max(entry["days"]) if entry["days"] else None - last_recalled_at = ( - datetime.fromisoformat(last_day) if last_day else None - ) - memory_record_db.update_memory_record( - memory_id, - tenant_id, - { - "recall_count": entry["hit_count"], - "grounded_count": entry["grounded_count"], - "query_hashes": sorted(entry["query_hashes"]), - "recall_days": sorted(entry["days"]), - "last_recalled_at": last_recalled_at, - }, - ) - memory_record_db.apply_dreaming_phase( - memory_id, tenant_id, phase="light" - ) - touched += 1 - return touched - - -_KEYWORD_STOPWORDS: Set[str] = { - "the", "a", "an", "and", "or", "but", "is", "are", "was", "were", - "be", "been", "being", "have", "has", "had", "do", "does", "did", - "of", "in", "on", "at", "by", "for", "with", "to", "from", - "i", "you", "he", "she", "it", "we", "they", - "的", "了", "是", "在", "和", "与", "及", "或", "我", "你", "他", "她", "它", - "我们", "你们", "他们", "这", "那", "这个", "那个", -} - - -def _tokenize(text: str) -> List[str]: - return [ - token.strip().lower() - for token in text.replace("\n", " ").split() - if token.strip() and token.strip().lower() not in _KEYWORD_STOPWORDS - ] - - -def run_rem_sleep( - *, - tenant_id: str, - user_id: str, - agent_id: Optional[str] = None, - max_keywords: int = 5, -) -> int: - """Extract concept tags from frequently appearing tokens. - - Returns the number of memory rows whose ``concept_tags`` were updated. - """ - rows = memory_record_db.list_memory_records( - tenant_id, - user_id=user_id, - agent_id=agent_id, - layer="agent", - memory_type="short_term", - status="active", - limit=500, - ) - touched = 0 - for row in rows: - tokens = _tokenize(row.get("content", "")) - if not tokens: - continue - counter = Counter(tokens) - top = [token for token, _ in counter.most_common(max_keywords)] - if not top: - continue - existing = list(row.get("concept_tags") or []) - merged = list(dict.fromkeys(existing + top))[:max_keywords] - memory_record_db.update_memory_record( - row["memory_id"], - tenant_id, - {"concept_tags": merged}, - ) - memory_record_db.apply_dreaming_phase( - row["memory_id"], tenant_id, phase="rem" - ) - touched += 1 - return touched - - -def run_deep_sleep( - *, - tenant_id: str, - user_id: str, - agent_id: Optional[str] = None, - min_score: float = MIN_PROMOTION_SCORE, - min_recall_count: int = MIN_RECALL_COUNT, - min_unique_queries: int = MIN_UNIQUE_QUERIES, -) -> List[Dict[str, Any]]: - """Promote agent memories that pass the promotion thresholds. - - Returns the list of promotion results (``memory_id``, ``score``, ``event``). - """ - eligible = memory_record_db.list_memories_for_dreaming( - tenant_id, - user_id=user_id, - layer="agent", - min_recall_count=min_recall_count, - window_days=LIGHT_SLEEP_WINDOW_DAYS, +def run_once(*, tenant_id: str, user_id: str, agent_id: str, **kwargs): + return get_memory_dreaming_service().run( + tenant_id=tenant_id, user_id=user_id, agent_id=agent_id, **kwargs ) - promoted: List[Dict[str, Any]] = [] - service = get_memory_record_service() - for row in eligible: - query_hashes = row.get("query_hashes") or [] - if len(set(query_hashes)) < min_unique_queries: - continue - score = compute_promotion_score(row) - if score < min_score: - continue - try: - service.create_memory( - tenant_id=tenant_id, - user_id=user_id, - content=row.get("content", ""), - layer="user", - memory_type="long_term", - agent_id=row.get("agent_id"), - conversation_id=row.get("conversation_id"), - concept_tags=row.get("concept_tags") or [], - idempotency_key=f"dreaming:{row['memory_id']}", - created_by="dreaming", - actor="dreaming", - ) - except MemoryRecordError as exc: - logger.warning( - "dreaming promotion skipped for %s: %s", row["memory_id"], exc - ) - continue - memory_record_db.apply_dreaming_phase( - row["memory_id"], tenant_id, phase="rem" - ) - promoted.append( - { - "memory_id": row["memory_id"], - "score": score, - "event": "PROMOTE", - } - ) - return promoted - - -# --------------------------------------------------------------------------- -# Manual entry points -# --------------------------------------------------------------------------- - - -def run_once(*, timeout_seconds: int = 1800) -> Dict[str, Any]: - """Execute one full Dreaming cycle across known tenants. - - This function is the single manual entry point: callers (e.g. an agent - timer introduced later) invoke ``run_once`` whenever they want a fresh - pass. Phase 2 does not ship a scheduler; the function is intentionally - synchronous and idempotent so it can be re-invoked safely. - - Args: - timeout_seconds: Soft cap on wall-clock runtime per call. Iteration - stops once the deadline is reached; partial state is returned. - - Returns: - Summary dict with tenant count, light/rem rows touched, and the - list of promotion events. - """ - started = time.time() - deadline = started + max(60, timeout_seconds) - - # ``list_distinct_tenants`` is intentionally conservative: we only run - # dreaming over tenants that have actually touched memory recently. - tenants = list_distinct_tenants() - summary: Dict[str, Any] = { - "tenants": len(tenants), - "light_rows": 0, - "rem_rows": 0, - "promotions": [], - } - - for tenant_id, user_id in tenants: - if time.time() >= deadline: - logger.warning("Dreaming run hit timeout; aborting remaining tenants") - break - try: - light = run_light_sleep(tenant_id=tenant_id, user_id=user_id) - rem = run_rem_sleep(tenant_id=tenant_id, user_id=user_id) - deep = run_deep_sleep(tenant_id=tenant_id, user_id=user_id) - summary["light_rows"] += light - summary["rem_rows"] += rem - summary["promotions"].extend(deep) - except Exception: - logger.exception( - "Dreaming iteration failed for tenant=%s user=%s", - tenant_id, - user_id, - ) - - summary["elapsed_seconds"] = time.time() - started - return summary - - -def list_distinct_tenants() -> List[Any]: - """Return ``(tenant_id, user_id)`` tuples with recent memory activity. - - Implementation: distinct pairs from ``memory_retrieval_hits_t``. When - no hits exist (fresh deployments) this returns ``[]`` and Dreaming - becomes a no-op, which is the intended behavior. - """ - try: - from database.client import get_db_session - from database.db_models import MemoryRetrievalHit - from sqlalchemy import distinct - - with get_db_session() as session: - rows = ( - session.query( - distinct(MemoryRetrievalHit.tenant_id), - distinct(MemoryRetrievalHit.user_id), - ) - .filter( - MemoryRetrievalHit.tenant_id.isnot(None), - MemoryRetrievalHit.user_id.isnot(None), - ) - .all() - ) - return [(t, u) for t, u in rows if t and u] - except Exception: - logger.exception("list_distinct_tenants failed") - return [] \ No newline at end of file diff --git a/backend/services/memory_dreaming_service.py b/backend/services/memory_dreaming_service.py new file mode 100644 index 0000000000..79a5b6e198 --- /dev/null +++ b/backend/services/memory_dreaming_service.py @@ -0,0 +1,237 @@ +"""Backend orchestration for the SDK Dreaming algorithm.""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional + +from consts.const import ( + LIGHT_SLEEP_WINDOW_DAYS, + MIN_PROMOTION_SCORE, + MIN_RECALL_COUNT, + MIN_UNIQUE_QUERIES, + RECENCY_HALF_LIFE_DAYS, +) +from database import memory_dreaming_db, memory_record_db, memory_retrieval_hit_db +from nexent.memory.dreaming import ( + DreamingThresholds, + build_candidate, + select_candidates, +) +from services.memory_record_service import get_memory_record_service + +logger = logging.getLogger("memory_dreaming_service") + + +class DreamingRunError(RuntimeError): + pass + + +class MemoryDreamingService: + def __init__(self, record_service: Any = None): + self.record_service = record_service or get_memory_record_service() + + def _run_light( + self, tenant_id: str, user_id: str, agent_id: str, window_days: int + ) -> Dict[int, Dict[str, Any]]: + stats = memory_retrieval_hit_db.aggregate_dreaming_stats( + tenant_id, + user_id, + agent_id, + since=datetime.utcnow() - timedelta(days=max(1, window_days)), + ) + by_id = {int(item["memory_id"]): item for item in stats} + for item in stats: + memory_record_db.update_memory_record( + item["memory_id"], + tenant_id, + { + "recall_count": item["hit_count"], + "daily_count": len(item["days"]), + "grounded_count": item["grounded_count"], + "last_recalled_at": item["last_recalled_at"], + "query_hashes": sorted(item["query_hashes"]), + "recall_days": sorted(item["days"]), + }, + ) + memory_record_db.apply_dreaming_phase( + item["memory_id"], tenant_id, phase="light" + ) + return by_id + + def _run_rem( + self, + tenant_id: str, + user_id: str, + agent_id: str, + stats: Dict[int, Dict[str, Any]], + ) -> List[Any]: + records = memory_record_db.list_memory_records( + tenant_id, + user_id=user_id, + agent_id=agent_id, + layer="agent", + memory_type="short_term", + status="active", + limit=1000, + ) + candidates = [] + for record in records: + evidence = stats.get(int(record["memory_id"]), {}) + candidate = build_candidate( + record, float(evidence.get("total_retrieval_score") or 0) + ) + memory_record_db.update_memory_record( + candidate.memory_id, + tenant_id, + {"concept_tags": candidate.concept_tags}, + ) + if not candidate.noise: + memory_record_db.apply_dreaming_phase( + candidate.memory_id, tenant_id, phase="rem" + ) + candidate.rem_hits += 1 + candidate.last_rem_at = datetime.utcnow() + candidates.append(candidate) + return candidates + + def _promote(self, decisions: List[Any]) -> List[Dict[str, Any]]: + results = [] + for decision in decisions: + candidate = decision.candidate + if decision.promote: + created = self.record_service.create_memory( + tenant_id=candidate.tenant_id, + user_id=candidate.user_id, + agent_id=candidate.agent_id, + content=candidate.content, + layer="user", + memory_type="long_term", + concept_tags=candidate.concept_tags, + idempotency_key=f"dreaming:{candidate.memory_id}", + created_by="dreaming", + actor="dreaming", + ) + event = created.get("event", "ADD") + else: + event = "DEFER" + results.append( + { + "memory_id": candidate.memory_id, + "score": decision.score, + "event": event, + "reason": decision.reason, + "archive_suggested": decision.archive_suggested, + } + ) + return results + + def run( + self, + *, + tenant_id: str, + user_id: str, + agent_id: str, + window_days: int = LIGHT_SLEEP_WINDOW_DAYS, + min_score: float = MIN_PROMOTION_SCORE, + min_recall_count: int = MIN_RECALL_COUNT, + min_unique_queries: int = MIN_UNIQUE_QUERIES, + ) -> Dict[str, Any]: + if not tenant_id or not user_id or not agent_id: + raise DreamingRunError("tenant_id, user_id and agent_id are required") + run_id = memory_dreaming_db.create_audit(tenant_id, user_id, agent_id) + with memory_dreaming_db.try_scope_lock( + tenant_id, user_id, agent_id + ) as acquired: + if not acquired: + result = { + "run_id": run_id, + "status": "skipped", + "reason": "lock_busy", + } + memory_dreaming_db.finish_audit( + run_id, status="skipped", result_json=result + ) + return result + try: + stats = self._run_light(tenant_id, user_id, agent_id, window_days) + memory_dreaming_db.update_audit( + run_id, + {"current_phase": "rem", "light_count": len(stats)}, + ) + candidates = self._run_rem(tenant_id, user_id, agent_id, stats) + memory_dreaming_db.update_audit( + run_id, + {"current_phase": "deep", "rem_count": len(candidates)}, + ) + decisions = select_candidates( + candidates, + thresholds=DreamingThresholds( + min_score=min_score, + min_recall_count=min_recall_count, + min_unique_queries=min_unique_queries, + ), + recency_half_life_days=RECENCY_HALF_LIFE_DAYS, + ) + results = self._promote(decisions) + promoted_count = sum( + item["event"] in {"ADD", "UPDATE"} for item in results + ) + result = { + "run_id": run_id, + "status": "completed", + "light_count": len(stats), + "rem_count": len(candidates), + "promoted_count": promoted_count, + "deferred_count": len(results) - promoted_count, + "decisions": results, + } + memory_dreaming_db.finish_audit( + run_id, + status="completed", + light_count=len(stats), + rem_count=len(candidates), + promoted_count=promoted_count, + deferred_count=len(results) - promoted_count, + result_json=result, + ) + return result + except Exception as exc: + logger.exception( + "Dreaming failed for tenant=%s user=%s agent=%s run=%s", + tenant_id, + user_id, + agent_id, + run_id, + ) + error = f"{type(exc).__name__}: Dreaming phase failed" + memory_dreaming_db.finish_audit(run_id, status="failed", error=error) + raise DreamingRunError(error) from exc + + def list_audits( + self, + tenant_id: str, + user_id: str, + *, + agent_id: Optional[str] = None, + run_id: Optional[int] = None, + limit: int = 100, + ) -> List[Dict[str, Any]]: + return memory_dreaming_db.list_audits( + tenant_id, + user_id, + agent_id=agent_id, + run_id=run_id, + limit=limit, + ) + + +_service: Optional[MemoryDreamingService] = None + + +def get_memory_dreaming_service() -> MemoryDreamingService: + global _service + if _service is None: + _service = MemoryDreamingService() + return _service diff --git a/backend/utils/memory_utils.py b/backend/utils/memory_utils.py new file mode 100644 index 0000000000..3bac960770 --- /dev/null +++ b/backend/utils/memory_utils.py @@ -0,0 +1,13 @@ +"""Compatibility helper for agent cleanup paths on the new Memory system.""" + +from typing import Any, Dict + + +def build_memory_config(_tenant_id: str) -> Dict[str, Any]: + """Return an empty legacy config. + + The removed Mem0 functions still accept this argument at a few guarded + cleanup call sites. New Memory services resolve tenant model configuration + through backend services instead of this utility. + """ + return {} diff --git a/deploy/sql/init.sql b/deploy/sql/init.sql index 9d64ab2423..cc939d923c 100644 --- a/deploy/sql/init.sql +++ b/deploy/sql/init.sql @@ -718,3 +718,30 @@ FOR EACH ROW EXECUTE FUNCTION nexent.update_memory_retrieval_hits_update_time(); COMMENT ON TRIGGER update_memory_retrieval_hits_update_time_trigger ON nexent.memory_retrieval_hits_t IS 'Trigger to call update_memory_retrieval_hits_update_time function before each update on memory_retrieval_hits_t table'; +-- Manual Dreaming run audit. Scope concurrency uses PostgreSQL transaction +-- advisory locks, so no persistent lock row is required. +CREATE TABLE IF NOT EXISTS nexent.memory_dreaming_audit_t ( + run_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + agent_id VARCHAR(100) NOT NULL, + trigger_source VARCHAR(30) NOT NULL DEFAULT 'manual', + status VARCHAR(30) NOT NULL DEFAULT 'running', + current_phase VARCHAR(30), + started_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + finished_at TIMESTAMP, + light_count INTEGER NOT NULL DEFAULT 0, + rem_count INTEGER NOT NULL DEFAULT 0, + promoted_count INTEGER NOT NULL DEFAULT 0, + deferred_count INTEGER NOT NULL DEFAULT 0, + result_json JSONB, + error TEXT, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) NOT NULL DEFAULT 'N' +); +CREATE INDEX IF NOT EXISTS idx_memory_dreaming_audit_scope + ON nexent.memory_dreaming_audit_t + (tenant_id, user_id, agent_id, started_at DESC); diff --git a/deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_audit.sql b/deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_audit.sql new file mode 100644 index 0000000000..4986a2b959 --- /dev/null +++ b/deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_audit.sql @@ -0,0 +1,33 @@ +-- Manual Dreaming run audit. Advisory locks are transaction-scoped and +-- therefore require no persistent lock table. +SET search_path TO nexent; +BEGIN; + +CREATE TABLE IF NOT EXISTS nexent.memory_dreaming_audit_t ( + run_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + agent_id VARCHAR(100) NOT NULL, + trigger_source VARCHAR(30) NOT NULL DEFAULT 'manual', + status VARCHAR(30) NOT NULL DEFAULT 'running', + current_phase VARCHAR(30), + started_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + finished_at TIMESTAMP, + light_count INTEGER NOT NULL DEFAULT 0, + rem_count INTEGER NOT NULL DEFAULT 0, + promoted_count INTEGER NOT NULL DEFAULT 0, + deferred_count INTEGER NOT NULL DEFAULT 0, + result_json JSONB, + error TEXT, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) NOT NULL DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS idx_memory_dreaming_audit_scope + ON nexent.memory_dreaming_audit_t + (tenant_id, user_id, agent_id, started_at DESC); + +COMMIT; diff --git a/sdk/nexent/memory/dreaming/__init__.py b/sdk/nexent/memory/dreaming/__init__.py new file mode 100644 index 0000000000..589f3f2bd8 --- /dev/null +++ b/sdk/nexent/memory/dreaming/__init__.py @@ -0,0 +1,23 @@ +"""Storage-independent Dreaming consolidation primitives.""" + +from .models import ( + DreamingCandidate, + DreamingDecision, + DreamingMetrics, + DreamingThresholds, +) +from .scoring import compute_metrics, score_candidate, select_candidates +from .service import analyze_rem_content, build_candidate + + +__all__ = [ + "DreamingCandidate", + "DreamingDecision", + "DreamingMetrics", + "DreamingThresholds", + "analyze_rem_content", + "build_candidate", + "compute_metrics", + "score_candidate", + "select_candidates", +] diff --git a/sdk/nexent/memory/dreaming/models.py b/sdk/nexent/memory/dreaming/models.py new file mode 100644 index 0000000000..1c46a3b506 --- /dev/null +++ b/sdk/nexent/memory/dreaming/models.py @@ -0,0 +1,58 @@ +"""Models shared by the three Dreaming phases.""" + +from __future__ import annotations + +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, Field + + +class DreamingCandidate(BaseModel): + memory_id: int + tenant_id: str + user_id: str + agent_id: str + content: str + recall_count: int = 0 + daily_count: int = 0 + grounded_count: int = 0 + total_retrieval_score: float = 0.0 + query_hashes: List[str] = Field(default_factory=list) + recall_days: List[str] = Field(default_factory=list) + concept_tags: List[str] = Field(default_factory=list) + light_hits: int = 0 + rem_hits: int = 0 + last_recalled_at: Optional[datetime] = None + last_light_at: Optional[datetime] = None + last_rem_at: Optional[datetime] = None + noise: bool = False + already_promoted: bool = False + + +class DreamingMetrics(BaseModel): + signal_count: int + context_diversity: int + frequency: float + relevance: float + query_diversity: float + recency: float + consolidation: float + conceptual_richness: float + phase_boost: float + + +class DreamingThresholds(BaseModel): + min_score: float = 0.72 + min_recall_count: int = 3 + min_unique_queries: int = 2 + include_promoted: bool = False + + +class DreamingDecision(BaseModel): + candidate: DreamingCandidate + metrics: DreamingMetrics + score: float + promote: bool + reason: str + archive_suggested: bool = False diff --git a/sdk/nexent/memory/dreaming/scoring.py b/sdk/nexent/memory/dreaming/scoring.py new file mode 100644 index 0000000000..7eafaa9c49 --- /dev/null +++ b/sdk/nexent/memory/dreaming/scoring.py @@ -0,0 +1,146 @@ +"""OpenClaw-compatible Deep Sleep scoring and deterministic selection.""" + +from __future__ import annotations + +import math +from datetime import datetime +from typing import Iterable, List, Optional + +from .models import ( + DreamingCandidate, + DreamingDecision, + DreamingMetrics, + DreamingThresholds, +) + + +WEIGHTS = { + "frequency": 0.24, + "relevance": 0.30, + "query_diversity": 0.15, + "recency": 0.15, + "consolidation": 0.10, + "conceptual_richness": 0.06, +} + + +def clamp_score(value: float) -> float: + return max(0.0, min(1.0, value)) + + +def _age_days(value: Optional[datetime], now: datetime) -> float: + if value is None: + return float("inf") + return max(0.0, (now - value).total_seconds() / 86400.0) + + +def _recency(value: Optional[datetime], now: datetime, half_life_days: float) -> float: + if value is None: + return 0.0 + decay_lambda = math.log(2) / max(1.0, half_life_days) + return clamp_score(math.exp(-decay_lambda * _age_days(value, now))) + + +def compute_metrics( + candidate: DreamingCandidate, + *, + now: Optional[datetime] = None, + recency_half_life_days: float = 14, +) -> DreamingMetrics: + now = now or datetime.utcnow() + signal_count = max(0, candidate.recall_count) + max(0, candidate.daily_count) + max(0, candidate.grounded_count) + unique_queries = len(set(candidate.query_hashes)) + unique_days = len(set(candidate.recall_days)) + context_diversity = max(unique_queries, unique_days) + frequency = clamp_score(math.log1p(signal_count) / math.log1p(10)) + relevance = clamp_score(candidate.total_retrieval_score / max(1, signal_count)) + query_diversity = clamp_score(context_diversity / 5) + recency = _recency(candidate.last_recalled_at, now, recency_half_life_days) + + if unique_days == 0: + consolidation = 0.0 + elif unique_days == 1: + consolidation = 0.2 + else: + parsed_days = sorted(datetime.fromisoformat(day).date() for day in set(candidate.recall_days)) + span_days = (parsed_days[-1] - parsed_days[0]).days + spacing = clamp_score(math.log1p(unique_days - 1) / math.log1p(4)) + span = clamp_score(span_days / 7) + consolidation = max( + clamp_score(0.55 * spacing + 0.45 * span), + clamp_score(candidate.grounded_count / 3), + ) + + conceptual_richness = clamp_score(len(set(candidate.concept_tags)) / 6) + light_strength = clamp_score(math.log1p(max(0, candidate.light_hits)) / math.log1p(6)) + rem_strength = clamp_score(math.log1p(max(0, candidate.rem_hits)) / math.log1p(6)) + phase_boost = clamp_score( + 0.06 * light_strength * _recency(candidate.last_light_at, now, 14) + + 0.09 * rem_strength * _recency(candidate.last_rem_at, now, 14) + ) + return DreamingMetrics( + signal_count=signal_count, + context_diversity=context_diversity, + frequency=frequency, + relevance=relevance, + query_diversity=query_diversity, + recency=recency, + consolidation=consolidation, + conceptual_richness=conceptual_richness, + phase_boost=phase_boost, + ) + + +def score_candidate( + candidate: DreamingCandidate, + *, + now: Optional[datetime] = None, + recency_half_life_days: float = 14, +) -> tuple[float, DreamingMetrics]: + metrics = compute_metrics(candidate, now=now, recency_half_life_days=recency_half_life_days) + score = sum(getattr(metrics, name) * weight for name, weight in WEIGHTS.items()) + return clamp_score(score + metrics.phase_boost), metrics + + +def select_candidates( + candidates: Iterable[DreamingCandidate], + *, + thresholds: Optional[DreamingThresholds] = None, + now: Optional[datetime] = None, + recency_half_life_days: float = 14, +) -> List[DreamingDecision]: + thresholds = thresholds or DreamingThresholds() + decisions: List[DreamingDecision] = [] + for candidate in candidates: + score, metrics = score_candidate(candidate, now=now, recency_half_life_days=recency_half_life_days) + reason = "eligible" + promote = True + if candidate.noise: + promote, reason = False, "noise" + elif candidate.already_promoted and not thresholds.include_promoted: + promote, reason = False, "already_promoted" + elif score < thresholds.min_score: + promote, reason = False, "score_below_threshold" + elif metrics.signal_count < thresholds.min_recall_count: + promote, reason = False, "recall_below_threshold" + elif metrics.context_diversity < thresholds.min_unique_queries: + promote, reason = False, "diversity_below_threshold" + decisions.append( + DreamingDecision( + candidate=candidate, + metrics=metrics, + score=score, + promote=promote, + reason=reason, + archive_suggested=promote, + ) + ) + return sorted( + decisions, + key=lambda item: ( + not item.promote, + -item.score, + -item.metrics.signal_count, + item.candidate.memory_id, + ), + ) diff --git a/sdk/nexent/memory/dreaming/service.py b/sdk/nexent/memory/dreaming/service.py new file mode 100644 index 0000000000..2d67e623d5 --- /dev/null +++ b/sdk/nexent/memory/dreaming/service.py @@ -0,0 +1,63 @@ +"""Lightweight REM analysis and candidate construction.""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Tuple + +from .models import DreamingCandidate + + +NOISE_PATTERNS = ( + r"\b(todo|today|temporary|this session|current task)\b", + r"(今日|今天|待办|临时|本轮|当前任务)", +) +CONCEPT_PATTERNS = { + "preference": (r"\b(prefer|preference|always use|likes?)\b", r"(偏好|喜欢|习惯|总是使用)"), + "persistent": (r"\b(always|persist|long[- ]term|stable)\b", r"(长期|稳定|持续|固定)"), + "build": (r"\b(build|compile|deploy|package)\b", r"(构建|编译|部署|打包)"), + "failure": (r"\b(error|failure|failed|exception|bug)\b", r"(错误|失败|异常|故障)"), + "transaction": (r"\b(transaction|commit|rollback|atomic)\b", r"(事务|提交|回滚|原子)"), + "routing": (r"\b(route|routing|gateway|proxy)\b", r"(路由|网关|代理)"), +} + + +def analyze_rem_content(content: str) -> Tuple[List[str], bool]: + normalized = " ".join(content.split()).lower() + noise = any(re.search(pattern, normalized, re.IGNORECASE) for pattern in NOISE_PATTERNS) + tags = [ + tag + for tag, patterns in CONCEPT_PATTERNS.items() + if any(re.search(pattern, normalized, re.IGNORECASE) for pattern in patterns) + ] + word_tags = re.findall(r"[\u4e00-\u9fff]{2,8}|[a-z][a-z0-9_-]{2,}", normalized) + for tag in word_tags: + if tag not in tags and len(tags) < 8: + tags.append(tag) + return tags, noise + + +def build_candidate(record: Dict[str, Any], total_retrieval_score: float) -> DreamingCandidate: + tags, noise = analyze_rem_content(str(record.get("content") or "")) + merged_tags = list(dict.fromkeys([*(record.get("concept_tags") or []), *tags])) + return DreamingCandidate( + memory_id=int(record["memory_id"]), + tenant_id=str(record["tenant_id"]), + user_id=str(record["user_id"]), + agent_id=str(record["agent_id"]), + content=str(record.get("content") or ""), + recall_count=int(record.get("recall_count") or 0), + daily_count=int(record.get("daily_count") or 0), + grounded_count=int(record.get("grounded_count") or 0), + total_retrieval_score=float(total_retrieval_score or 0), + query_hashes=list(record.get("query_hashes") or []), + recall_days=list(record.get("recall_days") or []), + concept_tags=merged_tags, + light_hits=int(record.get("light_hits") or 0), + rem_hits=int(record.get("rem_hits") or 0), + last_recalled_at=record.get("last_recalled_at"), + last_light_at=record.get("last_light_at"), + last_rem_at=record.get("last_rem_at"), + noise=noise, + already_promoted=bool(record.get("already_promoted", False)), + ) diff --git a/sdk/nexent/memory/memory_service.py b/sdk/nexent/memory/memory_service.py new file mode 100644 index 0000000000..f80dc27ad1 --- /dev/null +++ b/sdk/nexent/memory/memory_service.py @@ -0,0 +1,28 @@ +"""Legacy import bridge removed by the new Memory architecture. + +The target branch still imports these names from agent code. Keeping explicit +errors here lets applications start while preventing the removed Mem0/local-ES +implementation from silently appearing to persist data. Callers already +degrade on these exceptions; new code must use ``nexent.memory.service`` with +backend hooks. +""" + + +class LegacyMemoryApiRemoved(RuntimeError): + pass + + +async def add_memory_in_levels(**_kwargs): + raise LegacyMemoryApiRemoved( + "add_memory_in_levels was removed; use MemoryService.store_memory with a backend hook" + ) + + +async def search_memory_in_levels(**_kwargs): + raise LegacyMemoryApiRemoved( + "search_memory_in_levels was removed; use MemoryService.search_memory with a backend hook" + ) + + +async def clear_memory(**_kwargs): + raise LegacyMemoryApiRemoved("clear_memory was removed; use the backend memory record service") diff --git a/test/backend/apps/test_memory_dreaming_app.py b/test/backend/apps/test_memory_dreaming_app.py new file mode 100644 index 0000000000..f94a1ff1f3 --- /dev/null +++ b/test/backend/apps/test_memory_dreaming_app.py @@ -0,0 +1,86 @@ +from unittest.mock import MagicMock + +import httpx +import pytest +from fastapi import FastAPI + +from apps import memory_dreaming_app + + +def app(): + app = FastAPI() + app.include_router(memory_dreaming_app.router) + return app + + +@pytest.fixture +async def client(): + transport = httpx.ASGITransport(app=app()) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as value: + yield value + + +@pytest.mark.asyncio +async def test_ac009_missing_agent_id_is_422(client): + response = await client.post("/memory/dreaming/run", json={}) + assert response.status_code == 422 + + +@pytest.mark.asyncio +async def test_ac009_run_uses_authenticated_scope(monkeypatch, client): + service = MagicMock() + service.run.return_value = {"run_id": 1, "status": "completed"} + monkeypatch.setattr( + memory_dreaming_app, "get_memory_dreaming_service", lambda: service + ) + monkeypatch.setattr( + memory_dreaming_app, + "get_current_user_id", + lambda _authorization: ("user-1", "tenant-1"), + ) + response = await client.post( + "/memory/dreaming/run", + headers={"Authorization": "Bearer token"}, + json={"agent_id": "agent-1"}, + ) + assert response.status_code == 200 + assert response.json()["status"] == "completed" + assert service.run.call_args.kwargs["tenant_id"] == "tenant-1" + assert service.run.call_args.kwargs["user_id"] == "user-1" + assert service.run.call_args.kwargs["agent_id"] == "agent-1" + + +@pytest.mark.asyncio +async def test_ac009_audit_uses_authenticated_scope(monkeypatch, client): + service = MagicMock() + service.list_audits.return_value = [{"run_id": 2}] + monkeypatch.setattr( + memory_dreaming_app, "get_memory_dreaming_service", lambda: service + ) + monkeypatch.setattr( + memory_dreaming_app, + "get_current_user_id", + lambda _authorization: ("user-2", "tenant-2"), + ) + response = await client.get("/memory/dreaming/audit?agent_id=agent-2&run_id=2") + assert response.status_code == 200 + service.list_audits.assert_called_once_with( + "tenant-2", "user-2", agent_id="agent-2", run_id=2, limit=100 + ) + + +@pytest.mark.asyncio +async def test_ac008_service_failure_maps_to_500(monkeypatch, client): + service = MagicMock() + service.run.side_effect = memory_dreaming_app.DreamingRunError("failed") + monkeypatch.setattr( + memory_dreaming_app, "get_memory_dreaming_service", lambda: service + ) + monkeypatch.setattr( + memory_dreaming_app, + "get_current_user_id", + lambda _authorization: ("user", "tenant"), + ) + response = await client.post("/memory/dreaming/run", json={"agent_id": "agent"}) + assert response.status_code == 500 + assert response.json()["detail"] == "failed" diff --git a/test/backend/database/test_memory_dreaming_schema.py b/test/backend/database/test_memory_dreaming_schema.py new file mode 100644 index 0000000000..cc2500141f --- /dev/null +++ b/test/backend/database/test_memory_dreaming_schema.py @@ -0,0 +1,84 @@ +from pathlib import Path +from datetime import datetime + +from database import memory_retrieval_hit_db +from database.db_models import MemoryDreamingAudit, MemoryRecord, MemoryRetrievalHit +from database.memory_dreaming_db import advisory_lock_key + + +def test_ac010_orm_contract(): + assert MemoryRecord.__tablename__ == "memory_records_t" + assert MemoryRetrievalHit.__tablename__ == "memory_retrieval_hits_t" + columns = MemoryDreamingAudit.__table__.columns + for name in ( + "run_id", + "tenant_id", + "user_id", + "agent_id", + "status", + "current_phase", + "result_json", + "error", + ): + assert name in columns + + +def test_ac007_lock_key_is_stable_and_scope_specific(): + key = advisory_lock_key("tenant", "user", "agent") + assert key == advisory_lock_key("tenant", "user", "agent") + assert key != advisory_lock_key("tenant", "user", "other-agent") + assert -(2**63) <= key < 2**63 + + +def test_ac010_migration_and_fresh_install_match(): + root = Path(__file__).resolve().parents[3] + migration = ( + root / "deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_audit.sql" + ).read_text() + init_sql = (root / "deploy/sql/init.sql").read_text() + for token in ( + "memory_dreaming_audit_t", + "idx_memory_dreaming_audit_scope", + "result_json", + "promoted_count", + ): + assert token in migration + assert token in init_sql + assert "CREATE TABLE IF NOT EXISTS" in migration + assert "CREATE INDEX IF NOT EXISTS" in migration + + +def test_ac002_dreaming_stats_filter_agent_scope(monkeypatch): + monkeypatch.setattr( + memory_retrieval_hit_db, + "list_hits_for_user", + lambda *_args, **_kwargs: [ + { + "agent_id": "agent-1", + "memory_id": 1, + "day": "2026-07-22", + "query_hash": "q1", + "retrieval_score": 0.75, + "grounded": True, + "occurred_at": datetime(2026, 7, 22, 12), + }, + { + "agent_id": "agent-2", + "memory_id": 2, + "day": "2026-07-22", + "query_hash": "q2", + "retrieval_score": 1.0, + "grounded": True, + "occurred_at": datetime(2026, 7, 22, 13), + }, + ], + ) + rows = memory_retrieval_hit_db.aggregate_dreaming_stats( + "tenant", + "user", + "agent-1", + since=datetime(2026, 7, 20), + ) + assert len(rows) == 1 + assert rows[0]["memory_id"] == 1 + assert rows[0]["total_retrieval_score"] == 0.75 diff --git a/test/backend/services/test_memory_dreaming_scheduler.py b/test/backend/services/test_memory_dreaming_scheduler.py deleted file mode 100644 index 966ffb2be2..0000000000 --- a/test/backend/services/test_memory_dreaming_scheduler.py +++ /dev/null @@ -1,425 +0,0 @@ -"""Unit tests for ``backend.services.memory_dreaming_scheduler`` (Phase 2).""" - -import sys -import types -from datetime import datetime, timedelta -from unittest.mock import MagicMock - -import pytest - - -# Path setup -sys.path.insert( - 0, - __import__("os").path.join(__import__("os").path.dirname(__file__), "../../.."), -) - - -# Stub consts -consts_pkg = types.ModuleType("consts") -consts_pkg.AGENT_SHORT_TERM_HALF_LIFE_DAYS = 14 -consts_pkg.LIGHT_SLEEP_WINDOW_DAYS = 7 -consts_pkg.MIN_PROMOTION_SCORE = 0.72 -consts_pkg.MIN_RECALL_COUNT = 3 -consts_pkg.MIN_UNIQUE_QUERIES = 2 -consts_pkg.RECENCY_HALF_LIFE_DAYS = 14 -consts_mod = types.ModuleType("consts.const") -for name, value in vars(consts_pkg).items(): - if not name.startswith("_"): - setattr(consts_mod, name, value) -sys.modules["consts"] = types.ModuleType("consts") -sys.modules["consts.const"] = consts_mod - - -# Stub database -database_pkg = types.ModuleType("database") -database_pkg.memory_record_db = MagicMock(name="memory_record_db") -database_pkg.memory_retrieval_hit_db = MagicMock(name="memory_retrieval_hit_db") -sys.modules["database"] = database_pkg -sys.modules["backend.database"] = database_pkg - - -# Stub services.memory_record_service -memory_record_service_mod = types.ModuleType("services.memory_record_service") -memory_record_service_mod.MemoryRecordError = type("MemoryRecordError", (Exception,), {}) - - -class _RecordService: - pass - - -memory_record_service_mod.MemoryRecordService = _RecordService -memory_record_service_mod.get_memory_record_service = MagicMock( - name="get_memory_record_service" -) -sys.modules["services.memory_record_service"] = memory_record_service_mod - - -from backend.services import memory_dreaming_scheduler - - -def test_compute_promotion_score_no_signal(): - score = memory_dreaming_scheduler.compute_promotion_score({}) - assert 0.0 <= score <= 1.0 - - -def test_compute_promotion_score_increases_with_recall(): - low = memory_dreaming_scheduler.compute_promotion_score( - {"recall_count": 1, "daily_count": 1, "grounded_count": 1, "light_hits": 0, - "rem_hits": 0, "last_recalled_at": datetime.utcnow(), "concept_tags": [], - "query_hashes": []} - ) - high = memory_dreaming_scheduler.compute_promotion_score( - {"recall_count": 12, "daily_count": 5, "grounded_count": 4, - "light_hits": 3, "rem_hits": 3, - "last_recalled_at": datetime.utcnow(), "concept_tags": ["python"], - "query_hashes": ["a", "b", "c", "d", "e"]} - ) - assert high > low - - -def test_run_light_sleep_aggregates_into_rows(): - memory_dreaming_scheduler.memory_retrieval_hit_db.aggregate_memory_stats.return_value = [ - { - "memory_id": 1, - "hit_count": 5, - "grounded_count": 2, - "days": {"2026-07-13", "2026-07-12"}, - "query_hashes": {"q1", "q2"}, - } - ] - memory_dreaming_scheduler.memory_record_db.update_memory_record.return_value = True - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.return_value = True - - touched = memory_dreaming_scheduler.run_light_sleep( - tenant_id="t1", user_id="u1" - ) - - assert touched == 1 - memory_dreaming_scheduler.memory_record_db.update_memory_record.assert_called_once() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.assert_called_once_with( - 1, "t1", phase="light" - ) - - -def test_run_rem_sleep_writes_concept_tags(): - memory_dreaming_scheduler.memory_record_db.list_memory_records.return_value = [ - { - "memory_id": 1, - "tenant_id": "t1", - "user_id": "u1", - "content": "Python Python Java Python Java C++", - "layer": "agent", - "memory_type": "short_term", - "concept_tags": [], - } - ] - memory_dreaming_scheduler.memory_record_db.update_memory_record.return_value = True - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.return_value = True - - touched = memory_dreaming_scheduler.run_rem_sleep( - tenant_id="t1", user_id="u1" - ) - - assert touched == 1 - # Update payload should carry the new tags. - update_call = memory_dreaming_scheduler.memory_record_db.update_memory_record.call_args - payload = update_call.args[2] - assert "python" in payload["concept_tags"] - assert "java" in payload["concept_tags"] - - -def test_run_deep_sleep_skips_low_signal(): - memory_dreaming_scheduler.memory_record_db.list_memories_for_dreaming.return_value = [ - { - "memory_id": 1, - "content": "low signal", - "layer": "agent", - "recall_count": 1, - "daily_count": 0, - "grounded_count": 0, - "query_hashes": ["q1"], - "concept_tags": [], - "last_recalled_at": datetime.utcnow(), - "light_hits": 0, - "rem_hits": 0, - } - ] - memory_record_service_mod.get_memory_record_service.return_value.create_memory.return_value = { - "memory_id": 999, - "event": "ADD", - } - - promoted = memory_dreaming_scheduler.run_deep_sleep( - tenant_id="t1", user_id="u1", min_score=0.99 - ) - - assert promoted == [] - memory_record_service_mod.get_memory_record_service.return_value.create_memory.assert_not_called() - - -def test_run_deep_sleep_promotes_high_signal(): - memory_dreaming_scheduler.memory_record_db.list_memories_for_dreaming.return_value = [ - { - "memory_id": 1, - "content": "user prefers dark mode", - "layer": "agent", - "recall_count": 8, - "daily_count": 4, - "grounded_count": 2, - "query_hashes": ["q1", "q2", "q3"], - "concept_tags": ["preference"], - "last_recalled_at": datetime.utcnow(), - "light_hits": 2, - "rem_hits": 1, - "agent_id": "a1", - "conversation_id": "c1", - } - ] - memory_record_service_mod.get_memory_record_service.return_value.create_memory.return_value = { - "memory_id": 999, - "event": "ADD", - } - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.return_value = True - - promoted = memory_dreaming_scheduler.run_deep_sleep( - tenant_id="t1", user_id="u1", min_score=0.5 - ) - - assert len(promoted) == 1 - create_kwargs = memory_record_service_mod.get_memory_record_service.return_value.create_memory.call_args.kwargs - assert create_kwargs["layer"] == "user" - assert create_kwargs["memory_type"] == "long_term" - assert create_kwargs["actor"] == "dreaming" - - -def test_scoring_helpers_cover_bounds_and_decay(mocker): - assert memory_dreaming_scheduler._clamp01(-1.0) == 0.0 - assert memory_dreaming_scheduler._clamp01(2.0) == 1.0 - assert memory_dreaming_scheduler._clamp01(0.5) == 0.5 - assert memory_dreaming_scheduler._relevance(0, 10.0) == 0.0 - assert memory_dreaming_scheduler._relevance(2, 3.0) == 1.0 - assert memory_dreaming_scheduler._diversity(100) == 1.0 - assert memory_dreaming_scheduler._consolidation(0, 0) == 0.0 - assert memory_dreaming_scheduler._concept([]) == 0.0 - assert memory_dreaming_scheduler._phase_boost(0, 1) == 0.0 - assert memory_dreaming_scheduler._phase_boost(10, 10) == 0.05 - assert memory_dreaming_scheduler._normalize_weights({}) == {} - - future = datetime.utcnow() + timedelta(days=1) - old = datetime.utcnow() - timedelta(days=14) - assert memory_dreaming_scheduler._recency(future) == 1.0 - assert 0.49 < memory_dreaming_scheduler._recency(old) < 0.51 - assert memory_dreaming_scheduler._recency(None) == 0.0 - mocker.patch.object(memory_dreaming_scheduler, "RECENCY_HALF_LIFE_DAYS", 0) - assert 0.0 < memory_dreaming_scheduler._recency(old) <= 1.0 - - -def test_run_light_sleep_handles_empty_hit_days(): - memory_dreaming_scheduler.memory_retrieval_hit_db.aggregate_memory_stats.reset_mock() - memory_dreaming_scheduler.memory_record_db.update_memory_record.reset_mock() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.reset_mock() - memory_dreaming_scheduler.memory_retrieval_hit_db.aggregate_memory_stats.return_value = [ - { - "memory_id": 2, - "hit_count": 1, - "grounded_count": 0, - "days": set(), - "query_hashes": set(), - } - ] - - assert memory_dreaming_scheduler.run_light_sleep( - tenant_id="t1", user_id="u1", window_days=0 - ) == 1 - payload = memory_dreaming_scheduler.memory_record_db.update_memory_record.call_args.args[2] - assert payload["last_recalled_at"] is None - assert payload["query_hashes"] == [] - assert payload["recall_days"] == [] - - -def test_run_rem_sleep_skips_empty_content_and_merges_limited_tags(): - memory_dreaming_scheduler.memory_record_db.list_memory_records.reset_mock() - memory_dreaming_scheduler.memory_record_db.update_memory_record.reset_mock() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.reset_mock() - memory_dreaming_scheduler.memory_record_db.list_memory_records.return_value = [ - {"memory_id": 1, "content": "the and"}, - { - "memory_id": 2, - "content": "Python Python Java Java Rust", - "concept_tags": ["existing"], - }, - ] - - assert memory_dreaming_scheduler.run_rem_sleep( - tenant_id="t1", user_id="u1", max_keywords=2 - ) == 1 - payload = memory_dreaming_scheduler.memory_record_db.update_memory_record.call_args.args[2] - assert payload["concept_tags"] == ["existing", "python"] - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.assert_called_once_with( - 2, "t1", phase="rem" - ) - - -def test_run_deep_sleep_skips_duplicate_queries_and_promotion_errors(): - memory_dreaming_scheduler.memory_record_db.list_memories_for_dreaming.reset_mock() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.reset_mock() - service = memory_record_service_mod.get_memory_record_service.return_value - service.create_memory.reset_mock() - service.create_memory.side_effect = memory_record_service_mod.MemoryRecordError( - "already promoted" - ) - memory_dreaming_scheduler.memory_record_db.list_memories_for_dreaming.return_value = [ - { - "memory_id": 1, - "query_hashes": ["same", "same"], - "recall_count": 10, - }, - { - "memory_id": 2, - "query_hashes": ["q1", "q2"], - "recall_count": 10, - "daily_count": 5, - "grounded_count": 2, - "concept_tags": ["tag"], - "last_recalled_at": datetime.utcnow(), - "light_hits": 2, - "rem_hits": 2, - "content": "promote me", - }, - ] - - assert memory_dreaming_scheduler.run_deep_sleep( - tenant_id="t1", user_id="u1", min_score=0.0, min_unique_queries=2 - ) == [] - service.create_memory.assert_called_once() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.assert_not_called() - - -def test_run_once_aggregates_tenants_and_handles_iteration_errors(mocker): - mocker.patch.object( - memory_dreaming_scheduler, "list_distinct_tenants", - return_value=[("t1", "u1"), ("t2", "u2")], - ) - light = mocker.patch.object( - memory_dreaming_scheduler, "run_light_sleep", side_effect=[2, RuntimeError("db")] - ) - rem = mocker.patch.object(memory_dreaming_scheduler, "run_rem_sleep", return_value=3) - deep = mocker.patch.object( - memory_dreaming_scheduler, "run_deep_sleep", - return_value=[{"memory_id": 8, "score": 0.8, "event": "PROMOTE"}], - ) - - summary = memory_dreaming_scheduler.run_once(timeout_seconds=60) - - assert summary["tenants"] == 2 - assert summary["light_rows"] == 2 - assert summary["rem_rows"] == 3 - assert summary["promotions"] == [{"memory_id": 8, "score": 0.8, "event": "PROMOTE"}] - assert light.call_count == 2 - rem.assert_called_once_with(tenant_id="t1", user_id="u1") - deep.assert_called_once_with(tenant_id="t1", user_id="u1") - - -def test_run_once_stops_before_work_when_deadline_reached(mocker): - mocker.patch.object( - memory_dreaming_scheduler, "list_distinct_tenants", - return_value=[("t1", "u1")], - ) - mocker.patch.object( - memory_dreaming_scheduler.time, "time", side_effect=[100.0, 2000.0, 2000.0, 2000.0] - ) - light = mocker.patch.object(memory_dreaming_scheduler, "run_light_sleep") - - summary = memory_dreaming_scheduler.run_once(timeout_seconds=1) - - assert summary["tenants"] == 1 - assert summary["light_rows"] == 0 - assert summary["rem_rows"] == 0 - assert summary["promotions"] == [] - light.assert_not_called() - - -def test_list_distinct_tenants_returns_filtered_pairs_and_handles_errors(monkeypatch): - class Column: - def isnot(self, value): - return (self, value) - - class MemoryRetrievalHit: - tenant_id = Column() - user_id = Column() - - class Query: - def filter(self, *conditions): - self.conditions = conditions - return self - - def all(self): - return [("t1", "u1"), ("", "u2"), ("t2", None), ("t3", "u3")] - - class Session: - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def query(self, *columns): - self.columns = columns - return Query() - - client = types.ModuleType("database.client") - client.get_db_session = lambda: Session() - db_models = types.ModuleType("database.db_models") - db_models.MemoryRetrievalHit = MemoryRetrievalHit - sqlalchemy = types.ModuleType("sqlalchemy") - sqlalchemy.distinct = lambda value: ("distinct", value) - monkeypatch.setitem(sys.modules, "database.client", client) - monkeypatch.setitem(sys.modules, "database.db_models", db_models) - monkeypatch.setitem(sys.modules, "sqlalchemy", sqlalchemy) - - assert memory_dreaming_scheduler.list_distinct_tenants() == [("t1", "u1"), ("t3", "u3")] - - client.get_db_session = MagicMock(side_effect=RuntimeError("db unavailable")) - assert memory_dreaming_scheduler.list_distinct_tenants() == [] - - -def test_run_rem_sleep_skips_when_keyword_limit_is_zero(): - memory_dreaming_scheduler.memory_record_db.list_memory_records.reset_mock() - memory_dreaming_scheduler.memory_record_db.update_memory_record.reset_mock() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.reset_mock() - memory_dreaming_scheduler.memory_record_db.list_memory_records.return_value = [ - {"memory_id": 3, "content": "python java"}, - ] - - assert memory_dreaming_scheduler.run_rem_sleep( - tenant_id="t1", user_id="u1", max_keywords=0 - ) == 0 - memory_dreaming_scheduler.memory_record_db.update_memory_record.assert_not_called() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.assert_not_called() - - -def test_run_deep_sleep_skips_record_below_score_threshold(): - memory_dreaming_scheduler.memory_record_db.list_memories_for_dreaming.reset_mock() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.reset_mock() - service = memory_record_service_mod.get_memory_record_service.return_value - service.create_memory.reset_mock() - memory_dreaming_scheduler.memory_record_db.list_memories_for_dreaming.return_value = [ - { - "memory_id": 4, - "query_hashes": ["q1", "q2"], - "recall_count": 1, - "daily_count": 0, - "grounded_count": 0, - "concept_tags": [], - "last_recalled_at": None, - "light_hits": 0, - "rem_hits": 0, - }, - ] - - assert memory_dreaming_scheduler.run_deep_sleep( - tenant_id="t1", user_id="u1", min_score=0.99 - ) == [] - service.create_memory.assert_not_called() - memory_dreaming_scheduler.memory_record_db.apply_dreaming_phase.assert_not_called() diff --git a/test/backend/services/test_memory_dreaming_service.py b/test/backend/services/test_memory_dreaming_service.py new file mode 100644 index 0000000000..5cea0f474b --- /dev/null +++ b/test/backend/services/test_memory_dreaming_service.py @@ -0,0 +1,143 @@ +from contextlib import contextmanager +from datetime import datetime +from unittest.mock import MagicMock + +import pytest + +from services.memory_dreaming_service import DreamingRunError, MemoryDreamingService + + +@contextmanager +def lock(value): + yield value + + +def test_ac007_lock_busy_skips(monkeypatch): + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.create_audit", + lambda *_: 41, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.try_scope_lock", + lambda *_: lock(False), + ) + finish = MagicMock() + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.finish_audit", finish + ) + result = MemoryDreamingService(record_service=MagicMock()).run( + tenant_id="t", user_id="u", agent_id="a" + ) + assert result == {"run_id": 41, "status": "skipped", "reason": "lock_busy"} + finish.assert_called_once() + + +def test_ac001_ac006_full_run_and_idempotency_key(monkeypatch): + record = { + "memory_id": 7, + "tenant_id": "t", + "user_id": "u", + "agent_id": "a", + "content": "Always prefer stable transaction rollback behavior", + "recall_count": 3, + "daily_count": 2, + "grounded_count": 1, + "last_recalled_at": datetime.utcnow().isoformat(), + "query_hashes": ["q1", "q2"], + "recall_days": ["2026-07-22", "2026-07-23"], + "light_hits": 2, + "rem_hits": 2, + "last_light_at": datetime.utcnow().isoformat(), + "last_rem_at": datetime.utcnow().isoformat(), + "concept_tags": [], + } + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.create_audit", + lambda *_: 42, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.try_scope_lock", + lambda *_: lock(True), + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.update_audit", + lambda *_args, **_kwargs: True, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.finish_audit", + lambda *_args, **_kwargs: True, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_retrieval_hit_db.aggregate_dreaming_stats", + lambda *_args, **_kwargs: [ + { + "memory_id": 7, + "hit_count": 4, + "grounded_count": 1, + "days": {"2026-07-22", "2026-07-23"}, + "query_hashes": {"q1", "q2"}, + "total_retrieval_score": 3.8, + "last_recalled_at": datetime.utcnow(), + } + ], + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_record_db.list_memory_records", + lambda *_args, **_kwargs: [record], + ) + update_record = MagicMock(return_value=True) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_record_db.update_memory_record", + update_record, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_record_db.apply_dreaming_phase", + lambda *_args, **_kwargs: True, + ) + record_service = MagicMock() + record_service.create_memory.return_value = {"event": "ADD"} + result = MemoryDreamingService(record_service=record_service).run( + tenant_id="t", + user_id="u", + agent_id="a", + min_score=0, + min_recall_count=0, + min_unique_queries=0, + ) + assert result["status"] == "completed" + assert result["light_count"] == 1 + assert result["promoted_count"] == 1 + light_payload = update_record.call_args_list[0].args[2] + assert light_payload["recall_count"] == 4 + assert light_payload["daily_count"] == 2 + assert light_payload["grounded_count"] == 1 + assert light_payload["query_hashes"] == ["q1", "q2"] + assert ( + record_service.create_memory.call_args.kwargs["idempotency_key"] == "dreaming:7" + ) + assert record_service.create_memory.call_args.kwargs["layer"] == "user" + + +def test_ac008_failure_is_audited(monkeypatch): + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.create_audit", + lambda *_: 43, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.try_scope_lock", + lambda *_: lock(True), + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_retrieval_hit_db.aggregate_dreaming_stats", + MagicMock(side_effect=ValueError("bad data")), + ) + finish = MagicMock() + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.finish_audit", finish + ) + with pytest.raises(DreamingRunError): + MemoryDreamingService(record_service=MagicMock()).run( + tenant_id="t", user_id="u", agent_id="a" + ) + assert finish.call_args.kwargs["status"] == "failed" + assert "ValueError" in finish.call_args.kwargs["error"] diff --git a/test/sdk/memory/test_dreaming.py b/test/sdk/memory/test_dreaming.py new file mode 100644 index 0000000000..3d5f4538a7 --- /dev/null +++ b/test/sdk/memory/test_dreaming.py @@ -0,0 +1,132 @@ +from datetime import datetime, timedelta + +import pytest + +from nexent.memory.dreaming import ( + DreamingCandidate, + DreamingThresholds, + analyze_rem_content, + compute_metrics, + score_candidate, + select_candidates, +) + + +def candidate(**overrides): + values = { + "memory_id": 1, + "tenant_id": "tenant", + "user_id": "user", + "agent_id": "agent", + "content": "The user always prefers PostgreSQL transactions.", + "recall_count": 3, + "daily_count": 2, + "grounded_count": 1, + "total_retrieval_score": 4.2, + "query_hashes": ["q1", "q2"], + "recall_days": ["2026-07-20", "2026-07-22"], + "concept_tags": ["preference", "transaction"], + "light_hits": 2, + "rem_hits": 1, + "last_recalled_at": datetime(2026, 7, 22), + "last_light_at": datetime(2026, 7, 22), + "last_rem_at": datetime(2026, 7, 22), + } + values.update(overrides) + return DreamingCandidate(**values) + + +def test_ac004_openclaw_score_formula(): + score, metrics = score_candidate(candidate(), now=datetime(2026, 7, 23)) + assert score == pytest.approx(0.7268086998271306) + assert metrics.signal_count == 6 + assert metrics.context_diversity == 2 + assert metrics.relevance == pytest.approx(0.7) + assert metrics.consolidation == pytest.approx(0.36544353551179476) + + +@pytest.mark.parametrize( + ("changes", "thresholds", "reason"), + [ + ({"noise": True}, DreamingThresholds(min_score=0), "noise"), + ( + {"already_promoted": True}, + DreamingThresholds(min_score=0), + "already_promoted", + ), + ( + {"total_retrieval_score": 0}, + DreamingThresholds(min_score=0.7), + "score_below_threshold", + ), + ( + {"recall_count": 0, "daily_count": 0, "grounded_count": 0}, + DreamingThresholds(min_score=0, min_recall_count=1), + "recall_below_threshold", + ), + ( + {"query_hashes": [], "recall_days": []}, + DreamingThresholds(min_score=0, min_unique_queries=1), + "diversity_below_threshold", + ), + ], +) +def test_ac005_gates(changes, thresholds, reason): + decisions = select_candidates( + [candidate(**changes)], thresholds=thresholds, now=datetime(2026, 7, 23) + ) + assert decisions[0].promote is False + assert decisions[0].reason == reason + + +def test_ac005_stable_sorting(): + now = datetime(2026, 7, 23) + first = candidate(memory_id=9) + second = candidate(memory_id=2) + decisions = select_candidates( + [first, second], + thresholds=DreamingThresholds( + min_score=0, min_recall_count=0, min_unique_queries=0 + ), + now=now, + ) + assert [item.candidate.memory_id for item in decisions] == [2, 9] + + +def test_ac003_rem_patterns_and_noise(): + tags, noise = analyze_rem_content( + "I always prefer PostgreSQL transaction rollback." + ) + assert {"preference", "persistent", "transaction"} <= set(tags) + assert noise is False + _, noise = analyze_rem_content("Today's temporary TODO for this session") + assert noise is True + + +def test_metrics_boundaries_and_missing_dates(): + metrics = compute_metrics( + candidate( + recall_count=-2, + daily_count=0, + grounded_count=0, + total_retrieval_score=100, + recall_days=[], + last_recalled_at=None, + last_light_at=None, + last_rem_at=None, + ), + now=datetime(2026, 7, 23), + ) + assert metrics.frequency == 0 + assert metrics.relevance == 1 + assert metrics.recency == 0 + assert metrics.consolidation == 0 + assert metrics.phase_boost == 0 + + +def test_future_timestamps_are_clamped_to_full_recency(): + now = datetime(2026, 7, 23) + metrics = compute_metrics( + candidate(last_recalled_at=now + timedelta(days=1)), now=now + ) + assert metrics.recency == 1 From 44a3c428cca9b58c51daafd30b0a4082d6a930c9 Mon Sep 17 00:00:00 2001 From: Jinglong Wang Date: Thu, 23 Jul 2026 15:27:17 +0800 Subject: [PATCH 02/19] fix(memory): skip previously promoted Dreaming candidates --- backend/apps/memory_dreaming_app.py | 4 +- backend/services/memory_dreaming_service.py | 6 ++ test/backend/apps/test_memory_dreaming_app.py | 62 ++++++++----------- .../services/test_memory_dreaming_service.py | 57 +++++++++++++++++ 4 files changed, 91 insertions(+), 38 deletions(-) diff --git a/backend/apps/memory_dreaming_app.py b/backend/apps/memory_dreaming_app.py index d55a353365..c3b040d5b8 100644 --- a/backend/apps/memory_dreaming_app.py +++ b/backend/apps/memory_dreaming_app.py @@ -20,7 +20,7 @@ class DreamingRunRequest(BaseModel): @router.post("/run") -async def run_dreaming( +def run_dreaming( payload: DreamingRunRequest, authorization: Optional[str] = Header(None), ): @@ -38,7 +38,7 @@ async def run_dreaming( @router.get("/audit") -async def list_dreaming_audits( +def list_dreaming_audits( authorization: Optional[str] = Header(None), agent_id: Optional[str] = Query(default=None), run_id: Optional[int] = Query(default=None, ge=1), diff --git a/backend/services/memory_dreaming_service.py b/backend/services/memory_dreaming_service.py index 79a5b6e198..7364f801e6 100644 --- a/backend/services/memory_dreaming_service.py +++ b/backend/services/memory_dreaming_service.py @@ -82,6 +82,12 @@ def _run_rem( candidate = build_candidate( record, float(evidence.get("total_retrieval_score") or 0) ) + candidate.already_promoted = ( + memory_record_db.find_by_idempotency( + tenant_id, f"dreaming:{candidate.memory_id}" + ) + is not None + ) memory_record_db.update_memory_record( candidate.memory_id, tenant_id, diff --git a/test/backend/apps/test_memory_dreaming_app.py b/test/backend/apps/test_memory_dreaming_app.py index f94a1ff1f3..44d3cf3387 100644 --- a/test/backend/apps/test_memory_dreaming_app.py +++ b/test/backend/apps/test_memory_dreaming_app.py @@ -1,33 +1,18 @@ from unittest.mock import MagicMock -import httpx import pytest -from fastapi import FastAPI +from fastapi import HTTPException +from pydantic import ValidationError from apps import memory_dreaming_app -def app(): - app = FastAPI() - app.include_router(memory_dreaming_app.router) - return app +def test_ac009_missing_agent_id_is_rejected(): + with pytest.raises(ValidationError): + memory_dreaming_app.DreamingRunRequest() -@pytest.fixture -async def client(): - transport = httpx.ASGITransport(app=app()) - async with httpx.AsyncClient(transport=transport, base_url="http://test") as value: - yield value - - -@pytest.mark.asyncio -async def test_ac009_missing_agent_id_is_422(client): - response = await client.post("/memory/dreaming/run", json={}) - assert response.status_code == 422 - - -@pytest.mark.asyncio -async def test_ac009_run_uses_authenticated_scope(monkeypatch, client): +def test_ac009_run_uses_authenticated_scope(monkeypatch): service = MagicMock() service.run.return_value = {"run_id": 1, "status": "completed"} monkeypatch.setattr( @@ -38,20 +23,17 @@ async def test_ac009_run_uses_authenticated_scope(monkeypatch, client): "get_current_user_id", lambda _authorization: ("user-1", "tenant-1"), ) - response = await client.post( - "/memory/dreaming/run", - headers={"Authorization": "Bearer token"}, - json={"agent_id": "agent-1"}, + result = memory_dreaming_app.run_dreaming( + memory_dreaming_app.DreamingRunRequest(agent_id="agent-1"), + authorization="Bearer token", ) - assert response.status_code == 200 - assert response.json()["status"] == "completed" + assert result["status"] == "completed" assert service.run.call_args.kwargs["tenant_id"] == "tenant-1" assert service.run.call_args.kwargs["user_id"] == "user-1" assert service.run.call_args.kwargs["agent_id"] == "agent-1" -@pytest.mark.asyncio -async def test_ac009_audit_uses_authenticated_scope(monkeypatch, client): +def test_ac009_audit_uses_authenticated_scope(monkeypatch): service = MagicMock() service.list_audits.return_value = [{"run_id": 2}] monkeypatch.setattr( @@ -62,15 +44,19 @@ async def test_ac009_audit_uses_authenticated_scope(monkeypatch, client): "get_current_user_id", lambda _authorization: ("user-2", "tenant-2"), ) - response = await client.get("/memory/dreaming/audit?agent_id=agent-2&run_id=2") - assert response.status_code == 200 + result = memory_dreaming_app.list_dreaming_audits( + authorization="Bearer token", + agent_id="agent-2", + run_id=2, + limit=100, + ) + assert result == [{"run_id": 2}] service.list_audits.assert_called_once_with( "tenant-2", "user-2", agent_id="agent-2", run_id=2, limit=100 ) -@pytest.mark.asyncio -async def test_ac008_service_failure_maps_to_500(monkeypatch, client): +def test_ac008_service_failure_maps_to_500(monkeypatch): service = MagicMock() service.run.side_effect = memory_dreaming_app.DreamingRunError("failed") monkeypatch.setattr( @@ -81,6 +67,10 @@ async def test_ac008_service_failure_maps_to_500(monkeypatch, client): "get_current_user_id", lambda _authorization: ("user", "tenant"), ) - response = await client.post("/memory/dreaming/run", json={"agent_id": "agent"}) - assert response.status_code == 500 - assert response.json()["detail"] == "failed" + with pytest.raises(HTTPException) as exc: + memory_dreaming_app.run_dreaming( + memory_dreaming_app.DreamingRunRequest(agent_id="agent"), + authorization=None, + ) + assert exc.value.status_code == 500 + assert exc.value.detail == "failed" diff --git a/test/backend/services/test_memory_dreaming_service.py b/test/backend/services/test_memory_dreaming_service.py index 5cea0f474b..2c425e7461 100644 --- a/test/backend/services/test_memory_dreaming_service.py +++ b/test/backend/services/test_memory_dreaming_service.py @@ -4,6 +4,7 @@ import pytest +from nexent.memory.dreaming import DreamingThresholds, select_candidates from services.memory_dreaming_service import DreamingRunError, MemoryDreamingService @@ -85,6 +86,10 @@ def test_ac001_ac006_full_run_and_idempotency_key(monkeypatch): "services.memory_dreaming_service.memory_record_db.list_memory_records", lambda *_args, **_kwargs: [record], ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_record_db.find_by_idempotency", + lambda *_args, **_kwargs: None, + ) update_record = MagicMock(return_value=True) monkeypatch.setattr( "services.memory_dreaming_service.memory_record_db.update_memory_record", @@ -118,6 +123,58 @@ def test_ac001_ac006_full_run_and_idempotency_key(monkeypatch): assert record_service.create_memory.call_args.kwargs["layer"] == "user" +def test_ac006_already_promoted_candidate_is_not_written_again(monkeypatch): + service = MemoryDreamingService(record_service=MagicMock()) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_record_db.list_memory_records", + lambda *_args, **_kwargs: [ + { + "memory_id": 9, + "tenant_id": "t", + "user_id": "u", + "agent_id": "a", + "content": "Always prefer stable transaction behavior", + "recall_count": 10, + "daily_count": 5, + "grounded_count": 2, + "last_recalled_at": datetime.utcnow().isoformat(), + "query_hashes": ["q1", "q2", "q3"], + "recall_days": ["2026-07-21", "2026-07-22", "2026-07-23"], + "light_hits": 3, + "rem_hits": 3, + "last_light_at": datetime.utcnow().isoformat(), + "last_rem_at": datetime.utcnow().isoformat(), + "concept_tags": ["preference", "transaction"], + } + ], + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_record_db.find_by_idempotency", + lambda *_args, **_kwargs: {"memory_id": 99}, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_record_db.update_memory_record", + lambda *_args, **_kwargs: True, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_record_db.apply_dreaming_phase", + lambda *_args, **_kwargs: True, + ) + candidates = service._run_rem("t", "u", "a", {}) + decisions = select_candidates( + candidates, + thresholds=DreamingThresholds( + min_score=0, + min_recall_count=0, + min_unique_queries=0, + ), + ) + result = service._promote(decisions) + assert result[0]["event"] == "DEFER" + assert result[0]["reason"] == "already_promoted" + service.record_service.create_memory.assert_not_called() + + def test_ac008_failure_is_audited(monkeypatch): monkeypatch.setattr( "services.memory_dreaming_service.memory_dreaming_db.create_audit", From 32f172ea49a43b21d5e77bd0f5d757ed9e599215 Mon Sep 17 00:00:00 2001 From: Jinglong Wang Date: Thu, 23 Jul 2026 15:30:11 +0800 Subject: [PATCH 03/19] fix(memory): satisfy Dreaming quality checks --- backend/apps/memory_dreaming_app.py | 12 ++++++------ backend/database/memory_dreaming_db.py | 4 ++-- backend/services/memory_dreaming_scheduler.py | 2 +- backend/services/memory_dreaming_service.py | 10 +++++++--- sdk/nexent/memory/dreaming/scoring.py | 4 ++-- .../backend/services/test_memory_dreaming_service.py | 5 ++--- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/backend/apps/memory_dreaming_app.py b/backend/apps/memory_dreaming_app.py index c3b040d5b8..d20d660244 100644 --- a/backend/apps/memory_dreaming_app.py +++ b/backend/apps/memory_dreaming_app.py @@ -1,7 +1,7 @@ """Manual Dreaming run and audit endpoints.""" from http import HTTPStatus -from typing import Optional +from typing import Annotated, Optional from fastapi import APIRouter, Header, HTTPException, Query from pydantic import BaseModel, Field @@ -22,7 +22,7 @@ class DreamingRunRequest(BaseModel): @router.post("/run") def run_dreaming( payload: DreamingRunRequest, - authorization: Optional[str] = Header(None), + authorization: Annotated[Optional[str], Header()] = None, ): user_id, tenant_id = get_current_user_id(authorization) try: @@ -39,10 +39,10 @@ def run_dreaming( @router.get("/audit") def list_dreaming_audits( - authorization: Optional[str] = Header(None), - agent_id: Optional[str] = Query(default=None), - run_id: Optional[int] = Query(default=None, ge=1), - limit: int = Query(default=100, ge=1, le=500), + authorization: Annotated[Optional[str], Header()] = None, + agent_id: Annotated[Optional[str], Query()] = None, + run_id: Annotated[Optional[int], Query(ge=1)] = None, + limit: Annotated[int, Query(ge=1, le=500)] = 100, ): user_id, tenant_id = get_current_user_id(authorization) return get_memory_dreaming_service().list_audits( diff --git a/backend/database/memory_dreaming_db.py b/backend/database/memory_dreaming_db.py index 545a929833..e982766d8b 100644 --- a/backend/database/memory_dreaming_db.py +++ b/backend/database/memory_dreaming_db.py @@ -4,7 +4,7 @@ import hashlib from contextlib import contextmanager -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Dict, Iterator, List, Optional from sqlalchemy import text @@ -84,7 +84,7 @@ def finish_audit(run_id: int, *, status: str, **values: Any) -> bool: payload = { **values, "status": status, - "finished_at": datetime.utcnow(), + "finished_at": datetime.now(timezone.utc).replace(tzinfo=None), } if status != "failed": payload["current_phase"] = None diff --git a/backend/services/memory_dreaming_scheduler.py b/backend/services/memory_dreaming_scheduler.py index 3ec4572986..d6b0e4855e 100644 --- a/backend/services/memory_dreaming_scheduler.py +++ b/backend/services/memory_dreaming_scheduler.py @@ -2,7 +2,7 @@ try: from services.memory_dreaming_service import get_memory_dreaming_service -except (ImportError, ModuleNotFoundError): # package-style unit-test imports +except ImportError: # package-style unit-test imports from .memory_dreaming_service import get_memory_dreaming_service diff --git a/backend/services/memory_dreaming_service.py b/backend/services/memory_dreaming_service.py index 7364f801e6..cc13ff533c 100644 --- a/backend/services/memory_dreaming_service.py +++ b/backend/services/memory_dreaming_service.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional from consts.const import ( @@ -24,6 +24,10 @@ logger = logging.getLogger("memory_dreaming_service") +def _utcnow() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None) + + class DreamingRunError(RuntimeError): pass @@ -39,7 +43,7 @@ def _run_light( tenant_id, user_id, agent_id, - since=datetime.utcnow() - timedelta(days=max(1, window_days)), + since=_utcnow() - timedelta(days=max(1, window_days)), ) by_id = {int(item["memory_id"]): item for item in stats} for item in stats: @@ -98,7 +102,7 @@ def _run_rem( candidate.memory_id, tenant_id, phase="rem" ) candidate.rem_hits += 1 - candidate.last_rem_at = datetime.utcnow() + candidate.last_rem_at = _utcnow() candidates.append(candidate) return candidates diff --git a/sdk/nexent/memory/dreaming/scoring.py b/sdk/nexent/memory/dreaming/scoring.py index 7eafaa9c49..9f69280c76 100644 --- a/sdk/nexent/memory/dreaming/scoring.py +++ b/sdk/nexent/memory/dreaming/scoring.py @@ -3,7 +3,7 @@ from __future__ import annotations import math -from datetime import datetime +from datetime import datetime, timezone from typing import Iterable, List, Optional from .models import ( @@ -47,7 +47,7 @@ def compute_metrics( now: Optional[datetime] = None, recency_half_life_days: float = 14, ) -> DreamingMetrics: - now = now or datetime.utcnow() + now = now or datetime.now(timezone.utc).replace(tzinfo=None) signal_count = max(0, candidate.recall_count) + max(0, candidate.daily_count) + max(0, candidate.grounded_count) unique_queries = len(set(candidate.query_hashes)) unique_days = len(set(candidate.recall_days)) diff --git a/test/backend/services/test_memory_dreaming_service.py b/test/backend/services/test_memory_dreaming_service.py index 2c425e7461..9bca862e2a 100644 --- a/test/backend/services/test_memory_dreaming_service.py +++ b/test/backend/services/test_memory_dreaming_service.py @@ -192,9 +192,8 @@ def test_ac008_failure_is_audited(monkeypatch): monkeypatch.setattr( "services.memory_dreaming_service.memory_dreaming_db.finish_audit", finish ) + service = MemoryDreamingService(record_service=MagicMock()) with pytest.raises(DreamingRunError): - MemoryDreamingService(record_service=MagicMock()).run( - tenant_id="t", user_id="u", agent_id="a" - ) + service.run(tenant_id="t", user_id="u", agent_id="a") assert finish.call_args.kwargs["status"] == "failed" assert "ValueError" in finish.call_args.kwargs["error"] From e43a2b60ab6cf78e6e61d1c57b5ddd9c8482eb94 Mon Sep 17 00:00:00 2001 From: Jinglong Wang Date: Thu, 23 Jul 2026 15:52:16 +0800 Subject: [PATCH 04/19] test(memory): verify Dreaming against PostgreSQL --- ...st_memory_dreaming_postgres_integration.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 test/backend/database/test_memory_dreaming_postgres_integration.py diff --git a/test/backend/database/test_memory_dreaming_postgres_integration.py b/test/backend/database/test_memory_dreaming_postgres_integration.py new file mode 100644 index 0000000000..a1251ec2cd --- /dev/null +++ b/test/backend/database/test_memory_dreaming_postgres_integration.py @@ -0,0 +1,89 @@ +"""Opt-in PostgreSQL integration coverage for Dreaming advisory locks.""" + +import os + +import psycopg2 +import pytest + +from database.memory_dreaming_db import advisory_lock_key + +pytestmark = pytest.mark.skipif( + os.getenv("RUN_POSTGRES_INTEGRATION") != "1", + reason="set RUN_POSTGRES_INTEGRATION=1 with local PostgreSQL env", +) + + +def _connect(): + return psycopg2.connect( + host=os.getenv("DREAMING_TEST_POSTGRES_HOST", os.environ["POSTGRES_HOST"]), + port=os.getenv("DREAMING_TEST_POSTGRES_PORT", os.environ["POSTGRES_PORT"]), + user=os.getenv("DREAMING_TEST_POSTGRES_USER", os.environ["POSTGRES_USER"]), + password=os.getenv( + "DREAMING_TEST_POSTGRES_PASSWORD", + os.environ["NEXENT_POSTGRES_PASSWORD"], + ), + dbname=os.getenv("DREAMING_TEST_POSTGRES_DB", os.environ["POSTGRES_DB"]), + ) + + +def _try_lock(connection, lock_key): + with connection.cursor() as cursor: + cursor.execute("SELECT pg_try_advisory_xact_lock(%s)", (lock_key,)) + return cursor.fetchone()[0] + + +def test_ac007_real_postgres_scope_lock_is_non_blocking_and_released(): + same_scope = advisory_lock_key("dreaming-it", "user", "agent") + other_scope = advisory_lock_key("dreaming-it", "user", "other-agent") + + first = _connect() + second = _connect() + try: + assert _try_lock(first, same_scope) is True + assert _try_lock(second, same_scope) is False + assert _try_lock(second, other_scope) is True + + first.rollback() + assert _try_lock(second, same_scope) is True + finally: + first.rollback() + second.rollback() + first.close() + second.close() + + +def test_ac010_real_postgres_audit_schema_matches_orm_contract(): + connection = _connect() + try: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'nexent' + AND table_name = 'memory_dreaming_audit_t' + """ + ) + columns = {row[0] for row in cursor.fetchall()} + cursor.execute( + """ + SELECT indexname + FROM pg_indexes + WHERE schemaname = 'nexent' + AND tablename = 'memory_dreaming_audit_t' + """ + ) + indexes = {row[0] for row in cursor.fetchall()} + assert { + "run_id", + "tenant_id", + "user_id", + "agent_id", + "status", + "current_phase", + "result_json", + "error", + } <= columns + assert "idx_memory_dreaming_audit_scope" in indexes + finally: + connection.close() From ade5aa41800cd45fdcabddc586f4ddc3b7d3864a Mon Sep 17 00:00:00 2001 From: Jinglong Wang Date: Thu, 23 Jul 2026 15:56:07 +0800 Subject: [PATCH 05/19] test(memory): simplify Dreaming exception assertion --- test/backend/apps/test_memory_dreaming_app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/backend/apps/test_memory_dreaming_app.py b/test/backend/apps/test_memory_dreaming_app.py index 44d3cf3387..1842c1aa04 100644 --- a/test/backend/apps/test_memory_dreaming_app.py +++ b/test/backend/apps/test_memory_dreaming_app.py @@ -67,9 +67,10 @@ def test_ac008_service_failure_maps_to_500(monkeypatch): "get_current_user_id", lambda _authorization: ("user", "tenant"), ) + request = memory_dreaming_app.DreamingRunRequest(agent_id="agent") with pytest.raises(HTTPException) as exc: memory_dreaming_app.run_dreaming( - memory_dreaming_app.DreamingRunRequest(agent_id="agent"), + request, authorization=None, ) assert exc.value.status_code == 500 From b1cfc3dc58531f5ce6c316ab5e310c369d772f1c Mon Sep 17 00:00:00 2001 From: Jinglong Wang Date: Mon, 27 Jul 2026 19:24:42 +0800 Subject: [PATCH 06/19] feat(memory): complete Dreaming consolidation --- .gitignore | 1 + backend/agents/create_agent_info.py | 27 + backend/apps/config_app.py | 13 + backend/apps/memory_dreaming_app.py | 131 +++- backend/consts/const.py | 20 +- backend/data_process/app.py | 2 +- backend/database/db_models.py | 84 +++ backend/database/memory_dreaming_db.py | 335 +++++++++- backend/database/memory_record_db.py | 4 +- .../services/memory_dreaming_compressor.py | 540 +++++++++++++++ backend/services/memory_dreaming_scheduler.py | 134 +++- backend/services/memory_dreaming_service.py | 216 +++++- backend/utils/context_utils.py | 14 +- deploy/k8s/deploy.sh | 2 +- .../nexent/charts/nexent-common/values.yaml | 2 +- deploy/sql/init.sql | 172 +++++ ...2.4.0_0723_add_memory_dreaming_version.sql | 124 ++++ ...v2.4.0_0724_add_dreaming_lease_columns.sql | 12 + .../app/[locale]/memory/DreamingPanel.tsx | 442 +++++++++++++ .../app/[locale]/memory/MemoryManager.tsx | 43 +- frontend/const/modelConfig.ts | 12 +- frontend/package.json | 4 + frontend/playwright.config.ts | 12 + frontend/public/locales/en/common.json | 50 +- frontend/public/locales/zh/common.json | 50 +- frontend/services/api.ts | 14 +- frontend/services/memoryService.ts | 133 +++- frontend/tests/e2e/dreaming.spec.ts | 70 ++ sdk/nexent/core/agents/context/manager.py | 423 ++++++++++-- sdk/nexent/core/tools/store_memory_tool.py | 5 +- sdk/nexent/memory/dreaming/__init__.py | 14 + sdk/nexent/memory/dreaming/version_builder.py | 361 ++++++++++ test/backend/apps/test_memory_dreaming_app.py | 198 +++++- .../database/test_memory_dreaming_lease.py | 153 +++++ .../database/test_memory_dreaming_schema.py | 78 ++- .../test_memory_dreaming_compressor.py | 622 ++++++++++++++++++ .../test_memory_dreaming_model_integration.py | 73 ++ .../services/test_memory_dreaming_service.py | 65 +- .../test_long_term_memory_turn_compaction.py | 142 ++++ .../memory/test_dreaming_version_builder.py | 282 ++++++++ 40 files changed, 4899 insertions(+), 180 deletions(-) create mode 100644 backend/services/memory_dreaming_compressor.py create mode 100644 deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_version.sql create mode 100644 deploy/sql/migrations/v2.4.0_0724_add_dreaming_lease_columns.sql create mode 100644 frontend/app/[locale]/memory/DreamingPanel.tsx create mode 100644 frontend/playwright.config.ts create mode 100644 frontend/tests/e2e/dreaming.spec.ts create mode 100644 sdk/nexent/memory/dreaming/version_builder.py create mode 100644 test/backend/database/test_memory_dreaming_lease.py create mode 100644 test/backend/services/test_memory_dreaming_compressor.py create mode 100644 test/backend/services/test_memory_dreaming_model_integration.py create mode 100644 test/sdk/core/agents/test_long_term_memory_turn_compaction.py create mode 100644 test/sdk/memory/test_dreaming_version_builder.py diff --git a/.gitignore b/.gitignore index 78993904bb..adb702cfbb 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ agent_repository_frontend .tokensave .playwright-mcp/ +frontend/test-results/ diff --git a/backend/agents/create_agent_info.py b/backend/agents/create_agent_info.py index 045db63e99..368e6a4731 100644 --- a/backend/agents/create_agent_info.py +++ b/backend/agents/create_agent_info.py @@ -101,6 +101,19 @@ def _format_long_term_memory_prompt(search_context: Any, language: str) -> str: return "\n\n".join(sections) +def _get_active_dreaming_version( + tenant_id: str, user_id: str, agent_id: str +) -> Optional[Dict[str, Any]]: + """Keep the Dreaming repository optional in isolated SDK-style tests.""" + try: + from database.memory_dreaming_db import get_active_version + + return get_active_version(tenant_id, user_id, agent_id) + except ModuleNotFoundError: + logger.debug("Dreaming repository is unavailable in this runtime") + return None + + # Safe fallback for context-manager token_threshold when no capacity is known. # Used only when the resolver fails (uncataloged model with no operator-supplied # hard capacity). Sized to cover the typical 32K-context band shared by the @@ -830,6 +843,20 @@ async def create_agent_config( memory_context = build_memory_context( user_id, tenant_id, agent_id, skip_query=not allow_memory_search ) + if allow_memory_search and memory_context.user_config.memory_switch: + active_dreaming_version = _get_active_dreaming_version( + str(tenant_id), str(user_id), str(agent_id) + ) + if active_dreaming_version: + memory_list.append( + { + "memory": active_dreaming_version["published_content"], + "memory_level": "user", + "memory_type": "long_term", + "score": 1.0, + "dreaming_version_id": active_dreaming_version["version_id"], + } + ) # Append active memory tools if memory is enabled if memory_context.user_config.memory_switch: diff --git a/backend/apps/config_app.py b/backend/apps/config_app.py index 94cb18ae14..38fd58f834 100644 --- a/backend/apps/config_app.py +++ b/backend/apps/config_app.py @@ -64,6 +64,19 @@ async def sync_default_prompt_template_on_startup(): except Exception as exc: logger.error(f"Failed to sync system default prompt template: {str(exc)}") + +@app.on_event("startup") +async def start_dreaming_scheduler(): + from services.memory_dreaming_scheduler import dreaming_scheduler + await dreaming_scheduler.start() + + +@app.on_event("shutdown") +async def stop_dreaming_scheduler(): + from services.memory_dreaming_scheduler import dreaming_scheduler + await dreaming_scheduler.stop() + + app.include_router(model_manager_router) app.include_router(config_sync_router) app.include_router(agent_router) diff --git a/backend/apps/memory_dreaming_app.py b/backend/apps/memory_dreaming_app.py index d20d660244..3050c8a2eb 100644 --- a/backend/apps/memory_dreaming_app.py +++ b/backend/apps/memory_dreaming_app.py @@ -6,7 +6,16 @@ from fastapi import APIRouter, Header, HTTPException, Query from pydantic import BaseModel, Field +from consts.const import ( + DREAMING_COMPRESSION_MAX_ATTEMPTS, + DREAMING_LONG_TERM_MAX_CHARS, + DREAMING_SOURCE_LIMIT, +) +from database import memory_dreaming_db +from database.role_permission_db import check_role_permission +from database.user_tenant_db import get_user_tenant_by_user_id from services.memory_dreaming_service import ( + DreamingConflictError, DreamingRunError, get_memory_dreaming_service, ) @@ -17,20 +26,76 @@ class DreamingRunRequest(BaseModel): agent_id: str = Field(..., min_length=1) + target_user_id: Optional[str] = None + + +class DreamingVersionSwitchRequest(BaseModel): + agent_id: str = Field(..., min_length=1) + expected_active_version_id: int = Field(..., ge=1) + target_user_id: Optional[str] = None + + +def _resolve_target_user( + authorization: Optional[str], + target_user_id: Optional[str], + *, + tenant_capability: str, +) -> tuple[str, str]: + caller_user_id, tenant_id = get_current_user_id(authorization) + if not target_user_id or target_user_id == caller_user_id: + return caller_user_id, tenant_id + caller = get_user_tenant_by_user_id(caller_user_id) or {} + caller_role = str(caller.get("user_role") or "").upper() + if not check_role_permission( + caller_role, + permission_category="RESOURCE", + permission_type="DREAMING", + permission_subtype=tenant_capability, + ): + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, detail="Resource not found" + ) + target = get_user_tenant_by_user_id(target_user_id) or {} + if target.get("tenant_id") != tenant_id: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, detail="Resource not found" + ) + return target_user_id, tenant_id + + +@router.get("/parameters") +def get_dreaming_parameters( + authorization: Annotated[Optional[str], Header()] = None, +): + """Expose effective read-only build parameters for an authenticated user.""" + get_current_user_id(authorization) + return { + "source_limit": DREAMING_SOURCE_LIMIT, + "long_term_max_chars": DREAMING_LONG_TERM_MAX_CHARS, + "compression_max_attempts": DREAMING_COMPRESSION_MAX_ATTEMPTS, + } -@router.post("/run") + +@router.post("/run", status_code=HTTPStatus.ACCEPTED) def run_dreaming( payload: DreamingRunRequest, authorization: Annotated[Optional[str], Header()] = None, ): - user_id, tenant_id = get_current_user_id(authorization) + user_id, tenant_id = _resolve_target_user( + authorization, + payload.target_user_id, + tenant_capability="EDIT_TENANT", + ) try: - return get_memory_dreaming_service().run( - tenant_id=tenant_id, - user_id=user_id, - agent_id=payload.agent_id, + run_id = memory_dreaming_db.create_audit( + tenant_id, + user_id, + payload.agent_id, + trigger_source="manual", + status="queued", ) + return {"run_id": run_id, "status": "queued"} except DreamingRunError as exc: raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=str(exc) @@ -43,8 +108,13 @@ def list_dreaming_audits( agent_id: Annotated[Optional[str], Query()] = None, run_id: Annotated[Optional[int], Query(ge=1)] = None, limit: Annotated[int, Query(ge=1, le=500)] = 100, + target_user_id: Annotated[Optional[str], Query()] = None, ): - user_id, tenant_id = get_current_user_id(authorization) + user_id, tenant_id = _resolve_target_user( + authorization, + target_user_id, + tenant_capability="VIEW_TENANT", + ) return get_memory_dreaming_service().list_audits( tenant_id, user_id, @@ -52,3 +122,50 @@ def list_dreaming_audits( run_id=run_id, limit=limit, ) + + +@router.get("/versions") +def list_dreaming_versions( + agent_id: Annotated[str, Query(min_length=1)], + authorization: Annotated[Optional[str], Header()] = None, + limit: Annotated[int, Query(ge=1, le=500)] = 100, + target_user_id: Annotated[Optional[str], Query()] = None, +): + user_id, tenant_id = _resolve_target_user( + authorization, + target_user_id, + tenant_capability="VIEW_TENANT", + ) + return get_memory_dreaming_service().list_versions( + tenant_id, user_id, agent_id=agent_id, limit=limit + ) + + +@router.post("/versions/{version_id}/activate") +def activate_dreaming_version( + version_id: int, + payload: DreamingVersionSwitchRequest, + authorization: Annotated[Optional[str], Header()] = None, +): + user_id, tenant_id = _resolve_target_user( + authorization, + payload.target_user_id, + tenant_capability="EDIT_TENANT", + ) + actor_user_id, _ = get_current_user_id(authorization) + try: + version = get_memory_dreaming_service().activate_version( + tenant_id, + user_id, + agent_id=payload.agent_id, + version_id=version_id, + actor_user_id=actor_user_id, + expected_active_version_id=payload.expected_active_version_id, + ) + except DreamingConflictError as exc: + raise HTTPException(status_code=HTTPStatus.CONFLICT, detail=str(exc)) from exc + if version is None: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, detail="Version not found" + ) + return version diff --git a/backend/consts/const.py b/backend/consts/const.py index a1b84ad1eb..d123fb4efe 100644 --- a/backend/consts/const.py +++ b/backend/consts/const.py @@ -4,7 +4,10 @@ from dotenv import load_dotenv # Load environment variables -load_dotenv(override=True) +# Explicitly sourced deployment variables take precedence over a nearby +# developer .env file. This is required for tmux/K8s-local verification and +# avoids silently replacing operator-provided service addresses. +load_dotenv(override=False) # TODO: Analyze every variable if this is used # Test voice file path (WAV format for volcengine STT) @@ -352,10 +355,17 @@ class VectorDatabaseType(str, Enum): MIN_PROMOTION_SCORE = float(os.getenv("MIN_PROMOTION_SCORE", "0.72")) MIN_RECALL_COUNT = int(os.getenv("MIN_RECALL_COUNT", "3")) MIN_UNIQUE_QUERIES = int(os.getenv("MIN_UNIQUE_QUERIES", "2")) -# Scheduling/cron constants are intentionally not defined here: the -# background Dreaming scheduler is not part of Phase 2 (an agent-driven -# timer will be added in a later phase, at which point the cron expression -# and heartbeat can be reintroduced). +DREAMING_SOURCE_LIMIT = int(os.getenv("DREAMING_SOURCE_LIMIT", "10")) +DREAMING_LONG_TERM_MAX_CHARS = int( + os.getenv("DREAMING_LONG_TERM_MAX_CHARS", "10000") +) +DREAMING_COMPRESSION_MAX_ATTEMPTS = int( + os.getenv("DREAMING_COMPRESSION_MAX_ATTEMPTS", "2") +) +DREAMING_SCHEDULER_POLL_SECONDS = float(os.getenv("DREAMING_SCHEDULER_POLL_SECONDS", "5.0")) +DREAMING_SCHEDULER_LEASE_SECONDS = float(os.getenv("DREAMING_SCHEDULER_LEASE_SECONDS", "120.0")) +DREAMING_SCHEDULER_MAX_CONCURRENCY = int(os.getenv("DREAMING_SCHEDULER_MAX_CONCURRENCY", "1")) +DREAMING_SCHEDULER_ENABLED = os.getenv("DREAMING_SCHEDULER_ENABLED", "true").lower() in ("true", "1", "yes") # External provider retry / timeout PROVIDER_RETRY_MAX_ATTEMPTS = int(os.getenv("PROVIDER_RETRY_MAX_ATTEMPTS", "3")) diff --git a/backend/data_process/app.py b/backend/data_process/app.py index e70403c36d..875d015e98 100644 --- a/backend/data_process/app.py +++ b/backend/data_process/app.py @@ -48,7 +48,7 @@ task_routes={ f'{import_path}.process': {'queue': 'process_q'}, f'{import_path}.forward': {'queue': 'forward_q'}, - f'{import_path}.process_and_forward': {'queue': 'process_q'} + f'{import_path}.process_and_forward': {'queue': 'process_q'}, }, task_serializer='json', accept_content=['json'], diff --git a/backend/database/db_models.py b/backend/database/db_models.py index f47f9ec50b..9d6e416ba0 100644 --- a/backend/database/db_models.py +++ b/backend/database/db_models.py @@ -972,6 +972,90 @@ class MemoryDreamingAudit(TableBase): deferred_count = Column(Integer, nullable=False, default=0) result_json = Column(JSONB) error = Column(Text) + lock_owner = Column(String(100), nullable=True) + lock_until = Column(TIMESTAMP(timezone=False), nullable=True) + + +class MemoryDreamingVersion(TableBase): + """Immutable long-term memory artifact produced by one Dreaming run.""" + + __tablename__ = "memory_dreaming_version_t" + __table_args__ = ( + Index( + "idx_memory_dreaming_version_scope", + "tenant_id", + "user_id", + "agent_id", + "version_no", + unique=True, + ), + Index( + "uq_memory_dreaming_version_active_scope", + "tenant_id", + "user_id", + "agent_id", + unique=True, + postgresql_where=text("is_active AND delete_flag = 'N'"), + ), + Index("uq_memory_dreaming_version_run", "run_id", unique=True), + {"schema": SCHEMA}, + ) + + version_id = Column( + BigInteger, + Sequence("memory_dreaming_version_t_version_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + ) + tenant_id = Column(String(100), nullable=False) + user_id = Column(String(100), nullable=False) + agent_id = Column(String(100), nullable=False) + version_no = Column(Integer, nullable=False) + parent_version_id = Column(BigInteger) + run_id = Column(BigInteger, nullable=False) + is_active = Column(Boolean, nullable=False, default=False) + raw_content = Column(Text, nullable=False) + published_content = Column(Text, nullable=False) + published_units = Column(JSONB, nullable=False, default=list) + source_evidence_ids = Column(JSONB, nullable=False, default=list) + config_snapshot = Column(JSONB, nullable=False, default=dict) + raw_char_count = Column(Integer, nullable=False) + published_char_count = Column(Integer, nullable=False) + compression_status = Column(String(30), nullable=False) + compression_attempts = Column(Integer, nullable=False, default=0) + compression_audit = Column(JSONB, nullable=False, default=list) + omitted_evidence_ids = Column(JSONB, nullable=False, default=list) + mechanical_truncation = Column(Boolean, nullable=False, default=False) + + +class MemoryDreamingActivationAudit(TableBase): + """Append-only audit for active-version pointer changes.""" + + __tablename__ = "memory_dreaming_activation_audit_t" + __table_args__ = ( + Index( + "idx_memory_dreaming_activation_scope", + "tenant_id", + "user_id", + "agent_id", + "create_time", + ), + {"schema": SCHEMA}, + ) + + activation_id = Column( + BigInteger, + Sequence("memory_dreaming_activation_audit_t_activation_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + ) + tenant_id = Column(String(100), nullable=False) + user_id = Column(String(100), nullable=False) + agent_id = Column(String(100), nullable=False) + actor_user_id = Column(String(100), nullable=False) + from_version_id = Column(BigInteger) + to_version_id = Column(BigInteger, nullable=False) + reason = Column(String(100), nullable=False, default="user_switch") class McpRecord(TableBase): diff --git a/backend/database/memory_dreaming_db.py b/backend/database/memory_dreaming_db.py index e982766d8b..b5c4413317 100644 --- a/backend/database/memory_dreaming_db.py +++ b/backend/database/memory_dreaming_db.py @@ -7,10 +7,14 @@ from datetime import datetime, timezone from typing import Any, Dict, Iterator, List, Optional -from sqlalchemy import text +from sqlalchemy import func, text from .client import get_db_session -from .db_models import MemoryDreamingAudit +from .db_models import ( + MemoryDreamingActivationAudit, + MemoryDreamingAudit, + MemoryDreamingVersion, +) def advisory_lock_key(tenant_id: str, user_id: str, agent_id: str) -> int: @@ -38,21 +42,230 @@ def try_scope_lock(tenant_id: str, user_id: str, agent_id: str) -> Iterator[bool raise -def create_audit(tenant_id: str, user_id: str, agent_id: str) -> int: +def create_audit( + tenant_id: str, + user_id: str, + agent_id: str, + *, + trigger_source: str = "manual", + status: str = "running", +) -> int: with get_db_session() as session: row = MemoryDreamingAudit( tenant_id=tenant_id, user_id=user_id, agent_id=agent_id, - trigger_source="manual", - status="running", - current_phase="light", + trigger_source=trigger_source, + status=status, + current_phase=None if status == "queued" else "light", ) session.add(row) session.commit() return int(row.run_id) +def get_active_version( + tenant_id: str, user_id: str, agent_id: str +) -> Optional[Dict[str, Any]]: + with get_db_session() as session: + row = ( + session.query(MemoryDreamingVersion) + .filter( + MemoryDreamingVersion.tenant_id == tenant_id, + MemoryDreamingVersion.user_id == user_id, + MemoryDreamingVersion.agent_id == agent_id, + MemoryDreamingVersion.is_active.is_(True), + MemoryDreamingVersion.delete_flag == "N", + ) + .first() + ) + return _version_to_dict(row) if row else None + + +def create_and_activate_version( + *, + tenant_id: str, + user_id: str, + agent_id: str, + run_id: int, + parent_version_id: Optional[int], + raw_content: str, + published_content: str, + published_units: List[Dict[str, Any]], + source_evidence_ids: List[str], + config_snapshot: Dict[str, Any], + raw_char_count: int, + published_char_count: int, + compression_status: str, + compression_attempts: int, + omitted_evidence_ids: List[str], + mechanical_truncation: bool, + compression_audit: List[Dict[str, Any]], +) -> Dict[str, Any]: + """Atomically append and activate a version for one locked scope.""" + with get_db_session() as session: + existing = ( + session.query(MemoryDreamingVersion) + .filter( + MemoryDreamingVersion.run_id == run_id, + MemoryDreamingVersion.tenant_id == tenant_id, + MemoryDreamingVersion.user_id == user_id, + MemoryDreamingVersion.agent_id == agent_id, + MemoryDreamingVersion.delete_flag == "N", + ) + .first() + ) + if existing is not None: + return _version_to_dict(existing) + scope = ( + MemoryDreamingVersion.tenant_id == tenant_id, + MemoryDreamingVersion.user_id == user_id, + MemoryDreamingVersion.agent_id == agent_id, + MemoryDreamingVersion.delete_flag == "N", + ) + next_version = ( + session.query(func.coalesce(func.max(MemoryDreamingVersion.version_no), 0)) + .filter(*scope) + .scalar() + + 1 + ) + session.query(MemoryDreamingVersion).filter( + *scope, MemoryDreamingVersion.is_active.is_(True) + ).update({"is_active": False}, synchronize_session=False) + row = MemoryDreamingVersion( + tenant_id=tenant_id, + user_id=user_id, + agent_id=agent_id, + version_no=next_version, + parent_version_id=parent_version_id, + run_id=run_id, + is_active=True, + raw_content=raw_content, + published_content=published_content, + published_units=published_units, + source_evidence_ids=source_evidence_ids, + config_snapshot=config_snapshot, + raw_char_count=raw_char_count, + published_char_count=published_char_count, + compression_status=compression_status, + compression_attempts=compression_attempts, + omitted_evidence_ids=omitted_evidence_ids, + mechanical_truncation=mechanical_truncation, + compression_audit=compression_audit, + created_by="dreaming", + ) + session.add(row) + session.commit() + session.refresh(row) + return _version_to_dict(row) + + +def list_versions( + tenant_id: str, + user_id: str, + *, + agent_id: str, + limit: int = 100, +) -> List[Dict[str, Any]]: + with get_db_session() as session: + rows = ( + session.query(MemoryDreamingVersion) + .filter( + MemoryDreamingVersion.tenant_id == tenant_id, + MemoryDreamingVersion.user_id == user_id, + MemoryDreamingVersion.agent_id == agent_id, + MemoryDreamingVersion.delete_flag == "N", + ) + .order_by(MemoryDreamingVersion.version_no.desc()) + .limit(limit) + .all() + ) + return [_version_to_dict(row) for row in rows] + + +def activate_version( + tenant_id: str, + user_id: str, + agent_id: str, + version_id: int, + *, + actor_user_id: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Switch the active pointer without modifying immutable version content.""" + with get_db_session() as session: + rows = ( + session.query(MemoryDreamingVersion) + .filter( + MemoryDreamingVersion.tenant_id == tenant_id, + MemoryDreamingVersion.user_id == user_id, + MemoryDreamingVersion.agent_id == agent_id, + MemoryDreamingVersion.delete_flag == "N", + ) + .all() + ) + target = next((row for row in rows if row.version_id == version_id), None) + if target is None: + return None + current = next((row for row in rows if row.is_active), None) + if current is not None and current.version_id == version_id: + return _version_to_dict(target) + actor = actor_user_id or user_id + session.query(MemoryDreamingVersion).filter( + MemoryDreamingVersion.tenant_id == tenant_id, + MemoryDreamingVersion.user_id == user_id, + MemoryDreamingVersion.agent_id == agent_id, + MemoryDreamingVersion.delete_flag == "N", + MemoryDreamingVersion.is_active.is_(True), + ).update( + {"is_active": False, "updated_by": actor}, + synchronize_session=False, + ) + session.flush() + target.is_active = True + target.updated_by = actor + session.add( + MemoryDreamingActivationAudit( + tenant_id=tenant_id, + user_id=user_id, + agent_id=agent_id, + actor_user_id=actor, + from_version_id=current.version_id if current else None, + to_version_id=version_id, + reason="user_switch", + created_by=actor, + ) + ) + session.commit() + session.refresh(target) + return _version_to_dict(target) + + +def _version_to_dict(row: MemoryDreamingVersion) -> Dict[str, Any]: + return { + "version_id": row.version_id, + "tenant_id": row.tenant_id, + "user_id": row.user_id, + "agent_id": row.agent_id, + "version_no": row.version_no, + "parent_version_id": row.parent_version_id, + "run_id": row.run_id, + "is_active": row.is_active, + "raw_content": row.raw_content, + "published_content": row.published_content, + "published_units": row.published_units or [], + "source_evidence_ids": row.source_evidence_ids or [], + "config_snapshot": row.config_snapshot or {}, + "raw_char_count": row.raw_char_count, + "published_char_count": row.published_char_count, + "compression_status": row.compression_status, + "compression_attempts": row.compression_attempts, + "omitted_evidence_ids": row.omitted_evidence_ids or [], + "mechanical_truncation": row.mechanical_truncation, + "compression_audit": row.compression_audit or [], + "created_at": row.create_time.isoformat() if row.create_time else None, + } + + def update_audit(run_id: int, values: Dict[str, Any]) -> bool: allowed = { "status", @@ -130,3 +343,113 @@ def list_audits( } for row in rows ] + + +# --------------------------------------------------------------------------- +# Worker lease management +# --------------------------------------------------------------------------- + + +def claim_queued(owner_id: str, lease_seconds: float) -> Optional[Dict[str, Any]]: + """Atomically claim the oldest queued audit row and set a lease. + + Uses FOR UPDATE SKIP LOCKED so concurrent workers never block each other. + Returns the payload the executor needs (run_id, tenant_id, user_id, + agent_id, trigger_source) or None when no row is available. + """ + sql = text(""" + WITH candidate AS ( + SELECT run_id + FROM nexent.memory_dreaming_audit_t + WHERE status = 'queued' + AND delete_flag = 'N' + ORDER BY started_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + UPDATE nexent.memory_dreaming_audit_t AS audit + SET lock_owner = :owner_id, + lock_until = now() + (:lease_seconds * interval '1 second'), + status = 'running', + current_phase = 'light', + update_time = now() + FROM candidate + WHERE audit.run_id = candidate.run_id + RETURNING audit.run_id, + audit.tenant_id, + audit.user_id, + audit.agent_id, + audit.trigger_source + """) + with get_db_session() as session: + row = session.execute(sql, { + "owner_id": owner_id, + "lease_seconds": lease_seconds, + }).fetchone() + if row is None: + return None + return dict(row._mapping) + + +def renew_lease(run_id: int, owner_id: str, lease_seconds: float) -> bool: + """Extend the lease only when the caller still owns it and it has not expired.""" + sql = text(""" + UPDATE nexent.memory_dreaming_audit_t + SET lock_until = now() + (:lease_seconds * interval '1 second'), + update_time = now() + WHERE run_id = :run_id + AND lock_owner = :owner_id + AND lock_until > now() + AND delete_flag = 'N' + RETURNING run_id + """) + with get_db_session() as session: + renewed = session.execute(sql, { + "run_id": run_id, + "owner_id": owner_id, + "lease_seconds": lease_seconds, + }).scalar_one_or_none() + return renewed is not None + + +def release_lease(run_id: int, owner_id: str) -> bool: + """Clear the lease fields only when the caller owns the lock.""" + sql = text(""" + UPDATE nexent.memory_dreaming_audit_t + SET lock_owner = NULL, + lock_until = NULL, + update_time = now() + WHERE run_id = :run_id + AND lock_owner = :owner_id + AND delete_flag = 'N' + RETURNING run_id + """) + with get_db_session() as session: + released = session.execute(sql, { + "run_id": run_id, + "owner_id": owner_id, + }).scalar_one_or_none() + return released is not None + + +def recover_stale() -> int: + """Reap runs whose lease expired without completion. + + Marks them as failed and clears lock fields so they can be retried. + Safe to call on every worker startup. + """ + sql = text(""" + UPDATE nexent.memory_dreaming_audit_t + SET status = 'failed', + error = 'Worker lost — reaped by startup recovery', + lock_owner = NULL, + lock_until = NULL, + finished_at = now(), + update_time = now() + WHERE status = 'running' + AND lock_until < now() + AND delete_flag = 'N' + """) + with get_db_session() as session: + result = session.execute(sql) + return result.rowcount or 0 diff --git a/backend/database/memory_record_db.py b/backend/database/memory_record_db.py index d81f684b24..b92670b041 100644 --- a/backend/database/memory_record_db.py +++ b/backend/database/memory_record_db.py @@ -329,7 +329,9 @@ def list_memory_records( query = query.filter(MemoryRecord.delete_flag == "N") query = query.order_by(MemoryRecord.update_time.desc()) - query = query.limit(limit).offset(offset) + if limit is not None: + query = query.limit(limit) + query = query.offset(offset) result = [] for ( record, diff --git a/backend/services/memory_dreaming_compressor.py b/backend/services/memory_dreaming_compressor.py new file mode 100644 index 0000000000..e25b371a13 --- /dev/null +++ b/backend/services/memory_dreaming_compressor.py @@ -0,0 +1,540 @@ +"""Tenant-model semantic compressor for bounded Dreaming versions.""" + +from __future__ import annotations + +import json +import re + +from consts.const import MODEL_CONFIG_MAPPING +from nexent.core.models import OpenAIModel +from nexent.memory.dreaming import ( + DreamingCompressionOutput, + DreamingCompressionRequest, +) +from nexent.monitor import ( + AgentRunMetadata, + agent_monitoring_context, + set_monitoring_operation, +) +from utils.config_utils import get_model_name_from_config, tenant_config_manager + + +class TenantDreamingCompressor: + """Compress RAW memory with the tenant's configured default LLM.""" + + def __init__(self, tenant_id: str, user_id: str): + config = tenant_config_manager.get_model_config( + key=MODEL_CONFIG_MAPPING["llm"], tenant_id=tenant_id + ) + if not config: + raise RuntimeError("No tenant LLM is configured for Dreaming") + self.tenant_id = tenant_id + self.user_id = user_id + context_tokens = int( + config.get("max_input_tokens") + or config.get("context_window_tokens") + or 32_000 + ) + self.max_compression_input_chars = max(20_000, context_tokens * 3) + self.model = OpenAIModel( + model_id=get_model_name_from_config(config), + api_base=config.get("base_url", ""), + api_key=config.get("api_key", ""), + temperature=0.1, + top_p=0.9, + model_factory=config.get("model_factory"), + ssl_verify=config.get("ssl_verify", True), + display_name=config.get("display_name") or None, + timeout_seconds=config.get("timeout_seconds"), + stream=False, + ) + + def __call__( + self, request: DreamingCompressionRequest + ) -> DreamingCompressionOutput: + evidence_ids = sorted( + {evidence_id for unit in request.units for evidence_id in unit.evidence_ids} + ) + feedback = ", ".join(request.validation_feedback) or "none" + numeric_agent_id = ( + int(request.agent_id) + if request.agent_id and request.agent_id.isdigit() + else None + ) + metadata = AgentRunMetadata( + tenant_id=self.tenant_id, + user_id=self.user_id, + agent_id=numeric_agent_id, + extra_metadata={ + "dreaming_run_id": request.run_id, + "dreaming_agent_id": request.agent_id, + "dreaming_attempt": request.attempt, + }, + ) + with agent_monitoring_context(metadata): + input_limit = getattr(self, "max_compression_input_chars", 40_000) + + units = self._prepare_units_with_ids(request.units) + + if len(request.raw_content) > input_limit or len(units) > 12: + return self._map_reduce_extract( + request, units, evidence_ids, feedback + ) + + spans = self._extract_spans(request.raw_content, units, feedback) + + facts, span_feedback = self._validate_spans( + spans, request.raw_content, units + ) + if span_feedback: + raise ValueError( + f"Span validation failed: {span_feedback}" + ) + + facts.extend(self._required_literal_facts(units, len(facts))) + unique_facts = self._deterministic_dedup(facts) + self._require_source_coverage(unique_facts, units) + + output = self._format_facts( + unique_facts, request.max_chars, evidence_ids + ) + + # Stage 3: Coverage validation + all_fact_ids = [f["fact_id"] for f in unique_facts] + covered_fact_ids = output.metadata.get( + "covered_fact_ids", + self._count_covered_facts(output.content, unique_facts), + ) + output.metadata["covered_fact_ids"] = covered_fact_ids + + if len(all_fact_ids) > 0: + coverage = len(covered_fact_ids) / len(all_fact_ids) + if coverage < 0.95: + raise ValueError( + f"Fact coverage too low: {len(covered_fact_ids)}/{len(all_fact_ids)}={coverage:.2f}" + ) + + return output + + def _map_reduce_extract( + self, + request: DreamingCompressionRequest, + units: list[dict], + evidence_ids: list[str], + feedback: str, + ) -> DreamingCompressionOutput: + input_limit = getattr(self, "max_compression_input_chars", 40_000) + unit_models = request.units + chunks = self._chunk_units( + unit_models, + input_limit=max(10_000, input_limit // 2), + max_units=12, + ) + + all_facts = [] + fact_counter = 0 + for chunk in chunks: + chunk_units = [u for u in units if any( + m.unit_id == u["unit_id"] for m in chunk + )] + chunk_raw = "\n".join( + f"- {u['text'].strip()}" for u in chunk_units if u["text"].strip() + ) + spans = self._extract_spans(chunk_raw, chunk_units, feedback) + facts, span_fb = self._validate_spans(spans, chunk_raw, chunk_units) + if span_fb: + raise ValueError(f"Map chunk span validation failed: {span_fb}") + for f in facts: + f["fact_id"] = f"f{fact_counter:03d}" + fact_counter += 1 + all_facts.extend(facts) + + all_facts.extend( + self._required_literal_facts(units, fact_counter) + ) + unique_facts = self._deterministic_dedup(all_facts) + self._require_source_coverage(unique_facts, units) + output = self._format_facts(unique_facts, request.max_chars, evidence_ids) + + all_fact_ids = [f["fact_id"] for f in unique_facts] + covered_fact_ids = output.metadata.get( + "covered_fact_ids", + self._count_covered_facts(output.content, unique_facts), + ) + output.metadata["covered_fact_ids"] = covered_fact_ids + + if len(all_fact_ids) > 0: + coverage = len(covered_fact_ids) / len(all_fact_ids) + if coverage < 0.95: + raise ValueError( + f"Fact coverage too low: {len(covered_fact_ids)}/{len(all_fact_ids)}={coverage:.2f}" + ) + + return output + + @staticmethod + def _chunk_units( + units: list, *, input_limit: int, max_units: int = 12 + ) -> list[list]: + chunks: list[list] = [] + current: list = [] + current_chars = 0 + for unit in units: + unit_chars = len(unit.content) + 200 + if current and ( + current_chars + unit_chars > input_limit + or len(current) >= max_units + ): + chunks.append(current) + current = [] + current_chars = 0 + current.append(unit) + current_chars += unit_chars + if current: + chunks.append(current) + return chunks + + @staticmethod + def _prepare_units_with_ids( + units: list, + ) -> list[dict]: + """Pre-assign evidence IDs to each unit before LLM extraction.""" + prepared = [] + for unit in units: + prepared.append({ + "unit_id": unit.unit_id, + "text": unit.content, + "evidence_ids": list(unit.evidence_ids), + }) + return prepared + + @staticmethod + def _extract_spans_prompt(raw_content: str, units: list[dict]) -> str: + units_json = json.dumps( + [ + {"unit_id": unit["unit_id"], "text": unit["text"]} + for unit in units + ], + ensure_ascii=False, + ) + return ( + "Select all atomic facts from the memory-unit JSON below.\n\n" + "Rules:\n" + "1. Select every fact, including labels, identifiers, and numbers; " + "return one input unit_id and offsets indexing only that object's " + "text string (exclude JSON syntax and unit_id).\n" + "2. Do NOT generate or copy any text.\n" + "3. Each fact must be a contiguous substring of its selected unit.\n" + "4. Select the smallest complete atomic fact; split units that contain " + "multiple facts into separate spans.\n" + "5. Keep contradictory facts as separate spans.\n\n" + f"Memory units JSON:\n{units_json}\n\n" + "Return only a JSON array of objects with exactly these keys: " + "unit_id, start, end. start and end are character offsets in the " + "selected object's text value." + ) + + @staticmethod + def _compression_prompt(raw_content: str, units: list[dict]) -> str: + """Compatibility name for the approved information-extraction prompt.""" + return TenantDreamingCompressor._extract_spans_prompt(raw_content, units) + + def _extract_spans( + self, + raw_content: str, + units: list[dict], + feedback: str, + ) -> list[dict]: + """Call LLM to extract spans, parse JSON, return list of span dicts.""" + prompt = self._compression_prompt(raw_content, units) + if feedback and feedback != "none": + prompt += f"\n\nPrevious attempt feedback: {feedback}" + set_monitoring_operation("dreaming_semantic_compression_extract") + response = self.model.generate( + [ + { + "role": "system", + "content": ( + "You are an information extraction engine. " + "Your task is to select factual spans from RAW memory. " + "You do not summarize. You do not rewrite. You do not compress. " + "Return only unit-relative character offsets that exist exactly " + "in the selected source unit. " + "Output JSON only." + ), + }, + {"role": "user", "content": prompt}, + ] + ) + raw = str(response.content or "").strip() + raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=re.IGNORECASE) + return json.loads(raw) + + @staticmethod + def _validate_spans( + spans: list[dict], + raw_content: str, + units: list[dict], + ) -> tuple[list[dict], list[str]]: + """Validate spans and extract fact text. Returns (facts, feedback).""" + feedback = [] + valid_facts = [] + unit_evidence_map = {u["unit_id"]: u["evidence_ids"] for u in units} + unit_text_map = {u["unit_id"]: u["text"] for u in units} + + for i, span_obj in enumerate(spans): + start = span_obj.get("start") + end = span_obj.get("end") + + if start is None or end is None: + feedback.append(f"span_{i}_missing_offsets") + continue + + if not isinstance(start, int) or not isinstance(end, int): + feedback.append(f"span_{i}_non_integer_offsets") + continue + + unit_id = span_obj.get("unit_id") + legacy_unit_ids = span_obj.get("unit_ids") + if unit_id is None and isinstance(legacy_unit_ids, list): + if len(legacy_unit_ids) == 1: + unit_id = legacy_unit_ids[0] + if unit_id not in unit_text_map: + feedback.append(f"span_{i}_invalid_unit:{unit_id}") + continue + unit_text = unit_text_map[unit_id] + + if end > len(unit_text): + json_text_prefix = ( + json.dumps( + {"unit_id": unit_id, "text": ""}, + ensure_ascii=False, + )[:-2] + ) + normalized_start = start - len(json_text_prefix) + normalized_end = end - len(json_text_prefix) + if ( + normalized_start >= 0 + and normalized_start < normalized_end <= len(unit_text) + ): + start, end = normalized_start, normalized_end + + if start < 0 or end > len(unit_text): + feedback.append(f"span_{i}_out_of_bounds:{start},{end}") + continue + + if start >= end: + feedback.append(f"span_{i}_invalid_range:{start}>={end}") + continue + + fact_text = unit_text[start:end].strip() + + if not fact_text: + feedback.append(f"span_{i}_empty_text:{start},{end}") + continue + + unit_ids = [unit_id] + evidence_ids = sorted(set(unit_evidence_map.get(unit_id, []))) + + valid_facts.append({ + "fact_id": f"f{i:03d}", + "unit_ids": unit_ids, + "span": {"start": start, "end": end}, + "text": fact_text, + "evidence_ids": evidence_ids, + }) + + return valid_facts, feedback + + @staticmethod + def _deterministic_dedup(facts: list[dict]) -> list[dict]: + """Remove exact duplicates while preserving their source attribution.""" + by_text: dict[str, dict] = {} + order: list[str] = [] + + for fact_obj in facts: + normalized = re.sub(r"\s+", " ", fact_obj["text"].strip()) + if normalized in by_text: + retained = by_text[normalized] + retained["unit_ids"] = sorted( + set(retained["unit_ids"]) | set(fact_obj["unit_ids"]) + ) + retained["evidence_ids"] = sorted( + set(retained["evidence_ids"]) | set(fact_obj["evidence_ids"]) + ) + continue + by_text[normalized] = { + **fact_obj, + "unit_ids": sorted(set(fact_obj["unit_ids"])), + "evidence_ids": sorted(set(fact_obj["evidence_ids"])), + } + order.append(normalized) + + return [by_text[text] for text in order] + + @staticmethod + def _required_literal_facts( + units: list[dict], start_index: int + ) -> list[dict]: + """Extract validation-critical literals from authoritative unit text.""" + patterns = ( + r"https?://[^\s)\]}>,]+", + r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}", + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b", + r"(? None: + """Reject extraction that represents fewer than 95% of source units.""" + source_unit_ids = {unit["unit_id"] for unit in units} + covered_unit_ids = { + unit_id for fact in facts for unit_id in fact["unit_ids"] + } + if not source_unit_ids: + return + coverage = len(covered_unit_ids) / len(source_unit_ids) + if coverage < 0.95: + raise ValueError( + "Source unit coverage too low: " + f"{len(covered_unit_ids)}/{len(source_unit_ids)}={coverage:.2f}" + ) + + def _format_facts( + self, + facts: list[dict], + max_chars: int, + evidence_ids: list[str], + ) -> DreamingCompressionOutput: + """Format facts as bullet list. LLM only if over limit.""" + content = "\n".join(f"- {f['text']}" for f in facts) + + if len(content) <= max_chars: + return DreamingCompressionOutput( + content=content, + evidence_ids=evidence_ids, + metadata={ + "all_fact_ids": [f["fact_id"] for f in facts], + "covered_fact_ids": [f["fact_id"] for f in facts], + "fact_to_units_map": {f["fact_id"]: f["unit_ids"] for f in facts}, + }, + ) + + return self._lossless_formatting(facts, max_chars, evidence_ids) + + def _lossless_formatting( + self, + facts: list[dict], + max_chars: int, + evidence_ids: list[str], + ) -> DreamingCompressionOutput: + """LLM-based character-level shortening. No semantic changes.""" + facts_text = "\n".join( + f"[{fact['fact_id']}] {fact['text']}" for fact in facts + ) + prompt = ( + "You are a lossless formatter.\n\n" + "Input contains validated facts.\n" + "Your task: reduce character count while preserving every fact.\n\n" + "Allowed:\n" + "- Remove redundant whitespace\n" + "- Shorten connective words (e.g., 'in order to' → 'to')\n" + "- Change bullet formatting\n\n" + "Forbidden:\n" + "- Merge facts\n" + "- Reorder facts if it changes meaning\n" + "- Generalize facts\n" + "- Remove examples\n" + "- Remove identifiers\n" + "- Combine facts\n\n" + f"The following facts total {len(facts_text)} characters.\n" + f"Format them to fit within {max_chars} characters.\n\n" + f"Facts:\n{facts_text}\n\n" + "Return strict JSON with one entry for every supplied fact_id:\n" + '{"facts": [{"fact_id": "f001", "text": "shortened fact"}]}\n' + "Each fact_id must appear exactly once. Never combine multiple fact_ids." + ) + set_monitoring_operation("dreaming_semantic_compression_format") + response = self.model.generate( + [ + { + "role": "system", + "content": ( + "You are a lossless formatter. " + "You shorten text without changing meaning. " + "Output JSON only." + ), + }, + {"role": "user", "content": prompt}, + ] + ) + raw = str(response.content or "").strip() + raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=re.IGNORECASE) + payload = json.loads(raw) + formatted_facts = payload.get("facts") + if not isinstance(formatted_facts, list): + raise ValueError("Lossless formatting response must contain a facts list") + expected_ids = [fact["fact_id"] for fact in facts] + returned_ids = [ + item.get("fact_id") for item in formatted_facts + if isinstance(item, dict) + ] + if len(returned_ids) != len(set(returned_ids)): + raise ValueError("Lossless formatting returned duplicate fact_ids") + if set(returned_ids) != set(expected_ids): + raise ValueError( + "Lossless formatting fact_ids do not match the extracted facts" + ) + text_by_id = {} + for item in formatted_facts: + text = item.get("text") + if not isinstance(text, str) or not text.strip(): + raise ValueError("Lossless formatting returned an empty fact") + text_by_id[item["fact_id"]] = text.strip() + content = "\n".join(f"- {text_by_id[fact_id]}" for fact_id in expected_ids) + return DreamingCompressionOutput( + content=content, + evidence_ids=evidence_ids, + metadata={ + "all_fact_ids": expected_ids, + "covered_fact_ids": returned_ids, + "fact_to_units_map": {f["fact_id"]: f["unit_ids"] for f in facts}, + }, + ) + + @staticmethod + def _count_covered_facts( + output_content: str, + facts: list[dict], + ) -> list[str]: + """Return fact_ids whose text appears in output_content.""" + covered = [] + for fact_obj in facts: + if fact_obj["text"] in output_content: + covered.append(fact_obj["fact_id"]) + return covered diff --git a/backend/services/memory_dreaming_scheduler.py b/backend/services/memory_dreaming_scheduler.py index d6b0e4855e..341b687ca4 100644 --- a/backend/services/memory_dreaming_scheduler.py +++ b/backend/services/memory_dreaming_scheduler.py @@ -1,12 +1,130 @@ -"""Compatibility facade for the former Phase 2 Dreaming placeholder.""" +"""Backend adapter for the SDK's durable lease scheduler — dreaming jobs.""" -try: +import asyncio +import logging +from typing import Any, Dict, Hashable + +from consts.const import ( + DREAMING_SCHEDULER_ENABLED, + DREAMING_SCHEDULER_LEASE_SECONDS, + DREAMING_SCHEDULER_MAX_CONCURRENCY, + DREAMING_SCHEDULER_POLL_SECONDS, +) +from database import memory_dreaming_db +from nexent.scheduler import ClaimedJob, ExecutionLease, LeaseScheduler, SchedulerConfig + + +logger = logging.getLogger("memory_dreaming.scheduler") + + +class DreamingLeaseStore: + """Adapt synchronous PostgreSQL operations to the async scheduler contract.""" + + async def recover(self) -> None: + await asyncio.to_thread(memory_dreaming_db.recover_stale) + + async def claim_due( + self, + owner_id: str, + limit: int, + lease_seconds: float, + ) -> list[ClaimedJob[Dict[str, Any]]]: + row = await asyncio.to_thread( + memory_dreaming_db.claim_queued, + owner_id, + lease_seconds, + ) + if row is None: + return [] + return [ClaimedJob(job_id=row["run_id"], payload=row)] + + async def renew(self, job_id: Hashable, owner_id: str, lease_seconds: float) -> bool: + return await asyncio.to_thread( + memory_dreaming_db.renew_lease, + int(job_id), + owner_id, + lease_seconds, + ) + + async def release(self, job_id: Hashable, owner_id: str) -> bool: + return await asyncio.to_thread( + memory_dreaming_db.release_lease, + int(job_id), + owner_id, + ) + + +async def execute_dreaming( + job: ClaimedJob[Dict[str, Any]], + lease: ExecutionLease, +) -> None: + """Executor callback invoked by the SDK scheduler for each claimed dreaming job.""" + # Lazy import to avoid circular dependencies at module load time. from services.memory_dreaming_service import get_memory_dreaming_service -except ImportError: # package-style unit-test imports - from .memory_dreaming_service import get_memory_dreaming_service + + payload = job.payload + tenant_id = payload["tenant_id"] + user_id = payload["user_id"] + agent_id = payload["agent_id"] + trigger_source = payload.get("trigger_source", "scheduler") + + try: + await asyncio.to_thread( + get_memory_dreaming_service().run, + tenant_id=tenant_id, + user_id=user_id, + agent_id=agent_id, + run_id=int(job.job_id), + trigger_source=trigger_source, + ) + logger.info( + "Dreaming job completed: run_id=%s tenant=%s user=%s agent=%s", + job.job_id, + tenant_id, + user_id, + agent_id, + ) + except Exception: + logger.exception( + "Dreaming job failed: run_id=%s tenant=%s user=%s agent=%s", + job.job_id, + tenant_id, + user_id, + agent_id, + ) + raise + + +class DreamingScheduler: + """Application lifecycle wrapper around the reusable SDK scheduler.""" + + def __init__(self) -> None: + self._scheduler = LeaseScheduler( + store=DreamingLeaseStore(), + executor=execute_dreaming, + config=SchedulerConfig( + poll_interval_seconds=DREAMING_SCHEDULER_POLL_SECONDS, + lease_seconds=DREAMING_SCHEDULER_LEASE_SECONDS, + max_concurrency=DREAMING_SCHEDULER_MAX_CONCURRENCY, + ), + ) + + @property + def instance_id(self) -> str: + return self._scheduler.owner_id + + @property + def is_running(self) -> bool: + return self._scheduler.is_running + + async def start(self) -> None: + if not DREAMING_SCHEDULER_ENABLED: + logger.info("Dreaming scheduler disabled") + return + await self._scheduler.start() + + async def stop(self) -> None: + await self._scheduler.stop() -def run_once(*, tenant_id: str, user_id: str, agent_id: str, **kwargs): - return get_memory_dreaming_service().run( - tenant_id=tenant_id, user_id=user_id, agent_id=agent_id, **kwargs - ) +dreaming_scheduler = DreamingScheduler() diff --git a/backend/services/memory_dreaming_service.py b/backend/services/memory_dreaming_service.py index cc13ff533c..fce09f8472 100644 --- a/backend/services/memory_dreaming_service.py +++ b/backend/services/memory_dreaming_service.py @@ -7,6 +7,9 @@ from typing import Any, Dict, List, Optional from consts.const import ( + DREAMING_COMPRESSION_MAX_ATTEMPTS, + DREAMING_LONG_TERM_MAX_CHARS, + DREAMING_SOURCE_LIMIT, LIGHT_SLEEP_WINDOW_DAYS, MIN_PROMOTION_SCORE, MIN_RECALL_COUNT, @@ -15,9 +18,12 @@ ) from database import memory_dreaming_db, memory_record_db, memory_retrieval_hit_db from nexent.memory.dreaming import ( + DreamingMemoryUnit, DreamingThresholds, + build_dreaming_version, build_candidate, select_candidates, + units_from_decisions, ) from services.memory_record_service import get_memory_record_service @@ -32,9 +38,14 @@ class DreamingRunError(RuntimeError): pass +class DreamingConflictError(RuntimeError): + pass + + class MemoryDreamingService: - def __init__(self, record_service: Any = None): + def __init__(self, record_service: Any = None, compressor: Any = None): self.record_service = record_service or get_memory_record_service() + self.compressor = compressor def _run_light( self, tenant_id: str, user_id: str, agent_id: str, window_days: int @@ -78,7 +89,7 @@ def _run_rem( layer="agent", memory_type="short_term", status="active", - limit=1000, + limit=None, ) candidates = [] for record in records: @@ -106,36 +117,92 @@ def _run_rem( candidates.append(candidate) return candidates - def _promote(self, decisions: List[Any]) -> List[Dict[str, Any]]: - results = [] - for decision in decisions: - candidate = decision.candidate - if decision.promote: - created = self.record_service.create_memory( - tenant_id=candidate.tenant_id, - user_id=candidate.user_id, - agent_id=candidate.agent_id, - content=candidate.content, - layer="user", - memory_type="long_term", - concept_tags=candidate.concept_tags, - idempotency_key=f"dreaming:{candidate.memory_id}", - created_by="dreaming", - actor="dreaming", - ) - event = created.get("event", "ADD") - else: - event = "DEFER" - results.append( + def _build_version( + self, + tenant_id: str, + user_id: str, + agent_id: str, + run_id: int, + decisions: List[Any], + config_snapshot: Optional[Dict[str, Any]] = None, + ) -> Optional[Dict[str, Any]]: + active = memory_dreaming_db.get_active_version(tenant_id, user_id, agent_id) + parent_units = [ + DreamingMemoryUnit.model_validate(unit) + for unit in (active or {}).get("published_units", []) + ] + parent_unit_ids = {unit.unit_id for unit in parent_units} + parent_evidence_ids = { + evidence_id for unit in parent_units for evidence_id in unit.evidence_ids + } + new_units = units_from_decisions( + decisions, + source_limit=DREAMING_SOURCE_LIMIT, + excluded_evidence_ids=parent_evidence_ids, + ) + new_units = [ + unit + for unit in new_units + if unit.unit_id not in parent_unit_ids + and not set(unit.evidence_ids).issubset(parent_evidence_ids) + ] + if not new_units: + return None + result = build_dreaming_version( + parent_units=parent_units, + new_units=new_units, + max_chars=DREAMING_LONG_TERM_MAX_CHARS, + compressor=self.compressor or self._tenant_compressor(tenant_id, user_id), + max_attempts=DREAMING_COMPRESSION_MAX_ATTEMPTS, + run_id=run_id, + agent_id=agent_id, + ) + return memory_dreaming_db.create_and_activate_version( + tenant_id=tenant_id, + user_id=user_id, + agent_id=agent_id, + run_id=run_id, + parent_version_id=(active or {}).get("version_id"), + raw_content=result.raw_content, + published_content=result.published_content, + published_units=[ + unit.model_dump(mode="json") for unit in result.published_units + ], + source_evidence_ids=sorted( { - "memory_id": candidate.memory_id, - "score": decision.score, - "event": event, - "reason": decision.reason, - "archive_suggested": decision.archive_suggested, + evidence_id + for unit in [*parent_units, *new_units] + for evidence_id in unit.evidence_ids } - ) - return results + ), + config_snapshot=config_snapshot + or { + "source_limit": DREAMING_SOURCE_LIMIT, + "long_term_max_chars": DREAMING_LONG_TERM_MAX_CHARS, + "compression_max_attempts": DREAMING_COMPRESSION_MAX_ATTEMPTS, + }, + raw_char_count=result.raw_char_count, + published_char_count=result.published_char_count, + compression_status=result.compression_status, + compression_attempts=result.compression_attempts, + omitted_evidence_ids=result.omitted_evidence_ids, + mechanical_truncation=result.mechanical_truncation, + compression_audit=result.compression_audit, + ) + + @staticmethod + def _tenant_compressor(tenant_id: str, user_id: str): + instance = None + + def compress(request): + nonlocal instance + if instance is None: + from services.memory_dreaming_compressor import TenantDreamingCompressor + + instance = TenantDreamingCompressor(tenant_id, user_id) + return instance(request) + + return compress def run( self, @@ -147,10 +214,25 @@ def run( min_score: float = MIN_PROMOTION_SCORE, min_recall_count: int = MIN_RECALL_COUNT, min_unique_queries: int = MIN_UNIQUE_QUERIES, + run_id: Optional[int] = None, + trigger_source: str = "manual", ) -> Dict[str, Any]: if not tenant_id or not user_id or not agent_id: raise DreamingRunError("tenant_id, user_id and agent_id are required") - run_id = memory_dreaming_db.create_audit(tenant_id, user_id, agent_id) + if run_id is None: + if trigger_source == "manual": + run_id = memory_dreaming_db.create_audit(tenant_id, user_id, agent_id) + else: + run_id = memory_dreaming_db.create_audit( + tenant_id, + user_id, + agent_id, + trigger_source=trigger_source, + ) + else: + memory_dreaming_db.update_audit( + run_id, {"status": "running", "current_phase": "light"} + ) with memory_dreaming_db.try_scope_lock( tenant_id, user_id, agent_id ) as acquired: @@ -184,10 +266,37 @@ def run( ), recency_half_life_days=RECENCY_HALF_LIFE_DAYS, ) - results = self._promote(decisions) - promoted_count = sum( - item["event"] in {"ADD", "UPDATE"} for item in results + memory_dreaming_db.update_audit( + run_id, {"current_phase": "compression"} ) + version = self._build_version( + tenant_id, + user_id, + agent_id, + run_id, + decisions, + config_snapshot={ + "window_days": window_days, + "min_score": min_score, + "min_recall_count": min_recall_count, + "min_unique_queries": min_unique_queries, + "source_limit": DREAMING_SOURCE_LIMIT, + "long_term_max_chars": DREAMING_LONG_TERM_MAX_CHARS, + "compression_max_attempts": (DREAMING_COMPRESSION_MAX_ATTEMPTS), + }, + ) + results = [ + { + "memory_id": decision.candidate.memory_id, + "score": decision.score, + "evidence_ids": [str(decision.candidate.memory_id)], + "event": "SELECT" if decision.promote else "DEFER", + "reason": decision.reason, + "archive_suggested": decision.archive_suggested, + } + for decision in decisions + ] + promoted_count = sum(decision.promote for decision in decisions) result = { "run_id": run_id, "status": "completed", @@ -196,6 +305,7 @@ def run( "promoted_count": promoted_count, "deferred_count": len(results) - promoted_count, "decisions": results, + "version": version, } memory_dreaming_db.finish_audit( run_id, @@ -236,6 +346,44 @@ def list_audits( limit=limit, ) + def list_versions( + self, tenant_id: str, user_id: str, *, agent_id: str, limit: int = 100 + ) -> List[Dict[str, Any]]: + return memory_dreaming_db.list_versions( + tenant_id, user_id, agent_id=agent_id, limit=limit + ) + + def activate_version( + self, + tenant_id: str, + user_id: str, + *, + agent_id: str, + version_id: int, + actor_user_id: Optional[str] = None, + expected_active_version_id: Optional[int] = None, + ) -> Optional[Dict[str, Any]]: + with memory_dreaming_db.try_scope_lock( + tenant_id, user_id, agent_id + ) as acquired: + if not acquired: + raise DreamingConflictError("Dreaming scope is busy") + active = memory_dreaming_db.get_active_version(tenant_id, user_id, agent_id) + if ( + expected_active_version_id is not None + and (active or {}).get("version_id") != expected_active_version_id + ): + raise DreamingConflictError( + "Active Dreaming version changed; refresh and retry" + ) + return memory_dreaming_db.activate_version( + tenant_id, + user_id, + agent_id, + version_id, + actor_user_id=actor_user_id, + ) + _service: Optional[MemoryDreamingService] = None diff --git a/backend/utils/context_utils.py b/backend/utils/context_utils.py index 9e4d12097a..a7c669457e 100644 --- a/backend/utils/context_utils.py +++ b/backend/utils/context_utils.py @@ -391,7 +391,19 @@ def add_system( inputs.append(ContextItemInput( id=f"memory:{index}", type=ContextItemType.MEMORY, content=payload, source=(f"memory:{memory_search_query or 'run'}",), priority=90, - metadata={"render_group": "memory", "language": language, "authority": "retrieved"}, + metadata={ + "render_group": "memory", + "language": language, + "authority": "retrieved", + **( + { + "version_id": payload["dreaming_version_id"], + "memory_type": "long_term", + } + if payload.get("dreaming_version_id") is not None + else {} + ), + }, )) if duty: diff --git a/deploy/k8s/deploy.sh b/deploy/k8s/deploy.sh index b765845b22..564baf41ab 100755 --- a/deploy/k8s/deploy.sh +++ b/deploy/k8s/deploy.sh @@ -492,7 +492,7 @@ render_k8s_runtime_config_values() { printf ' celeryWorkerPrefetchMultiplier: %s\n' "$(yaml_quote "$(env_or_default CELERY_WORKER_PREFETCH_MULTIPLIER "1")")" printf ' celeryTaskTimeLimit: %s\n' "$(yaml_quote "$(env_or_default CELERY_TASK_TIME_LIMIT "3600")")" printf ' elasticsearchRequestTimeout: %s\n' "$(yaml_quote "$(env_or_default ELASTICSEARCH_REQUEST_TIMEOUT "30")")" - printf ' queues: %s\n' "$(yaml_quote "$(env_or_default QUEUES "process_q,forward_q")")" + printf ' queues: %s\n' "$(yaml_quote "$(env_or_default QUEUES "process_q,process_part_q,forward_q,dreaming_q")")" printf ' workerName: %s\n' "$(yaml_quote "$(env_or_default WORKER_NAME "")")" printf ' workerConcurrency: %s\n' "$(yaml_quote "$(env_or_default WORKER_CONCURRENCY "4")")" echo " oauth:" diff --git a/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml index 446bf9cd31..a43784bd20 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml @@ -124,7 +124,7 @@ config: celeryWorkerPrefetchMultiplier: "1" celeryTaskTimeLimit: "3600" elasticsearchRequestTimeout: "30" - queues: "process_q,forward_q" + queues: "process_q,process_part_q,forward_q,dreaming_q" workerName: "" workerConcurrency: "4" telemetry: diff --git a/deploy/sql/init.sql b/deploy/sql/init.sql index cc939d923c..2b8016388a 100644 --- a/deploy/sql/init.sql +++ b/deploy/sql/init.sql @@ -230,6 +230,58 @@ COMMENT ON COLUMN "knowledge_record_t"."updated_by" IS 'Last updater ID, audit f COMMENT ON COLUMN "knowledge_record_t"."created_by" IS 'Creator ID, audit field'; COMMENT ON TABLE "knowledge_record_t" IS 'Records knowledge base description and status information'; +-- Create the ag_prompt_template_t table +CREATE TABLE IF NOT EXISTS nexent.ag_prompt_template_t ( + template_id SERIAL PRIMARY KEY, + template_name VARCHAR(100) NOT NULL, + description VARCHAR(500), + template_type VARCHAR(50) NOT NULL DEFAULT 'agent_generate', + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + template_content_zh JSONB NOT NULL, + template_content_en JSONB, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE OR REPLACE FUNCTION update_ag_prompt_template_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER update_ag_prompt_template_update_time_trigger +BEFORE UPDATE ON nexent.ag_prompt_template_t +FOR EACH ROW +EXECUTE FUNCTION update_ag_prompt_template_update_time(); + +COMMENT ON TABLE nexent.ag_prompt_template_t IS 'Prompt template table for user-defined business logic generation prompts'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.template_id IS 'Prompt template ID'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.template_name IS 'Prompt template name'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.description IS 'Prompt template description'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.template_type IS 'Prompt template type'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.tenant_id IS 'Tenant ID'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.user_id IS 'User ID'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.template_content_zh IS 'Chinese prompt template content'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.template_content_en IS 'English prompt template content'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.created_by IS 'Creator'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.updated_by IS 'Updater'; +COMMENT ON COLUMN nexent.ag_prompt_template_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_prompt_template_user_name_active +ON nexent.ag_prompt_template_t (tenant_id, user_id, template_name) +WHERE delete_flag = 'N'; + +CREATE INDEX IF NOT EXISTS idx_ag_prompt_template_t_user +ON nexent.ag_prompt_template_t (tenant_id, user_id, template_type); + -- Create the ag_tool_info_t table CREATE TABLE IF NOT EXISTS nexent.ag_tool_info_t ( tool_id SERIAL PRIMARY KEY NOT NULL, @@ -736,6 +788,8 @@ CREATE TABLE IF NOT EXISTS nexent.memory_dreaming_audit_t ( deferred_count INTEGER NOT NULL DEFAULT 0, result_json JSONB, error TEXT, + lock_owner VARCHAR(100), + lock_until TIMESTAMP, create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, created_by VARCHAR(100), @@ -745,3 +799,121 @@ CREATE TABLE IF NOT EXISTS nexent.memory_dreaming_audit_t ( CREATE INDEX IF NOT EXISTS idx_memory_dreaming_audit_scope ON nexent.memory_dreaming_audit_t (tenant_id, user_id, agent_id, started_at DESC); + +-- Immutable, switchable Dreaming long-term memory versions. +CREATE TABLE IF NOT EXISTS nexent.memory_dreaming_version_t ( + version_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + agent_id VARCHAR(100) NOT NULL, + version_no INTEGER NOT NULL, + parent_version_id BIGINT, + run_id BIGINT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT FALSE, + raw_content TEXT NOT NULL, + published_content TEXT NOT NULL, + published_units JSONB NOT NULL DEFAULT '[]'::jsonb, + source_evidence_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + config_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb, + raw_char_count INTEGER NOT NULL, + published_char_count INTEGER NOT NULL, + compression_status VARCHAR(30) NOT NULL, + compression_attempts INTEGER NOT NULL DEFAULT 0, + compression_audit JSONB NOT NULL DEFAULT '[]'::jsonb, + omitted_evidence_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + mechanical_truncation BOOLEAN NOT NULL DEFAULT FALSE, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) NOT NULL DEFAULT 'N' +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_dreaming_version_scope + ON nexent.memory_dreaming_version_t + (tenant_id, user_id, agent_id, version_no); +CREATE UNIQUE INDEX IF NOT EXISTS uq_memory_dreaming_version_active_scope + ON nexent.memory_dreaming_version_t + (tenant_id, user_id, agent_id) + WHERE is_active AND delete_flag = 'N'; +CREATE UNIQUE INDEX IF NOT EXISTS uq_memory_dreaming_version_run + ON nexent.memory_dreaming_version_t (run_id); +CREATE INDEX IF NOT EXISTS idx_memory_dreaming_version_history + ON nexent.memory_dreaming_version_t + (tenant_id, user_id, agent_id, create_time DESC); +CREATE OR REPLACE FUNCTION nexent.prevent_memory_dreaming_version_content_update() +RETURNS TRIGGER AS $$ +BEGIN + IF OLD.version_no IS DISTINCT FROM NEW.version_no + OR OLD.parent_version_id IS DISTINCT FROM NEW.parent_version_id + OR OLD.run_id IS DISTINCT FROM NEW.run_id + OR OLD.raw_content IS DISTINCT FROM NEW.raw_content + OR OLD.published_content IS DISTINCT FROM NEW.published_content + OR OLD.published_units IS DISTINCT FROM NEW.published_units + OR OLD.source_evidence_ids IS DISTINCT FROM NEW.source_evidence_ids + OR OLD.config_snapshot IS DISTINCT FROM NEW.config_snapshot + OR OLD.raw_char_count IS DISTINCT FROM NEW.raw_char_count + OR OLD.published_char_count IS DISTINCT FROM NEW.published_char_count + OR OLD.compression_status IS DISTINCT FROM NEW.compression_status + OR OLD.compression_attempts IS DISTINCT FROM NEW.compression_attempts + OR OLD.compression_audit IS DISTINCT FROM NEW.compression_audit + OR OLD.omitted_evidence_ids IS DISTINCT FROM NEW.omitted_evidence_ids + OR OLD.mechanical_truncation IS DISTINCT FROM NEW.mechanical_truncation THEN + RAISE EXCEPTION 'Dreaming version content is immutable'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_memory_dreaming_version_immutable + ON nexent.memory_dreaming_version_t; +CREATE TRIGGER trg_memory_dreaming_version_immutable +BEFORE UPDATE ON nexent.memory_dreaming_version_t +FOR EACH ROW EXECUTE FUNCTION nexent.prevent_memory_dreaming_version_content_update(); + +CREATE TABLE IF NOT EXISTS nexent.memory_dreaming_activation_audit_t ( + activation_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + agent_id VARCHAR(100) NOT NULL, + actor_user_id VARCHAR(100) NOT NULL, + from_version_id BIGINT, + to_version_id BIGINT NOT NULL, + reason VARCHAR(100) NOT NULL DEFAULT 'user_switch', + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) NOT NULL DEFAULT 'N' +); +CREATE INDEX IF NOT EXISTS idx_memory_dreaming_activation_scope + ON nexent.memory_dreaming_activation_audit_t + (tenant_id, user_id, agent_id, create_time DESC); + +CREATE TABLE IF NOT EXISTS nexent.role_permission_t ( + role_permission_id SERIAL PRIMARY KEY, + user_role VARCHAR(30) NOT NULL, + permission_category VARCHAR(30), + permission_type VARCHAR(30), + permission_subtype VARCHAR(30), + parent_key VARCHAR(100) +); + +INSERT INTO nexent.role_permission_t ( + role_permission_id, + user_role, + permission_category, + permission_type, + permission_subtype +) +VALUES + (1004, 'SU', 'RESOURCE', 'DREAMING', 'VIEW_TENANT'), + (1005, 'SU', 'RESOURCE', 'DREAMING', 'EDIT_TENANT'), + (1116, 'ADMIN', 'RESOURCE', 'DREAMING', 'VIEW_TENANT'), + (1117, 'ADMIN', 'RESOURCE', 'DREAMING', 'EDIT_TENANT'), + (1514, 'ASSET_OWNER', 'RESOURCE', 'DREAMING', 'VIEW_TENANT'), + (1515, 'ASSET_OWNER', 'RESOURCE', 'DREAMING', 'EDIT_TENANT') +ON CONFLICT (role_permission_id) DO UPDATE SET + user_role = EXCLUDED.user_role, + permission_category = EXCLUDED.permission_category, + permission_type = EXCLUDED.permission_type, + permission_subtype = EXCLUDED.permission_subtype; diff --git a/deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_version.sql b/deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_version.sql new file mode 100644 index 0000000000..631315ebc0 --- /dev/null +++ b/deploy/sql/migrations/v2.4.0_0723_add_memory_dreaming_version.sql @@ -0,0 +1,124 @@ +-- Immutable Dreaming long-term memory versions and active-version pointer. +SET search_path TO nexent; +BEGIN; + +CREATE TABLE IF NOT EXISTS nexent.memory_dreaming_version_t ( + version_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + agent_id VARCHAR(100) NOT NULL, + version_no INTEGER NOT NULL, + parent_version_id BIGINT, + run_id BIGINT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT FALSE, + raw_content TEXT NOT NULL, + published_content TEXT NOT NULL, + published_units JSONB NOT NULL DEFAULT '[]'::jsonb, + source_evidence_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + config_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb, + raw_char_count INTEGER NOT NULL, + published_char_count INTEGER NOT NULL, + compression_status VARCHAR(30) NOT NULL, + compression_attempts INTEGER NOT NULL DEFAULT 0, + compression_audit JSONB NOT NULL DEFAULT '[]'::jsonb, + omitted_evidence_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + mechanical_truncation BOOLEAN NOT NULL DEFAULT FALSE, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) NOT NULL DEFAULT 'N' +); + +ALTER TABLE nexent.memory_dreaming_version_t + ADD COLUMN IF NOT EXISTS compression_audit JSONB NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE nexent.memory_dreaming_version_t + ADD COLUMN IF NOT EXISTS source_evidence_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS config_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_dreaming_version_scope + ON nexent.memory_dreaming_version_t + (tenant_id, user_id, agent_id, version_no); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_memory_dreaming_version_active_scope + ON nexent.memory_dreaming_version_t + (tenant_id, user_id, agent_id) + WHERE is_active AND delete_flag = 'N'; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_memory_dreaming_version_run + ON nexent.memory_dreaming_version_t (run_id); + +CREATE INDEX IF NOT EXISTS idx_memory_dreaming_version_history + ON nexent.memory_dreaming_version_t + (tenant_id, user_id, agent_id, create_time DESC); + +CREATE OR REPLACE FUNCTION nexent.prevent_memory_dreaming_version_content_update() +RETURNS TRIGGER AS $$ +BEGIN + IF OLD.version_no IS DISTINCT FROM NEW.version_no + OR OLD.parent_version_id IS DISTINCT FROM NEW.parent_version_id + OR OLD.run_id IS DISTINCT FROM NEW.run_id + OR OLD.raw_content IS DISTINCT FROM NEW.raw_content + OR OLD.published_content IS DISTINCT FROM NEW.published_content + OR OLD.published_units IS DISTINCT FROM NEW.published_units + OR OLD.source_evidence_ids IS DISTINCT FROM NEW.source_evidence_ids + OR OLD.config_snapshot IS DISTINCT FROM NEW.config_snapshot + OR OLD.raw_char_count IS DISTINCT FROM NEW.raw_char_count + OR OLD.published_char_count IS DISTINCT FROM NEW.published_char_count + OR OLD.compression_status IS DISTINCT FROM NEW.compression_status + OR OLD.compression_attempts IS DISTINCT FROM NEW.compression_attempts + OR OLD.compression_audit IS DISTINCT FROM NEW.compression_audit + OR OLD.omitted_evidence_ids IS DISTINCT FROM NEW.omitted_evidence_ids + OR OLD.mechanical_truncation IS DISTINCT FROM NEW.mechanical_truncation THEN + RAISE EXCEPTION 'Dreaming version content is immutable'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_memory_dreaming_version_immutable + ON nexent.memory_dreaming_version_t; +CREATE TRIGGER trg_memory_dreaming_version_immutable +BEFORE UPDATE ON nexent.memory_dreaming_version_t +FOR EACH ROW EXECUTE FUNCTION nexent.prevent_memory_dreaming_version_content_update(); + +CREATE TABLE IF NOT EXISTS nexent.memory_dreaming_activation_audit_t ( + activation_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + agent_id VARCHAR(100) NOT NULL, + actor_user_id VARCHAR(100) NOT NULL, + from_version_id BIGINT, + to_version_id BIGINT NOT NULL, + reason VARCHAR(100) NOT NULL DEFAULT 'user_switch', + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) NOT NULL DEFAULT 'N' +); +CREATE INDEX IF NOT EXISTS idx_memory_dreaming_activation_scope + ON nexent.memory_dreaming_activation_audit_t + (tenant_id, user_id, agent_id, create_time DESC); + +INSERT INTO nexent.role_permission_t ( + role_permission_id, + user_role, + permission_category, + permission_type, + permission_subtype +) +VALUES + (1004, 'SU', 'RESOURCE', 'DREAMING', 'VIEW_TENANT'), + (1005, 'SU', 'RESOURCE', 'DREAMING', 'EDIT_TENANT'), + (1116, 'ADMIN', 'RESOURCE', 'DREAMING', 'VIEW_TENANT'), + (1117, 'ADMIN', 'RESOURCE', 'DREAMING', 'EDIT_TENANT'), + (1514, 'ASSET_OWNER', 'RESOURCE', 'DREAMING', 'VIEW_TENANT'), + (1515, 'ASSET_OWNER', 'RESOURCE', 'DREAMING', 'EDIT_TENANT') +ON CONFLICT (role_permission_id) DO UPDATE SET + user_role = EXCLUDED.user_role, + permission_category = EXCLUDED.permission_category, + permission_type = EXCLUDED.permission_type, + permission_subtype = EXCLUDED.permission_subtype; + +COMMIT; diff --git a/deploy/sql/migrations/v2.4.0_0724_add_dreaming_lease_columns.sql b/deploy/sql/migrations/v2.4.0_0724_add_dreaming_lease_columns.sql new file mode 100644 index 0000000000..9fc379eadf --- /dev/null +++ b/deploy/sql/migrations/v2.4.0_0724_add_dreaming_lease_columns.sql @@ -0,0 +1,12 @@ +-- Add worker lease columns to the Dreaming audit table so the executor can +-- claim, renew, and release row-level leases with FOR UPDATE SKIP LOCKED. +SET search_path TO nexent; +BEGIN; + +ALTER TABLE nexent.memory_dreaming_audit_t + ADD COLUMN IF NOT EXISTS lock_owner VARCHAR(100); + +ALTER TABLE nexent.memory_dreaming_audit_t + ADD COLUMN IF NOT EXISTS lock_until TIMESTAMP; + +COMMIT; diff --git a/frontend/app/[locale]/memory/DreamingPanel.tsx b/frontend/app/[locale]/memory/DreamingPanel.tsx new file mode 100644 index 0000000000..245f854948 --- /dev/null +++ b/frontend/app/[locale]/memory/DreamingPanel.tsx @@ -0,0 +1,442 @@ +"use client"; + +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { + Alert, + App, + Button, + Card, + Empty, + List, + Progress, + Select, + Space, + Spin, + Tag, + Timeline, + Typography, +} from "antd"; +import { Brain, History, Play, RotateCcw } from "lucide-react"; +import { useTranslation } from "react-i18next"; + +import { useAuthorizationContext } from "@/components/providers/AuthorizationProvider"; +import { + activateDreamingVersion, + DreamingAudit, + DreamingVersion, + fetchDreamingAgents, + fetchDreamingAudits, + fetchDreamingParameters, + fetchDreamingVersions, + runDreaming, +} from "@/services/memoryService"; +import type { DreamingParameters } from "@/services/memoryService"; +import { getTenantUsers, TenantUser } from "@/services/tenantService"; + +const phaseProgress: Record = { + light: 20, + rem: 45, + deep: 70, + compression: 90, +}; + +export function DreamingPanel() { + const { message } = App.useApp(); + const { t } = useTranslation("common"); + const { user, hasPermission } = useAuthorizationContext(); + const [agents, setAgents] = useState>( + [] + ); + const [agentId, setAgentId] = useState(); + const [tenantUsers, setTenantUsers] = useState([]); + const [targetUserId, setTargetUserId] = useState(); + const [audits, setAudits] = useState([]); + const [versions, setVersions] = useState([]); + const [parameters, setParameters] = useState(); + const [loading, setLoading] = useState(true); + const [triggering, setTriggering] = useState(false); + + const refresh = useCallback( + async (selectedAgent: string) => { + const target = + targetUserId && targetUserId !== user?.id ? targetUserId : undefined; + const [nextAudits, nextVersions] = await Promise.all([ + fetchDreamingAudits(selectedAgent, 20, target), + fetchDreamingVersions(selectedAgent, 20, target), + ]); + setAudits(nextAudits); + setVersions(nextVersions); + }, + [targetUserId, user?.id] + ); + + useEffect(() => { + if (user?.id && !targetUserId) setTargetUserId(user.id); + }, [targetUserId, user?.id]); + + useEffect(() => { + if (!user?.tenantId || !hasPermission("DREAMING:VIEW_TENANT")) { + setTenantUsers([]); + return; + } + getTenantUsers(user.tenantId) + .then(({ users }) => setTenantUsers(users)) + .catch(() => message.error(t("dreaming.error.loadUsers"))); + }, [hasPermission, message, t, user?.tenantId]); + + useEffect(() => { + Promise.all([fetchDreamingAgents(), fetchDreamingParameters()]) + .then(([options, effectiveParameters]) => { + setAgents(options); + setParameters(effectiveParameters); + if (options.length) setAgentId(options[0].value); + }) + .catch(() => message.error(t("dreaming.error.loadAgents"))) + .finally(() => setLoading(false)); + }, [message, t]); + + useEffect(() => { + if (!agentId) return; + setLoading(true); + refresh(agentId) + .catch(() => message.error(t("dreaming.error.loadStatus"))) + .finally(() => setLoading(false)); + }, [agentId, message, refresh]); + + const activeRun = useMemo( + () => audits.find((run) => ["queued", "running"].includes(run.status)), + [audits] + ); + const latestRun = audits[0]; + const selectedIsSelf = !targetUserId || targetUserId === user?.id; + const canEditTarget = selectedIsSelf || hasPermission("DREAMING:EDIT_TENANT"); + const queueDelayed = + activeRun?.status === "queued" && + !!activeRun.started_at && + Date.now() - new Date(activeRun.started_at).getTime() > 60_000; + useEffect(() => { + if (!agentId || !activeRun) return; + const timer = window.setInterval(() => refresh(agentId), 2000); + return () => window.clearInterval(timer); + }, [activeRun, agentId, refresh]); + + const trigger = async () => { + if (!agentId) return; + setTriggering(true); + try { + await runDreaming(agentId, selectedIsSelf ? undefined : targetUserId); + message.success(t("dreaming.run.queued")); + await refresh(agentId); + } catch { + message.error(t("dreaming.error.trigger")); + } finally { + setTriggering(false); + } + }; + + const activate = async (version: DreamingVersion) => { + if (!agentId) return; + const activeVersion = versions.find((candidate) => candidate.is_active); + if (!activeVersion) return; + try { + await activateDreamingVersion( + agentId, + version.version_id, + activeVersion.version_id, + selectedIsSelf ? undefined : targetUserId + ); + message.success( + t("dreaming.version.switched", { version: version.version_no }) + ); + await refresh(agentId); + } catch { + message.error(t("dreaming.error.activate")); + } + }; + + if (loading && !agentId) return ; + + return ( +
+ +
+
+ + + Dreaming + + + {t("dreaming.description")} + +
+ {parameters && ( + + + {t("dreaming.parameter.sourceLimit", { + count: parameters.source_limit, + })} + + + {t("dreaming.parameter.maxChars", { + count: parameters.long_term_max_chars, + })} + + + {t("dreaming.parameter.compressionRetries", { + count: parameters.compression_max_attempts, + })} + + + )} +
+
+ + {tenantUsers.length > 1 && ( + + + +
+ + + {activeRun && ( + + {queueDelayed && ( +
+ {t("dreaming.run.queueDelayedDescription")} +
+ )} +
+ {t("dreaming.run.currentPhase")}:{" "} + {activeRun.current_phase + ? t(`dreaming.phase.${activeRun.current_phase}`, { + defaultValue: activeRun.current_phase, + }) + : t("dreaming.phase.queued")} +
+ +
+ } + /> + )} + {!activeRun && latestRun?.status === "failed" && ( + + )} + {!activeRun && latestRun?.status === "completed" && ( + + )} + {!activeRun && latestRun?.status === "skipped" && ( + + )} + + + {versions.find((version) => version.is_active) ? ( + (() => { + const active = versions.find((version) => version.is_active)!; + return ( +
+ + Active V{active.version_no} + + {t("dreaming.characters", { + count: active.published_char_count, + })} + + + {active.compression_status} + + + {active.mechanical_truncation && ( + + )} + + {active.published_content} + +
+ ); + })() + ) : ( + + )} + + + + {latestRun?.result?.decisions?.length ? ( + ( + + + + {t(`dreaming.decision.${decision.event.toLowerCase()}`)} + + + {t("dreaming.decision.memory", { + id: decision.memory_id, + })} + + + {t("dreaming.decision.score", { + score: decision.score.toFixed(3), + })} + + + } + description={ +
+
{decision.reason}
+ + {t("dreaming.decision.evidence", { + ids: ( + decision.evidence_ids || [ + String(decision.memory_id), + ] + ).join(", "), + })} + +
+ } + /> + + )} + /> + ) : ( + + )} + + + + + {t("dreaming.history.title")} + + } + > + {versions.length ? ( + ({ + color: version.is_active ? "green" : "gray", + children: ( +
+
+ + V{version.version_no} + {version.is_active && ( + {t("dreaming.version.current")} + )} + {version.compression_status} + +
+ {t("dreaming.version.lengths", { + raw: version.raw_char_count, + published: version.published_char_count, + })} +
+
+ {!version.is_active && canEditTarget && ( + + )} +
+ ), + }))} + /> + ) : ( + + )} + + + ); +} diff --git a/frontend/app/[locale]/memory/MemoryManager.tsx b/frontend/app/[locale]/memory/MemoryManager.tsx index b3d5ddb908..3ca04db6c1 100644 --- a/frontend/app/[locale]/memory/MemoryManager.tsx +++ b/frontend/app/[locale]/memory/MemoryManager.tsx @@ -24,6 +24,7 @@ import { import type { Dayjs } from "dayjs"; import { Bot, + Brain, Building2, Clock3, Edit3, @@ -37,6 +38,7 @@ import Link from "next/link"; import { useTranslation } from "react-i18next"; import { Can } from "@/components/permission/Can"; +import { DreamingPanel } from "./DreamingPanel"; import { loadMemoryConfig, setMemorySwitch, @@ -55,7 +57,7 @@ import { const { Text, Title, Paragraph } = Typography; -type TabKey = "base" | MemoryScope; +type TabKey = "base" | "dreaming" | MemoryScope; type MemoryForm = { memory_type: MemoryType; status: MemoryStatus; @@ -135,7 +137,8 @@ export function MemoryManager() { const [editing, setEditing] = useState(null); const [form] = Form.useForm(); - const scope = activeTab === "base" ? null : activeTab; + const scope = + activeTab === "base" || activeTab === "dreaming" ? null : activeTab; const records = scope ? recordsByScope[scope] : []; const refreshRecords = useCallback( @@ -435,15 +438,15 @@ export function MemoryManager() { : "编辑" } > - + } + > + + + + {t("dreaming.schedule.enabled")} + + setSchedule({ ...schedule, enabled })} + /> + + ({ + value, + label: t(`dreaming.schedule.weekday.${value}`), + }))} + /> + )} + {scheduleMode !== "interval" ? ( + + value && setScheduleTime(value.format("HH:mm")) + } + /> + ) : ( + + setIntervalHours(value || 1)} + /> + + {t("dreaming.schedule.hours")} + + + )} + +
+ + {schedule.next_fire_at + ? t("dreaming.schedule.nextFire", { + time: new Date(schedule.next_fire_at).toLocaleString(), + }) + : t("dreaming.schedule.disabledHint")} + +
+ + )} + {activeRun && ( { }); } +export async function fetchDreamingSchedule( + agentId: string, + targetUserId?: string +): Promise { + const params = new URLSearchParams({ agent_id: agentId }); + if (targetUserId) params.set("target_user_id", targetUserId); + return requestJson( + `${API_ENDPOINTS.memory.dreaming.schedule}?${params.toString()}`, + { headers: getAuthHeaders() } + ); +} + +export async function saveDreamingSchedule( + schedule: Omit & { target_user_id?: string } +): Promise { + return requestJson(API_ENDPOINTS.memory.dreaming.schedule, { + method: "PUT", + headers: getAuthHeaders(), + body: JSON.stringify(schedule), + }); +} + export async function runDreaming(agentId: string, targetUserId?: string) { return requestJson(API_ENDPOINTS.memory.dreaming.run, { method: "POST", diff --git a/test/backend/apps/test_memory_dreaming_app.py b/test/backend/apps/test_memory_dreaming_app.py index f9f5bd07fd..0eccecd820 100644 --- a/test/backend/apps/test_memory_dreaming_app.py +++ b/test/backend/apps/test_memory_dreaming_app.py @@ -253,3 +253,63 @@ def test_ac016_admin_without_tenant_capability_is_denied(monkeypatch): permission_type="DREAMING", permission_subtype="VIEW_TENANT", ) + + +def test_ac033_schedule_defaults_disabled(monkeypatch): + monkeypatch.setattr( + memory_dreaming_app, + "get_current_user_id", + lambda _authorization: ("user-1", "tenant-1"), + ) + monkeypatch.setattr( + memory_dreaming_app.memory_dreaming_db, "get_schedule", lambda *_args: None + ) + + result = memory_dreaming_app.get_dreaming_schedule( + agent_id="agent-1", authorization="Bearer token" + ) + + assert result["enabled"] is False + assert result["cron_expr"] == "0 3 * * *" + assert result["next_fire_at"] is None + + +def test_ac033_schedule_is_validated_and_saved(monkeypatch): + monkeypatch.setattr( + memory_dreaming_app, + "get_current_user_id", + lambda _authorization: ("user-1", "tenant-1"), + ) + saved = MagicMock(return_value={"enabled": True, "next_fire_at": "future"}) + monkeypatch.setattr( + memory_dreaming_app.memory_dreaming_db, "upsert_schedule", saved + ) + payload = memory_dreaming_app.DreamingScheduleRequest( + agent_id="agent-1", + enabled=True, + rule_type="CRON", + timezone="Asia/Shanghai", + cron_expr="30 3 * * *", + ) + + result = memory_dreaming_app.put_dreaming_schedule(payload, "Bearer token") + + assert result["enabled"] is True + kwargs = saved.call_args.kwargs + assert kwargs["rule_type"] == "CRON" + assert kwargs["next_fire_at"] is not None + + +@pytest.mark.parametrize( + "values", + [ + {"rule_type": "CRON", "cron_expr": "bad cron"}, + {"rule_type": "INTERVAL", "interval_seconds": 3599}, + {"rule_type": "CRON", "cron_expr": "0 3 * * *", "timezone": "Mars/Olympus"}, + ], +) +def test_ac034_invalid_schedule_is_rejected(values): + with pytest.raises(ValidationError): + memory_dreaming_app.DreamingScheduleRequest( + agent_id="agent-1", enabled=True, **values + ) diff --git a/test/backend/database/test_memory_dreaming_db.py b/test/backend/database/test_memory_dreaming_db.py new file mode 100644 index 0000000000..a27f435fcb --- /dev/null +++ b/test/backend/database/test_memory_dreaming_db.py @@ -0,0 +1,601 @@ +"""Unit tests for memory_dreaming_db CRUD and audit functions.""" + +from contextlib import contextmanager +from datetime import datetime +from unittest.mock import MagicMock, patch + +import pytest + +from database import memory_dreaming_db +from database.db_models import ( + MemoryDreamingActivationAudit, + MemoryDreamingAudit, + MemoryDreamingVersion, +) + + +def _mock_session(monkeypatch): + session = MagicMock() + + @contextmanager + def _ctx(): + yield session + + monkeypatch.setattr(memory_dreaming_db, "get_db_session", _ctx) + return session + + +def _make_version_row(**overrides): + row = MagicMock(spec=MemoryDreamingVersion) + row.version_id = overrides.get("version_id", 1) + row.tenant_id = overrides.get("tenant_id", "t") + row.user_id = overrides.get("user_id", "u") + row.agent_id = overrides.get("agent_id", "a") + row.version_no = overrides.get("version_no", 1) + row.parent_version_id = overrides.get("parent_version_id", None) + row.run_id = overrides.get("run_id", 10) + row.is_active = overrides.get("is_active", True) + row.raw_content = overrides.get("raw_content", "raw") + row.published_content = overrides.get("published_content", "pub") + row.published_units = overrides.get("published_units", []) + row.source_evidence_ids = overrides.get("source_evidence_ids", []) + row.config_snapshot = overrides.get("config_snapshot", {}) + row.raw_char_count = overrides.get("raw_char_count", 3) + row.published_char_count = overrides.get("published_char_count", 3) + row.compression_status = overrides.get("compression_status", "not_needed") + row.compression_attempts = overrides.get("compression_attempts", 0) + row.omitted_evidence_ids = overrides.get("omitted_evidence_ids", []) + row.mechanical_truncation = overrides.get("mechanical_truncation", False) + row.compression_audit = overrides.get("compression_audit", []) + row.create_time = overrides.get("create_time", datetime(2026, 7, 25)) + return row + + +# --------------------------------------------------------------------------- +# try_scope_lock +# --------------------------------------------------------------------------- + + +def test_try_scope_lock_acquired(monkeypatch): + session = _mock_session(monkeypatch) + session.execute.return_value.scalar.return_value = True + + with memory_dreaming_db.try_scope_lock("t", "u", "a") as acquired: + assert acquired is True + + session.commit.assert_called_once() + + +def test_try_scope_lock_not_acquired(monkeypatch): + session = _mock_session(monkeypatch) + session.execute.return_value.scalar.return_value = False + + with memory_dreaming_db.try_scope_lock("t", "u", "a") as acquired: + assert acquired is False + + session.commit.assert_called_once() + + +def test_try_scope_lock_rollback_on_exception(monkeypatch): + session = _mock_session(monkeypatch) + session.execute.return_value.scalar.return_value = True + + with pytest.raises(ValueError), memory_dreaming_db.try_scope_lock("t", "u", "a"): + raise ValueError("boom") + + session.rollback.assert_called_once() + session.commit.assert_not_called() + + +# --------------------------------------------------------------------------- +# create_audit +# --------------------------------------------------------------------------- + + +def test_create_audit_default(monkeypatch): + session = _mock_session(monkeypatch) + + added_row = None + + def capture_add(row): + nonlocal added_row + added_row = row + row.run_id = 42 + + session.add.side_effect = capture_add + + result = memory_dreaming_db.create_audit("t", "u", "a") + + assert result == 42 + assert added_row.tenant_id == "t" + assert added_row.user_id == "u" + assert added_row.agent_id == "a" + assert added_row.trigger_source == "manual" + assert added_row.status == "running" + assert added_row.current_phase == "light" + session.commit.assert_called_once() + + +def test_create_audit_queued_status(monkeypatch): + session = _mock_session(monkeypatch) + + added_row = None + + def capture_add(row): + nonlocal added_row + added_row = row + row.run_id = 99 + + session.add.side_effect = capture_add + + result = memory_dreaming_db.create_audit( + "t", "u", "a", trigger_source="scheduler", status="queued" + ) + + assert result == 99 + assert added_row.trigger_source == "scheduler" + assert added_row.status == "queued" + assert added_row.current_phase is None + + +# --------------------------------------------------------------------------- +# get_active_version +# --------------------------------------------------------------------------- + + +def test_get_active_version_returns_dict(monkeypatch): + session = _mock_session(monkeypatch) + row = _make_version_row(version_id=5, is_active=True) + session.query.return_value.filter.return_value.first.return_value = row + + result = memory_dreaming_db.get_active_version("t", "u", "a") + + assert result is not None + assert result["version_id"] == 5 + assert result["is_active"] is True + + +def test_get_active_version_returns_none(monkeypatch): + session = _mock_session(monkeypatch) + session.query.return_value.filter.return_value.first.return_value = None + + result = memory_dreaming_db.get_active_version("t", "u", "a") + + assert result is None + + +# --------------------------------------------------------------------------- +# create_and_activate_version +# --------------------------------------------------------------------------- + + +def test_create_and_activate_version_returns_existing(monkeypatch): + session = _mock_session(monkeypatch) + existing = _make_version_row(version_id=7) + session.query.return_value.filter.return_value.first.return_value = existing + + result = memory_dreaming_db.create_and_activate_version( + tenant_id="t", + user_id="u", + agent_id="a", + run_id=10, + parent_version_id=None, + raw_content="raw", + published_content="pub", + published_units=[], + source_evidence_ids=[], + config_snapshot={}, + raw_char_count=3, + published_char_count=3, + compression_status="not_needed", + compression_attempts=0, + omitted_evidence_ids=[], + mechanical_truncation=False, + compression_audit=[], + ) + + assert result["version_id"] == 7 + session.add.assert_not_called() + + +def test_create_and_activate_version_creates_new(monkeypatch): + session = _mock_session(monkeypatch) + + # First query: check for existing → None + # Second query: get max version_no → 3 + query_mock = MagicMock() + filter_mock = MagicMock() + + call_count = [0] + + def query_side_effect(model): + call_count[0] += 1 + return query_mock + + session.query.side_effect = query_side_effect + + # First filter call: check existing → None + # Second filter call: max version_no → 3 + # Third filter call: deactivate old active + filter_count = [0] + + def filter_side_effect(*args, **kwargs): + filter_count[0] += 1 + if filter_count[0] == 1: + # Check for existing row + result = MagicMock() + result.first.return_value = None + return result + elif filter_count[0] == 2: + # Max version_no + result = MagicMock() + result.scalar.return_value = 3 + return result + else: + # Deactivate old active versions + result = MagicMock() + return result + + query_mock.filter.side_effect = filter_side_effect + + added_row = None + + def capture_add(row): + nonlocal added_row + added_row = row + row.version_id = 100 + row.version_no = 4 + + session.add.side_effect = capture_add + + result = memory_dreaming_db.create_and_activate_version( + tenant_id="t", + user_id="u", + agent_id="a", + run_id=10, + parent_version_id=3, + raw_content="raw content", + published_content="pub content", + published_units=[{"unit_id": "u1"}], + source_evidence_ids=["1"], + config_snapshot={"key": "val"}, + raw_char_count=11, + published_char_count=11, + compression_status="semantic", + compression_attempts=1, + omitted_evidence_ids=[], + mechanical_truncation=False, + compression_audit=[{"attempt": 1, "outcome": "accepted"}], + ) + + assert added_row is not None + assert added_row.tenant_id == "t" + assert added_row.version_no == 4 + assert added_row.parent_version_id == 3 + assert added_row.is_active is True + assert added_row.created_by == "dreaming" + session.commit.assert_called_once() + + +# --------------------------------------------------------------------------- +# list_versions +# --------------------------------------------------------------------------- + + +def test_list_versions(monkeypatch): + session = _mock_session(monkeypatch) + rows = [ + _make_version_row(version_id=1, version_no=2), + _make_version_row(version_id=2, version_no=1), + ] + query_chain = MagicMock() + session.query.return_value = query_chain + query_chain.filter.return_value = query_chain + query_chain.order_by.return_value = query_chain + query_chain.limit.return_value = query_chain + query_chain.all.return_value = rows + + result = memory_dreaming_db.list_versions("t", "u", agent_id="a", limit=50) + + assert len(result) == 2 + assert result[0]["version_id"] == 1 + assert result[1]["version_id"] == 2 + + +# --------------------------------------------------------------------------- +# activate_version +# --------------------------------------------------------------------------- + + +def test_activate_version_not_found(monkeypatch): + session = _mock_session(monkeypatch) + query_chain = MagicMock() + session.query.return_value = query_chain + query_chain.filter.return_value = query_chain + query_chain.all.return_value = [] + + result = memory_dreaming_db.activate_version("t", "u", "a", 999) + + assert result is None + + +def test_activate_version_already_active(monkeypatch): + session = _mock_session(monkeypatch) + row = _make_version_row(version_id=5, is_active=True) + query_chain = MagicMock() + session.query.return_value = query_chain + query_chain.filter.return_value = query_chain + query_chain.all.return_value = [row] + + result = memory_dreaming_db.activate_version("t", "u", "a", 5) + + assert result["version_id"] == 5 + session.commit.assert_not_called() + + +def test_activate_version_switch(monkeypatch): + session = _mock_session(monkeypatch) + current = _make_version_row(version_id=5, is_active=True) + target = _make_version_row(version_id=10, is_active=False) + query_chain = MagicMock() + session.query.return_value = query_chain + + filter_count = [0] + + def filter_side_effect(*args, **kwargs): + filter_count[0] += 1 + result = MagicMock() + if filter_count[0] == 1: + result.all.return_value = [current, target] + return result + + query_chain.filter.side_effect = filter_side_effect + + result = memory_dreaming_db.activate_version( + "t", "u", "a", 10, actor_user_id="admin" + ) + + assert result["version_id"] == 10 + assert target.is_active is True + assert target.updated_by == "admin" + session.add.assert_called_once() + audit_row = session.add.call_args[0][0] + assert isinstance(audit_row, MemoryDreamingActivationAudit) + assert audit_row.from_version_id == 5 + assert audit_row.to_version_id == 10 + assert audit_row.actor_user_id == "admin" + session.commit.assert_called_once() + + +def test_activate_version_switch_no_current_active(monkeypatch): + session = _mock_session(monkeypatch) + target = _make_version_row(version_id=10, is_active=False) + query_chain = MagicMock() + session.query.return_value = query_chain + + filter_count = [0] + + def filter_side_effect(*args, **kwargs): + filter_count[0] += 1 + result = MagicMock() + if filter_count[0] == 1: + result.all.return_value = [target] + return result + + query_chain.filter.side_effect = filter_side_effect + + result = memory_dreaming_db.activate_version("t", "u", "a", 10) + + assert result["version_id"] == 10 + audit_row = session.add.call_args[0][0] + assert audit_row.from_version_id is None + assert audit_row.created_by == "u" + + +# --------------------------------------------------------------------------- +# _version_to_dict +# --------------------------------------------------------------------------- + + +def test_version_to_dict_handles_none_fields(): + row = MagicMock(spec=MemoryDreamingVersion) + row.version_id = 1 + row.tenant_id = "t" + row.user_id = "u" + row.agent_id = "a" + row.version_no = 1 + row.parent_version_id = None + row.run_id = 10 + row.is_active = True + row.raw_content = "raw" + row.published_content = "pub" + row.published_units = None + row.source_evidence_ids = None + row.config_snapshot = None + row.raw_char_count = 3 + row.published_char_count = 3 + row.compression_status = "not_needed" + row.compression_attempts = 0 + row.omitted_evidence_ids = None + row.mechanical_truncation = False + row.compression_audit = None + row.create_time = None + + result = memory_dreaming_db._version_to_dict(row) + + assert result["published_units"] == [] + assert result["source_evidence_ids"] == [] + assert result["config_snapshot"] == {} + assert result["omitted_evidence_ids"] == [] + assert result["compression_audit"] == [] + assert result["created_at"] is None + + +# --------------------------------------------------------------------------- +# update_audit +# --------------------------------------------------------------------------- + + +def test_update_audit_success(monkeypatch): + session = _mock_session(monkeypatch) + row = MagicMock(spec=MemoryDreamingAudit) + row.run_id = 42 + session.query.return_value.filter.return_value.first.return_value = row + + result = memory_dreaming_db.update_audit( + 42, {"status": "completed", "light_count": 5} + ) + + assert result is True + assert row.status == "completed" + assert row.light_count == 5 + session.commit.assert_called_once() + + +def test_update_audit_not_found(monkeypatch): + session = _mock_session(monkeypatch) + session.query.return_value.filter.return_value.first.return_value = None + + result = memory_dreaming_db.update_audit(999, {"status": "failed"}) + + assert result is False + session.commit.assert_not_called() + + +def test_update_audit_ignores_disallowed_keys(monkeypatch): + session = _mock_session(monkeypatch) + row = MagicMock(spec=MemoryDreamingAudit) + session.query.return_value.filter.return_value.first.return_value = row + + result = memory_dreaming_db.update_audit( + 42, {"status": "completed", "tenant_id": "hacked"} + ) + + assert result is True + assert row.status == "completed" + assert not hasattr(row, "tenant_id") or row.tenant_id != "hacked" + + +# --------------------------------------------------------------------------- +# finish_audit +# --------------------------------------------------------------------------- + + +def test_finish_audit_completed(monkeypatch): + mock_update = MagicMock(return_value=True) + monkeypatch.setattr(memory_dreaming_db, "update_audit", mock_update) + + result = memory_dreaming_db.finish_audit( + 42, status="completed", light_count=3, rem_count=2 + ) + + assert result is True + call_values = mock_update.call_args[0][1] + assert call_values["status"] == "completed" + assert call_values["light_count"] == 3 + assert call_values["current_phase"] is None + assert "finished_at" in call_values + + +def test_finish_audit_failed(monkeypatch): + mock_update = MagicMock(return_value=True) + monkeypatch.setattr(memory_dreaming_db, "update_audit", mock_update) + + result = memory_dreaming_db.finish_audit( + 42, status="failed", error="something broke" + ) + + assert result is True + call_values = mock_update.call_args[0][1] + assert call_values["status"] == "failed" + assert call_values["error"] == "something broke" + # Failed status should NOT clear current_phase + assert "current_phase" not in call_values + + +# --------------------------------------------------------------------------- +# list_audits +# --------------------------------------------------------------------------- + + +def test_list_audits_with_filters(monkeypatch): + session = _mock_session(monkeypatch) + audit_row = MagicMock() + audit_row.run_id = 1 + audit_row.tenant_id = "t" + audit_row.user_id = "u" + audit_row.agent_id = "a" + audit_row.trigger_source = "manual" + audit_row.status = "completed" + audit_row.current_phase = None + audit_row.started_at = datetime(2026, 7, 25) + audit_row.finished_at = datetime(2026, 7, 25, 1) + audit_row.light_count = 3 + audit_row.rem_count = 2 + audit_row.promoted_count = 1 + audit_row.deferred_count = 1 + audit_row.result_json = {"status": "completed"} + audit_row.error = None + + query_chain = MagicMock() + session.query.return_value = query_chain + query_chain.filter.return_value = query_chain + query_chain.order_by.return_value = query_chain + query_chain.limit.return_value = query_chain + query_chain.all.return_value = [audit_row] + + result = memory_dreaming_db.list_audits( + "t", "u", agent_id="a", run_id=1, limit=50 + ) + + assert len(result) == 1 + assert result[0]["run_id"] == 1 + assert result[0]["status"] == "completed" + assert result[0]["started_at"] is not None + assert result[0]["finished_at"] is not None + + +def test_list_audits_no_optional_filters(monkeypatch): + session = _mock_session(monkeypatch) + query_chain = MagicMock() + session.query.return_value = query_chain + query_chain.filter.return_value = query_chain + query_chain.order_by.return_value = query_chain + query_chain.limit.return_value = query_chain + query_chain.all.return_value = [] + + result = memory_dreaming_db.list_audits("t", "u") + + assert result == [] + + +def test_list_audits_none_datetime_fields(monkeypatch): + session = _mock_session(monkeypatch) + audit_row = MagicMock() + audit_row.run_id = 1 + audit_row.tenant_id = "t" + audit_row.user_id = "u" + audit_row.agent_id = "a" + audit_row.trigger_source = "manual" + audit_row.status = "queued" + audit_row.current_phase = None + audit_row.started_at = None + audit_row.finished_at = None + audit_row.light_count = 0 + audit_row.rem_count = 0 + audit_row.promoted_count = 0 + audit_row.deferred_count = 0 + audit_row.result_json = None + audit_row.error = None + + query_chain = MagicMock() + session.query.return_value = query_chain + query_chain.filter.return_value = query_chain + query_chain.order_by.return_value = query_chain + query_chain.limit.return_value = query_chain + query_chain.all.return_value = [audit_row] + + result = memory_dreaming_db.list_audits("t", "u") + + assert result[0]["started_at"] is None + assert result[0]["finished_at"] is None diff --git a/test/backend/database/test_memory_dreaming_schema.py b/test/backend/database/test_memory_dreaming_schema.py index 3802f88088..c054b39863 100644 --- a/test/backend/database/test_memory_dreaming_schema.py +++ b/test/backend/database/test_memory_dreaming_schema.py @@ -5,6 +5,7 @@ from database.db_models import ( MemoryDreamingActivationAudit, MemoryDreamingAudit, + MemoryDreamingSchedule, MemoryDreamingVersion, MemoryRecord, MemoryRetrievalHit, @@ -124,6 +125,40 @@ def test_ac012_dreaming_scheduler_is_wired_for_deployment(): assert "dreaming_q" not in const_py.split("QUEUES")[1].split("\n")[0] +def test_ac033_schedule_orm_and_sql_contract(): + columns = MemoryDreamingSchedule.__table__.columns + for name in ( + "schedule_id", + "tenant_id", + "user_id", + "agent_id", + "enabled", + "rule_type", + "timezone", + "start_at", + "cron_expr", + "interval_seconds", + "next_fire_at", + "last_fire_at", + "fire_count", + ): + assert name in columns + + root = Path(__file__).resolve().parents[3] + migration = ( + root / "deploy/sql/migrations/v2.4.0_0727_add_memory_dreaming_schedule.sql" + ).read_text() + init_sql = (root / "deploy/sql/init.sql").read_text() + for token in ( + "memory_dreaming_schedule_t", + "uq_memory_dreaming_schedule_scope", + "idx_memory_dreaming_schedule_due", + "interval_seconds >= 3600", + ): + assert token in migration + assert token in init_sql + + def test_ac002_dreaming_stats_filter_agent_scope(monkeypatch): monkeypatch.setattr( memory_retrieval_hit_db, diff --git a/test/backend/services/test_memory_dreaming_compressor.py b/test/backend/services/test_memory_dreaming_compressor.py index 531e321761..dc81ea16db 100644 --- a/test/backend/services/test_memory_dreaming_compressor.py +++ b/test/backend/services/test_memory_dreaming_compressor.py @@ -636,3 +636,130 @@ def generate(self, messages): max_chars=10_000, ) assert not any("missing_critical_literals" in f for f in feedback) + + +def test_strip_json_fence_no_newline(): + assert _strip_json_fence("```json") == "```json" + + +def test_strip_json_fence_none_input(): + assert _strip_json_fence(None) == "" + + +def test_validate_spans_empty_fact_text(): + units = [{"unit_id": "u1", "text": " ", "evidence_ids": ["1"]}] + spans = [{"unit_id": "u1", "start": 0, "end": 3}] + facts, feedback = TenantDreamingCompressor._validate_spans(spans, " ", units) + assert any("empty_text" in f for f in feedback) + + +def test_require_source_coverage_empty_source(): + facts = [] + units = [] + TenantDreamingCompressor._require_source_coverage(facts, units) + + +def test_lossless_formatter_rejects_non_list_response(): + class Model: + def generate(self, _messages): + return SimpleNamespace(content=json.dumps({"facts": "not a list"})) + + compressor = TenantDreamingCompressor.__new__(TenantDreamingCompressor) + compressor.model = Model() + facts = [ + { + "fact_id": "f001", + "text": "fact one", + "unit_ids": ["u1"], + "evidence_ids": ["1"], + } + ] + with pytest.raises(ValueError, match="must contain a facts list"): + compressor._lossless_formatting(facts, 10, ["1"]) + + +def test_lossless_formatter_rejects_duplicate_fact_ids(): + class Model: + def generate(self, _messages): + return SimpleNamespace( + content=json.dumps( + { + "facts": [ + {"fact_id": "f001", "text": "first"}, + {"fact_id": "f001", "text": "duplicate"}, + ] + } + ) + ) + + compressor = TenantDreamingCompressor.__new__(TenantDreamingCompressor) + compressor.model = Model() + facts = [ + { + "fact_id": "f001", + "text": "fact one", + "unit_ids": ["u1"], + "evidence_ids": ["1"], + } + ] + with pytest.raises(ValueError, match="duplicate fact_ids"): + compressor._lossless_formatting(facts, 10, ["1"]) + + +def test_lossless_formatter_rejects_empty_fact_text(): + class Model: + def generate(self, _messages): + return SimpleNamespace( + content=json.dumps( + {"facts": [{"fact_id": "f001", "text": " "}]} + ) + ) + + compressor = TenantDreamingCompressor.__new__(TenantDreamingCompressor) + compressor.model = Model() + facts = [ + { + "fact_id": "f001", + "text": "fact one", + "unit_ids": ["u1"], + "evidence_ids": ["1"], + } + ] + with pytest.raises(ValueError, match="empty fact"): + compressor._lossless_formatting(facts, 10, ["1"]) + + +def test_span_validation_rejects_missing_unit_id_with_legacy(): + spans = [{"start": 0, "end": 5, "unit_ids": ["u1", "u2"]}] + units = [{"unit_id": "u1", "text": "Hello", "evidence_ids": ["1"]}] + _, feedback = TenantDreamingCompressor._validate_spans(spans, "Hello", units) + assert any("invalid_unit" in f or "missing_offsets" in f for f in feedback) + + +def test_fact_coverage_too_low_raises_in_main_path(): + class Model: + def generate(self, messages): + return SimpleNamespace( + content=json.dumps([{"unit_id": "u1", "start": 0, "end": 5}]) + ) + + compressor = TenantDreamingCompressor.__new__(TenantDreamingCompressor) + compressor.tenant_id = "t" + compressor.user_id = "u" + compressor.model = Model() + compressor.max_compression_input_chars = 40_000 + + units = [ + DreamingMemoryUnit(unit_id="u1", content="Hello world", evidence_ids=["1"]), + DreamingMemoryUnit(unit_id="u2", content="Another fact", evidence_ids=["2"]), + ] + + with pytest.raises(ValueError, match="Source unit coverage too low"): + compressor( + DreamingCompressionRequest( + raw_content="- Hello world\n- Another fact", + units=units, + max_chars=10_000, + attempt=1, + ) + ) diff --git a/test/backend/services/test_memory_dreaming_scheduler.py b/test/backend/services/test_memory_dreaming_scheduler.py new file mode 100644 index 0000000000..7b56ce1fb4 --- /dev/null +++ b/test/backend/services/test_memory_dreaming_scheduler.py @@ -0,0 +1,301 @@ +"""Unit tests for memory_dreaming_scheduler (DreamingLeaseStore, execute_dreaming, DreamingScheduler).""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +@pytest.fixture +def mock_consts(monkeypatch): + monkeypatch.setattr( + "services.memory_dreaming_scheduler.DREAMING_SCHEDULER_ENABLED", True + ) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.DREAMING_SCHEDULER_LEASE_SECONDS", 120.0 + ) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.DREAMING_SCHEDULER_MAX_CONCURRENCY", 2 + ) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.DREAMING_SCHEDULER_POLL_SECONDS", 30.0 + ) + + +# --------------------------------------------------------------------------- +# DreamingLeaseStore +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_lease_store_recover(monkeypatch): + from services.memory_dreaming_scheduler import DreamingLeaseStore + + mock_recover = MagicMock(return_value=3) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.memory_dreaming_db.recover_stale", + mock_recover, + ) + + store = DreamingLeaseStore() + await store.recover() + + mock_recover.assert_called_once() + + +@pytest.mark.asyncio +async def test_lease_store_claim_due_returns_job(monkeypatch): + from services.memory_dreaming_scheduler import DreamingLeaseStore + + row = { + "run_id": 42, + "tenant_id": "t", + "user_id": "u", + "agent_id": "a", + "trigger_source": "manual", + } + monkeypatch.setattr( + "services.memory_dreaming_scheduler.memory_dreaming_db.claim_queued", + lambda owner_id, lease_seconds: row, + ) + materialize = MagicMock(return_value=1) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.memory_dreaming_db.materialize_due_schedules", + materialize, + ) + + store = DreamingLeaseStore() + jobs = await store.claim_due("worker-1", 1, 120.0) + + assert len(jobs) == 1 + assert jobs[0].job_id == 42 + assert jobs[0].payload == row + materialize.assert_called_once_with(1) + + +@pytest.mark.asyncio +async def test_lease_store_claim_due_returns_empty(monkeypatch): + from services.memory_dreaming_scheduler import DreamingLeaseStore + + monkeypatch.setattr( + "services.memory_dreaming_scheduler.memory_dreaming_db.claim_queued", + lambda owner_id, lease_seconds: None, + ) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.memory_dreaming_db.materialize_due_schedules", + lambda limit: 0, + ) + + store = DreamingLeaseStore() + jobs = await store.claim_due("worker-1", 1, 120.0) + + assert jobs == [] + + +@pytest.mark.asyncio +async def test_lease_store_renew(monkeypatch): + from services.memory_dreaming_scheduler import DreamingLeaseStore + + mock_renew = MagicMock(return_value=True) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.memory_dreaming_db.renew_lease", + mock_renew, + ) + + store = DreamingLeaseStore() + result = await store.renew(42, "worker-1", 120.0) + + assert result is True + mock_renew.assert_called_once_with(42, "worker-1", 120.0) + + +@pytest.mark.asyncio +async def test_lease_store_release(monkeypatch): + from services.memory_dreaming_scheduler import DreamingLeaseStore + + mock_release = MagicMock(return_value=True) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.memory_dreaming_db.release_lease", + mock_release, + ) + + store = DreamingLeaseStore() + result = await store.release(42, "worker-1") + + assert result is True + mock_release.assert_called_once_with(42, "worker-1") + + +# --------------------------------------------------------------------------- +# execute_dreaming +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_execute_dreaming_success(monkeypatch): + from nexent.scheduler import ClaimedJob + from services.memory_dreaming_scheduler import execute_dreaming + + mock_run = MagicMock() + mock_service = MagicMock() + mock_service.run = mock_run + monkeypatch.setattr( + "services.memory_dreaming_service.get_memory_dreaming_service", + lambda: mock_service, + ) + + job = ClaimedJob( + job_id=42, + payload={ + "tenant_id": "t", + "user_id": "u", + "agent_id": "a", + "trigger_source": "scheduler", + }, + ) + lease = MagicMock() + + await execute_dreaming(job, lease) + + mock_run.assert_called_once_with( + tenant_id="t", + user_id="u", + agent_id="a", + run_id=42, + trigger_source="scheduler", + ) + + +@pytest.mark.asyncio +async def test_execute_dreaming_default_trigger_source(monkeypatch): + from nexent.scheduler import ClaimedJob + from services.memory_dreaming_scheduler import execute_dreaming + + mock_run = MagicMock() + mock_service = MagicMock() + mock_service.run = mock_run + monkeypatch.setattr( + "services.memory_dreaming_service.get_memory_dreaming_service", + lambda: mock_service, + ) + + job = ClaimedJob( + job_id=10, + payload={ + "tenant_id": "t", + "user_id": "u", + "agent_id": "a", + }, + ) + lease = MagicMock() + + await execute_dreaming(job, lease) + + mock_run.assert_called_once_with( + tenant_id="t", + user_id="u", + agent_id="a", + run_id=10, + trigger_source="scheduler", + ) + + +@pytest.mark.asyncio +async def test_execute_dreaming_failure_raises(monkeypatch): + from nexent.scheduler import ClaimedJob + from services.memory_dreaming_scheduler import execute_dreaming + + mock_service = MagicMock() + mock_service.run = MagicMock(side_effect=RuntimeError("model down")) + monkeypatch.setattr( + "services.memory_dreaming_service.get_memory_dreaming_service", + lambda: mock_service, + ) + + job = ClaimedJob( + job_id=42, + payload={ + "tenant_id": "t", + "user_id": "u", + "agent_id": "a", + "trigger_source": "manual", + }, + ) + lease = MagicMock() + + with pytest.raises(RuntimeError, match="model down"): + await execute_dreaming(job, lease) + + +# --------------------------------------------------------------------------- +# DreamingScheduler +# --------------------------------------------------------------------------- + + +def test_dreaming_scheduler_properties(monkeypatch, mock_consts): + with patch("services.memory_dreaming_scheduler.LeaseScheduler") as MockScheduler: + mock_instance = MagicMock() + mock_instance.owner_id = "worker-abc" + mock_instance.is_running = True + MockScheduler.return_value = mock_instance + + from services.memory_dreaming_scheduler import DreamingScheduler + + scheduler = DreamingScheduler() + + assert scheduler.instance_id == "worker-abc" + assert scheduler.is_running is True + + +@pytest.mark.asyncio +async def test_dreaming_scheduler_start_disabled(monkeypatch): + monkeypatch.setattr( + "services.memory_dreaming_scheduler.DREAMING_SCHEDULER_ENABLED", False + ) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.DREAMING_SCHEDULER_LEASE_SECONDS", 120.0 + ) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.DREAMING_SCHEDULER_MAX_CONCURRENCY", 2 + ) + monkeypatch.setattr( + "services.memory_dreaming_scheduler.DREAMING_SCHEDULER_POLL_SECONDS", 30.0 + ) + + with patch("services.memory_dreaming_scheduler.LeaseScheduler") as MockScheduler: + mock_instance = AsyncMock() + MockScheduler.return_value = mock_instance + + from services.memory_dreaming_scheduler import DreamingScheduler + + scheduler = DreamingScheduler() + await scheduler.start() + + mock_instance.start.assert_not_called() + + +@pytest.mark.asyncio +async def test_dreaming_scheduler_start_enabled(monkeypatch, mock_consts): + with patch("services.memory_dreaming_scheduler.LeaseScheduler") as MockScheduler: + mock_instance = AsyncMock() + MockScheduler.return_value = mock_instance + + from services.memory_dreaming_scheduler import DreamingScheduler + + scheduler = DreamingScheduler() + await scheduler.start() + + mock_instance.start.assert_called_once() + + +@pytest.mark.asyncio +async def test_dreaming_scheduler_stop(monkeypatch, mock_consts): + with patch("services.memory_dreaming_scheduler.LeaseScheduler") as MockScheduler: + mock_instance = AsyncMock() + MockScheduler.return_value = mock_instance + + from services.memory_dreaming_scheduler import DreamingScheduler + + scheduler = DreamingScheduler() + await scheduler.stop() + + mock_instance.stop.assert_called_once() diff --git a/test/backend/services/test_memory_dreaming_service.py b/test/backend/services/test_memory_dreaming_service.py index 5506e9acd0..0a6d72a891 100644 --- a/test/backend/services/test_memory_dreaming_service.py +++ b/test/backend/services/test_memory_dreaming_service.py @@ -244,3 +244,170 @@ def test_ac022_stale_active_version_switch_is_rejected(monkeypatch): ) activate.assert_not_called() + + +def test_run_rejects_empty_ids(): + service = MemoryDreamingService(record_service=MagicMock()) + with pytest.raises(DreamingRunError, match="required"): + service.run(tenant_id="", user_id="u", agent_id="a") + with pytest.raises(DreamingRunError, match="required"): + service.run(tenant_id="t", user_id="", agent_id="a") + with pytest.raises(DreamingRunError, match="required"): + service.run(tenant_id="t", user_id="u", agent_id="") + + +def test_run_with_preexisting_run_id(monkeypatch): + update = MagicMock(return_value=True) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.update_audit", update + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.try_scope_lock", + lambda *_: lock(False), + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.finish_audit", + lambda *_args, **_kwargs: True, + ) + + service = MemoryDreamingService(record_service=MagicMock()) + result = service.run( + tenant_id="t", user_id="u", agent_id="a", run_id=100 + ) + + assert result["status"] == "skipped" + update.assert_called_once_with( + 100, {"status": "running", "current_phase": "light"} + ) + + +def test_run_non_manual_trigger_creates_audit(monkeypatch): + create_audit = MagicMock(return_value=55) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.create_audit", + create_audit, + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.try_scope_lock", + lambda *_: lock(False), + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.finish_audit", + lambda *_args, **_kwargs: True, + ) + + service = MemoryDreamingService(record_service=MagicMock()) + service.run( + tenant_id="t", user_id="u", agent_id="a", trigger_source="scheduler" + ) + + create_audit.assert_called_once_with( + "t", "u", "a", trigger_source="scheduler" + ) + + +def test_list_audits_delegates(monkeypatch): + mock_list = MagicMock(return_value=[{"run_id": 1}]) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.list_audits", mock_list + ) + + service = MemoryDreamingService(record_service=MagicMock()) + result = service.list_audits("t", "u", agent_id="a", run_id=1, limit=50) + + assert result == [{"run_id": 1}] + mock_list.assert_called_once_with("t", "u", agent_id="a", run_id=1, limit=50) + + +def test_list_versions_delegates(monkeypatch): + mock_list = MagicMock(return_value=[{"version_id": 2}]) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.list_versions", mock_list + ) + + service = MemoryDreamingService(record_service=MagicMock()) + result = service.list_versions("t", "u", agent_id="a", limit=50) + + assert result == [{"version_id": 2}] + mock_list.assert_called_once_with("t", "u", agent_id="a", limit=50) + + +def test_activate_version_lock_busy(monkeypatch): + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.try_scope_lock", + lambda *_args: lock(False), + ) + + service = MemoryDreamingService(record_service=MagicMock()) + with pytest.raises(DreamingConflictError, match="busy"): + service.activate_version( + "t", "u", agent_id="a", version_id=10 + ) + + +def test_activate_version_success(monkeypatch): + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.try_scope_lock", + lambda *_args: lock(True), + ) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.get_active_version", + lambda *_args: {"version_id": 5}, + ) + activate = MagicMock(return_value={"version_id": 10, "is_active": True}) + monkeypatch.setattr( + "services.memory_dreaming_service.memory_dreaming_db.activate_version", + activate, + ) + + service = MemoryDreamingService(record_service=MagicMock()) + result = service.activate_version( + "t", "u", + agent_id="a", + version_id=10, + actor_user_id="admin", + expected_active_version_id=5, + ) + + assert result["version_id"] == 10 + activate.assert_called_once_with( + "t", "u", "a", 10, actor_user_id="admin" + ) + + +def test_get_memory_dreaming_service_singleton(monkeypatch): + import services.memory_dreaming_service as mod + monkeypatch.setattr(mod, "_service", None) + + svc1 = mod.get_memory_dreaming_service() + svc2 = mod.get_memory_dreaming_service() + + assert svc1 is svc2 + assert isinstance(svc1, MemoryDreamingService) + + monkeypatch.setattr(mod, "_service", None) + + +def test_tenant_compressor_lazy_init(monkeypatch): + mock_compressor_cls = MagicMock() + mock_instance = MagicMock(return_value="compressed") + mock_compressor_cls.return_value = mock_instance + + import sys + + fake_module = MagicMock() + fake_module.TenantDreamingCompressor = mock_compressor_cls + monkeypatch.setitem(sys.modules, "services.memory_dreaming_compressor", fake_module) + + service = MemoryDreamingService(record_service=MagicMock()) + compress = service._tenant_compressor("t", "u") + + request = MagicMock() + result = compress(request) + + assert result == "compressed" + mock_compressor_cls.assert_called_once_with("t", "u") + mock_instance.assert_called_once_with(request) + + result2 = compress(request) + assert mock_compressor_cls.call_count == 1 diff --git a/test/sdk/memory/test_dreaming_version_builder.py b/test/sdk/memory/test_dreaming_version_builder.py index d8f4ff5306..e4548be700 100644 --- a/test/sdk/memory/test_dreaming_version_builder.py +++ b/test/sdk/memory/test_dreaming_version_builder.py @@ -1,5 +1,7 @@ from datetime import datetime +import pytest + from nexent.memory.dreaming import ( DreamingCandidate, DreamingCompressionOutput, @@ -280,3 +282,91 @@ def test_ac044_coverage_validation_acceptance(): compressed_fact_ids=compressed_fact_ids, ) assert not any("fact_coverage" in f for f in feedback) + + +def test_units_from_decisions_negative_source_limit(): + with pytest.raises(ValueError, match="non-negative"): + units_from_decisions([], source_limit=-1) + + +def test_units_from_decisions_zero_source_limit(): + decisions = select_candidates( + [candidate(memory_id=1)], + thresholds=DreamingThresholds( + min_score=0, min_recall_count=0, min_unique_queries=0 + ), + now=datetime(2026, 7, 23), + ) + result = units_from_decisions(decisions, source_limit=0) + assert result == [] + + +def test_build_dreaming_version_invalid_max_chars(): + with pytest.raises(ValueError, match="positive"): + build_dreaming_version(parent_units=[], new_units=[], max_chars=0) + + +def test_build_dreaming_version_negative_max_attempts(): + with pytest.raises(ValueError, match="non-negative"): + build_dreaming_version(parent_units=[], new_units=[], max_chars=100, max_attempts=-1) + + +def test_build_dreaming_version_compressor_exception(): + def failing_compressor(request): + raise RuntimeError("model unavailable") + + result = build_dreaming_version( + parent_units=[], + new_units=[ + DreamingMemoryUnit( + unit_id="new", + content="important fact " * 20, + evidence_ids=["1"], + is_new=True, + ) + ], + max_chars=20, + compressor=failing_compressor, + max_attempts=2, + ) + + assert result.compression_status == "mechanical_fallback" + assert result.compression_audit[0]["outcome"] == "model_error" + assert "compressor_error" in result.compression_audit[0]["validation"][0] + + +def test_validate_compression_empty_content(): + from nexent.memory.dreaming.version_builder import _validate_compression + + output = DreamingCompressionOutput( + content=" ", + evidence_ids=["1"], + ) + feedback = _validate_compression( + output, + required_evidence={"1"}, + required_literals=set(), + max_chars=10_000, + ) + assert "content_empty" in feedback + + +def test_truncate_at_sentence_short_text(): + from nexent.memory.dreaming.version_builder import _truncate_at_sentence + + assert _truncate_at_sentence("short", 100) == "short" + + +def test_truncate_at_sentence_zero_limit(): + from nexent.memory.dreaming.version_builder import _truncate_at_sentence + + assert _truncate_at_sentence("some text", 0) == "" + + +def test_truncate_at_sentence_word_boundary(): + from nexent.memory.dreaming.version_builder import _truncate_at_sentence + + text = "This is a long sentence without period marks" + result = _truncate_at_sentence(text, 30) + assert len(result) <= 30 + assert result == "This is a long sentence" From e387ba2e843bc8178c15e70ded2853d4accd1522 Mon Sep 17 00:00:00 2001 From: Jinglong Wang Date: Mon, 27 Jul 2026 20:33:35 +0800 Subject: [PATCH 11/19] fix(memory): show dreaming schedule without agents --- .../app/[locale]/memory/DreamingPanel.tsx | 189 ++++++++++-------- frontend/public/locales/en/common.json | 1 + frontend/public/locales/zh/common.json | 1 + 3 files changed, 104 insertions(+), 87 deletions(-) diff --git a/frontend/app/[locale]/memory/DreamingPanel.tsx b/frontend/app/[locale]/memory/DreamingPanel.tsx index 61ee226f1c..fb165dc759 100644 --- a/frontend/app/[locale]/memory/DreamingPanel.tsx +++ b/frontend/app/[locale]/memory/DreamingPanel.tsx @@ -142,6 +142,15 @@ export function DreamingPanel() { [audits] ); const latestRun = audits[0]; + const displayedSchedule: DreamingSchedule = schedule || { + agent_id: agentId || "", + enabled: false, + rule_type: "CRON", + timezone: "Asia/Shanghai", + cron_expr: "0 3 * * *", + interval_seconds: null, + fire_count: 0, + }; const selectedIsSelf = !targetUserId || targetUserId === user?.id; const canEditTarget = selectedIsSelf || hasPermission("DREAMING:EDIT_TENANT"); const queueDelayed = @@ -291,96 +300,102 @@ export function DreamingPanel() { - {agentId && schedule && ( - - - {t("dreaming.schedule.title")} - - } - extra={ - - } - > - - - - {t("dreaming.schedule.enabled")} - - setSchedule({ ...schedule, enabled })} - /> - + + + {t("dreaming.schedule.title")} + + } + extra={ + + } + > + {!agentId && ( + + )} + + + {t("dreaming.schedule.enabled")} + + setSchedule({ ...displayedSchedule, enabled }) + } + /> + + ({ + value, + label: t(`dreaming.schedule.weekday.${value}`), + }))} /> - {scheduleMode === "weekly" && ( - } > - {!agentId && ( - - )} {t("dreaming.schedule.enabled")} setSchedule({ ...displayedSchedule, enabled }) } @@ -340,7 +316,7 @@ export function DreamingPanel() {