From 72ebca042ffd932fde6a5fff298a73c4e05e0978 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sat, 26 Sep 2026 16:47:11 -0400 Subject: [PATCH] Fold AgentModelFreeApproach and AgentSessionMixin into AgentBaseApproach AgentModelFreeApproach had become the base of every agent arm, the EMPIRIC line and the model-free arm alike, and its task solver only raised. AgentSessionMixin had that class as its only host. The two become one abstract base, AgentBaseApproach, which has no approach name of its own, so the unused agent_model_free name is gone. Behavior is unchanged: the arms inherit the same methods, the log directory is still created on first use, and checkpoints keep their format. --- docs/protocol/design.md | 6 +- predicators/agent_sdk/config.py | 2 +- predicators/agent_sdk/local_sandbox.py | 2 +- predicators/agent_sdk/session_manager.py | 9 +- predicators/agent_sdk/tools/context.py | 8 +- predicators/agent_sdk/tools/registry.py | 4 +- predicators/approaches/agent_base_approach.py | 488 ++++++++++++++++++ .../approaches/agent_continual_approach.py | 16 +- .../approaches/agent_model_free_approach.py | 253 --------- predicators/approaches/agent_session_mixin.py | 299 ----------- .../approaches/agent_sim_learning_approach.py | 7 +- .../approaches/bilevel_planning_approach.py | 2 +- .../approaches/continual_play_mixin.py | 11 +- tests/agent_sdk/test_continual_tools.py | 8 +- tests/agent_sdk/test_tool_registry.py | 17 +- tests/run/test_public_observation_boundary.py | 8 +- tests/test_agent_sdk_tools.py | 2 +- tests/test_docker_option_plan.py | 2 +- 18 files changed, 541 insertions(+), 603 deletions(-) create mode 100644 predicators/approaches/agent_base_approach.py delete mode 100644 predicators/approaches/agent_model_free_approach.py delete mode 100644 predicators/approaches/agent_session_mixin.py diff --git a/docs/protocol/design.md b/docs/protocol/design.md index 8a36619c0a..dbce0eacab 100644 --- a/docs/protocol/design.md +++ b/docs/protocol/design.md @@ -295,7 +295,7 @@ The prompt gives no schedule and no certification rule. - `predicators/agent_sdk/tools/env_tools.py`: the `env.*` namespace, the observation formatter, the ledger footer, and the recording writer. - `predicators/agent_sdk/tools/skill_tools.py`: the `skills.*` namespace over the existing option grounding, `option_plan_to_policy`, and the divergence check on expected outcomes. In code the skills stay `_Option` objects; the agent-facing name is skill. - `predicators/agent_sdk/prompts/play_system.md` and `play_query.md`: the continual prompts, with golden renders like the existing templates. -- `predicators/approaches/agent_continual_approach.py`: `AgentSessionMixin` plus the tool context, the session loop, session resume, and `learn.run` as a sub-session launcher over the existing synthesis code. +- `predicators/approaches/agent_continual_approach.py`: `AgentBaseApproach` plus the tool context, the session loop, session resume, and `learn.run` as a sub-session launcher over the existing synthesis code. - `scripts/aggregate_scorecards.py`: scorecards to tables and curves, parameterised by the aggregation chosen later. - `scripts/configs/empiric/benchmark.yaml`: the seven arms (`approaches.yaml`) on the five settings (`envs.yaml`) with the shared flags (`common.yaml`); a launch names its round with `--round` or a `ROUND` key, which suffixes every experiment id, and `--envs`, `--approaches` and `--seeds` pick a subset; one job per env and arm, one array task per seed. @@ -450,8 +450,8 @@ First PyBullet result, the oracle arm on boil (train task then test task), job 2 Step 2, the agent arm, landed the same day: -- `predicators/approaches/continual_play_mixin.py`: `ContinualPlayMixin`, the play loop, mixed in front of each arm's phased base class (the way `AgentSessionMixin` adds its concern) so the arms keep the learning and session machinery of the phased classes without a class hierarchy of their own. It implements `play_level`: each session is a fresh context over the journal; after a session it deploys what the session wrote, records it in `attempts.md`, rebuilds the arm's data from the recorded episodes, and checkpoints. -- `predicators/approaches/agent_continual_approach.py`: the arms. `AgentContinualApproach` is C1's learner (hybrid simulator, parameter fit, predicate invention) on `AgentSimPredicateInventionApproach`; `AgentContinualModelFreeApproach` is the model-free baseline on `AgentModelFreeApproach`. There is no separate learning session (merged 2026-09-04): `AgentContinualApproach` attaches the model workbench (the phased synthesis `run_python` with the `sim` probe over the agent's `simulator.py` / `predicates.py`) to every play session, so the same session that acts also writes, fits and validates the model; `sim.predicates()` installs the invented predicates for the session's observation, Wait targets and divergence checks as soon as it loads them, and after the session the arm reloads the files and deploys the fit. +- `predicators/approaches/continual_play_mixin.py`: `ContinualPlayMixin`, the play loop, mixed in front of each arm's base class so the arms keep its learning and session machinery without a class hierarchy of their own. It implements `play_level`: each session is a fresh context over the journal; after a session it deploys what the session wrote, records it in `attempts.md`, rebuilds the arm's data from the recorded episodes, and checkpoints. +- `predicators/approaches/agent_continual_approach.py`: the arms. `AgentContinualApproach` is C1's learner (hybrid simulator, parameter fit, predicate invention) on `AgentSimPredicateInventionApproach`; `AgentContinualModelFreeApproach` is the model-free baseline on `AgentBaseApproach`. There is no separate learning session (merged 2026-09-04): `AgentContinualApproach` attaches the model workbench (the phased synthesis `run_python` with the `sim` probe over the agent's `simulator.py` / `predicates.py`) to every play session, so the same session that acts also writes, fits and validates the model; `sim.predicates()` installs the invented predicates for the session's observation, Wait targets and divergence checks as soon as it loads them, and after the session the arm reloads the files and deploys the fit. - `predicators/agent_sdk/tools/continual_tools.py`: the play tools `env_observe`, `env_step`, `env_reset`, `give_up`, `skills_list`, `skills_invoke`, `skills_execute_plan`, thin text adapters over `ProtocolSession`. Every result ends with the ledger line and the context line. The model tools (`run_python` with `sim`) are attached by the model-based arm. Ending the run cannot happen inside a running SDK turn, so `give_up` records a request the arm acts on when the agent stops. - `predicators/agent_sdk/prompts/play_system.md` and `play_query.md`, rendered by `agent_sdk/play_prompts.py`: the rules of section 4, the tools, the skill grammar, the sandbox, learning, the journal protocol and the context protocol; a round's query carries the level, the ledger and the context line, the observation with a render, the skills, the predicates, the model status, the journal and the attempts record (a continuation carries the observation, the ledger, the context line and the model status only). The model arm's system prompt also carries the contract of its model files, `prompts/play_model_contract.md`: the `simulator.py` API (the three top-level names, the rule signature by observability, the physics-command channel, `ParamSpec`) organised around the kinds of process the ground-truth simulators model, the rules for writing conditions, the hidden-state block under partial observability, and the `predicates.py` surface, so a run's agent never has to reverse-engineer the API from docstrings. diff --git a/predicators/agent_sdk/config.py b/predicators/agent_sdk/config.py index 4b20e3c85c..1ef3773f79 100644 --- a/predicators/agent_sdk/config.py +++ b/predicators/agent_sdk/config.py @@ -20,7 +20,7 @@ class SessionConfig: """Agent session construction: model, budgets, and sandbox choice. Consumed by the three session managers and by - ``AgentSessionMixin._ensure_agent_session``, which builds one + ``AgentBaseApproach._ensure_agent_session``, which builds one instance per session and passes it to whichever manager it constructs. """ diff --git a/predicators/agent_sdk/local_sandbox.py b/predicators/agent_sdk/local_sandbox.py index a82062f688..e12209276d 100644 --- a/predicators/agent_sdk/local_sandbox.py +++ b/predicators/agent_sdk/local_sandbox.py @@ -26,7 +26,7 @@ Usage ----- When the ``agent_sdk_use_local_sandbox`` flag is ``True``, the -``AgentSessionMixin`` creates a ``LocalSandboxSessionManager`` in place +``AgentBaseApproach`` creates a ``LocalSandboxSessionManager`` in place of the normal ``AgentSessionManager``:: manager = LocalSandboxSessionManager(...) diff --git a/predicators/agent_sdk/session_manager.py b/predicators/agent_sdk/session_manager.py index 55a940705b..6eb9add669 100644 --- a/predicators/agent_sdk/session_manager.py +++ b/predicators/agent_sdk/session_manager.py @@ -18,11 +18,10 @@ class SessionManagerProtocol(Protocol): """Structural interface shared by the three session managers. - Implemented by :class:`AgentSessionManager` (in-process, no - sandbox), ``LocalSandboxSessionManager`` (in-process, sandbox cwd + - hooks), and ``DockerSessionManager`` (stateless container per - query). Consumers - ``AgentSessionMixin`` and the agent explorers - - must depend on this, not a concrete manager. + Implemented by :class:`AgentSessionManager` (in-process, no sandbox) + and ``LocalSandboxSessionManager`` (in-process, sandbox cwd + + hooks). Consumers such as ``AgentBaseApproach`` must depend on this, + not a concrete manager. """ session_id: Optional[str] diff --git a/predicators/agent_sdk/tools/context.py b/predicators/agent_sdk/tools/context.py index fe957abbf8..d97e62cd8b 100644 --- a/predicators/agent_sdk/tools/context.py +++ b/predicators/agent_sdk/tools/context.py @@ -145,10 +145,10 @@ class ToolContext: # label output by phase (e.g. attempt-log headers) can attribute # entries to the learning cycle instead of "pre-test phase". learn_cycle_index: Optional[int] = None - # Managed by AgentSessionMixin: populated from - # `_build_synthesis_mcp_tools` at session-open, reset to [] for - # solve sessions. Approaches should not write to this directly — - # override the builder hook instead. + # Dynamic SdkMcpTool instances the approach attaches for a session + # (the play tools, a round's run_python); + # AgentBaseApproach._ensure_agent_session checks their names + # against the declared tool list. extra_mcp_tools: list = field(default_factory=list) # Extra Claude Agent SDK ``HookMatcher`` instances applied to the # next session that's started. Read once at session start, then diff --git a/predicators/agent_sdk/tools/registry.py b/predicators/agent_sdk/tools/registry.py index 6932f3b75b..d3fe2e58b0 100644 --- a/predicators/agent_sdk/tools/registry.py +++ b/predicators/agent_sdk/tools/registry.py @@ -70,8 +70,8 @@ def list_session_tool_names( Args: mcp_filter: Subset of ``ALL_TOOL_NAMES`` to keep. ``None`` (the default) lists every MCP tool. - extra_mcp_tools: Synthesis tools supplied for the session - (e.g. by ``_build_synthesis_mcp_tools``). Their names are + extra_mcp_tools: Dynamic tools the approach attached for the + session (``ToolContext.extra_mcp_tools``). Their names are read off each tool's ``name`` attribute. include_builtin: Whether to include the Claude built-in tools (``Bash``, ``Read``, ``Write``, …). diff --git a/predicators/approaches/agent_base_approach.py b/predicators/approaches/agent_base_approach.py new file mode 100644 index 0000000000..a5db724dc7 --- /dev/null +++ b/predicators/approaches/agent_base_approach.py @@ -0,0 +1,488 @@ +"""The base of the agent arms: the agent session, its tool context, the +recorded trajectories and the checkpoints. + +The arms themselves (``agent_continual_approach`` and its siblings) add +the play loop of the continual protocol through ``ContinualPlayMixin``; +this class and the simulator-learning classes between it and the arms +carry the machinery they share. None of them solves a task on its own. +""" +import datetime +import logging +import os +from typing import Any, Callable, Dict, List, Optional, Set, cast + +import dill as pkl +from gym.spaces import Box + +from predicators import utils +from predicators.agent_sdk.config import SessionConfig +from predicators.agent_sdk.session_manager import AgentSessionManager, \ + SessionManagerProtocol, run_async_sync, run_query_sync +from predicators.agent_sdk.tools import ALL_TOOL_NAMES, ToolContext, \ + create_mcp_tools, get_allowed_tool_list +from predicators.approaches import ApproachFailure +from predicators.approaches.base_approach import BaseApproach +from predicators.option_model import _OptionModelBase, create_option_model +from predicators.settings import CFG +from predicators.structs import Action, Dataset, LowLevelTrajectory, \ + ParameterizedOption, Predicate, State, Task, Type + +logger = logging.getLogger(__name__) + + +class AgentBaseApproach(BaseApproach): + """The agent session, tool context, trajectories and checkpoints the agent + arms share. + + Subclasses must override: + - get_name() + - _get_agent_system_prompt() + + And may optionally override: + - _get_solve_tool_names() -- complete tool surface for + play sessions. May mix static MCP tool names with + names of dynamic ``SdkMcpTool`` instances. ``None`` = all + static MCP tools, ``[]`` = none. + - _get_synthesis_tool_names() -- complete tool surface for + synthesis sessions (``_learning_mode=True``). Same shape / + semantics as the solve hook, independent value. + + Dynamic ``SdkMcpTool`` instances are supplied by the approach + directly: it assigns them to ``ctx.extra_mcp_tools`` before + opening a synthesis session and clears the field afterwards. The + session setup asserts the instance names line up with the names + declared in :meth:`_get_synthesis_tool_names`. + """ + + # ``_learning_mode`` is flipped by the sim-learning approach around + # synthesis sessions; the class default keeps play-only arms working + # without declaring it. + _learning_mode: bool = False + # Phase tag the live ``_agent_session`` was created with; a query + # under a different phase closes and rebuilds the session so the + # saved prompt, tools, and CLAUDE.md always match the active phase. + _agent_session_phase: Optional[str] = None + # Session-transcript count an auto-resumed run inherits from its + # checkpoint (see AgentSimLearningApproach._extra_save_state), so + # transcript ids continue across the lineage. + _resume_query_count: int = 0 + + def __init__(self, + initial_predicates: Set[Predicate], + initial_options: Set[ParameterizedOption], + types: Set[Type], + action_space: Box, + train_tasks: List[Task], + *args: Any, + option_model: Optional[_OptionModelBase] = None, + **kwargs: Any) -> None: + super().__init__(initial_predicates, initial_options, types, + action_space, train_tasks, *args, **kwargs) + self._offline_dataset = Dataset([]) + self._online_trajectories: List[LowLevelTrajectory] = [] + self._option_model: Optional[_OptionModelBase] = ( + option_model if option_model is not None else + self._create_planner_option_model()) + # Terminate Wait on atom change using the approach's predicates (which + # may include invented ones), looked up lazily so the lambda picks up + # predicates invented after __init__. + if self._option_model is not None and \ + CFG.wait_option_terminate_on_atom_change: + cast( # pylint: disable=protected-access + Any, self._option_model)._abstract_function = ( + lambda s: utils.abstract(s, self._get_all_predicates())) + self._run_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + + # The tool context the MCP tools read, and the lazily created + # agent session. + self._tool_context = ToolContext( + types=self._types, + predicates=self._initial_predicates, + options=initial_options, + train_tasks=train_tasks, + ) + self._agent_session: Optional[SessionManagerProtocol] = None + self._agent_session_id: Optional[str] = None + + # Capture the underlying env once at construction. The initial option + # model wraps ``env.simulate`` (a bound method), so ``__self__`` is the + # env. A later model may rebuild ``_option_model`` with a plain + # learned simulator that has no ``__self__``; pinning the env + # reference here keeps scene rendering (the probe's sim.render) + # working in every round. + env_self = getattr(getattr(self._option_model, '_simulator', None), + '__self__', None) + if env_self is not None: + self._tool_context.env = env_self + + @property + def is_learning_based(self) -> bool: + return True + + # ------------------------------------------------------------------ # + # Overridable helpers (for subclass customisation) + # ------------------------------------------------------------------ # + + def _get_agent_system_prompt(self) -> str: + """Return the system prompt for the agent session.""" + raise NotImplementedError + + def _get_solve_tool_names(self) -> Optional[List[str]]: + """Return the complete tool surface for play sessions. + + May mix static MCP tool names with names of dynamic + ``SdkMcpTool`` instances. ``None`` means "all static MCP tools"; + override to subset. + """ + return None + + def _get_synthesis_tool_names(self) -> Optional[List[str]]: + """Return the complete tool surface for the synthesis session. + + Selected when ``_learning_mode`` is True. Independent of the + solve list — the two phases may share names or be disjoint. Each + name must back either a static MCP tool (member of + ``ALL_TOOL_NAMES``) or a dynamic ``SdkMcpTool`` instance the + approach attaches via ``ctx.extra_mcp_tools``. Default ``[]`` + means no tools (approaches with no synthesis phase need not + override). + """ + return [] + + def _get_sandbox_reference_files(self) -> Dict[str, str]: + """Document public control semantics without exporting implementation. + + Maps destination paths (relative to ``/sandbox/reference/``) to + source paths (relative to the repo root). + """ + return {"skills.md": "predicators/agent_sdk/prompts/public_skills.md"} + + def _get_all_options(self) -> Set[ParameterizedOption]: + """Return the full set of options available for planning.""" + return self._initial_options + + def _get_all_predicates(self) -> Set[Predicate]: + """Return the full set of predicates for abstraction.""" + return self._initial_predicates + + def _get_all_trajectories(self) -> List[LowLevelTrajectory]: + """Return all trajectories (offline + online).""" + return self._offline_dataset.trajectories + self._online_trajectories + + def _create_planner_option_model(self) -> Optional[_OptionModelBase]: + """Build the option model the tools roll plans out through. + + Honors two CFG knobs: + + * ``agent_planner_use_simulator`` -- when False, returns ``None`` + so the agent has no simulator to roll plans out in. + * ``agent_planner_use_base_simulator`` -- when True (and a + simulator is used), wraps the *base* env + (``skip_residual_dynamics=True``), denying the planner the delayed + ``_domain_specific_step`` dynamics; otherwise wraps the real env. + """ + if not CFG.agent_planner_use_simulator: + return None + return create_option_model( + CFG.option_model_name, + skip_residual_dynamics=CFG.agent_planner_use_base_simulator) + + def _solve(self, task: Task, timeout: int) -> Callable[[State], Action]: + raise ApproachFailure( + f"{self.get_name()} has no task solver: the agent arms play " + "levels under the continual protocol " + "(ContinualPlayMixin.play_level).") + + # ------------------------------------------------------------------ # + # Agent session + # ------------------------------------------------------------------ # + + def _ensure_agent_session(self) -> None: + """Create the agent session manager if needed. + + When ``SessionConfig.use_local_sandbox`` is ``True``, creates a + ``LocalSandboxSessionManager`` that runs the agent in a sandbox + directory with its built-in tools (Bash, Read, Write, ...). + Otherwise creates the in-process ``AgentSessionManager``. + """ + phase = "synthesis" if self._learning_mode else "solve" + self._tool_context.phase = phase + if self._agent_session is not None: + if self._agent_session_phase == phase: + return + # A live session from another phase would keep that phase's + # prompt tag and tool surface; rebuild so they match. + self._close_agent_session() + + # Session config is read once here (at session-construction + # time, never import time) and handed to whichever manager is + # built below. + config = SessionConfig.from_cfg() + + # Pick the declared tool surface by phase. ``_learning_mode`` is + # the same signal the system-prompt branch reads, so tools and + # prompt stay in sync. Each approach declares its solve and + # synthesis tool sets independently — they may be disjoint. + # ``tool_names`` is the *complete* declared list (may mix static + # MCP names with names of dynamic SdkMcpTool instances). + if self._learning_mode: + tool_names = self._get_synthesis_tool_names() # pylint: disable=assignment-from-none + else: + tool_names = self._get_solve_tool_names() # pylint: disable=assignment-from-none + + # Sanity: every dynamic name in the declared list must have a + # backing tool attached to ``ctx.extra_mcp_tools``. Static MCP + # names (``ALL_TOOL_NAMES``) are excluded — they're materialized + # downstream by ``create_mcp_tools``. Catches typos and missing + # builder hooks before the agent silently fails to invoke a + # declared-but-missing tool. + declared = set(tool_names or ()) + dynamic_declared = declared - set(ALL_TOOL_NAMES) + if dynamic_declared: + attached = list(self._tool_context.extra_mcp_tools or ()) + built = {getattr(t, "name", "") for t in attached} + missing = dynamic_declared - built + phase_for_msg = "synthesis" if self._learning_mode else "solve" + assert not missing, ( + f"Dynamic tool name(s) {sorted(missing)} declared in " + f"_get_{phase_for_msg}_tool_names but no matching tool " + f"attached to ctx.extra_mcp_tools — add them to the " + f"builder or drop the names.") + + approach_name = self.get_name() + if tool_names is None: + logger.info( + "[%s] %s session tool surface: ALL static MCP tools " + "(no subset declared).", approach_name, phase) + else: + attached_names = { + getattr(t, "name", "") + for t in (self._tool_context.extra_mcp_tools or ()) + } + static = sorted( + n for n in tool_names + if n in set(ALL_TOOL_NAMES) and n not in attached_names) + dynamic = sorted( + n for n in tool_names + if n not in set(ALL_TOOL_NAMES) or n in attached_names) + lines = [ + f"[{approach_name}] {phase} session tool surface " + f"({len(tool_names)} total):" + ] + for n in static: + lines.append(f" static {n}") + for n in dynamic: + lines.append(f" dynamic {n}") + logger.info("\n".join(lines)) + + session: SessionManagerProtocol + if config.use_local_sandbox: + from predicators.agent_sdk.local_sandbox import \ + LocalSandboxSessionManager # pylint: disable=import-outside-toplevel + session = LocalSandboxSessionManager( + system_prompt=self._get_agent_system_prompt(), + log_dir=self._get_log_dir(), + model_name=config.model_name, + tool_context=self._tool_context, + tool_names=tool_names, + extra_reference_files=self._get_sandbox_reference_files(), + phase=phase, + config=config, + query_count_floor=self._resume_query_count, + ) + else: + from claude_agent_sdk import \ + create_sdk_mcp_server # pylint: disable=import-outside-toplevel + + tools = create_mcp_tools(self._tool_context, tool_names=tool_names) + mcp_server = create_sdk_mcp_server( + name="predicator_tools", + version="1.0.0", + tools=tools, + ) + + session = AgentSessionManager( + system_prompt=self._get_agent_system_prompt(), + mcp_server=mcp_server, + log_dir=self._get_log_dir(), + model_name=config.model_name, + allowed_tools=get_allowed_tool_list(tool_names), + tool_context=self._tool_context, + config=config, + ) + + self._agent_session = session + self._agent_session_phase = phase + if self._agent_session_id is not None: + session.session_id = self._agent_session_id + + # Save the system prompt to the log directory, one file per phase. + # Sandbox sessions write the FULL prompt (approach prompt + sandbox + # suffix) as full_system_prompt_{phase}.md during sandbox setup; for + # in-process sessions the approach prompt IS the full prompt, so + # save it under the same name here and nothing else. + if not config.use_local_sandbox: + log_dir = self._get_log_dir() + prompt_path = os.path.join(log_dir, + f"full_system_prompt_{phase}.md") + with open(prompt_path, "w", encoding="utf-8") as f: + f.write(self._get_agent_system_prompt()) + + def _get_log_dir(self) -> str: + """Return the per-run log directory, creating it. + + Always absolute: agent tool code (``run_python``) executes with + the sandbox as its working directory, so any path that harness + code may touch mid-call must not depend on the process cwd. + """ + if hasattr(CFG, 'log_file') and CFG.log_file: + log_dir = os.path.abspath(CFG.log_file) + else: + log_dir = os.path.abspath(os.path.join("logs", self.get_name())) + os.makedirs(log_dir, exist_ok=True) + logger.info("Logging agent queries/responses to: %s", log_dir) + return log_dir + + def _close_agent_session(self) -> None: + """Close and discard the current agent session, if one exists.""" + if self._agent_session is None: + return + session = self._agent_session + self._agent_session = None + self._agent_session_phase = None + try: + run_async_sync(session.close()) + except Exception: # pylint: disable=broad-except + pass + + def _query_agent_sync(self, message: str, + **query_kwargs: Any) -> List[Dict[str, Any]]: + """Synchronous wrapper for async agent query. + + Extra kwargs (e.g. ``kind="learn"``) are forwarded to the + session's ``query`` method for log-file tagging. + """ + self._ensure_agent_session() + assert self._agent_session is not None + return run_query_sync(self._agent_session, message, **query_kwargs) + + def _sync_tool_context(self) -> None: + """Push current approach state into the shared ToolContext. + + The MCP tools (run_python and the play tools) read from the + ToolContext dataclass, not the approach directly. This keeps + them in sync after mutations (e.g. new trajectories collected, + options added). Subclasses should call super() and then set + additional fields (e.g. skill_factory_context). + """ + self._tool_context.types = self._types + # The agent's predicate vocabulary, not the raw env set: tools + # abstract states, list predicates, and parse plan annotations + # from this, so stripped predicates (agent_sim_learning + # allowlist) must not leak in and invented ones must appear. + self._tool_context.predicates = self._get_all_predicates() + self._tool_context.options = self._initial_options + self._tool_context.show_option_source = False + self._tool_context.gt_options_ref_path = None + self._tool_context.train_tasks = self._train_tasks + self._tool_context.offline_trajectories = \ + self._offline_dataset.trajectories + self._tool_context.online_trajectories = self._online_trajectories + self._tool_context.log_dir = self._get_log_dir() + self._tool_context.option_model = self._option_model + # Wire the active-experiment info-gain scorer when a learning subclass + # exposes one and info-seeking exploration is on. Syncing the bound + # method (not a snapshot) keeps it pointed at the latest fit/ensemble. + # getattr guard: non-learning approaches lack it. + if CFG.agent_explorer_info_seeking: + self._tool_context.atom_disagreement_fn = getattr( + self, "score_atom_disagreement", None) + else: + self._tool_context.atom_disagreement_fn = None + all_trajs = (self._offline_dataset.trajectories + + self._online_trajectories) + if all_trajs: + self._tool_context.example_state = all_trajs[0].states[0] + + # Refresh env from the option model only if extraction succeeds. After + # sim learning, ``_simulator`` may be a plain lambda with no + # ``__self__``; don't clobber the env reference seeded in ``__init__`` + # in that case. + if self._option_model is not None and \ + hasattr(self._option_model, '_simulator'): + env_self = getattr( + self._option_model._simulator, # pylint: disable=protected-access + '__self__', + None) + if env_self is not None: + self._tool_context.env = env_self + + # ------------------------------------------------------------------ # + # Save / Load + # ------------------------------------------------------------------ # + + # Filename suffix for the pickled approach state. Subclasses that + # persist extra fields override this so their saves don't collide + # with the base planner's. + _save_suffix: str = "AgentPlanner" + + def _extra_save_state(self) -> Dict[str, Any]: + """Subclass hook: extra (key -> value) pairs to persist. + + Merged into the base save dict; restored by the matching + :meth:`_load_extra_save_state`. + """ + return {} + + def _load_extra_save_state(self, save_dict: Dict[str, Any]) -> None: + """Subclass hook: restore fields written by _extra_save_state. + + Called after the base fields are restored and ``_run_id`` has + been refreshed, but before the tool context is re-synced. + """ + + def save(self, online_learning_cycle: Optional[int] = None) -> None: + """Save approach state to disk; the continual runner names the + checkpoint after the level it closes (``online_learning_cycle``).""" + save_path = utils.get_approach_save_path_str() + path = f"{save_path}_{online_learning_cycle}.{self._save_suffix}" + save_dict = { + "offline_dataset": + self._offline_dataset, + "online_trajectories": + self._online_trajectories, + "run_id": + self._run_id, + "agent_session_id": + (self._agent_session.session_id if self._agent_session else None), + **self._extra_save_state(), + } + with open(path, "wb") as f: + pkl.dump(save_dict, f) + logger.info("[Run %s] Saved approach to %s", self._run_id, path) + + def load(self, online_learning_cycle: Optional[int] = None) -> None: + save_path = utils.get_approach_load_path_str() + path = f"{save_path}_{online_learning_cycle}.{self._save_suffix}" + with open(path, "rb") as f: + save_dict = pkl.load(f) + + self._offline_dataset = save_dict["offline_dataset"] + self._online_trajectories = save_dict["online_trajectories"] + self._agent_session_id = save_dict.get("agent_session_id") + + # New run_id for continued execution (each run gets its own dir), but + # log the original run_id for reference. + original_run_id = save_dict.get("run_id", "unknown") + self._run_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + + self._load_extra_save_state(save_dict) + + # Re-sync tool context (subclass fields are restored first). + self._sync_tool_context() + + logger.info( + "[Run %s] Loaded from previous run %s: %d offline, %d online " + "trajectories", self._run_id, original_run_id, + len(self._offline_dataset.trajectories), + len(self._online_trajectories)) diff --git a/predicators/approaches/agent_continual_approach.py b/predicators/approaches/agent_continual_approach.py index a07a43f84e..717d6182f0 100644 --- a/predicators/approaches/agent_continual_approach.py +++ b/predicators/approaches/agent_continual_approach.py @@ -1,7 +1,7 @@ """The LLM agent arms of the continual protocol (docs/protocol/design.md). Both arms mix :class:`ContinualPlayMixin` (the play loop) in front of -the phased approach class that holds their machinery: +the approach class that holds their machinery: * ``AgentContinualApproach`` (``agent_continual``) is C1's learner (hybrid simulator synthesis, parameter fit, predicate invention) on @@ -14,7 +14,7 @@ reloads the files, deploys the fit and installs the invented predicates. * ``AgentContinualModelFreeApproach`` (``agent_continual_model_free``) - is the model-free baseline on ``AgentModelFreeApproach``: the same env + is the model-free baseline on ``AgentBaseApproach``: the same env and skill tools, sandbox and journal, but no belief model, no ``sim`` and no ``run_python``. What it knows comes from the recorded data and the real environment. @@ -43,8 +43,7 @@ from predicators.agent_sdk.tools.sandbox_guard import \ _screen_text_for_sandbox_escape from predicators.agent_sdk.tools.synthesis import create_synthesis_tools -from predicators.approaches.agent_model_free_approach import \ - AgentModelFreeApproach +from predicators.approaches.agent_base_approach import AgentBaseApproach from predicators.approaches.agent_sim_learning_approach import \ count_residual_hits, residual_hint_from_hits, \ resolve_kept_predicate_names @@ -681,8 +680,7 @@ def _refresh_workbench(self) -> None: fingerprint = tuple(len(t.actions) for t in bench.trajectories) if fingerprint != bench.fingerprint: # New data invalidates the memoized whole fits and - # explainability verdicts, as the phased learn hook does - # when its data arrives (the caches key trajectories by + # explainability verdicts (the caches key trajectories by # segment lengths, so a grown episode must not answer from # its shorter self). bench.fingerprint = fingerprint @@ -816,14 +814,14 @@ def _load_extra_save_state(self, save_dict: Dict[str, Any]) -> None: class AgentContinualModelFreeApproach(ContinualPlayMixin, ScenePackageMixin, - AgentModelFreeApproach): + AgentBaseApproach): """The model-free baseline of the continual protocol: the env and skill tools, the sandbox and the journal, and nothing else. No belief model, no ``sim``, no ``run_python`` and no learning session; the agent's own code in the sandbox reads the recorded - data. Run with ``agent_planner_use_simulator`` off, as the phased - ``agent_model_free`` arm is, so no simulator is built at all. + data. No simulator is built at all (see + :meth:`_create_planner_option_model`). """ _save_suffix = "AgentContinualModelFree" diff --git a/predicators/approaches/agent_model_free_approach.py b/predicators/approaches/agent_model_free_approach.py deleted file mode 100644 index 9c9b08913e..0000000000 --- a/predicators/approaches/agent_model_free_approach.py +++ /dev/null @@ -1,253 +0,0 @@ -"""The base of the agent arms: the agent session, its tool context, the -recorded trajectories and the checkpoints. - -The arms themselves (``agent_continual_approach`` and its siblings) add -the play loop of the continual protocol through ``ContinualPlayMixin``; -this class and the simulator-learning classes between it and the arms -carry the machinery they share. None of them solves a task on its own. -""" -import datetime -import logging -import os -from typing import Any, Callable, Dict, List, Optional, Set, cast - -import dill as pkl -from gym.spaces import Box - -from predicators import utils -from predicators.approaches import ApproachFailure -from predicators.approaches.agent_session_mixin import AgentSessionMixin -from predicators.approaches.base_approach import BaseApproach -from predicators.option_model import _OptionModelBase, create_option_model -from predicators.settings import CFG -from predicators.structs import Action, Dataset, LowLevelTrajectory, \ - ParameterizedOption, Predicate, State, Task, Type - - -class AgentModelFreeApproach(AgentSessionMixin, BaseApproach): - """The agent session, tool context, trajectories and checkpoints the agent - arms share.""" - - def __init__(self, - initial_predicates: Set[Predicate], - initial_options: Set[ParameterizedOption], - types: Set[Type], - action_space: Box, - train_tasks: List[Task], - *args: Any, - option_model: Optional[_OptionModelBase] = None, - **kwargs: Any) -> None: - super().__init__(initial_predicates, initial_options, types, - action_space, train_tasks, *args, **kwargs) - self._offline_dataset = Dataset([]) - self._online_trajectories: List[LowLevelTrajectory] = [] - self._option_model: Optional[_OptionModelBase] = ( - option_model if option_model is not None else - self._create_planner_option_model()) - # Terminate Wait on atom change using the approach's predicates (which - # may include invented ones), looked up lazily so the lambda picks up - # predicates invented after __init__. - if self._option_model is not None and \ - CFG.wait_option_terminate_on_atom_change: - cast( # pylint: disable=protected-access - Any, self._option_model)._abstract_function = ( - lambda s: utils.abstract(s, self._get_all_predicates())) - self._run_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - - # Initializes _tool_context and _agent_session_id (see mixin). - self._init_agent_session_state(self._types, self._initial_predicates, - initial_options, train_tasks) - - # Capture the underlying env once at construction. The initial option - # model wraps ``env.simulate`` (a bound method), so ``__self__`` is the - # env. A later model may rebuild ``_option_model`` with a plain - # learned simulator that has no ``__self__``; pinning the env - # reference here keeps scene rendering (the probe's sim.render) - # working in every round. - env_self = getattr(getattr(self._option_model, '_simulator', None), - '__self__', None) - if env_self is not None: - self._tool_context.env = env_self - - @classmethod - def get_name(cls) -> str: - return "agent_model_free" - - @property - def is_learning_based(self) -> bool: - return True - - def _get_log_dir(self) -> str: - """Return per-run log directory (created by configure_logging).""" - log_dir = super()._get_log_dir() - os.makedirs(log_dir, exist_ok=True) - logging.info("Logging agent queries/responses to: %s", log_dir) - return log_dir - - # ------------------------------------------------------------------ # - # Overridable helpers (for subclass customisation) - # ------------------------------------------------------------------ # - - def _get_all_options(self) -> Set[ParameterizedOption]: - """Return the full set of options available for planning.""" - return self._initial_options - - def _get_all_predicates(self) -> Set[Predicate]: - """Return the full set of predicates for abstraction.""" - return self._initial_predicates - - def _get_all_trajectories(self) -> List[LowLevelTrajectory]: - """Return all trajectories (offline + online).""" - return self._offline_dataset.trajectories + self._online_trajectories - - def _create_planner_option_model(self) -> Optional[_OptionModelBase]: - """Build the option model the tools roll plans out through. - - Honors two CFG knobs: - - * ``agent_planner_use_simulator`` -- when False, returns ``None`` - so the agent has no simulator to roll plans out in. - * ``agent_planner_use_base_simulator`` -- when True (and a - simulator is used), wraps the *base* env - (``skip_residual_dynamics=True``), denying the planner the delayed - ``_domain_specific_step`` dynamics; otherwise wraps the real env. - """ - if not CFG.agent_planner_use_simulator: - return None - return create_option_model( - CFG.option_model_name, - skip_residual_dynamics=CFG.agent_planner_use_base_simulator) - - def _get_sandbox_reference_files(self) -> Dict[str, str]: - """Document public control semantics without exporting - implementation.""" - return {"skills.md": "predicators/agent_sdk/prompts/public_skills.md"} - - def _solve(self, task: Task, timeout: int) -> Callable[[State], Action]: - raise ApproachFailure( - f"{self.get_name()} has no task solver: the agent arms play " - "levels under the continual protocol " - "(ContinualPlayMixin.play_level).") - - def _sync_tool_context(self) -> None: - """Push current approach state into the shared ToolContext. - - The MCP tools (run_python and the play tools) read from the - ToolContext dataclass, not the approach directly. This keeps - them in sync after mutations (e.g. new trajectories collected, - options added). Subclasses should call super() and then set - additional fields (e.g. skill_factory_context). - """ - self._tool_context.types = self._types - # The agent's predicate vocabulary, not the raw env set: tools - # abstract states, list predicates, and parse plan annotations - # from this, so stripped predicates (agent_sim_learning - # allowlist) must not leak in and invented ones must appear. - self._tool_context.predicates = self._get_all_predicates() - self._tool_context.options = self._initial_options - self._tool_context.show_option_source = False - self._tool_context.gt_options_ref_path = None - self._tool_context.train_tasks = self._train_tasks - self._tool_context.offline_trajectories = \ - self._offline_dataset.trajectories - self._tool_context.online_trajectories = self._online_trajectories - self._tool_context.log_dir = self._get_log_dir() - self._tool_context.option_model = self._option_model - # Wire the active-experiment info-gain scorer when a learning subclass - # exposes one and info-seeking exploration is on. Syncing the bound - # method (not a snapshot) keeps it pointed at the latest fit/ensemble. - # getattr guard: non-learning approaches lack it. - if CFG.agent_explorer_info_seeking: - self._tool_context.atom_disagreement_fn = getattr( - self, "score_atom_disagreement", None) - else: - self._tool_context.atom_disagreement_fn = None - all_trajs = (self._offline_dataset.trajectories + - self._online_trajectories) - if all_trajs: - self._tool_context.example_state = all_trajs[0].states[0] - - # Refresh env from the option model only if extraction succeeds. After - # sim learning, ``_simulator`` may be a plain lambda with no - # ``__self__``; don't clobber the env reference seeded in ``__init__`` - # in that case. - if self._option_model is not None and \ - hasattr(self._option_model, '_simulator'): - env_self = getattr( - self._option_model._simulator, # pylint: disable=protected-access - '__self__', - None) - if env_self is not None: - self._tool_context.env = env_self - - # ------------------------------------------------------------------ # - # Save / Load - # ------------------------------------------------------------------ # - - # Filename suffix for the pickled approach state. Subclasses that - # persist extra fields override this so their saves don't collide - # with the base planner's. - _save_suffix: str = "AgentPlanner" - - def _extra_save_state(self) -> Dict[str, Any]: - """Subclass hook: extra (key -> value) pairs to persist. - - Merged into the base save dict; restored by the matching - :meth:`_load_extra_save_state`. - """ - return {} - - def _load_extra_save_state(self, save_dict: Dict[str, Any]) -> None: - """Subclass hook: restore fields written by _extra_save_state. - - Called after the base fields are restored and ``_run_id`` has - been refreshed, but before the tool context is re-synced. - """ - - def save(self, online_learning_cycle: Optional[int] = None) -> None: - """Save approach state to disk; the continual runner names the - checkpoint after the level it closes (``online_learning_cycle``).""" - save_path = utils.get_approach_save_path_str() - path = f"{save_path}_{online_learning_cycle}.{self._save_suffix}" - save_dict = { - "offline_dataset": - self._offline_dataset, - "online_trajectories": - self._online_trajectories, - "run_id": - self._run_id, - "agent_session_id": - (self._agent_session.session_id if self._agent_session else None), - **self._extra_save_state(), - } - with open(path, "wb") as f: - pkl.dump(save_dict, f) - logging.info("[Run %s] Saved approach to %s", self._run_id, path) - - def load(self, online_learning_cycle: Optional[int] = None) -> None: - save_path = utils.get_approach_load_path_str() - path = f"{save_path}_{online_learning_cycle}.{self._save_suffix}" - with open(path, "rb") as f: - save_dict = pkl.load(f) - - self._offline_dataset = save_dict["offline_dataset"] - self._online_trajectories = save_dict["online_trajectories"] - # pylint: disable=attribute-defined-outside-init - # (_agent_session_id is initialized via the agent-session mixin.) - self._agent_session_id = save_dict.get("agent_session_id") - - # New run_id for continued execution (each run gets its own dir), but - # log the original run_id for reference. - original_run_id = save_dict.get("run_id", "unknown") - self._run_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - - self._load_extra_save_state(save_dict) - - # Re-sync tool context (subclass fields are restored first). - self._sync_tool_context() - - logging.info( - "[Run %s] Loaded from previous run %s: %d offline, %d online " - "trajectories", self._run_id, original_run_id, - len(self._offline_dataset.trajectories), - len(self._online_trajectories)) diff --git a/predicators/approaches/agent_session_mixin.py b/predicators/approaches/agent_session_mixin.py deleted file mode 100644 index c83dc77fee..0000000000 --- a/predicators/approaches/agent_session_mixin.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Mixin providing shared agent session infrastructure. - -Extracts common code for ToolContext initialization, lazy -AgentSessionManager creation and async-to-sync bridging shared by -AgentModelFreeApproach and its subclasses. -""" -import logging -import os -from typing import Any, Dict, List, Optional, Set - -from gym.spaces import Box - -from predicators.agent_sdk.config import SessionConfig -from predicators.agent_sdk.session_manager import AgentSessionManager, \ - SessionManagerProtocol, run_async_sync, run_query_sync -from predicators.agent_sdk.tools import ALL_TOOL_NAMES, ToolContext, \ - create_mcp_tools, get_allowed_tool_list -from predicators.settings import CFG -from predicators.structs import ParameterizedOption, Predicate, Task, Type - -logger = logging.getLogger(__name__) - - -class AgentSessionMixin: - """Mixin that provides shared agent session infrastructure. - - Subclasses must override: - - _get_agent_system_prompt() - - And may optionally override: - - _get_solve_tool_names() -- complete tool surface for - play sessions. May mix static MCP tool names with - names of dynamic ``SdkMcpTool`` instances. ``None`` = all - static MCP tools, ``[]`` = none. - - _get_synthesis_tool_names() -- complete tool surface for - synthesis sessions (``_learning_mode=True``). Same shape / - semantics as the solve hook, independent value. - - Dynamic ``SdkMcpTool`` instances are supplied by the approach - directly: it assigns them to ``ctx.extra_mcp_tools`` before - opening a synthesis session and clears the field afterwards. The - mixin asserts the instance names line up with the names declared - in :meth:`_get_synthesis_tool_names`. - """ - - _log_subdir: str = "agent" # fallback; _get_log_dir prefers get_name() - - # --- Host-class contract ------------------------------------------ # - # The mixin is mixed into BaseApproach subclasses; these declare the - # host attributes it reads, so a typo fails type-checking instead of - # silently hitting a getattr default. ``_learning_mode`` is flipped - # by the sim-learning approach around synthesis sessions; the class - # default keeps plain solve-only hosts working without declaring it. - _learning_mode: bool = False - # Phase tag the live ``_agent_session`` was created with; a query - # under a different phase closes and rebuilds the session so the - # saved prompt, tools, and CLAUDE.md always match the active phase. - _agent_session_phase: Optional[str] = None - # Session-transcript count an auto-resumed run inherits from its - # checkpoint (see AgentSimLearningApproach._extra_save_state), so - # transcript ids continue across the lineage. - _resume_query_count: int = 0 - _types: Set[Type] - _action_space: Box - _train_tasks: List[Any] - - # ------------------------------------------------------------------ # - # Initialization - # ------------------------------------------------------------------ # - - def _init_agent_session_state( - self, - types: Set[Type], - predicates: Set[Predicate], - options: Set[ParameterizedOption], - train_tasks: List[Task], - ) -> None: - """Initialize ToolContext and lazy agent session placeholders.""" - self._tool_context = ToolContext( - types=types, - predicates=predicates, - options=options, - train_tasks=train_tasks, - ) - self._agent_session: Optional[SessionManagerProtocol] = None - self._agent_session_id: Optional[str] = None - - # ------------------------------------------------------------------ # - # Customization hooks (override in subclasses) - # ------------------------------------------------------------------ # - - def _get_agent_system_prompt(self) -> str: - """Return the system prompt for the agent session.""" - raise NotImplementedError - - def _get_solve_tool_names(self) -> Optional[List[str]]: - """Return the complete tool surface for play sessions. - - May mix static MCP tool names with names of dynamic - ``SdkMcpTool`` instances. ``None`` means "all static MCP tools"; - override to subset. - """ - return None - - def _get_synthesis_tool_names(self) -> Optional[List[str]]: - """Return the complete tool surface for the synthesis session. - - Selected when ``_learning_mode`` is True. Independent of the - solve list — the two phases may share names or be disjoint. Each - name must back either a static MCP tool (member of - ``ALL_TOOL_NAMES``) or a dynamic ``SdkMcpTool`` instance the - approach attaches via ``ctx.extra_mcp_tools``. Default ``[]`` - means no tools (approaches with no synthesis phase need not - override). - """ - return [] - - def _get_sandbox_reference_files(self) -> Dict[str, str]: - """Return extra reference files for the sandbox. - - Maps destination paths (relative to ``/sandbox/reference/``) to - source paths (relative to the repo root). Override in - subclasses to provide approach-specific reference material. - """ - return {} - - # ------------------------------------------------------------------ # - # Shared implementations - # ------------------------------------------------------------------ # - - def _ensure_agent_session(self) -> None: - """Create the agent session manager if needed. - - When ``SessionConfig.use_local_sandbox`` is ``True``, creates a - ``LocalSandboxSessionManager`` that runs the agent in a sandbox - directory with its built-in tools (Bash, Read, Write, ...). - Otherwise creates the in-process ``AgentSessionManager``. - """ - phase = "synthesis" if self._learning_mode else "solve" - self._tool_context.phase = phase - if self._agent_session is not None: - if self._agent_session_phase == phase: - return - # A live session from another phase would keep that phase's - # prompt tag and tool surface; rebuild so they match. - self._close_agent_session() - - # Session config is read once here (at session-construction - # time, never import time) and handed to whichever manager is - # built below. - config = SessionConfig.from_cfg() - - # Pick the declared tool surface by phase. ``_learning_mode`` is - # the same signal the system-prompt branch reads, so tools and - # prompt stay in sync. Each approach declares its solve and - # synthesis tool sets independently — they may be disjoint. - # ``tool_names`` is the *complete* declared list (may mix static - # MCP names with names of dynamic SdkMcpTool instances). - if self._learning_mode: - tool_names = self._get_synthesis_tool_names() # pylint: disable=assignment-from-none - else: - tool_names = self._get_solve_tool_names() # pylint: disable=assignment-from-none - - # Sanity: every dynamic name in the declared list must have a - # backing tool attached to ``ctx.extra_mcp_tools``. Static MCP - # names (``ALL_TOOL_NAMES``) are excluded — they're materialized - # downstream by ``create_mcp_tools``. Catches typos and missing - # builder hooks before the agent silently fails to invoke a - # declared-but-missing tool. - declared = set(tool_names or ()) - dynamic_declared = declared - set(ALL_TOOL_NAMES) - if dynamic_declared: - attached = list(self._tool_context.extra_mcp_tools or ()) - built = {getattr(t, "name", "") for t in attached} - missing = dynamic_declared - built - phase_for_msg = "synthesis" if self._learning_mode else "solve" - assert not missing, ( - f"Dynamic tool name(s) {sorted(missing)} declared in " - f"_get_{phase_for_msg}_tool_names but no matching tool " - f"attached to ctx.extra_mcp_tools — add them to the " - f"builder or drop the names.") - - approach_name = getattr(type(self), "get_name", - lambda: type(self).__name__)() - if tool_names is None: - logger.info( - "[%s] %s session tool surface: ALL static MCP tools " - "(no subset declared).", approach_name, phase) - else: - attached_names = { - getattr(t, "name", "") - for t in (self._tool_context.extra_mcp_tools or ()) - } - static = sorted( - n for n in tool_names - if n in set(ALL_TOOL_NAMES) and n not in attached_names) - dynamic = sorted( - n for n in tool_names - if n not in set(ALL_TOOL_NAMES) or n in attached_names) - lines = [ - f"[{approach_name}] {phase} session tool surface " - f"({len(tool_names)} total):" - ] - for n in static: - lines.append(f" static {n}") - for n in dynamic: - lines.append(f" dynamic {n}") - logger.info("\n".join(lines)) - - session: SessionManagerProtocol - if config.use_local_sandbox: - from predicators.agent_sdk.local_sandbox import \ - LocalSandboxSessionManager # pylint: disable=import-outside-toplevel - session = LocalSandboxSessionManager( - system_prompt=self._get_agent_system_prompt(), - log_dir=self._get_log_dir(), - model_name=config.model_name, - tool_context=self._tool_context, - tool_names=tool_names, - extra_reference_files=self._get_sandbox_reference_files(), - phase=phase, - config=config, - query_count_floor=self._resume_query_count, - ) - else: - from claude_agent_sdk import \ - create_sdk_mcp_server # pylint: disable=import-outside-toplevel - - tools = create_mcp_tools(self._tool_context, tool_names=tool_names) - mcp_server = create_sdk_mcp_server( - name="predicator_tools", - version="1.0.0", - tools=tools, - ) - - session = AgentSessionManager( - system_prompt=self._get_agent_system_prompt(), - mcp_server=mcp_server, - log_dir=self._get_log_dir(), - model_name=config.model_name, - allowed_tools=get_allowed_tool_list(tool_names), - tool_context=self._tool_context, - config=config, - ) - - self._agent_session = session - self._agent_session_phase = phase - if self._agent_session_id is not None: - session.session_id = self._agent_session_id - - # Save the system prompt to the log directory, one file per phase. - # Sandbox sessions write the FULL prompt (approach prompt + sandbox - # suffix) as full_system_prompt_{phase}.md during sandbox setup; for - # in-process sessions the approach prompt IS the full prompt, so - # save it under the same name here and nothing else. - if not config.use_local_sandbox: - log_dir = self._get_log_dir() - os.makedirs(log_dir, exist_ok=True) - prompt_path = os.path.join(log_dir, - f"full_system_prompt_{phase}.md") - with open(prompt_path, "w", encoding="utf-8") as f: - f.write(self._get_agent_system_prompt()) - - def _get_log_dir(self) -> str: - """Return the log directory, using the approach name. - - Always absolute: agent tool code (``run_python``) executes with - the sandbox as its working directory, so any path that harness - code may touch mid-call must not depend on the process cwd. - """ - if hasattr(CFG, 'log_file') and CFG.log_file: - return os.path.abspath(CFG.log_file) - name = ( - self.get_name() # type: ignore[attr-defined] - if hasattr(self, 'get_name') else self._log_subdir) - return os.path.abspath(os.path.join("logs", name)) - - def _close_agent_session(self) -> None: - """Close and discard the current agent session, if one exists.""" - if self._agent_session is None: - return - session = self._agent_session - self._agent_session = None - self._agent_session_phase = None - try: - run_async_sync(session.close()) - except Exception: # pylint: disable=broad-except - pass - - def _query_agent_sync(self, message: str, - **query_kwargs: Any) -> List[Dict[str, Any]]: - """Synchronous wrapper for async agent query. - - Extra kwargs (e.g. ``kind="learn"``) are forwarded to the - session's ``query`` method for log-file tagging. - """ - self._ensure_agent_session() - assert self._agent_session is not None - return run_query_sync(self._agent_session, message, **query_kwargs) diff --git a/predicators/approaches/agent_sim_learning_approach.py b/predicators/approaches/agent_sim_learning_approach.py index e5227a1915..b8f9b32e76 100644 --- a/predicators/approaches/agent_sim_learning_approach.py +++ b/predicators/approaches/agent_sim_learning_approach.py @@ -32,8 +32,7 @@ _SnapshotTarget, evaluate_states_with, finalize_versioned_snapshot, \ make_write_snapshot_hook from predicators.agent_sdk.tools.digests import render_trajectory_digest -from predicators.approaches.agent_model_free_approach import \ - AgentModelFreeApproach +from predicators.approaches.agent_base_approach import AgentBaseApproach from predicators.approaches.synthesis_validation import \ build_candidate_option_model, carry_over_params from predicators.code_sim_learning.active_experiment import laplace_ensemble, \ @@ -175,7 +174,7 @@ def residual_hint_from_hits(hits: Dict[Tuple[str, str], int], return {t: sorted(fs) for t, fs in out.items()} -class AgentSimLearningApproach(AgentModelFreeApproach): +class AgentSimLearningApproach(AgentBaseApproach): """The agent-written simulator, its parameters and their belief. Loads the simulator the agent writes, deploys its parameters (the @@ -518,7 +517,7 @@ def _build_synthesis_session_hooks( # ── Learning ──────────────────────────────────────────────── # ── Checkpointing ──────────────────────────────────────────── - # The base checkpoint (AgentModelFreeApproach.save/load) persists + # The base checkpoint (AgentBaseApproach.save/load) persists # the datasets + cycle counter. This approach's real state is split # between plain fitted values (pickled below) and the sandbox # artifacts the agent wrote (simulator.py / predicates.py / ...), diff --git a/predicators/approaches/bilevel_planning_approach.py b/predicators/approaches/bilevel_planning_approach.py index 53e7c6a2ab..4f95bc9dc0 100644 --- a/predicators/approaches/bilevel_planning_approach.py +++ b/predicators/approaches/bilevel_planning_approach.py @@ -51,7 +51,7 @@ def __init__(self, # this, Wait runs to max_num_steps_option_rollout during # refinement and the step is rejected for "exceeded individual # horizon", even when the expected atoms have already become - # true. Mirrors AgentModelFreeApproach.__init__. + # true. Mirrors AgentBaseApproach.__init__. # Looked up lazily so subclasses whose _get_current_predicates # depends on attributes set after super().__init__() (e.g. # GrammarSearchInventionApproach._learned_predicates) don't break, diff --git a/predicators/approaches/continual_play_mixin.py b/predicators/approaches/continual_play_mixin.py index 0706ca1610..bbe4456389 100644 --- a/predicators/approaches/continual_play_mixin.py +++ b/predicators/approaches/continual_play_mixin.py @@ -29,14 +29,13 @@ A level can span several rounds if the agent returns before settling it. Why a mixin. The arms' model and session machinery live in the -approach classes below them (``AgentModelFreeApproach`` and its +approach classes below them (``AgentBaseApproach`` and its ``AgentSimPredicateInventionApproach`` descendant), where the simulator loading, the parameter belief, predicate invention, the sandbox and the session managers are implemented. An arm keeps that class as its base -and mixes this loop in front of it, the way ``AgentSessionMixin`` adds -its concerns. The mixin has no base class of its own, so there is no -diamond, and what it needs from its host is declared below as the host -contract. +and mixes this loop in front of it. The mixin has no base class of its +own, so there is no diamond, and what it needs from its host is +declared below as the host contract. The harness never chooses for the agent: whether to act, reset, model or give up is decided inside the conversation; the loop only services @@ -105,7 +104,7 @@ class ContinualPlayMixin: """ # -- Host contract ------------------------------------------------- - # The host is an AgentModelFreeApproach (or a descendant); these + # The host is an AgentBaseApproach (or a descendant); these # declare what the loop reads and calls on it, so a typo fails # type-checking instead of surfacing at run time. if TYPE_CHECKING: diff --git a/tests/agent_sdk/test_continual_tools.py b/tests/agent_sdk/test_continual_tools.py index 7094c167a4..e66b16f805 100644 --- a/tests/agent_sdk/test_continual_tools.py +++ b/tests/agent_sdk/test_continual_tools.py @@ -19,8 +19,8 @@ PRIMITIVE_TOOL_NAMES, PlayState, build_continual_tools, context_status, \ format_observation, parse_plan_lines, play_tool_names from predicators.approaches import create_approach -from predicators.approaches.agent_model_free_approach import \ - AgentModelFreeApproach +from predicators.approaches.agent_continual_approach import \ + AgentContinualModelFreeApproach from predicators.envs import create_new_env from predicators.ground_truth_models import get_gt_options from predicators.run.continual import ContinualRun, ProtocolSession, RunEnded @@ -149,9 +149,9 @@ def body(session: ProtocolSession) -> None: "pybullet_balloons" ]) def test_shared_controller_references_are_public_api(env_name: str) -> None: - """The MF parent supplies API documentation instead of engine source.""" + """The MF arm supplies API documentation instead of engine source.""" utils.reset_config({"env": env_name}) - approach = object.__new__(AgentModelFreeApproach) + approach = object.__new__(AgentContinualModelFreeApproach) references = approach._get_sandbox_reference_files() # pylint: disable=protected-access assert references == { "skills.md": "predicators/agent_sdk/prompts/public_skills.md" diff --git a/tests/agent_sdk/test_tool_registry.py b/tests/agent_sdk/test_tool_registry.py index e581cca5db..0c267f7183 100644 --- a/tests/agent_sdk/test_tool_registry.py +++ b/tests/agent_sdk/test_tool_registry.py @@ -14,7 +14,7 @@ from predicators.agent_sdk.tools import ALL_TOOL_NAMES, BUILTIN_TOOLS, \ MCP_SERVER_NAME, SYNTHESIS_TOOL_NAMES, ToolContext, create_mcp_tools, \ create_synthesis_tools, get_allowed_tool_list, list_session_tool_names -from predicators.approaches.agent_session_mixin import AgentSessionMixin +from predicators.approaches.agent_base_approach import AgentBaseApproach def _required_names(names: Optional[List[str]]) -> List[str]: @@ -129,18 +129,25 @@ def test_list_session_tool_names_filters_and_combines() -> None: } +class _Approach(AgentBaseApproach): # pylint: disable=abstract-method + """The smallest concrete agent approach, for the base's tool hooks.""" + + @classmethod + def get_name(cls) -> str: + return "test_agent" + + def test_synthesis_tool_names_default_is_empty() -> None: """No synthesis MCP filter by default — approaches with no synthesis phase get an empty allowlist for free.""" - obj = AgentSessionMixin() + obj = object.__new__(_Approach) assert not obj._get_synthesis_tool_names() def test_solve_and_synthesis_tool_names_are_independent() -> None: """Subclasses can declare disjoint solve / synthesis tool sets.""" - # pylint: disable=abstract-method - class _Approach(AgentSessionMixin): + class _Arm(_Approach): # pylint: disable=abstract-method def _get_solve_tool_names(self) -> Optional[List[str]]: return ["run_python", "skills_execute_plan"] @@ -148,7 +155,7 @@ def _get_solve_tool_names(self) -> Optional[List[str]]: def _get_synthesis_tool_names(self) -> Optional[List[str]]: return ["run_python"] - obj = _Approach() + obj = object.__new__(_Arm) assert obj._get_solve_tool_names() == ["run_python", "skills_execute_plan"] assert obj._get_synthesis_tool_names() == ["run_python"] diff --git a/tests/run/test_public_observation_boundary.py b/tests/run/test_public_observation_boundary.py index 66a8d4f7a5..85dd580baa 100644 --- a/tests/run/test_public_observation_boundary.py +++ b/tests/run/test_public_observation_boundary.py @@ -5,8 +5,8 @@ import numpy as np from predicators import utils -from predicators.approaches.agent_model_free_approach import \ - AgentModelFreeApproach +from predicators.approaches.agent_continual_approach import \ + AgentContinualModelFreeApproach from predicators.envs import create_new_env from predicators.envs.pybullet_balloons import BalloonsEvaluator from predicators.run.episode import EpisodeRunner, EpisodeState @@ -29,9 +29,9 @@ def test_recording_objects_have_only_public_schema() -> None: def test_skill_reference_is_public_documentation() -> None: - """The same baseline parent supplies API docs instead of implementation.""" + """The model-free arm supplies API docs instead of implementation.""" utils.reset_config({"env": "pybullet_bridge"}) - approach = object.__new__(AgentModelFreeApproach) + approach = object.__new__(AgentContinualModelFreeApproach) references = approach._get_sandbox_reference_files() assert references assert all(path.endswith('.md') for path in references.values()) diff --git a/tests/test_agent_sdk_tools.py b/tests/test_agent_sdk_tools.py index 0f874e005e..354c4bfd84 100644 --- a/tests/test_agent_sdk_tools.py +++ b/tests/test_agent_sdk_tools.py @@ -25,7 +25,7 @@ _CFG_OVERRIDES = { "env": "pybullet_domino", - "approach": "agent_model_free", + "approach": "agent_continual_model_free", "seed": 0, "use_gui": False, "domino_restricted_push": True, diff --git a/tests/test_docker_option_plan.py b/tests/test_docker_option_plan.py index c94db7ea2a..6ffb449635 100644 --- a/tests/test_docker_option_plan.py +++ b/tests/test_docker_option_plan.py @@ -31,7 +31,7 @@ # Flags of the retired phased mf_agent config. _CFG_OVERRIDES = { "env": "pybullet_domino", - "approach": "agent_model_free", + "approach": "agent_continual_model_free", "seed": 0, "use_gui": False, "domino_restricted_push": True,