Group agents by engine in the directory tree

README.md

@@ -70,7 +70,7 @@ covers pointing it at a mirror.
 
 | Command | What it does | Touches GCP |
 |---|---|---|
-| `gete init <name>` | Scaffold an agent (and the project if there is none) | no |
+| `gete init [<engine>/]<name>` | Scaffold an agent (and the project if there is none); below an engine's directory when one is given | no |
 | `gete validate [--check-secrets] [--import-check]` | Schemas and rules; optionally secrets' versions and a deployment-shaped import | `--check-secrets` reads |
 | `gete run <name>` | Local conversation; tokens from `GETE_TOKEN_<CONNECTION>` | calls the model |
 | `gete graph [name...]` | Mermaid diagram of agents, engines, tools, connections | no |
@@ -168,6 +168,30 @@ since accepting both would let an id copied from the agent next door land on
 the wrong engine unnoticed. Each id is written once, so a rebuilt engine is
 one line, and `gete graph` and `gete register` say the name.
 
+With the engines named, the tree itself can say where each agent is listed.
+A directory below `agents/` that carries an engine's name holds the agents
+listed in that engine, and they declare no `registration` block:
+
+```
+agents/
+  sales/
+    partner-review/agent.yaml    listed in sales
+    mail-triage/agent.yaml       listed in sales
+  support/
+    ticket-digest/agent.yaml     listed in support
+  local-only/agent.yaml          listed nowhere, as before
+```
+
+`gete init sales/partner-review` starts an agent there. The name still
+matches the agent's own directory, so the module address, service account,
+and archive do not change when an agent moves into a group: the generated
+module call points at the new path, and nothing is recreated. Groups go one
+level deep. A directory below `agents/` with no `agent.yaml` that names no
+engine is reported by `validate` rather than stepped over, since a misspelt
+engine would otherwise drop every agent below it from the project, and the
+next apply would destroy them. A `registration` block inside a group is
+accepted while it agrees with the directory, and reported once it does not.
+
 ### Tools from an OpenAPI description
 
 A service that publishes an OpenAPI description but runs no MCP server can be

docs/quickstart.md

@@ -213,8 +213,9 @@ registration:
 ```
 
 With several engines, name them once in `gete.yaml` under
-`gemini_enterprise.engines` and write the name here instead; the README
-shows the shape. Then:
+`gemini_enterprise.engines` and write the name here instead, or move the
+agent below `agents/<engine name>/` and leave the block out; the README
+shows both. Then:
 
 ```sh
 gete register

src/gete/cli/__init__.py

@@ -182,7 +182,12 @@ def register(names: tuple[str, ...], notice: Path, reset: tuple[str, ...]) -> No
 @main.command()
 @click.argument("name")
 def init(name: str) -> None:
-    """Create agents/NAME from a template; create gete.yaml too if there is none."""
+    """Create agents/NAME from a template; create gete.yaml too if there is none.
+
+    NAME may be ENGINE/NAME, with ENGINE named in gete.yaml under
+    gemini_enterprise.engines: the agent is then listed in that engine by
+    where it sits, and declares no registration block.
+    """
     from gete.scaffold import init_agent, init_project
 
     try:
@@ -194,6 +199,10 @@ def init(name: str) -> None:
     try:
         written.extend(init_agent(root, name))
     except GeteError as error:
+        # The project files are on disk by now; say so, since a refused
+        # group is fixed by editing the gete.yaml just written.
+        for path in written:
+            click.echo(str(path))
         click.echo(str(error), err=True)
         sys.exit(1)
     for path in written:

src/gete/declaration.py

@@ -58,6 +58,9 @@ class Agent:
 
     directory: Path
     data: Mapping[str, Any]
+    # The engine whose directory the agent sits in, agents/<engine>/<name>.
+    # None for an agent directly below agents_dir.
+    group: str | None = None
 
     @property
     def path(self) -> Path:
@@ -129,8 +132,8 @@ def instruction_text(self) -> str:
         return path.read_text(encoding="utf-8") if path else self.instruction
 
     @property
-    def engine(self) -> str | None:
-        """What registration.gemini_enterprise.engine says, or None when unlisted.
+    def registration_engine(self) -> str | None:
+        """What registration.gemini_enterprise.engine says, or None when absent.
 
         A name from gemini_enterprise.engines when gete.yaml declares them,
         the engine's id otherwise; Project.engine_id tells which.
@@ -139,6 +142,17 @@ def engine(self) -> str | None:
         engine = registration.get("gemini_enterprise", {}).get("engine")
         return str(engine) if engine else None
 
+    @property
+    def engine(self) -> str | None:
+        """The engine the agent is listed in, or None when unlisted.
+
+        The group's engine for an agent below agents/<engine>/, what the
+        registration block says otherwise. The directory is what the tree
+        shows, so it wins over a block that disagrees; validate reports the
+        disagreement.
+        """
+        return self.group or self.registration_engine
+
     @property
     def source(self) -> Path | None:
         value = self.data.get("source")
@@ -162,8 +176,9 @@ class Project:
     path: Path
     data: Mapping[str, Any]
     agents: tuple[Agent, ...]
-    # Agents whose agent.yaml did not pass the schema. They are reported and
-    # left out of agents, so the rules never see an unexpected shape.
+    # Agents whose agent.yaml did not pass the schema, and directories that
+    # are neither an agent nor an engine's group. They are reported and left
+    # out of agents, so the rules never see an unexpected shape.
     problems: tuple[Problem, ...] = ()
 
     @property
@@ -230,32 +245,113 @@ def find_project_file(start: Path) -> Path:
 
 
 def load_project(path: Path) -> Project:
-    """Read gete.yaml and every agents/*/agent.yaml below it.
-
-    gete.yaml must pass its schema; nothing can be checked without it. Agents
-    that fail theirs are collected as problems rather than raised, so one broken
-    agent does not hide the state of the others.
+    """Read gete.yaml and the agents below it.
+
+    An agent is agents/<name>/agent.yaml, or agents/<engine>/<name>/agent.yaml
+    once gete.yaml names the engines: the directory then says where the agent
+    is listed. gete.yaml must pass its schema; nothing can be checked without
+    it. Agents that fail theirs are collected as problems rather than raised,
+    so one broken agent does not hide the state of the others. A directory
+    that is neither is a problem too, not something to step over: skipped
+    quietly, a misspelt engine would take every agent below it out of the
+    project, and the next apply would destroy them.
     """
     data = read_yaml(path)
     validate_document("gete", data, source=path)
     project = Project(path=path, data=data, agents=())
     agents: list[Agent] = []
     problems: list[Problem] = []
-    if project.agents_dir.is_dir():
-        for directory in sorted(project.agents_dir.iterdir()):
-            agent_file = directory / AGENT_FILE
-            if not agent_file.is_file():
-                continue
-            agent_data = read_yaml(agent_file)
-            found = schema_problems("agent", agent_data)
-            if found:
-                source = project.display(agent_file)
-                problems.extend(Problem(source, message) for message in found)
-                continue
-            agents.append(Agent(directory=directory, data=agent_data))
+    engines = project.engines or {}
+    for directory in visible_directories(project.agents_dir):
+        if (directory / AGENT_FILE).is_file():
+            _load_agent(project, directory, None, agents, problems)
+            if directory.name in engines:
+                problems.extend(_agents_below_an_agent(project, directory))
+        elif directory.name in engines:
+            for member in visible_directories(directory):
+                if (member / AGENT_FILE).is_file():
+                    _load_agent(project, member, directory.name, agents, problems)
+                else:
+                    problems.append(
+                        Problem(
+                            project.display(member),
+                            f"no {AGENT_FILE}; a directory below an engine's "
+                            "group is an agent, and groups do not nest",
+                        )
+                    )
+        else:
+            problems.append(Problem(project.display(directory), _not_a_group(project)))
     return Project(path=path, data=data, agents=tuple(agents), problems=tuple(problems))
 
 
+def visible_directories(parent: Path) -> list[Path]:
+    """The directories below parent, in name order; hidden ones are nobody's.
+
+    Hidden entries hold tooling (a venv, a VCS directory) and stay out of the
+    archive for the same reason, so they are not reported as strays either.
+    """
+    if not parent.is_dir():
+        return []
+    return sorted(
+        path
+        for path in parent.iterdir()
+        if path.is_dir() and not path.name.startswith(".")
+    )
+
+
+def _agents_below_an_agent(project: Project, directory: Path) -> list[Problem]:
+    """Agents put below an agent that carries an engine's name.
+
+    The directory holds its own agent.yaml, so it is that agent and not the
+    engine's group, and an agent placed below it would be stepped over
+    without a word: the one thing this loader must not do.
+    """
+    return [
+        Problem(
+            project.display(member),
+            f"below {project.display(directory)}, which holds its own "
+            f"{AGENT_FILE} and so is the agent {directory.name!r}, not the group "
+            f"of the engine {directory.name!r}; an agent and an engine's group "
+            "cannot share a directory",
+        )
+        for member in visible_directories(directory)
+        if (member / AGENT_FILE).is_file()
+    ]
+
+
+def _load_agent(
+    project: Project,
+    directory: Path,
+    group: str | None,
+    agents: list[Agent],
+    problems: list[Problem],
+) -> None:
+    agent_file = directory / AGENT_FILE
+    agent_data = read_yaml(agent_file)
+    found = schema_problems("agent", agent_data)
+    if found:
+        source = project.display(agent_file)
+        problems.extend(Problem(source, message) for message in found)
+        return
+    agents.append(Agent(directory=directory, data=agent_data, group=group))
+
+
+def _not_a_group(project: Project) -> str:
+    """Why a directory without agent.yaml is not read as an engine's group."""
+    if project.engines is None:
+        return (
+            f"no {AGENT_FILE}, and {PROJECT_FILE} names no engines; a directory "
+            "here is an agent, or an engine's group of agents once "
+            "gemini_enterprise.engines names the engine"
+        )
+    names = ", ".join(project.engines)
+    return (
+        f"no {AGENT_FILE}, and no engine of that name under "
+        f"gemini_enterprise.engines in {PROJECT_FILE} ({names}); a directory "
+        "here is an agent, or an engine's group of agents"
+    )
+
+
 def resolve(project: Project, agent: Agent) -> dict[str, Any]:
     """Fold gete.yaml into one agent's declaration.
 

src/gete/scaffold.py

@@ -3,9 +3,15 @@
 import re
 from pathlib import Path
 from string import Template
+from typing import Any
 
 from gete._yaml import read_yaml
-from gete.declaration import AGENT_FILE, DEFAULT_AGENTS_DIR, PROJECT_FILE
+from gete.declaration import (
+    AGENT_FILE,
+    DEFAULT_AGENTS_DIR,
+    PROJECT_FILE,
+    visible_directories,
+)
 from gete.errors import DeclarationError
 from gete.templates import template_text
 
@@ -26,17 +32,56 @@ def init_project(root: Path) -> list[Path]:
 
 
 def init_agent(root: Path, name: str) -> list[Path]:
-    """Write agents/<name>/agent.yaml and instruction.md unless they exist."""
+    """Write agents/<name>/agent.yaml and instruction.md unless they exist.
+
+    <engine>/<name> puts the agent below the engine's directory, where it is
+    listed in that engine. The engine has to be named in gete.yaml already:
+    validate would refuse the directory otherwise, and a scaffold that fails
+    its own validator teaches the wrong shape.
+    """
+    *groups, name = name.split("/")
+    if len(groups) > 1:
+        raise DeclarationError(
+            f"{'/'.join(groups)!r}: agents group one level deep, as <engine>/<name>"
+        )
     if len(name) > 63 or not _LABEL.match(name):
         raise DeclarationError(
             f"{name!r} is not a valid agent name: lowercase letters, digits, hyphens"
         )
-    agents_dir = root / DEFAULT_AGENTS_DIR
+    declared: dict[str, Any] = {}
     project_file = root / PROJECT_FILE
     if project_file.is_file():
         declared = read_yaml(project_file) or {}
-        agents_dir = root / declared.get("agents_dir", DEFAULT_AGENTS_DIR)
-    directory = agents_dir / name
+    agents_dir = Path(declared.get("agents_dir", DEFAULT_AGENTS_DIR))
+    directory = root / agents_dir
+    engines = (declared.get("gemini_enterprise") or {}).get("engines") or {}
+    # An agent and an engine's group cannot share a directory: the loader
+    # reads a directory with agent.yaml as the agent and steps over nothing
+    # below it, so either shape written here would hide agents.
+    if groups:
+        if groups[0] not in engines:
+            named = f" ({', '.join(engines)})" if engines else ""
+            raise DeclarationError(
+                f"{groups[0]!r} is not an engine named under "
+                f"gemini_enterprise.engines in {PROJECT_FILE}{named}; name it "
+                "there first, and agents below that directory are listed in it"
+            )
+        directory = directory / groups[0]
+        if (directory / AGENT_FILE).is_file():
+            raise DeclarationError(
+                f"{groups[0]!r} is an agent, not the engine's group: "
+                f"{agents_dir / groups[0] / AGENT_FILE} exists, and an agent and "
+                "an engine's group cannot share a directory"
+            )
+    elif name in engines and any(
+        (member / AGENT_FILE).is_file()
+        for member in visible_directories(directory / name)
+    ):
+        raise DeclarationError(
+            f"{name!r} is the engine's group and holds agents; an agent and an "
+            "engine's group cannot share a directory"
+        )
+    directory = directory / name
     return [
         path
         for path, text in (

src/gete/templates/agent.yaml

@@ -17,6 +17,7 @@ instruction: ./instruction.md
 
 # Where the agent is listed: the id from the Gemini Enterprise console URL,
 # or its name once gete.yaml names the engines under gemini_enterprise.engines.
+# An agent below agents/<engine name>/ is listed there and needs no block.
 # registration:
 #   gemini_enterprise:
 #     engine: my-app_1234567890

src/gete/validate.py

@@ -111,6 +111,18 @@ def _agent_problems(
             f"name {agent.name!r} differs from directory {agent.directory.name!r}; "
             "the module, service account, and archive are named after the directory"
         )
+    # Possible once agents sit in groups: two directories, one name, and both
+    # would claim the module, the service account, and the reasoning engine.
+    twins = [
+        project.display(other.directory)
+        for other in project.agents
+        if other.name == agent.name and other.directory != agent.directory
+    ]
+    if twins:
+        found.append(
+            f"name {agent.name!r} is also {', '.join(twins)}; the module, "
+            "service account, and reasoning engine are named after it"
+        )
     account = f"{agent.name}{SERVICE_ACCOUNT_SUFFIX}"
     if not (
         MIN_SERVICE_ACCOUNT_ID_LENGTH <= len(account) <= MAX_SERVICE_ACCOUNT_ID_LENGTH
@@ -184,11 +196,22 @@ def _agent_problems(
     found.extend(
         f"connections: {message}" for message in elimination_problems(known, registry)
     )
-    if agent.engine is not None:
+    written = agent.registration_engine
+    if written is not None and agent.group is None:
         try:
-            project.engine_id(agent.engine)
+            project.engine_id(written)
         except DeclarationError as error:
             found.append(str(error))
+    elif written is not None and written != agent.group:
+        # The block an agent moved into a group brought along, and then
+        # edited: the directory is what the tree shows, so it is the one
+        # that counts, and the block has nothing left to say. Whether what
+        # it says is a name or an id makes no difference, so one line.
+        found.append(
+            f"registration.gemini_enterprise.engine: {written!r} differs from "
+            f"the directory, which lists the agent in {agent.group!r}; an "
+            "agent below an engine's group needs no registration block"
+        )
     declared_shared: Mapping[str, Any] = project.data.get("shared_credentials", {})
     for name in agent.shared_credentials:
         credential = SHARED_CREDENTIALS.get(name)

tests/conftest.py

@@ -43,7 +43,8 @@ def write_agent(
     ) -> Path:
         agent_dir = self.agents_dir / directory
         agent_dir.mkdir(parents=True, exist_ok=True)
-        body = {**MINIMAL_AGENT, "name": directory, **(document or {})}
+        # The directory may be <engine>/<name>; the name is the last part.
+        body = {**MINIMAL_AGENT, "name": Path(directory).name, **(document or {})}
         (agent_dir / "agent.yaml").write_text(yaml.safe_dump(body, sort_keys=False))
         if instruction is not None:
             (agent_dir / "instruction.md").write_text(instruction)