Let the assistant work from attached files, pictures, notes and web pages

backend/alembic/versions/20261021_1400_add_assistant_session_attachments.py

@@ -0,0 +1,30 @@
+"""Material attached to an assistant conversation: files, notes and web pages, as text.
+
+A server default so the conversations that already exist read as having none.
+
+Revision ID: 8a4c2e9f1d37
+Revises: 5f1b8c3d2a90
+Create Date: 2026-10-21 14:00:00.000000+00:00
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+revision: str = '8a4c2e9f1d37'
+down_revision: Union[str, None] = '5f1b8c3d2a90'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table('assistant_session', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('attachments', sqlmodel.sql.sqltypes.AutoString(),
+                                      nullable=False, server_default='[]'))
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('assistant_session', schema=None) as batch_op:
+        batch_op.drop_column('attachments')

backend/app/assistant/loop.py

@@ -12,7 +12,7 @@
 
 from app.assistant import tools as T
 from app.assistant.tools import Working, _issues_text
-from app.providers.base import ChatProvider, Reply
+from app.providers.base import ChatProvider, Image, Reply
 
 logger = logging.getLogger(__name__)
 
@@ -42,9 +42,9 @@ def _first_line(text: str) -> str:
 def run(provider: ChatProvider, working: Working, *, system: Sequence[str],
         history: Sequence[Tuple[str, str]], prompt: str, on_step: Step,
         on_reply: Callable[[Reply], None], cancelled: Callable[[], bool],
-        max_rounds: int) -> Outcome:
+        max_rounds: int, images: Sequence[Image] = ()) -> Outcome:
     conversation = provider.conversation(system=system, tools=T.SPECS, history=history,
-                                         prompt=prompt)
+                                         prompt=prompt, images=images)
     outcome = Outcome(reply="")
     nudges = 0
     for _ in range(max_rounds):

backend/app/assistant/prompt.py

@@ -1,8 +1,10 @@
-"""What the assistant is told, in two blocks.
+"""What the assistant is told, in three blocks.
 
 1. `REFERENCE`: who it is and how a VideoSpec works. Fixed text, so a provider that caches
    prompts reads it at the cached rate on every turn of every session.
-2. `context(...)`: this project as it stands. Fixed for one turn, so the rounds of a turn
+2. The material attached to the conversation (`attachments.material`), when there is
+   any. Fixed until something is attached or removed, so it caches across turns.
+3. `context(...)`: this project as it stands. Fixed for one turn, so the rounds of a turn
    share its cache too.
 
 The reference is written for a capable model: it says what the format means and what

backend/app/assistant/worker.py

@@ -19,6 +19,7 @@
 from sqlmodel import Session
 
 from app.assistant import loop
+from app.assistant.attachments import images_of, material
 from app.assistant.prompt import REFERENCE, context
 from app.assistant.tools import Working
 from app.clock import utcnow
@@ -27,7 +28,7 @@
 from app.generation import service as generation_service
 from app.jobs.runner import JobQueue, readable_error, set_fields
 from app.models.assistant import AssistantSession, AssistantTurn
-from app.providers import build_provider
+from app.providers import active_provider, build_provider
 from app.providers.base import ProviderError, Reply
 from app.spec.model import dump_spec
 from app.spec.validation import parse_spec
@@ -82,6 +83,8 @@ def _run(turn_id: str) -> None:
         prompt, focus = turn.prompt, turn.focus
         base = json.loads(turn.base_spec or "{}")
         history = history_for(chat, turn_id) if chat else []
+        attachments = json.loads(chat.attachments or "[]") if chat else []
+        chat_id = chat.id if chat else ""
 
     token = _tokens.get(turn_id)
     gam = GamClient(token)
@@ -112,8 +115,12 @@ def on_reply(reply: Reply) -> None:
     try:
         with queue.session() as session:
             provider = build_provider(session, user_id)
+            row = active_provider(session, user_id)
+            sees_images = bool(row and row.supports_images)
         if provider is None:
             raise ProviderError("No AI provider is set up. Add one in Settings first.")
+        attached = material(attachments, sees_images=sees_images)
+        images = images_of(chat_id, attachments) if sees_images else []
         working = Working(base, gam=gam, max_generations=settings.assistant_max_generations)
 
         def generate(data, key, image_prompt, base_ids, outputs):
@@ -127,8 +134,10 @@ def generate(data, key, image_prompt, base_ids, outputs):
                 return row.model_label, row.estimate
 
         working.generate = generate
-        system = [REFERENCE, context(base, focus=focus, from_scratch=project_id is None)]
+        system = [REFERENCE, attached,
+                  context(base, focus=focus, from_scratch=project_id is None)]
         outcome = loop.run(provider, working, system=system, history=history, prompt=prompt,
+                           images=images,
                            on_step=on_step, on_reply=on_reply,
                            cancelled=lambda: queue.is_cancelled(turn_id),
                            max_rounds=settings.assistant_max_rounds)

backend/app/config.py

@@ -98,6 +98,17 @@ class Settings(BaseSettings):
     # Pictures one request may generate on its own (each is a paid call to fal.ai through
     # GAM). Past it the assistant leaves placeholders for the user to generate.
     assistant_max_generations: int = 6
+    # Material attached to a conversation (files, notes, web pages), as text. Each turn
+    # reads all of it on every round, so the limits are about cost as much as context
+    # length: 150k characters is roughly 40k tokens, cached after the first round.
+    assistant_attachment_max_chars: int = 60_000
+    assistant_attachments_max_chars: int = 150_000
+    assistant_max_attachments: int = 12
+    # Of those, pictures. Each is sent to the model on every round of every turn, at
+    # roughly 1,500 tokens apiece at full size.
+    assistant_max_images: int = 6
+    # The largest file, or web page, read for an attachment.
+    assistant_attachment_max_mb: int = 20
     job_heartbeat_seconds: int = 30
     job_stale_minutes: int = 40
 

backend/app/models/assistant.py

@@ -26,6 +26,10 @@ class AssistantSession(SQLModel, table=True):
     title: str = ""
     # JSON: [{"role": "user"|"assistant", "text", "turn_id", "at"}]
     messages: str = "[]"
+    # JSON: [{"id", "kind": "file"|"note"|"url", "title", "source", "text", "chars",
+    # "truncated", "added_at"}]: material attached to work from, as text taken when it
+    # was attached (assistant/attachments.py).
+    attachments: str = "[]"
     created_at: datetime = Field(default_factory=utcnow)
     updated_at: datetime = Field(default_factory=utcnow)
 

backend/app/notes/__init__.py

@@ -0,0 +1 @@
+"""Gecko Notes, as GVC reads it: notes attached to the assistant as context."""