M4/M5: the video assistant

README.md

@@ -28,8 +28,15 @@ GAM by what is said in it. Then preview and render it:
 - text, lower thirds and pictures on screen, timed to the scenes and lines they belong to;
 - end credits from the library's attribution, and a description with chapters and credits.
 
-A finished render can be saved back into GAM. AI drafting and the full storyboard
-editor are next (M3, M4).
+A finished render can be saved back into GAM.
+
+**The assistant (M4/M5).** A panel beside every page that knows the whole video format.
+Describe a video and it drafts one from your GAM library, or ask for any change to the
+one open. What it would change is listed for you to apply or discard. It runs in the
+background, so a request finishes with the page closed. Set up its AI provider
+(Anthropic, OpenAI, DeepSeek, Ollama or a custom endpoint) under Settings.
+
+The full storyboard editor is next (M3).
 
 ## Read first
 

backend/alembic/versions/20261016_1016_add_ai_provider_and_usage_event_tables.py

@@ -0,0 +1,87 @@
+"""Add the aiprovider and usageevent tables (M4/M5, the assistant's first half).
+
+Both are copied from GAM's, which came from gecko-notes. `usageevent.project_id` and
+`turn_id` are plain indexed columns, not foreign keys: deleting a project must not
+delete the record of what was spent on it.
+
+Revision ID: 4a8c2f6e9d13
+Revises: 7e2d5a91b3c6
+Create Date: 2026-10-16 10:16:48.213291+00:00
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+revision: str = '4a8c2f6e9d13'
+down_revision: Union[str, None] = '7e2d5a91b3c6'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table('aiprovider',
+    sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('user_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('provider_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('api_key', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('base_url', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('model', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('max_tokens', sa.Integer(), nullable=False),
+    sa.Column('supports_images', sa.Boolean(), nullable=False),
+    sa.Column('use_anthropic_api', sa.Boolean(), nullable=False),
+    sa.Column('extra_params', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('enabled', sa.Boolean(), nullable=False),
+    sa.Column('is_active', sa.Boolean(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    with op.batch_alter_table('aiprovider', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_aiprovider_user_id'), ['user_id'], unique=False)
+
+    op.create_table('usageevent',
+    sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('user_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('project_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('kind', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('provider', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('model', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('units', sa.Integer(), nullable=False),
+    sa.Column('unit_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('cost', sa.Float(), nullable=True),
+    sa.Column('currency', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('cost_estimated', sa.Boolean(), nullable=True),
+    sa.Column('turn_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    with op.batch_alter_table('usageevent', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_usageevent_created_at'), ['created_at'], unique=False)
+        batch_op.create_index(batch_op.f('ix_usageevent_kind'), ['kind'], unique=False)
+        batch_op.create_index(batch_op.f('ix_usageevent_project_id'), ['project_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_usageevent_provider'), ['provider'], unique=False)
+        batch_op.create_index(batch_op.f('ix_usageevent_turn_id'), ['turn_id'], unique=False)
+        batch_op.create_index('ix_usageevent_user_created', ['user_id', 'created_at'], unique=False)
+        batch_op.create_index(batch_op.f('ix_usageevent_user_id'), ['user_id'], unique=False)
+
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('usageevent', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_usageevent_user_id'))
+        batch_op.drop_index('ix_usageevent_user_created')
+        batch_op.drop_index(batch_op.f('ix_usageevent_turn_id'))
+        batch_op.drop_index(batch_op.f('ix_usageevent_provider'))
+        batch_op.drop_index(batch_op.f('ix_usageevent_project_id'))
+        batch_op.drop_index(batch_op.f('ix_usageevent_kind'))
+        batch_op.drop_index(batch_op.f('ix_usageevent_created_at'))
+
+    op.drop_table('usageevent')
+    with op.batch_alter_table('aiprovider', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_aiprovider_user_id'))
+
+    op.drop_table('aiprovider')

backend/alembic/versions/20261016_1027_add_assistant_session_and_turn_tables.py

@@ -0,0 +1,84 @@
+"""Add the assistant_session and assistant_turn tables (M4/M5).
+
+A turn is a job, so it carries the job fields the runner reads (status, stage,
+progress, detail, error_message, updated_at) next to everything the turn is: its
+prompt, the draft it started from, its steps, its reply and its proposal.
+
+Revision ID: 9b3e7d215f60
+Revises: 4a8c2f6e9d13
+Create Date: 2026-10-16 10:27:22.137896+00:00
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+revision: str = '9b3e7d215f60'
+down_revision: Union[str, None] = '4a8c2f6e9d13'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table('assistant_session',
+    sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('user_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('project_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('title', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('messages', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    with op.batch_alter_table('assistant_session', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_assistant_session_project_id'), ['project_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_assistant_session_user_id'), ['user_id'], unique=False)
+
+    op.create_table('assistant_turn',
+    sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('session_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('user_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('project_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('stage', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('progress', sa.Integer(), nullable=False),
+    sa.Column('detail', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('error_message', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('prompt', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('base_spec', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('base_version', sa.Integer(), nullable=True),
+    sa.Column('focus', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('steps', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('reply', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('proposed_spec', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('changes', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('outcome', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('cost', sa.Float(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['session_id'], ['assistant_session.id'], ),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    with op.batch_alter_table('assistant_turn', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_assistant_turn_project_id'), ['project_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_assistant_turn_session_id'), ['session_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_assistant_turn_user_id'), ['user_id'], unique=False)
+
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('assistant_turn', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_assistant_turn_user_id'))
+        batch_op.drop_index(batch_op.f('ix_assistant_turn_session_id'))
+        batch_op.drop_index(batch_op.f('ix_assistant_turn_project_id'))
+
+    op.drop_table('assistant_turn')
+    with op.batch_alter_table('assistant_session', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_assistant_session_user_id'))
+        batch_op.drop_index(batch_op.f('ix_assistant_session_project_id'))
+
+    op.drop_table('assistant_session')

backend/app/assistant/__init__.py

@@ -0,0 +1 @@
+"""The video assistant: tools over a working copy of the spec, a loop, a job (M4/M5)."""

backend/app/assistant/loop.py

@@ -0,0 +1,87 @@
+"""One turn: the model and the tools, round after round, until the model is done.
+
+Pure of the database and the job queue (`worker.py` owns those), so a whole turn can be
+run in a test against a scripted provider.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Callable, List, Optional, Sequence, Tuple
+
+from app.assistant import tools as T
+from app.assistant.tools import Working, _issues_text
+from app.providers.base import ChatProvider, Reply
+
+logger = logging.getLogger(__name__)
+
+# Times the loop tells a model that stopped with the video still invalid to fix it.
+# Past that it would only be spending the user's money to repeat itself.
+MAX_REPAIR_NUDGES = 2
+
+Step = Callable[[str, str], None]
+
+
+class Stopped(Exception):
+    """The user pressed Stop."""
+
+
+@dataclass
+class Outcome:
+    reply: str
+    # Set when the turn ran out of rounds or output before finishing on its own.
+    cut_short: Optional[str] = None
+    replies: List[Reply] = field(default_factory=list)
+
+
+def _first_line(text: str) -> str:
+    return text.split("\n", 1)[0].strip()
+
+
+def run(provider: ChatProvider, working: Working, *, system: Sequence[str],
+        history: Sequence[Tuple[str, str]], prompt: str, on_step: Step,
+        on_reply: Callable[[Reply], None], cancelled: Callable[[], bool],
+        max_rounds: int) -> Outcome:
+    conversation = provider.conversation(system=system, tools=T.SPECS, history=history,
+                                         prompt=prompt)
+    outcome = Outcome(reply="")
+    nudges = 0
+    for _ in range(max_rounds):
+        if cancelled():
+            raise Stopped()
+        reply = conversation.ask()
+        outcome.replies.append(reply)
+        on_reply(reply)
+        if reply.text:
+            outcome.reply = reply.text
+        if not reply.tool_calls:
+            if reply.stop == "max_tokens":
+                outcome.cut_short = ("The reply hit the provider's output limit. Raise the "
+                                     "maximum response length in Settings, or ask for less "
+                                     "at once.")
+                return outcome
+            if working.issues and working.changes and nudges < MAX_REPAIR_NUDGES:
+                nudges += 1
+                on_step("issue", f"{len(working.issues)} issue(s) left to fix")
+                conversation.tell("The video still has issues, so it can't be proposed yet. "
+                                  "Fix these, then finish:\n" + _issues_text(working.issues))
+                continue
+            return outcome
+        if reply.text:
+            on_step("think", reply.text)
+        results = []
+        for call in reply.tool_calls:
+            if cancelled():
+                raise Stopped()
+            if call.name == "search_assets":
+                on_step("search", f"Searching the library: {call.arguments.get('query', '')!r}")
+            result = T.execute(working, call)
+            results.append(result)
+            if call.name in T.READS:
+                continue
+            on_step("issue" if result.is_error else "edit", _first_line(result.content))
+        conversation.answer(results)
+    outcome.cut_short = (f"Stopped after {max_rounds} rounds without finishing. What it had "
+                         "done so far is below; ask again to carry on.")
+    return outcome