Generate placeholders' pictures and clips through GAM

README.md

@@ -36,6 +36,11 @@ one open. What it would change is listed for you to apply or discard. It runs in
 background, so a request finishes with the page closed. Set up its AI provider
 (Anthropic, OpenAI, DeepSeek, Ollama or a custom endpoint) under Settings.
 
+**Generated pictures and clips.** A placeholder for a picture or a clip has a Generate
+button: GAM makes it on fal.ai with your key (set in GAM's Settings), and it drops into
+the video when done, even if you closed the page meanwhile. The assistant may generate
+pictures itself; clips cost more, so it leaves those for you to start.
+
 The full storyboard editor is next (M3).
 
 ## Read first

backend/alembic/versions/20261016_2353_add_generation_table.py

@@ -0,0 +1,63 @@
+"""Add the generation table: GVC's record of each generation it asked GAM for.
+
+The job runs in GAM; a row is how GVC follows it and knows which placeholder it fills.
+`gam_job_id` is GAM's id, so no foreign key.
+
+Revision ID: c6d1a4e8b207
+Revises: 9b3e7d215f60
+Create Date: 2026-10-16 23:53:58.001597+00:00
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+revision: str = 'c6d1a4e8b207'
+down_revision: Union[str, None] = '9b3e7d215f60'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table('generation',
+    sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('user_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('project_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('turn_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('asset_key', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('gam_job_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('kind', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('prompt', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('model_label', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('params', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('stage', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('progress', sa.Integer(), nullable=False),
+    sa.Column('detail', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('error', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('result_ids', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('used_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('estimate', sa.Float(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    with op.batch_alter_table('generation', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_generation_gam_job_id'), ['gam_job_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_generation_project_id'), ['project_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_generation_turn_id'), ['turn_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_generation_user_id'), ['user_id'], unique=False)
+
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('generation', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_generation_user_id'))
+        batch_op.drop_index(batch_op.f('ix_generation_turn_id'))
+        batch_op.drop_index(batch_op.f('ix_generation_project_id'))
+        batch_op.drop_index(batch_op.f('ix_generation_gam_job_id'))
+
+    op.drop_table('generation')

backend/app/assistant/prompt.py

@@ -98,6 +98,20 @@
 `add_placeholder` with a clear description of what to find or generate. Never invent a \
 gam_id or an asset key that isn't in the registry.
 
+## Generating pictures
+`generate_image` makes a still with AI (through the user's asset library) and puts it in \
+the video; it fills in by itself a few seconds later. It costs money, so:
+- Search the library first. Generate only what a library can't have: an illustration, a \
+reconstruction of something never photographed, an abstract or title background, a \
+picture to fill a gap the user asked you to fill.
+- Write the prompt for one frame of video: the subject, the composition, the light and \
+the style, matched to the video's aspect (16:9 is wide). No text in the picture; put \
+words in an overlay instead.
+- Never generate video. When a scene needs footage the library doesn't have, leave an \
+`add_placeholder` of kind generate with media_type video and a good prompt, and tell \
+the user they can generate it from the scene with the Generate button.
+- Music and sound effects can't be generated; find them in the library.
+
 ## Pacing
 Narration reads at about 15 characters a second. A 3-minute video is about 2,500 characters \
 of narration over 10-20 segments. Keep beats to one or two sentences each so pictures can \

backend/app/assistant/tools.py

@@ -86,6 +86,13 @@ class Working:
     changes: List[str] = field(default_factory=list)
     hits: Dict[str, Hit] = field(default_factory=dict)
     issues: List[SpecIssue] = field(default_factory=list)
+    # Starts one image generation in GAM for a placeholder in `data` and says what it
+    # cost: (model label, estimated cost). Set by the worker; None in a turn that can't
+    # generate (no GAM), and then the tool says so.
+    generate: Optional[Callable[[Dict[str, Any], str, str, List[str], int],
+                                Tuple[str, Optional[float]]]] = None
+    max_generations: int = 6
+    generated: int = 0
 
     def __post_init__(self) -> None:
         self.data = copy.deepcopy(self.data)
@@ -415,6 +422,61 @@ def add_placeholder(w: Working, segment_id: str, kind: str, text: str,
                                f"{'find' if kind == 'search' else 'generate'}: {text!r}")
 
 
+def generate_image(w: Working, prompt: str, segment_id: Optional[str] = None,
+                   use_as: str = "visual", base_gam_ids: Optional[List[str]] = None,
+                   outputs: int = 1, label: Optional[str] = None) -> str:
+    """A picture made for the video, started now in GAM and filled in when it's done.
+
+    Images only. Video costs far more and takes minutes, so the user starts that
+    themselves from the placeholder (their call, recorded in the plan of attack).
+    """
+    if w.generate is None:
+        raise ToolFailed("Generating isn't available in this turn.")
+    if w.generated >= w.max_generations:
+        raise ToolFailed(f"That's the {w.max_generations} pictures this request may generate. "
+                         "Leave the rest as placeholders with add_placeholder; the user can "
+                         "generate them from the video.")
+    if not prompt or not prompt.strip():
+        raise ToolFailed("Describe the picture to generate.")
+    if use_as not in ("visual", "overlay", "none"):
+        raise ToolFailed('use_as is "visual", "overlay" or "none".')
+    if use_as != "none" and not segment_id:
+        raise ToolFailed("Say which segment the picture is for, or use_as \"none\".")
+    candidate = copy.deepcopy(w.data)
+    assets = candidate.setdefault("assets", {})
+    base = f"a_{_slug(label or prompt) or 'generated'}"
+    key, n = base, 2
+    while key in assets:
+        key, n = f"{base}_{n}", n + 1
+    ref: Dict[str, Any] = {"from": "generate", "type": "image", "prompt": prompt.strip()}
+    if label:
+        ref["label"] = label
+    assets[key] = ref
+    where = "kept in the registry"
+    if segment_id:
+        segment = candidate["segments"][w.index_of(segment_id)]
+        if use_as == "visual":
+            segment["visual"] = {"asset": key}
+            where = f"as the picture of {w.name_of(segment)}"
+        elif use_as == "overlay":
+            taken = w.all_overlay_ids()
+            segment.setdefault("overlays", []).append(
+                {"id": w.new_id("o_", taken), "type": "image", "asset": key,
+                 "position": "top-right", "scale_pct": 30})
+            where = f"on screen over {w.name_of(segment)}"
+    result = w.commit(candidate, f"Generating a picture {where}: {prompt.strip()[:80]!r}")
+    try:
+        model, cost = w.generate(w.data, key, prompt.strip(), list(base_gam_ids or []),
+                                 max(1, min(4, outputs)))
+    except Exception as exc:  # noqa: BLE001 - said to the model; the placeholder stays
+        return (f"{result} But the generation couldn't start ({exc}); the placeholder "
+                f"{key} is there for the user to generate later.")
+    w.generated += 1
+    price = f", about ${cost:.3f}" if cost is not None else ""
+    return (f"{result} Started with {model}{price}; it fills placeholder {key} by itself "
+            "in a few seconds.")
+
+
 def add_overlay(w: Working, segment_id: str, overlay: Dict[str, Any],
                 gam_id: Optional[str] = None) -> str:
     if not isinstance(overlay, dict):
@@ -607,6 +669,19 @@ def run_estimate(w: Working) -> str:
               "motion, background, cards…), output, voice, chrome, credits.",
               {"type": "object", "properties": {"changes": _OBJ}, "required": ["changes"]}),
      set_project),
+    (ToolSpec("generate_image",
+              "Generate a still picture with AI and put it in the video (as a segment's "
+              "picture, an image overlay, or just in the registry). It starts at once and "
+              "costs money, so search the library first and generate only what a library "
+              "can't have. base_gam_ids: library pictures to edit or restyle. Never use "
+              "this for video; leave video as a placeholder for the user.",
+              {"type": "object", "properties": {
+                  "prompt": _STR, "segment_id": _STR,
+                  "use_as": {"type": "string", "enum": ["visual", "overlay", "none"]},
+                  "base_gam_ids": {"type": "array", "items": _STR},
+                  "outputs": {"type": "integer", "minimum": 1, "maximum": 4},
+                  "label": _STR},
+               "required": ["prompt"]}), generate_image),
     (ToolSpec("estimate", "How long the video runs, its narration length, and what would "
               "stop a full render.", {"type": "object", "properties": {}}), run_estimate),
 ]

backend/app/assistant/worker.py

@@ -24,6 +24,7 @@
 from app.clock import utcnow
 from app.config import settings
 from app.gam.client import GamClient
+from app.generation import service as generation_service
 from app.jobs.runner import JobQueue, readable_error, set_fields
 from app.models.assistant import AssistantSession, AssistantTurn
 from app.providers import build_provider
@@ -113,7 +114,19 @@ def on_reply(reply: Reply) -> None:
             provider = build_provider(session, user_id)
         if provider is None:
             raise ProviderError("No AI provider is set up. Add one in Settings first.")
-        working = Working(base, gam=gam)
+        working = Working(base, gam=gam, max_generations=settings.assistant_max_generations)
+
+        def generate(data, key, image_prompt, base_ids, outputs):
+            with queue.session() as session:
+                row = generation_service.start(
+                    session, gam, user_id=user_id, project_id=project_id, turn_id=turn_id,
+                    spec=data, key=key, prompt=image_prompt, outputs=outputs,
+                    base_gam_ids=base_ids)
+                # Read inside the session: the commit expired the row, and a detached
+                # row can't reload.
+                return row.model_label, row.estimate
+
+        working.generate = generate
         system = [REFERENCE, context(base, focus=focus, from_scratch=project_id is None)]
         outcome = loop.run(provider, working, system=system, history=history, prompt=prompt,
                            on_step=on_step, on_reply=on_reply,

backend/app/config.py

@@ -95,6 +95,9 @@ class Settings(BaseSettings):
     # Earlier messages a turn is given, newest kept. Older ones are still in the
     # session; the spec itself, not the chat, is the record of what was decided.
     assistant_history_messages: int = 20
+    # Pictures one request may generate on its own (each is a paid call to fal.ai through
+    # GAM). Past it the assistant leaves placeholders for the user to generate.
+    assistant_max_generations: int = 6
     job_heartbeat_seconds: int = 30
     job_stale_minutes: int = 40
 

backend/app/gam/client.py

@@ -12,7 +12,7 @@
 from __future__ import annotations
 
 import logging
-from typing import IO, Any, Dict, Optional
+from typing import IO, Any, Dict, List, Optional
 
 import httpx
 
@@ -190,3 +190,43 @@ def update(self, asset_id: str, changes: Dict[str, Any]) -> Optional[GamAssetRea
             code, message = self._detail(response)
             raise GamError(code, message)
         return to_asset(response.json()["data"])
+
+    # ── generation (GAM's M8) ────────────────────────────────────────────────
+    # GAM runs the generation on fal.ai with the user's own key and keeps the outputs in
+    # their library. A job is followed by its id; GVC never talks to fal.
+
+    def _generation_error(self, response: httpx.Response) -> GamError:
+        code, message = self._detail(response)
+        if code == "fal_key_missing":
+            message = ("Generating needs a fal.ai API key in the asset library's settings: "
+                       f"{settings.gam_base_url.rstrip('/')}/settings")
+        return GamError(code, message, status_code=400 if response.status_code < 500 else 502)
+
+    def generation_models(self) -> List[Dict[str, Any]]:
+        response = self._request("GET", "/api/generate/models")
+        if response.status_code != 200:
+            raise self._generation_error(response)
+        return list(response.json().get("data") or [])
+
+    def start_generation(self, *, model_id: str, prompt: str, base_ids: List[str],
+                         params: Dict[str, Any]) -> Dict[str, Any]:
+        response = self._request("POST", "/api/generate", json={
+            "model_id": model_id, "prompt": prompt, "base_asset_ids": base_ids,
+            "params": params,
+        })
+        if response.status_code not in (200, 201, 202):
+            raise self._generation_error(response)
+        return response.json()["data"]
+
+    def generation_job(self, job_id: str) -> Optional[Dict[str, Any]]:
+        response = self._request("GET", f"/api/activity/enrichment/{job_id}")
+        if response.status_code == 404:
+            return None
+        if response.status_code != 200:
+            raise self._generation_error(response)
+        return response.json()["data"]
+
+    def cancel_generation(self, job_id: str) -> None:
+        response = self._request("DELETE", f"/api/activity/enrichment/{job_id}")
+        if response.status_code not in (200, 204, 404):
+            raise self._generation_error(response)

backend/app/generation/__init__.py

@@ -0,0 +1,7 @@
+"""Generating pictures and clips for `generate` placeholders, through GAM (GAM's M8).
+
+GVC owns no fal.ai code and no key: GAM generates with the user's own fal key, keeps the
+outputs in their library, and records the spend. GVC picks a model, sets the options
+from the video (its aspect above all), keeps a row per generation so it can be followed,
+and puts the result into the placeholder.
+"""

backend/app/generation/catalogue.py

@@ -0,0 +1,106 @@
+"""Choosing a model from GAM's catalogue, its options, and what a run will roughly cost.
+
+GAM matches every option against the model's declared values *with its type*: Kling's
+duration is the string "5", Veo's is "8s". So choices are always made from the
+catalogue's own lists, never constructed.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Optional
+
+# A placeholder's type, as the generation kind GAM runs. With base images an image
+# becomes an edit of them (image→image); a video is always made from a picture.
+KINDS = {"image": "text_to_image", "video": "image_to_video"}
+
+# FLUX names its sizes rather than its ratios; these are the ones for each output aspect,
+# and their size in pixels, for the per-megapixel price.
+_SIZE_FOR_ASPECT = {"16:9": "landscape_16_9", "9:16": "portrait_16_9", "1:1": "square_hd"}
+_SIZE_PIXELS = {
+    "square_hd": (1024, 1024), "square": (512, 512), "portrait_4_3": (768, 1024),
+    "portrait_16_9": (576, 1024), "landscape_4_3": (1024, 768), "landscape_16_9": (1024, 576),
+}
+
+MAX_IMAGE_OUTPUTS = 4
+
+
+class CatalogueError(Exception):
+    """No model can do what was asked; the message says why."""
+
+
+def kind_for(asset_type: str, has_base: bool) -> str:
+    if asset_type == "image":
+        return "image_to_image" if has_base else "text_to_image"
+    if asset_type == "video":
+        return "image_to_video"
+    raise CatalogueError("Only pictures and video can be generated; music and sound "
+                         "effects have to come from your library.")
+
+
+def pick_model(models: List[Dict[str, Any]], kind: str,
+               preferred: Optional[str] = None) -> Dict[str, Any]:
+    """The model asked for if it can do this kind, else the catalogue's first of the kind
+    (GAM lists them in its own preferred order)."""
+    of_kind = [m for m in models if m.get("kind") == kind and m.get("is_active", True)]
+    if preferred:
+        for model in of_kind:
+            if model.get("id") == preferred:
+                return model
+        raise CatalogueError("That model can't do this kind of generation.")
+    if not of_kind:
+        raise CatalogueError(f"The asset library has no active model for {kind.replace('_', ' ')}.")
+    return of_kind[0]
+
+
+def _seconds(value: Any) -> Optional[float]:
+    match = re.search(r"\d+(\.\d+)?", str(value))
+    return float(match.group()) if match else None
+
+
+def default_params(model: Dict[str, Any], *, aspect: str, outputs: int = 1,
+                   duration: Optional[str] = None) -> Dict[str, Any]:
+    """Options for one run, chosen from the model's own lists."""
+    options = model.get("options") or {}
+    params: Dict[str, Any] = {}
+    ratios = options.get("aspect_ratios") or []
+    sizes = options.get("image_sizes") or []
+    if aspect in ratios:
+        params["aspect_ratio"] = aspect
+    elif _SIZE_FOR_ASPECT.get(aspect) in sizes:
+        params["image_size"] = _SIZE_FOR_ASPECT[aspect]
+    durations = options.get("durations") or []
+    if durations:
+        # A requested duration is matched by its number, so "6" finds Veo's "6s".
+        wanted = _seconds(duration) if duration is not None else None
+        chosen = next((d for d in durations if wanted is not None and _seconds(d) == wanted),
+                      durations[0])
+        params["duration"] = chosen
+    most = max(1, int(options.get("max_outputs") or 1))
+    params["num_outputs"] = max(1, min(outputs, most, MAX_IMAGE_OUTPUTS))
+    return params
+
+
+def estimate(model: Dict[str, Any], params: Dict[str, Any]) -> Optional[float]:
+    """The list price of one run, from the catalogue. An estimate, never a bill: GAM
+    records what fal actually charged once it has run."""
+    price, unit = model.get("unit_price"), model.get("price_unit")
+    if price is None or not unit:
+        return None
+    count = params.get("num_outputs", 1)
+    if unit == "image" or unit == "video":
+        return round(price * count, 4)
+    if unit == "second":
+        return round(price * (_seconds(params.get("duration")) or 5.0) * count, 4)
+    if unit == "megapixel":
+        width, height = _SIZE_PIXELS.get(params.get("image_size"), (1024, 1024))
+        return round(price * width * height / 1_000_000 * count, 4)
+    return None
+
+
+def price_line(model: Dict[str, Any]) -> str:
+    """The catalogue's price as a person reads it: "$0.07 a second"."""
+    price, unit = model.get("unit_price"), model.get("price_unit")
+    if price is None or not unit:
+        return "price unknown"
+    return f"${price:g} a {unit}"