Make a clip from a prompt alone: generate its picture, then animate it

backend/alembic/versions/20261021_1200_add_generation_picture_step.py

@@ -0,0 +1,34 @@
+"""A clip made from a picture generated for it: the picture, and the clip run waiting.
+
+`then` holds the clip's run (model, prompt, params) while the picture is being made,
+and is cleared once the clip has been started from it. `picture_id` is the GAM asset
+the picture became, kept so a failed clip can be tried again from it.
+
+Revision ID: 5f1b8c3d2a90
+Revises: c6d1a4e8b207
+Create Date: 2026-10-21 12:00:00.000000+00:00
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+revision: str = '5f1b8c3d2a90'
+down_revision: Union[str, None] = 'c6d1a4e8b207'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table('generation', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('then', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+        batch_op.add_column(sa.Column('picture_id', sqlmodel.sql.sqltypes.AutoString(),
+                                      nullable=True))
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('generation', schema=None) as batch_op:
+        batch_op.drop_column('picture_id')
+        batch_op.drop_column('then')

backend/app/generation/service.py

@@ -52,40 +52,96 @@ def start(session: Session, gam: GamClient, *, user_id: str, project_id: Optiona
           turn_id: Optional[str], spec: Dict[str, Any], key: str,
           prompt: Optional[str] = None, model_id: Optional[str] = None, outputs: int = 1,
           duration: Optional[str] = None,
-          base_gam_ids: Optional[List[str]] = None) -> Generation:
+          base_gam_ids: Optional[List[str]] = None, make_picture: bool = False,
+          picture_prompt: Optional[str] = None,
+          picture_model_id: Optional[str] = None) -> Generation:
+    """Start a placeholder's generation in GAM.
+
+    `make_picture` makes a clip from a prompt alone: GAM has no text-to-video, so a
+    picture is generated first and the clip is started from it by `refresh`, once the
+    picture is done. One row follows both, so the editor sees one generation for the
+    placeholder, and its price is the two together.
+    """
     ref = placeholder(spec, key)
     prompt = (prompt or ref.get("prompt") or "").strip()
     if not prompt:
         raise GenerationRefused("no_prompt", "Say what to generate first.")
-    bases = list(base_gam_ids or [])
-    if not bases:
+    chained = make_picture and ref["type"] == "video"
+    bases = [] if chained else list(base_gam_ids or [])
+    if not bases and not chained:
         bases = [gid for gid in (_gam_id_of(spec, k) for k in ref.get("base_assets") or []) if gid]
-    if ref["type"] == "video" and not bases:
+    if ref["type"] == "video" and not bases and not chained:
         raise GenerationRefused("needs_base_image",
-                                "A clip is made from a picture: choose the picture to start from.")
+                                "A clip is made from a picture: choose the picture to start from, "
+                                "or have one made from the prompt.")
+    aspect = ref.get("aspect") or (spec.get("output") or {}).get("aspect") or "16:9"
     try:
-        kind = kind_for(ref["type"], bool(bases))
-        model = pick_model(gam.generation_models(), kind, model_id)
+        models = gam.generation_models()
+        kind = kind_for(ref["type"], bool(bases) or chained)
+        model = pick_model(models, kind, model_id)
+        picture_model = pick_model(models, "text_to_image", picture_model_id) if chained else None
     except CatalogueError as exc:
         raise GenerationRefused("no_model", str(exc))
-    aspect = ref.get("aspect") or (spec.get("output") or {}).get("aspect") or "16:9"
     params = default_params(model, aspect=aspect, outputs=outputs if ref["type"] == "image" else 1,
                             duration=duration)
-    job = gam.start_generation(model_id=model["id"], prompt=prompt,
-                               base_ids=bases[:1] if kind == "image_to_video" else bases,
-                               params=params)
+    label = model.get("label") or model.get("endpoint_id", "")
+    cost = estimate(model, params)
+    then = None
+    if picture_model is not None:
+        picture_params = default_params(picture_model, aspect=aspect, outputs=1)
+        then = json.dumps({"model_id": model["id"], "model_label": label, "prompt": prompt,
+                           "params": params})
+        picture_cost = estimate(picture_model, picture_params)
+        cost = None if cost is None or picture_cost is None else round(cost + picture_cost, 4)
+        label = f"{picture_model.get('label') or picture_model['id']} → {label}"
+        job = gam.start_generation(model_id=picture_model["id"],
+                                   prompt=(picture_prompt or "").strip() or prompt,
+                                   base_ids=[], params=picture_params)
+    else:
+        job = gam.start_generation(model_id=model["id"], prompt=prompt,
+                                   base_ids=bases[:1] if kind == "image_to_video" else bases,
+                                   params=params)
     row = Generation(id=str(uuid.uuid4()), user_id=user_id, project_id=project_id,
                      turn_id=turn_id, asset_key=key, gam_job_id=job["id"], kind=kind,
-                     prompt=prompt, model_label=model.get("label") or model.get("endpoint_id", ""),
-                     params=json.dumps(params), status=job.get("status") or "queued",
-                     stage=job.get("stage") or "", progress=job.get("progress") or 0,
-                     estimate=estimate(model, params))
+                     prompt=prompt, model_label=label, params=json.dumps(params),
+                     status=job.get("status") or "queued",
+                     stage=_stage(job.get("stage") or "", then), progress=job.get("progress") or 0,
+                     estimate=cost, then=then)
     session.add(row)
     session.commit()
     session.refresh(row)
     return row
 
 
+def _stage(stage: str, then: Optional[str]) -> str:
+    """While the picture for a clip is being made, the stage says so."""
+    if not then:
+        return stage
+    return f"Making the picture · {stage}" if stage else "Making the picture"
+
+
+def _start_clip(gam: GamClient, row: Generation, ids: List[str]) -> None:
+    """The picture is done: start the clip from it, on the same row."""
+    then = json.loads(row.then or "{}")
+    row.then = None
+    if not ids:
+        row.status, row.error = "error", "The picture came back empty, so no clip was made."
+        return
+    row.picture_id = ids[0]
+    try:
+        job = gam.start_generation(model_id=then["model_id"], prompt=then["prompt"],
+                                   base_ids=[row.picture_id], params=then["params"])
+    except GamError as exc:
+        row.status = "error"
+        row.error = f"The picture was made, but the clip couldn't be started: {exc.message}"
+        return
+    row.gam_job_id = job["id"]
+    row.status = job.get("status") or "queued"
+    row.stage = job.get("stage") or "Animating the picture"
+    row.progress = int(job.get("progress") or 0)
+    row.detail, row.error, row.result_ids = "", None, "[]"
+
+
 def refresh(session: Session, gam: GamClient, rows: List[Generation]) -> None:
     """Ask GAM about every unfinished generation. Best effort: GAM being unreachable
     leaves the rows as they were, to be caught up on the next read."""
@@ -100,15 +156,25 @@ def refresh(session: Session, gam: GamClient, rows: List[Generation]) -> None:
             return
         if job is None:
             row.status, row.error = "error", "The asset library no longer has this generation."
+            row.then = None
         else:
-            row.status = job.get("status") or row.status
-            row.stage = job.get("stage") or ""
-            row.progress = int(job.get("progress") or 0)
-            row.detail = job.get("detail") or ""
-            row.error = job.get("error_message")
             ids = job.get("result_asset_ids") or ([job["result_asset_id"]]
                                                   if job.get("result_asset_id") else [])
-            row.result_ids = json.dumps(ids)
+            if row.then and job.get("status") == "done":
+                # The picture's outputs are never the placeholder's: only the clip's are.
+                _start_clip(gam, row, ids)
+            else:
+                row.status = job.get("status") or row.status
+                row.stage = _stage(job.get("stage") or "", row.then)
+                row.progress = int(job.get("progress") or 0)
+                row.detail = job.get("detail") or ""
+                row.error = job.get("error_message")
+                if row.then and row.error:
+                    row.error = f"Making the picture failed: {row.error}"
+                if row.then is None:
+                    row.result_ids = json.dumps(ids)
+                if row.status in FINAL:
+                    row.then = None
         row.updated_at = utcnow()
         session.add(row)
         changed = True

backend/app/models/generation.py

@@ -37,5 +37,10 @@ class Generation(SQLModel, table=True):
     # The output in the placeholder now; None until one has been put there.
     used_id: Optional[str] = None
     estimate: Optional[float] = None   # list price, USD
+    # A clip made from a picture generated for it: while the picture is being made,
+    # the clip's run (JSON: model_id, model_label, prompt, params), started from the
+    # picture by `refresh` and then cleared. `picture_id` is the picture, once made.
+    then: Optional[str] = None
+    picture_id: Optional[str] = None
     created_at: datetime = Field(default_factory=utcnow)
     updated_at: datetime = Field(default_factory=utcnow)

backend/app/routers/generations.py

@@ -31,6 +31,10 @@ class GenerationCreate(BaseModel):
     outputs: int = Field(default=1, ge=1, le=4)
     duration: Optional[str] = None
     base_gam_ids: List[str] = []
+    # A clip from the prompt alone: generate its picture first, then animate that.
+    make_picture: bool = False
+    picture_prompt: Optional[str] = Field(default=None, max_length=4000)
+    picture_model_id: Optional[str] = None
     # The editor's draft: a placeholder made by hand is generated before it is saved.
     draft_spec: Optional[Dict[str, Any]] = None
 
@@ -55,6 +59,8 @@ class GenerationRead(BaseModel):
     used_id: Optional[str] = None
     # A list price from GAM's catalogue: an estimate, never a bill.
     estimate: Optional[float] = None
+    # The picture a clip was made from, when it was generated for it.
+    picture_id: Optional[str] = None
 
 
 class GenerationModelRead(BaseModel):
@@ -74,7 +80,7 @@ def read(row: Generation) -> GenerationRead:
         prompt=row.prompt, model_label=row.model_label, status=row.status, stage=row.stage,
         progress=row.progress, detail=row.detail, error=row.error,
         result_ids=json.loads(row.result_ids or "[]"), used_id=row.used_id,
-        estimate=row.estimate,
+        estimate=row.estimate, picture_id=row.picture_id,
     )
 
 
@@ -122,7 +128,9 @@ def start_generation(project_id: str, body: GenerationCreate, user: CurrentUser,
         row = service.start(session, gam, user_id=user.id, project_id=project.id, turn_id=None,
                             spec=spec, key=body.asset_key, prompt=body.prompt,
                             model_id=body.model_id, outputs=body.outputs, duration=body.duration,
-                            base_gam_ids=body.base_gam_ids)
+                            base_gam_ids=body.base_gam_ids, make_picture=body.make_picture,
+                            picture_prompt=body.picture_prompt,
+                            picture_model_id=body.picture_model_id)
     except service.GenerationRefused as exc:
         raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                             detail={"code": exc.code, "message": exc.message})

backend/tests/test_generation.py

@@ -182,3 +182,77 @@ def test_deleting_the_project_deletes_its_generations(auth_client, session, proj
     start(auth_client, project["id"], asset_key="a_dam")
     assert auth_client.delete(f"/api/projects/{project['id']}").status_code == 204
     assert session.exec(select(Generation)).all() == []
+
+
+# ── a clip from the prompt alone: picture, then animate ──────────────────────
+
+NO_BASE = {**SPEC, "assets": {**SPEC["assets"], "a_flood": {
+    "from": "generate", "type": "video", "prompt": "Water rising"}}}
+
+
+def _list(client, project_id):
+    return client.get(f"/api/projects/{project_id}/generations").json()["data"]
+
+
+def test_a_clip_from_a_prompt_makes_its_picture_first_then_animates_it(
+        auth_client, project, siblings):
+    res = start(auth_client, project["id"], asset_key="a_flood", draft_spec=NO_BASE,
+                make_picture=True, picture_prompt="A flooded valley", model_id="m-veo",
+                duration="8")
+    assert res.status_code == 202, res.text
+    row = res.json()["data"]
+    [picture] = siblings.generations
+    assert picture["model_id"] == "m-flux" and picture["prompt"] == "A flooded valley"
+    assert picture["base_asset_ids"] == []
+    assert picture["params"] == {"image_size": "landscape_16_9", "num_outputs": 1}
+    assert row["stage"].startswith("Making the picture")
+    assert row["model_label"] == "FLUX.1 [schnell] → Veo 3.1 Fast"
+    # Both runs, priced together.
+    assert row["estimate"] == pytest.approx(0.1 * 8 + 0.003 * 1024 * 576 / 1e6, abs=1e-3)
+
+    siblings.gen_jobs["job-1"].update(status="processing", stage="Generating", progress=50)
+    listed = _list(auth_client, project["id"])[0]
+    assert listed["stage"] == "Making the picture · Generating"
+    assert listed["result_ids"] == [] and listed["status"] == "processing"
+
+    # The picture is done: its output is never offered for the clip's placeholder.
+    siblings.gen_jobs["job-1"].update(status="done", result_asset_ids=["pic-1"])
+    listed = _list(auth_client, project["id"])[0]
+    assert listed["status"] == "queued" and listed["result_ids"] == []
+    assert listed["picture_id"] == "pic-1"
+    clip = siblings.generations[1]
+    assert clip["model_id"] == "m-veo" and clip["prompt"] == "Water rising"
+    assert clip["base_asset_ids"] == ["pic-1"]
+    assert clip["params"]["duration"] == "8s"
+
+    siblings.gen_jobs["job-2"].update(status="done", result_asset_ids=["clip-1"])
+    listed = _list(auth_client, project["id"])[0]
+    assert listed["status"] == "done" and listed["result_ids"] == ["clip-1"]
+    assert len(siblings.generations) == 2
+
+
+def test_a_failed_picture_says_so_and_starts_no_clip(auth_client, project, siblings):
+    start(auth_client, project["id"], asset_key="a_flood", draft_spec=NO_BASE, make_picture=True)
+    siblings.gen_jobs["job-1"].update(status="error", error_message="Content refused")
+    listed = _list(auth_client, project["id"])[0]
+    assert listed["status"] == "error"
+    assert listed["error"] == "Making the picture failed: Content refused"
+    assert len(siblings.generations) == 1
+
+
+def test_a_clip_that_cannot_start_keeps_its_picture(auth_client, project, siblings):
+    start(auth_client, project["id"], asset_key="a_flood", draft_spec=NO_BASE, make_picture=True)
+    siblings.gen_jobs["job-1"].update(status="done", result_asset_ids=["pic-1"])
+    siblings.fal_key = False
+    listed = _list(auth_client, project["id"])[0]
+    assert listed["status"] == "error" and listed["picture_id"] == "pic-1"
+    assert listed["error"].startswith("The picture was made, but the clip couldn't be started")
+
+
+def test_stopping_while_the_picture_is_made_starts_no_clip(auth_client, project, siblings):
+    row = start(auth_client, project["id"], asset_key="a_flood", draft_spec=NO_BASE,
+                make_picture=True).json()["data"]
+    auth_client.delete(f"/api/generations/{row['id']}")
+    siblings.gen_jobs["job-1"].update(status="done", result_asset_ids=["pic-1"])
+    assert _list(auth_client, project["id"])[0]["status"] == "cancelled"
+    assert len(siblings.generations) == 1

docs/plan-of-attack.md

@@ -146,6 +146,13 @@ worker for it:
   price; what fal.ai charged is in GAM's usage.
 - Images and clips only: GAM has no audio models, so music and sound placeholders stay
   "find it in your library".
+- **A clip from the prompt alone.** GAM's video models all start from a picture, and it
+  has no text-to-video. So "Made from: a new picture" generates the picture first and
+  animates it: one `generation` row follows both runs. It holds the clip's run in `then`
+  until the picture is done, and `refresh` then starts the clip from it. The picture is
+  kept on the row (`picture_id`) and offered again on "Try again", so a failed clip
+  doesn't pay for its picture twice. The picture's output is never put in the
+  placeholder; only the clip's is.
 - **The assistant generates pictures, never video** (the user's call: video costs far
   more). `generate_image` adds the placeholder and starts it at once, at most
   `ASSISTANT_MAX_GENERATIONS` (6) per turn; for footage it leaves a video placeholder for

frontend/src/api/generations.ts

@@ -28,6 +28,8 @@ export interface Generation {
   used_id: string | null
   /** List price × units, before it runs; what fal.ai charged is in GAM's usage. */
   estimate: number | null
+  /** The picture a clip was made from, when it was generated for it first. */
+  picture_id: string | null
   created_at: string
   updated_at: string
 }
@@ -58,6 +60,11 @@ export interface GenerationStart {
   outputs?: number
   duration?: string
   base_gam_ids?: string[]
+  /** A clip from the prompt alone: GAM makes its picture first, then animates it. */
+  make_picture?: boolean
+  /** What the picture shows, when that differs from the clip's prompt. */
+  picture_prompt?: string
+  picture_model_id?: string
   /** The editor's unsaved draft, so a placeholder added since the last save counts. */
   draft_spec?: VideoSpec
 }

frontend/src/components/GenerateDialog.test.tsx

@@ -119,6 +119,35 @@ describe('GenerateDialog', () => {
     })
   })
 
+  it('makes a clip from the prompt alone, its picture generated first', async () => {
+    const start = vi.spyOn(generationsApi, 'start').mockResolvedValue({} as never)
+    render(<GenerateDialog assetKey="a_flood" onClose={vi.fn()} />)
+    await screen.findByRole('option', { name: /Veo/ })
+    await userEvent.selectOptions(
+      screen.getByRole('combobox', { name: 'Made from' }),
+      'new'
+    )
+    await userEvent.selectOptions(
+      screen.getByRole('combobox', { name: 'Picture model' }),
+      'm-edit'
+    )
+    await userEvent.type(
+      screen.getByRole('textbox', { name: 'Picture prompt' }),
+      'A flooded valley'
+    )
+    // Nano's $0.039 picture plus four seconds of Veo at $0.10.
+    expect(screen.getByText(/About \$0\.44/)).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: /Generate/ }))
+    const [, body] = start.mock.calls[0]
+    expect(body).toMatchObject({
+      model_id: 'm-veo',
+      make_picture: true,
+      picture_model_id: 'm-edit',
+      picture_prompt: 'A flooded valley',
+    })
+    expect(body.base_gam_ids).toBeUndefined()
+  })
+
   it('says why it could not start, and stays open', async () => {
     vi.spyOn(generationsApi, 'start').mockRejectedValue(new Error('no key'))
     const onClose = vi.fn()