diff --git a/backend/alembic/versions/20261021_1200_add_generation_picture_step.py b/backend/alembic/versions/20261021_1200_add_generation_picture_step.py new file mode 100644 index 0000000..7abdfa1 --- /dev/null +++ b/backend/alembic/versions/20261021_1200_add_generation_picture_step.py @@ -0,0 +1,34 @@ +"""A clip made from a picture generated for it: the picture, and the clip run waiting. + +`then` holds the clip's run (model, prompt, params) while the picture is being made, +and is cleared once the clip has been started from it. `picture_id` is the GAM asset +the picture became, kept so a failed clip can be tried again from it. + +Revision ID: 5f1b8c3d2a90 +Revises: c6d1a4e8b207 +Create Date: 2026-10-21 12:00:00.000000+00:00 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = '5f1b8c3d2a90' +down_revision: Union[str, None] = 'c6d1a4e8b207' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table('generation', schema=None) as batch_op: + batch_op.add_column(sa.Column('then', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + batch_op.add_column(sa.Column('picture_id', sqlmodel.sql.sqltypes.AutoString(), + nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table('generation', schema=None) as batch_op: + batch_op.drop_column('picture_id') + batch_op.drop_column('then') diff --git a/backend/app/generation/service.py b/backend/app/generation/service.py index e711e48..3714154 100644 --- a/backend/app/generation/service.py +++ b/backend/app/generation/service.py @@ -52,40 +52,96 @@ def start(session: Session, gam: GamClient, *, user_id: str, project_id: Optiona turn_id: Optional[str], spec: Dict[str, Any], key: str, prompt: Optional[str] = None, model_id: Optional[str] = None, outputs: int = 1, duration: Optional[str] = None, - base_gam_ids: Optional[List[str]] = None) -> Generation: + base_gam_ids: Optional[List[str]] = None, make_picture: bool = False, + picture_prompt: Optional[str] = None, + picture_model_id: Optional[str] = None) -> Generation: + """Start a placeholder's generation in GAM. + + `make_picture` makes a clip from a prompt alone: GAM has no text-to-video, so a + picture is generated first and the clip is started from it by `refresh`, once the + picture is done. One row follows both, so the editor sees one generation for the + placeholder, and its price is the two together. + """ ref = placeholder(spec, key) prompt = (prompt or ref.get("prompt") or "").strip() if not prompt: raise GenerationRefused("no_prompt", "Say what to generate first.") - bases = list(base_gam_ids or []) - if not bases: + chained = make_picture and ref["type"] == "video" + bases = [] if chained else list(base_gam_ids or []) + if not bases and not chained: bases = [gid for gid in (_gam_id_of(spec, k) for k in ref.get("base_assets") or []) if gid] - if ref["type"] == "video" and not bases: + if ref["type"] == "video" and not bases and not chained: raise GenerationRefused("needs_base_image", - "A clip is made from a picture: choose the picture to start from.") + "A clip is made from a picture: choose the picture to start from, " + "or have one made from the prompt.") + aspect = ref.get("aspect") or (spec.get("output") or {}).get("aspect") or "16:9" try: - kind = kind_for(ref["type"], bool(bases)) - model = pick_model(gam.generation_models(), kind, model_id) + models = gam.generation_models() + kind = kind_for(ref["type"], bool(bases) or chained) + model = pick_model(models, kind, model_id) + picture_model = pick_model(models, "text_to_image", picture_model_id) if chained else None except CatalogueError as exc: raise GenerationRefused("no_model", str(exc)) - aspect = ref.get("aspect") or (spec.get("output") or {}).get("aspect") or "16:9" params = default_params(model, aspect=aspect, outputs=outputs if ref["type"] == "image" else 1, duration=duration) - job = gam.start_generation(model_id=model["id"], prompt=prompt, - base_ids=bases[:1] if kind == "image_to_video" else bases, - params=params) + label = model.get("label") or model.get("endpoint_id", "") + cost = estimate(model, params) + then = None + if picture_model is not None: + picture_params = default_params(picture_model, aspect=aspect, outputs=1) + then = json.dumps({"model_id": model["id"], "model_label": label, "prompt": prompt, + "params": params}) + picture_cost = estimate(picture_model, picture_params) + cost = None if cost is None or picture_cost is None else round(cost + picture_cost, 4) + label = f"{picture_model.get('label') or picture_model['id']} → {label}" + job = gam.start_generation(model_id=picture_model["id"], + prompt=(picture_prompt or "").strip() or prompt, + base_ids=[], params=picture_params) + else: + job = gam.start_generation(model_id=model["id"], prompt=prompt, + base_ids=bases[:1] if kind == "image_to_video" else bases, + params=params) row = Generation(id=str(uuid.uuid4()), user_id=user_id, project_id=project_id, turn_id=turn_id, asset_key=key, gam_job_id=job["id"], kind=kind, - prompt=prompt, model_label=model.get("label") or model.get("endpoint_id", ""), - params=json.dumps(params), status=job.get("status") or "queued", - stage=job.get("stage") or "", progress=job.get("progress") or 0, - estimate=estimate(model, params)) + prompt=prompt, model_label=label, params=json.dumps(params), + status=job.get("status") or "queued", + stage=_stage(job.get("stage") or "", then), progress=job.get("progress") or 0, + estimate=cost, then=then) session.add(row) session.commit() session.refresh(row) return row +def _stage(stage: str, then: Optional[str]) -> str: + """While the picture for a clip is being made, the stage says so.""" + if not then: + return stage + return f"Making the picture · {stage}" if stage else "Making the picture" + + +def _start_clip(gam: GamClient, row: Generation, ids: List[str]) -> None: + """The picture is done: start the clip from it, on the same row.""" + then = json.loads(row.then or "{}") + row.then = None + if not ids: + row.status, row.error = "error", "The picture came back empty, so no clip was made." + return + row.picture_id = ids[0] + try: + job = gam.start_generation(model_id=then["model_id"], prompt=then["prompt"], + base_ids=[row.picture_id], params=then["params"]) + except GamError as exc: + row.status = "error" + row.error = f"The picture was made, but the clip couldn't be started: {exc.message}" + return + row.gam_job_id = job["id"] + row.status = job.get("status") or "queued" + row.stage = job.get("stage") or "Animating the picture" + row.progress = int(job.get("progress") or 0) + row.detail, row.error, row.result_ids = "", None, "[]" + + def refresh(session: Session, gam: GamClient, rows: List[Generation]) -> None: """Ask GAM about every unfinished generation. Best effort: GAM being unreachable leaves the rows as they were, to be caught up on the next read.""" @@ -100,15 +156,25 @@ def refresh(session: Session, gam: GamClient, rows: List[Generation]) -> None: return if job is None: row.status, row.error = "error", "The asset library no longer has this generation." + row.then = None else: - row.status = job.get("status") or row.status - row.stage = job.get("stage") or "" - row.progress = int(job.get("progress") or 0) - row.detail = job.get("detail") or "" - row.error = job.get("error_message") ids = job.get("result_asset_ids") or ([job["result_asset_id"]] if job.get("result_asset_id") else []) - row.result_ids = json.dumps(ids) + if row.then and job.get("status") == "done": + # The picture's outputs are never the placeholder's: only the clip's are. + _start_clip(gam, row, ids) + else: + row.status = job.get("status") or row.status + row.stage = _stage(job.get("stage") or "", row.then) + row.progress = int(job.get("progress") or 0) + row.detail = job.get("detail") or "" + row.error = job.get("error_message") + if row.then and row.error: + row.error = f"Making the picture failed: {row.error}" + if row.then is None: + row.result_ids = json.dumps(ids) + if row.status in FINAL: + row.then = None row.updated_at = utcnow() session.add(row) changed = True diff --git a/backend/app/models/generation.py b/backend/app/models/generation.py index 9b9be81..eb4b5a0 100644 --- a/backend/app/models/generation.py +++ b/backend/app/models/generation.py @@ -37,5 +37,10 @@ class Generation(SQLModel, table=True): # The output in the placeholder now; None until one has been put there. used_id: Optional[str] = None estimate: Optional[float] = None # list price, USD + # A clip made from a picture generated for it: while the picture is being made, + # the clip's run (JSON: model_id, model_label, prompt, params), started from the + # picture by `refresh` and then cleared. `picture_id` is the picture, once made. + then: Optional[str] = None + picture_id: Optional[str] = None created_at: datetime = Field(default_factory=utcnow) updated_at: datetime = Field(default_factory=utcnow) diff --git a/backend/app/routers/generations.py b/backend/app/routers/generations.py index 225de33..22da6dc 100644 --- a/backend/app/routers/generations.py +++ b/backend/app/routers/generations.py @@ -31,6 +31,10 @@ class GenerationCreate(BaseModel): outputs: int = Field(default=1, ge=1, le=4) duration: Optional[str] = None base_gam_ids: List[str] = [] + # A clip from the prompt alone: generate its picture first, then animate that. + make_picture: bool = False + picture_prompt: Optional[str] = Field(default=None, max_length=4000) + picture_model_id: Optional[str] = None # The editor's draft: a placeholder made by hand is generated before it is saved. draft_spec: Optional[Dict[str, Any]] = None @@ -55,6 +59,8 @@ class GenerationRead(BaseModel): used_id: Optional[str] = None # A list price from GAM's catalogue: an estimate, never a bill. estimate: Optional[float] = None + # The picture a clip was made from, when it was generated for it. + picture_id: Optional[str] = None class GenerationModelRead(BaseModel): @@ -74,7 +80,7 @@ def read(row: Generation) -> GenerationRead: prompt=row.prompt, model_label=row.model_label, status=row.status, stage=row.stage, progress=row.progress, detail=row.detail, error=row.error, result_ids=json.loads(row.result_ids or "[]"), used_id=row.used_id, - estimate=row.estimate, + estimate=row.estimate, picture_id=row.picture_id, ) @@ -122,7 +128,9 @@ def start_generation(project_id: str, body: GenerationCreate, user: CurrentUser, row = service.start(session, gam, user_id=user.id, project_id=project.id, turn_id=None, spec=spec, key=body.asset_key, prompt=body.prompt, model_id=body.model_id, outputs=body.outputs, duration=body.duration, - base_gam_ids=body.base_gam_ids) + base_gam_ids=body.base_gam_ids, make_picture=body.make_picture, + picture_prompt=body.picture_prompt, + picture_model_id=body.picture_model_id) except service.GenerationRefused as exc: raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={"code": exc.code, "message": exc.message}) diff --git a/backend/tests/test_generation.py b/backend/tests/test_generation.py index 62387f1..8d12d7f 100644 --- a/backend/tests/test_generation.py +++ b/backend/tests/test_generation.py @@ -182,3 +182,77 @@ def test_deleting_the_project_deletes_its_generations(auth_client, session, proj start(auth_client, project["id"], asset_key="a_dam") assert auth_client.delete(f"/api/projects/{project['id']}").status_code == 204 assert session.exec(select(Generation)).all() == [] + + +# ── a clip from the prompt alone: picture, then animate ────────────────────── + +NO_BASE = {**SPEC, "assets": {**SPEC["assets"], "a_flood": { + "from": "generate", "type": "video", "prompt": "Water rising"}}} + + +def _list(client, project_id): + return client.get(f"/api/projects/{project_id}/generations").json()["data"] + + +def test_a_clip_from_a_prompt_makes_its_picture_first_then_animates_it( + auth_client, project, siblings): + res = start(auth_client, project["id"], asset_key="a_flood", draft_spec=NO_BASE, + make_picture=True, picture_prompt="A flooded valley", model_id="m-veo", + duration="8") + assert res.status_code == 202, res.text + row = res.json()["data"] + [picture] = siblings.generations + assert picture["model_id"] == "m-flux" and picture["prompt"] == "A flooded valley" + assert picture["base_asset_ids"] == [] + assert picture["params"] == {"image_size": "landscape_16_9", "num_outputs": 1} + assert row["stage"].startswith("Making the picture") + assert row["model_label"] == "FLUX.1 [schnell] → Veo 3.1 Fast" + # Both runs, priced together. + assert row["estimate"] == pytest.approx(0.1 * 8 + 0.003 * 1024 * 576 / 1e6, abs=1e-3) + + siblings.gen_jobs["job-1"].update(status="processing", stage="Generating", progress=50) + listed = _list(auth_client, project["id"])[0] + assert listed["stage"] == "Making the picture · Generating" + assert listed["result_ids"] == [] and listed["status"] == "processing" + + # The picture is done: its output is never offered for the clip's placeholder. + siblings.gen_jobs["job-1"].update(status="done", result_asset_ids=["pic-1"]) + listed = _list(auth_client, project["id"])[0] + assert listed["status"] == "queued" and listed["result_ids"] == [] + assert listed["picture_id"] == "pic-1" + clip = siblings.generations[1] + assert clip["model_id"] == "m-veo" and clip["prompt"] == "Water rising" + assert clip["base_asset_ids"] == ["pic-1"] + assert clip["params"]["duration"] == "8s" + + siblings.gen_jobs["job-2"].update(status="done", result_asset_ids=["clip-1"]) + listed = _list(auth_client, project["id"])[0] + assert listed["status"] == "done" and listed["result_ids"] == ["clip-1"] + assert len(siblings.generations) == 2 + + +def test_a_failed_picture_says_so_and_starts_no_clip(auth_client, project, siblings): + start(auth_client, project["id"], asset_key="a_flood", draft_spec=NO_BASE, make_picture=True) + siblings.gen_jobs["job-1"].update(status="error", error_message="Content refused") + listed = _list(auth_client, project["id"])[0] + assert listed["status"] == "error" + assert listed["error"] == "Making the picture failed: Content refused" + assert len(siblings.generations) == 1 + + +def test_a_clip_that_cannot_start_keeps_its_picture(auth_client, project, siblings): + start(auth_client, project["id"], asset_key="a_flood", draft_spec=NO_BASE, make_picture=True) + siblings.gen_jobs["job-1"].update(status="done", result_asset_ids=["pic-1"]) + siblings.fal_key = False + listed = _list(auth_client, project["id"])[0] + assert listed["status"] == "error" and listed["picture_id"] == "pic-1" + assert listed["error"].startswith("The picture was made, but the clip couldn't be started") + + +def test_stopping_while_the_picture_is_made_starts_no_clip(auth_client, project, siblings): + row = start(auth_client, project["id"], asset_key="a_flood", draft_spec=NO_BASE, + make_picture=True).json()["data"] + auth_client.delete(f"/api/generations/{row['id']}") + siblings.gen_jobs["job-1"].update(status="done", result_asset_ids=["pic-1"]) + assert _list(auth_client, project["id"])[0]["status"] == "cancelled" + assert len(siblings.generations) == 1 diff --git a/docs/plan-of-attack.md b/docs/plan-of-attack.md index 120c0f2..1444fb2 100644 --- a/docs/plan-of-attack.md +++ b/docs/plan-of-attack.md @@ -146,6 +146,13 @@ worker for it: price; what fal.ai charged is in GAM's usage. - Images and clips only: GAM has no audio models, so music and sound placeholders stay "find it in your library". +- **A clip from the prompt alone.** GAM's video models all start from a picture, and it + has no text-to-video. So "Made from: a new picture" generates the picture first and + animates it: one `generation` row follows both runs. It holds the clip's run in `then` + until the picture is done, and `refresh` then starts the clip from it. The picture is + kept on the row (`picture_id`) and offered again on "Try again", so a failed clip + doesn't pay for its picture twice. The picture's output is never put in the + placeholder; only the clip's is. - **The assistant generates pictures, never video** (the user's call: video costs far more). `generate_image` adds the placeholder and starts it at once, at most `ASSISTANT_MAX_GENERATIONS` (6) per turn; for footage it leaves a video placeholder for diff --git a/frontend/src/api/generations.ts b/frontend/src/api/generations.ts index dfc4a4d..26695c9 100644 --- a/frontend/src/api/generations.ts +++ b/frontend/src/api/generations.ts @@ -28,6 +28,8 @@ export interface Generation { used_id: string | null /** List price × units, before it runs; what fal.ai charged is in GAM's usage. */ estimate: number | null + /** The picture a clip was made from, when it was generated for it first. */ + picture_id: string | null created_at: string updated_at: string } @@ -58,6 +60,11 @@ export interface GenerationStart { outputs?: number duration?: string base_gam_ids?: string[] + /** A clip from the prompt alone: GAM makes its picture first, then animates it. */ + make_picture?: boolean + /** What the picture shows, when that differs from the clip's prompt. */ + picture_prompt?: string + picture_model_id?: string /** The editor's unsaved draft, so a placeholder added since the last save counts. */ draft_spec?: VideoSpec } diff --git a/frontend/src/components/GenerateDialog.test.tsx b/frontend/src/components/GenerateDialog.test.tsx index dcd57ee..5f6f316 100644 --- a/frontend/src/components/GenerateDialog.test.tsx +++ b/frontend/src/components/GenerateDialog.test.tsx @@ -119,6 +119,35 @@ describe('GenerateDialog', () => { }) }) + it('makes a clip from the prompt alone, its picture generated first', async () => { + const start = vi.spyOn(generationsApi, 'start').mockResolvedValue({} as never) + render() + await screen.findByRole('option', { name: /Veo/ }) + await userEvent.selectOptions( + screen.getByRole('combobox', { name: 'Made from' }), + 'new' + ) + await userEvent.selectOptions( + screen.getByRole('combobox', { name: 'Picture model' }), + 'm-edit' + ) + await userEvent.type( + screen.getByRole('textbox', { name: 'Picture prompt' }), + 'A flooded valley' + ) + // Nano's $0.039 picture plus four seconds of Veo at $0.10. + expect(screen.getByText(/About \$0\.44/)).toBeInTheDocument() + await userEvent.click(screen.getByRole('button', { name: /Generate/ })) + const [, body] = start.mock.calls[0] + expect(body).toMatchObject({ + model_id: 'm-veo', + make_picture: true, + picture_model_id: 'm-edit', + picture_prompt: 'A flooded valley', + }) + expect(body.base_gam_ids).toBeUndefined() + }) + it('says why it could not start, and stays open', async () => { vi.spyOn(generationsApi, 'start').mockRejectedValue(new Error('no key')) const onClose = vi.fn() diff --git a/frontend/src/components/GenerateDialog.tsx b/frontend/src/components/GenerateDialog.tsx index 43dd4a3..23a5f80 100644 --- a/frontend/src/components/GenerateDialog.tsx +++ b/frontend/src/components/GenerateDialog.tsx @@ -16,6 +16,9 @@ interface Props { onClose: () => void } +/** "Made from" when the clip's picture is to be generated from the prompt first. */ +const NEW_PICTURE = 'new' + /** The project's pictures, as GAM ids: what a clip can be made from. */ function picturesIn(spec: VideoSpec): { id: string; name: string }[] { const seen = new Set() @@ -50,6 +53,7 @@ export default function GenerateDialog({ assetKey, onClose }: Props) { const draft = useProjectStore((s) => s.draft) const edit = useProjectStore((s) => s.edit) const models = useGenerationStore((s) => s.models) + const rows = useGenerationStore((s) => s.rows) const loadModels = useGenerationStore((s) => s.loadModels) const start = useGenerationStore((s) => s.start) const starting = useGenerationStore((s) => s.starting) @@ -57,14 +61,28 @@ export default function GenerateDialog({ assetKey, onClose }: Props) { const ref = draft?.assets?.[assetKey] as GenerateRef | undefined const isVideo = ref?.type === 'video' - const pictures = useMemo(() => (draft ? picturesIn(draft) : []), [draft]) + const pictures = useMemo(() => { + const inVideo = draft ? picturesIn(draft) : [] + // A picture made for this clip before, so trying again doesn't pay for it twice. + const made = rows + .filter((r) => r.asset_key === assetKey && r.picture_id) + .map((r) => r.picture_id as string) + .filter((id, i, all) => all.indexOf(id) === i && !inVideo.some((p) => p.id === id)) + .map((id, i) => ({ + id, + name: `Picture made for this clip${i ? ` (${i + 1})` : ''}`, + })) + return [...made.reverse(), ...inVideo] + }, [draft, rows, assetKey]) const [prompt, setPrompt] = useState(ref?.prompt ?? '') + const [picturePrompt, setPicturePrompt] = useState('') const [base, setBase] = useState( () => (draft && ref?.base_assets?.map((k) => gamIdOf(draft, k)).find(Boolean)) ?? pictures[0]?.id ?? - '' + NEW_PICTURE ) + const makePicture = isVideo && base === NEW_PICTURE const imageBases = !isVideo && draft ? (ref?.base_assets ?? []).flatMap((k) => gamIdOf(draft, k) ?? []) @@ -73,6 +91,11 @@ export default function GenerateDialog({ assetKey, onClose }: Props) { const choices = (models ?? []).filter((m) => m.kind === kind) const [modelId, setModelId] = useState('') const model = choices.find((m) => m.id === modelId) ?? choices[0] + const pictureModels = (models ?? []).filter((m) => m.kind === 'text_to_image') + const [pictureModelId, setPictureModelId] = useState('') + const pictureModel = makePicture + ? (pictureModels.find((m) => m.id === pictureModelId) ?? pictureModels[0]) + : undefined const [outputs, setOutputs] = useState(1) const [duration, setDuration] = useState('') @@ -84,7 +107,11 @@ export default function GenerateDialog({ assetKey, onClose }: Props) { const maxOutputs = isVideo ? 1 : Math.min(4, model?.options.max_outputs ?? 4) const durations = (model?.options.durations ?? []).map(String) const seconds = duration || durations[0] - const cost = model ? estimateCost(model, Math.min(outputs, maxOutputs), seconds) : null + const runCost = model + ? estimateCost(model, Math.min(outputs, maxOutputs), seconds) + : null + const pictureCost = pictureModel ? estimateCost(pictureModel, 1) : 0 + const cost = runCost === null || pictureCost === null ? null : runCost + pictureCost async function generate() { if (!draft || !ref) return @@ -103,14 +130,26 @@ export default function GenerateDialog({ assetKey, onClose }: Props) { model_id: model?.id, outputs: Math.min(outputs, maxOutputs), ...(isVideo && seconds ? { duration: seconds.replace(/s$/, '') } : {}), - ...(isVideo && base ? { base_gam_ids: [base] } : {}), + ...(makePicture + ? { + make_picture: true, + picture_model_id: pictureModel?.id, + ...(picturePrompt.trim() ? { picture_prompt: picturePrompt.trim() } : {}), + } + : isVideo && base + ? { base_gam_ids: [base] } + : {}), draft_spec: next, }) if (ok) onClose() } const ready = - canGenerate(ref) && prompt.trim() !== '' && !!model && (!isVideo || !!base) + canGenerate(ref) && + prompt.trim() !== '' && + !!model && + (!isVideo || !!base) && + (!makePicture || !!pictureModel) return (
Made from - {pictures.length === 0 ? ( -

- A clip is made from a picture. Add one to the video first: find one, - or generate one. -

- ) : ( - - )} + )} + {makePicture && + (pictureModels.length === 0 ? ( +

+ The asset library has no picture models switched on, so the clip needs a + picture from the video. +

+ ) : ( + <> +