From 62f4896eedd518c41243351f38146f4b9d1f90e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 00:06:29 +0000 Subject: [PATCH] Generate placeholders' pictures and clips through GAM A `generate` placeholder for a picture or a clip now has a Generate button. GAM's M8 makes it on fal.ai with the user's own key; GVC has no fal code and no worker. - A `generation` row per run. Reading a project's generations refreshes the unfinished ones from GAM with the reader's token, so a run started and left lands the next time the project is opened. No token is stored. - A finished run's first output goes into the editor's draft (the placeholder's `result`, status ready), once; the other outputs stay offered beside it. - Options come from the model's own catalogue entry (aspect ratio or FLUX image size from the video's aspect, durations as GAM spells them), with GAM's list price as the estimate. - The assistant's new `generate_image` tool generates pictures, at most ASSISTANT_MAX_GENERATIONS a turn, and never video: for footage it leaves a video placeholder for the Generate button. Its rows follow the turn into a new project. - Music and sound placeholders are refused: GAM has no audio models. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016fiRJAt34ag62iPiuMkNhm --- README.md | 5 + .../20261016_2353_add_generation_table.py | 63 ++++ backend/app/assistant/prompt.py | 14 + backend/app/assistant/tools.py | 75 +++++ backend/app/assistant/worker.py | 15 +- backend/app/config.py | 3 + backend/app/gam/client.py | 42 ++- backend/app/generation/__init__.py | 7 + backend/app/generation/catalogue.py | 106 +++++++ backend/app/generation/service.py | 125 ++++++++ backend/app/main.py | 2 + backend/app/models/__init__.py | 3 +- backend/app/models/generation.py | 41 +++ backend/app/routers/assistant.py | 7 + backend/app/routers/generations.py | 178 ++++++++++++ backend/app/services/projects.py | 4 + backend/app/spec/model.py | 8 + backend/tests/fake_siblings.py | 54 ++++ backend/tests/test_assistant_tools.py | 34 +++ backend/tests/test_assistant_turns.py | 40 +++ backend/tests/test_generation.py | 184 ++++++++++++ docs/plan-of-attack.md | 23 +- frontend/src/api/generations.ts | 101 +++++++ frontend/src/api/projects.ts | 8 +- frontend/src/components/AssistantPanel.tsx | 17 +- .../src/components/GenerateDialog.test.tsx | 134 +++++++++ frontend/src/components/GenerateDialog.tsx | 269 ++++++++++++++++++ frontend/src/components/SceneList.tsx | 3 + frontend/src/components/VisualStrip.test.tsx | 98 ++++++- frontend/src/components/VisualStrip.tsx | 90 +++++- frontend/src/spec/footage.ts | 2 +- frontend/src/spec/generation.test.ts | 69 +++++ frontend/src/spec/generation.ts | 78 +++++ frontend/src/stores/auth.ts | 2 + frontend/src/stores/generations.test.ts | 121 ++++++++ frontend/src/stores/generations.ts | 183 ++++++++++++ frontend/src/views/ProjectView.tsx | 71 ++++- 37 files changed, 2263 insertions(+), 16 deletions(-) create mode 100644 backend/alembic/versions/20261016_2353_add_generation_table.py create mode 100644 backend/app/generation/__init__.py create mode 100644 backend/app/generation/catalogue.py create mode 100644 backend/app/generation/service.py create mode 100644 backend/app/models/generation.py create mode 100644 backend/app/routers/generations.py create mode 100644 backend/tests/test_generation.py create mode 100644 frontend/src/api/generations.ts create mode 100644 frontend/src/components/GenerateDialog.test.tsx create mode 100644 frontend/src/components/GenerateDialog.tsx create mode 100644 frontend/src/spec/generation.test.ts create mode 100644 frontend/src/spec/generation.ts create mode 100644 frontend/src/stores/generations.test.ts create mode 100644 frontend/src/stores/generations.ts diff --git a/README.md b/README.md index a3d7e67..47b986c 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,11 @@ one open. What it would change is listed for you to apply or discard. It runs in background, so a request finishes with the page closed. Set up its AI provider (Anthropic, OpenAI, DeepSeek, Ollama or a custom endpoint) under Settings. +**Generated pictures and clips.** A placeholder for a picture or a clip has a Generate +button: GAM makes it on fal.ai with your key (set in GAM's Settings), and it drops into +the video when done, even if you closed the page meanwhile. The assistant may generate +pictures itself; clips cost more, so it leaves those for you to start. + The full storyboard editor is next (M3). ## Read first diff --git a/backend/alembic/versions/20261016_2353_add_generation_table.py b/backend/alembic/versions/20261016_2353_add_generation_table.py new file mode 100644 index 0000000..833875f --- /dev/null +++ b/backend/alembic/versions/20261016_2353_add_generation_table.py @@ -0,0 +1,63 @@ +"""Add the generation table: GVC's record of each generation it asked GAM for. + +The job runs in GAM; a row is how GVC follows it and knows which placeholder it fills. +`gam_job_id` is GAM's id, so no foreign key. + +Revision ID: c6d1a4e8b207 +Revises: 9b3e7d215f60 +Create Date: 2026-10-16 23:53:58.001597+00:00 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = 'c6d1a4e8b207' +down_revision: Union[str, None] = '9b3e7d215f60' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table('generation', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('user_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('project_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('turn_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('asset_key', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('gam_job_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('kind', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('prompt', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('model_label', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('params', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('stage', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('progress', sa.Integer(), nullable=False), + sa.Column('detail', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('error', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('result_ids', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('used_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('estimate', sa.Float(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('generation', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_generation_gam_job_id'), ['gam_job_id'], unique=False) + batch_op.create_index(batch_op.f('ix_generation_project_id'), ['project_id'], unique=False) + batch_op.create_index(batch_op.f('ix_generation_turn_id'), ['turn_id'], unique=False) + batch_op.create_index(batch_op.f('ix_generation_user_id'), ['user_id'], unique=False) + + + +def downgrade() -> None: + with op.batch_alter_table('generation', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_generation_user_id')) + batch_op.drop_index(batch_op.f('ix_generation_turn_id')) + batch_op.drop_index(batch_op.f('ix_generation_project_id')) + batch_op.drop_index(batch_op.f('ix_generation_gam_job_id')) + + op.drop_table('generation') diff --git a/backend/app/assistant/prompt.py b/backend/app/assistant/prompt.py index beedf5a..3f643b2 100644 --- a/backend/app/assistant/prompt.py +++ b/backend/app/assistant/prompt.py @@ -98,6 +98,20 @@ `add_placeholder` with a clear description of what to find or generate. Never invent a \ gam_id or an asset key that isn't in the registry. +## Generating pictures +`generate_image` makes a still with AI (through the user's asset library) and puts it in \ +the video; it fills in by itself a few seconds later. It costs money, so: +- Search the library first. Generate only what a library can't have: an illustration, a \ +reconstruction of something never photographed, an abstract or title background, a \ +picture to fill a gap the user asked you to fill. +- Write the prompt for one frame of video: the subject, the composition, the light and \ +the style, matched to the video's aspect (16:9 is wide). No text in the picture; put \ +words in an overlay instead. +- Never generate video. When a scene needs footage the library doesn't have, leave an \ +`add_placeholder` of kind generate with media_type video and a good prompt, and tell \ +the user they can generate it from the scene with the Generate button. +- Music and sound effects can't be generated; find them in the library. + ## Pacing Narration reads at about 15 characters a second. A 3-minute video is about 2,500 characters \ of narration over 10-20 segments. Keep beats to one or two sentences each so pictures can \ diff --git a/backend/app/assistant/tools.py b/backend/app/assistant/tools.py index 690e3c9..0cf21bb 100644 --- a/backend/app/assistant/tools.py +++ b/backend/app/assistant/tools.py @@ -86,6 +86,13 @@ class Working: changes: List[str] = field(default_factory=list) hits: Dict[str, Hit] = field(default_factory=dict) issues: List[SpecIssue] = field(default_factory=list) + # Starts one image generation in GAM for a placeholder in `data` and says what it + # cost: (model label, estimated cost). Set by the worker; None in a turn that can't + # generate (no GAM), and then the tool says so. + generate: Optional[Callable[[Dict[str, Any], str, str, List[str], int], + Tuple[str, Optional[float]]]] = None + max_generations: int = 6 + generated: int = 0 def __post_init__(self) -> None: self.data = copy.deepcopy(self.data) @@ -415,6 +422,61 @@ def add_placeholder(w: Working, segment_id: str, kind: str, text: str, f"{'find' if kind == 'search' else 'generate'}: {text!r}") +def generate_image(w: Working, prompt: str, segment_id: Optional[str] = None, + use_as: str = "visual", base_gam_ids: Optional[List[str]] = None, + outputs: int = 1, label: Optional[str] = None) -> str: + """A picture made for the video, started now in GAM and filled in when it's done. + + Images only. Video costs far more and takes minutes, so the user starts that + themselves from the placeholder (their call, recorded in the plan of attack). + """ + if w.generate is None: + raise ToolFailed("Generating isn't available in this turn.") + if w.generated >= w.max_generations: + raise ToolFailed(f"That's the {w.max_generations} pictures this request may generate. " + "Leave the rest as placeholders with add_placeholder; the user can " + "generate them from the video.") + if not prompt or not prompt.strip(): + raise ToolFailed("Describe the picture to generate.") + if use_as not in ("visual", "overlay", "none"): + raise ToolFailed('use_as is "visual", "overlay" or "none".') + if use_as != "none" and not segment_id: + raise ToolFailed("Say which segment the picture is for, or use_as \"none\".") + candidate = copy.deepcopy(w.data) + assets = candidate.setdefault("assets", {}) + base = f"a_{_slug(label or prompt) or 'generated'}" + key, n = base, 2 + while key in assets: + key, n = f"{base}_{n}", n + 1 + ref: Dict[str, Any] = {"from": "generate", "type": "image", "prompt": prompt.strip()} + if label: + ref["label"] = label + assets[key] = ref + where = "kept in the registry" + if segment_id: + segment = candidate["segments"][w.index_of(segment_id)] + if use_as == "visual": + segment["visual"] = {"asset": key} + where = f"as the picture of {w.name_of(segment)}" + elif use_as == "overlay": + taken = w.all_overlay_ids() + segment.setdefault("overlays", []).append( + {"id": w.new_id("o_", taken), "type": "image", "asset": key, + "position": "top-right", "scale_pct": 30}) + where = f"on screen over {w.name_of(segment)}" + result = w.commit(candidate, f"Generating a picture {where}: {prompt.strip()[:80]!r}") + try: + model, cost = w.generate(w.data, key, prompt.strip(), list(base_gam_ids or []), + max(1, min(4, outputs))) + except Exception as exc: # noqa: BLE001 - said to the model; the placeholder stays + return (f"{result} But the generation couldn't start ({exc}); the placeholder " + f"{key} is there for the user to generate later.") + w.generated += 1 + price = f", about ${cost:.3f}" if cost is not None else "" + return (f"{result} Started with {model}{price}; it fills placeholder {key} by itself " + "in a few seconds.") + + def add_overlay(w: Working, segment_id: str, overlay: Dict[str, Any], gam_id: Optional[str] = None) -> str: if not isinstance(overlay, dict): @@ -607,6 +669,19 @@ def run_estimate(w: Working) -> str: "motion, background, cards…), output, voice, chrome, credits.", {"type": "object", "properties": {"changes": _OBJ}, "required": ["changes"]}), set_project), + (ToolSpec("generate_image", + "Generate a still picture with AI and put it in the video (as a segment's " + "picture, an image overlay, or just in the registry). It starts at once and " + "costs money, so search the library first and generate only what a library " + "can't have. base_gam_ids: library pictures to edit or restyle. Never use " + "this for video; leave video as a placeholder for the user.", + {"type": "object", "properties": { + "prompt": _STR, "segment_id": _STR, + "use_as": {"type": "string", "enum": ["visual", "overlay", "none"]}, + "base_gam_ids": {"type": "array", "items": _STR}, + "outputs": {"type": "integer", "minimum": 1, "maximum": 4}, + "label": _STR}, + "required": ["prompt"]}), generate_image), (ToolSpec("estimate", "How long the video runs, its narration length, and what would " "stop a full render.", {"type": "object", "properties": {}}), run_estimate), ] diff --git a/backend/app/assistant/worker.py b/backend/app/assistant/worker.py index 6d610f5..05bf8f2 100644 --- a/backend/app/assistant/worker.py +++ b/backend/app/assistant/worker.py @@ -24,6 +24,7 @@ from app.clock import utcnow from app.config import settings from app.gam.client import GamClient +from app.generation import service as generation_service from app.jobs.runner import JobQueue, readable_error, set_fields from app.models.assistant import AssistantSession, AssistantTurn from app.providers import build_provider @@ -113,7 +114,19 @@ def on_reply(reply: Reply) -> None: provider = build_provider(session, user_id) if provider is None: raise ProviderError("No AI provider is set up. Add one in Settings first.") - working = Working(base, gam=gam) + working = Working(base, gam=gam, max_generations=settings.assistant_max_generations) + + def generate(data, key, image_prompt, base_ids, outputs): + with queue.session() as session: + row = generation_service.start( + session, gam, user_id=user_id, project_id=project_id, turn_id=turn_id, + spec=data, key=key, prompt=image_prompt, outputs=outputs, + base_gam_ids=base_ids) + # Read inside the session: the commit expired the row, and a detached + # row can't reload. + return row.model_label, row.estimate + + working.generate = generate system = [REFERENCE, context(base, focus=focus, from_scratch=project_id is None)] outcome = loop.run(provider, working, system=system, history=history, prompt=prompt, on_step=on_step, on_reply=on_reply, diff --git a/backend/app/config.py b/backend/app/config.py index 6ad5c64..0510c7a 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -95,6 +95,9 @@ class Settings(BaseSettings): # Earlier messages a turn is given, newest kept. Older ones are still in the # session; the spec itself, not the chat, is the record of what was decided. assistant_history_messages: int = 20 + # Pictures one request may generate on its own (each is a paid call to fal.ai through + # GAM). Past it the assistant leaves placeholders for the user to generate. + assistant_max_generations: int = 6 job_heartbeat_seconds: int = 30 job_stale_minutes: int = 40 diff --git a/backend/app/gam/client.py b/backend/app/gam/client.py index 694cc45..9ec74a3 100644 --- a/backend/app/gam/client.py +++ b/backend/app/gam/client.py @@ -12,7 +12,7 @@ from __future__ import annotations import logging -from typing import IO, Any, Dict, Optional +from typing import IO, Any, Dict, List, Optional import httpx @@ -190,3 +190,43 @@ def update(self, asset_id: str, changes: Dict[str, Any]) -> Optional[GamAssetRea code, message = self._detail(response) raise GamError(code, message) return to_asset(response.json()["data"]) + + # ── generation (GAM's M8) ──────────────────────────────────────────────── + # GAM runs the generation on fal.ai with the user's own key and keeps the outputs in + # their library. A job is followed by its id; GVC never talks to fal. + + def _generation_error(self, response: httpx.Response) -> GamError: + code, message = self._detail(response) + if code == "fal_key_missing": + message = ("Generating needs a fal.ai API key in the asset library's settings: " + f"{settings.gam_base_url.rstrip('/')}/settings") + return GamError(code, message, status_code=400 if response.status_code < 500 else 502) + + def generation_models(self) -> List[Dict[str, Any]]: + response = self._request("GET", "/api/generate/models") + if response.status_code != 200: + raise self._generation_error(response) + return list(response.json().get("data") or []) + + def start_generation(self, *, model_id: str, prompt: str, base_ids: List[str], + params: Dict[str, Any]) -> Dict[str, Any]: + response = self._request("POST", "/api/generate", json={ + "model_id": model_id, "prompt": prompt, "base_asset_ids": base_ids, + "params": params, + }) + if response.status_code not in (200, 201, 202): + raise self._generation_error(response) + return response.json()["data"] + + def generation_job(self, job_id: str) -> Optional[Dict[str, Any]]: + response = self._request("GET", f"/api/activity/enrichment/{job_id}") + if response.status_code == 404: + return None + if response.status_code != 200: + raise self._generation_error(response) + return response.json()["data"] + + def cancel_generation(self, job_id: str) -> None: + response = self._request("DELETE", f"/api/activity/enrichment/{job_id}") + if response.status_code not in (200, 204, 404): + raise self._generation_error(response) diff --git a/backend/app/generation/__init__.py b/backend/app/generation/__init__.py new file mode 100644 index 0000000..3d2ed67 --- /dev/null +++ b/backend/app/generation/__init__.py @@ -0,0 +1,7 @@ +"""Generating pictures and clips for `generate` placeholders, through GAM (GAM's M8). + +GVC owns no fal.ai code and no key: GAM generates with the user's own fal key, keeps the +outputs in their library, and records the spend. GVC picks a model, sets the options +from the video (its aspect above all), keeps a row per generation so it can be followed, +and puts the result into the placeholder. +""" diff --git a/backend/app/generation/catalogue.py b/backend/app/generation/catalogue.py new file mode 100644 index 0000000..1dd39d8 --- /dev/null +++ b/backend/app/generation/catalogue.py @@ -0,0 +1,106 @@ +"""Choosing a model from GAM's catalogue, its options, and what a run will roughly cost. + +GAM matches every option against the model's declared values *with its type*: Kling's +duration is the string "5", Veo's is "8s". So choices are always made from the +catalogue's own lists, never constructed. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +# A placeholder's type, as the generation kind GAM runs. With base images an image +# becomes an edit of them (image→image); a video is always made from a picture. +KINDS = {"image": "text_to_image", "video": "image_to_video"} + +# FLUX names its sizes rather than its ratios; these are the ones for each output aspect, +# and their size in pixels, for the per-megapixel price. +_SIZE_FOR_ASPECT = {"16:9": "landscape_16_9", "9:16": "portrait_16_9", "1:1": "square_hd"} +_SIZE_PIXELS = { + "square_hd": (1024, 1024), "square": (512, 512), "portrait_4_3": (768, 1024), + "portrait_16_9": (576, 1024), "landscape_4_3": (1024, 768), "landscape_16_9": (1024, 576), +} + +MAX_IMAGE_OUTPUTS = 4 + + +class CatalogueError(Exception): + """No model can do what was asked; the message says why.""" + + +def kind_for(asset_type: str, has_base: bool) -> str: + if asset_type == "image": + return "image_to_image" if has_base else "text_to_image" + if asset_type == "video": + return "image_to_video" + raise CatalogueError("Only pictures and video can be generated; music and sound " + "effects have to come from your library.") + + +def pick_model(models: List[Dict[str, Any]], kind: str, + preferred: Optional[str] = None) -> Dict[str, Any]: + """The model asked for if it can do this kind, else the catalogue's first of the kind + (GAM lists them in its own preferred order).""" + of_kind = [m for m in models if m.get("kind") == kind and m.get("is_active", True)] + if preferred: + for model in of_kind: + if model.get("id") == preferred: + return model + raise CatalogueError("That model can't do this kind of generation.") + if not of_kind: + raise CatalogueError(f"The asset library has no active model for {kind.replace('_', ' ')}.") + return of_kind[0] + + +def _seconds(value: Any) -> Optional[float]: + match = re.search(r"\d+(\.\d+)?", str(value)) + return float(match.group()) if match else None + + +def default_params(model: Dict[str, Any], *, aspect: str, outputs: int = 1, + duration: Optional[str] = None) -> Dict[str, Any]: + """Options for one run, chosen from the model's own lists.""" + options = model.get("options") or {} + params: Dict[str, Any] = {} + ratios = options.get("aspect_ratios") or [] + sizes = options.get("image_sizes") or [] + if aspect in ratios: + params["aspect_ratio"] = aspect + elif _SIZE_FOR_ASPECT.get(aspect) in sizes: + params["image_size"] = _SIZE_FOR_ASPECT[aspect] + durations = options.get("durations") or [] + if durations: + # A requested duration is matched by its number, so "6" finds Veo's "6s". + wanted = _seconds(duration) if duration is not None else None + chosen = next((d for d in durations if wanted is not None and _seconds(d) == wanted), + durations[0]) + params["duration"] = chosen + most = max(1, int(options.get("max_outputs") or 1)) + params["num_outputs"] = max(1, min(outputs, most, MAX_IMAGE_OUTPUTS)) + return params + + +def estimate(model: Dict[str, Any], params: Dict[str, Any]) -> Optional[float]: + """The list price of one run, from the catalogue. An estimate, never a bill: GAM + records what fal actually charged once it has run.""" + price, unit = model.get("unit_price"), model.get("price_unit") + if price is None or not unit: + return None + count = params.get("num_outputs", 1) + if unit == "image" or unit == "video": + return round(price * count, 4) + if unit == "second": + return round(price * (_seconds(params.get("duration")) or 5.0) * count, 4) + if unit == "megapixel": + width, height = _SIZE_PIXELS.get(params.get("image_size"), (1024, 1024)) + return round(price * width * height / 1_000_000 * count, 4) + return None + + +def price_line(model: Dict[str, Any]) -> str: + """The catalogue's price as a person reads it: "$0.07 a second".""" + price, unit = model.get("unit_price"), model.get("price_unit") + if price is None or not unit: + return "price unknown" + return f"${price:g} a {unit}" diff --git a/backend/app/generation/service.py b/backend/app/generation/service.py new file mode 100644 index 0000000..e711e48 --- /dev/null +++ b/backend/app/generation/service.py @@ -0,0 +1,125 @@ +"""Starting, following and stopping generations for `generate` placeholders.""" + +from __future__ import annotations + +import json +import logging +import uuid +from typing import Any, Dict, List, Optional + +from sqlmodel import Session + +from app.clock import utcnow +from app.gam.client import GamClient, GamError +from app.generation.catalogue import CatalogueError, default_params, estimate, kind_for, pick_model +from app.models.generation import Generation + +logger = logging.getLogger(__name__) + +FINAL = ("done", "error", "cancelled") + + +class GenerationRefused(Exception): + """The placeholder can't be generated as asked; the message says why.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + self.message = message + + +def placeholder(spec: Dict[str, Any], key: str) -> Dict[str, Any]: + ref = (spec.get("assets") or {}).get(key) + if not isinstance(ref, dict) or ref.get("from") != "generate": + raise GenerationRefused("not_a_placeholder", + f"{key!r} is not a picture or clip waiting to be generated.") + if ref.get("type") not in ("image", "video"): + raise GenerationRefused("cannot_generate", + "Only pictures and video can be generated; music and sound " + "effects have to come from your library.") + return ref + + +def _gam_id_of(spec: Dict[str, Any], key: str) -> Optional[str]: + ref = (spec.get("assets") or {}).get(key) or {} + if ref.get("from") == "gam": + return ref.get("id") + result = ref.get("result") or {} + return result.get("id") + + +def start(session: Session, gam: GamClient, *, user_id: str, project_id: Optional[str], + turn_id: Optional[str], spec: Dict[str, Any], key: str, + prompt: Optional[str] = None, model_id: Optional[str] = None, outputs: int = 1, + duration: Optional[str] = None, + base_gam_ids: Optional[List[str]] = None) -> Generation: + ref = placeholder(spec, key) + prompt = (prompt or ref.get("prompt") or "").strip() + if not prompt: + raise GenerationRefused("no_prompt", "Say what to generate first.") + bases = list(base_gam_ids or []) + if not bases: + bases = [gid for gid in (_gam_id_of(spec, k) for k in ref.get("base_assets") or []) if gid] + if ref["type"] == "video" and not bases: + raise GenerationRefused("needs_base_image", + "A clip is made from a picture: choose the picture to start from.") + try: + kind = kind_for(ref["type"], bool(bases)) + model = pick_model(gam.generation_models(), kind, model_id) + except CatalogueError as exc: + raise GenerationRefused("no_model", str(exc)) + aspect = ref.get("aspect") or (spec.get("output") or {}).get("aspect") or "16:9" + params = default_params(model, aspect=aspect, outputs=outputs if ref["type"] == "image" else 1, + duration=duration) + job = gam.start_generation(model_id=model["id"], prompt=prompt, + base_ids=bases[:1] if kind == "image_to_video" else bases, + params=params) + row = Generation(id=str(uuid.uuid4()), user_id=user_id, project_id=project_id, + turn_id=turn_id, asset_key=key, gam_job_id=job["id"], kind=kind, + prompt=prompt, model_label=model.get("label") or model.get("endpoint_id", ""), + params=json.dumps(params), status=job.get("status") or "queued", + stage=job.get("stage") or "", progress=job.get("progress") or 0, + estimate=estimate(model, params)) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def refresh(session: Session, gam: GamClient, rows: List[Generation]) -> None: + """Ask GAM about every unfinished generation. Best effort: GAM being unreachable + leaves the rows as they were, to be caught up on the next read.""" + changed = False + for row in rows: + if row.status in FINAL: + continue + try: + job = gam.generation_job(row.gam_job_id) + except GamError as exc: + logger.info("Could not follow generation %s: %s", row.id, exc.message) + return + if job is None: + row.status, row.error = "error", "The asset library no longer has this generation." + else: + row.status = job.get("status") or row.status + row.stage = job.get("stage") or "" + row.progress = int(job.get("progress") or 0) + row.detail = job.get("detail") or "" + row.error = job.get("error_message") + ids = job.get("result_asset_ids") or ([job["result_asset_id"]] + if job.get("result_asset_id") else []) + row.result_ids = json.dumps(ids) + row.updated_at = utcnow() + session.add(row) + changed = True + if changed: + session.commit() + + +def cancel(session: Session, gam: GamClient, row: Generation) -> None: + if row.status in FINAL: + return + gam.cancel_generation(row.gam_job_id) + row.status, row.stage, row.updated_at = "cancelled", "", utcnow() + session.add(row) + session.commit() diff --git a/backend/app/main.py b/backend/app/main.py index b0001b3..e71c1ca 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -22,6 +22,7 @@ from app.gam.client import GamError from app.routers import assistant as assistant_router from app.routers import gam as gam_router +from app.routers import generations as generations_router from app.routers import projects as projects_router from app.routers import providers as providers_router from app.routers import renders as renders_router @@ -154,6 +155,7 @@ def me(user: CurrentUser, session: Session = Depends(get_session)) -> DataRespon app.include_router(renders_router.router, prefix="/api", tags=["renders"]) app.include_router(providers_router.router, prefix="/api/providers", tags=["providers"]) app.include_router(assistant_router.router, prefix="/api/assistant", tags=["assistant"]) +app.include_router(generations_router.router, prefix="/api", tags=["generations"]) app.include_router(usage_router.router, prefix="/api/usage", tags=["usage"]) # Not under /api: these URLs go straight into