fix(extension): make captcha and token capture work on flow.google.com

flow-agent/flow_engine/generators/i2v.py

@@ -11,6 +11,14 @@
 log = logging.getLogger("flow_engine.generators.i2v")
 
 
+def _safe_error_text(value) -> str | None:
+    """Return a bounded scalar error without serializing arbitrary responses."""
+    if not isinstance(value, (str, int, float)):
+        return None
+    text = " ".join(str(value).split())
+    return text[:500] or None
+
+
 async def upload_image(
     bridge,
     image_path: str,
@@ -52,11 +60,20 @@ async def upload_image(
     log.info("Uploading image: %s", os.path.basename(image_path))
     result = await bridge.api_request(ENDPOINTS["upload_image"], body)
 
-    status = result.get("status", 0)
-    data = result.get("data", {})
+    status = result.get("status", 0) if isinstance(result, dict) else 0
+    data = result.get("data", {}) if isinstance(result, dict) else {}
     if status != 200:
-        err = data.get("error", {}).get("message", "Unknown") if isinstance(data, dict) else str(data)
-        err_msg = f"Image upload failed: {err}"
+        nested = data.get("error", {}) if isinstance(data, dict) else {}
+        nested = nested if isinstance(nested, dict) else {}
+        message = _safe_error_text(nested.get("message"))
+        google_status = _safe_error_text(nested.get("status"))
+        top_level_error = _safe_error_text(result.get("error")) if isinstance(result, dict) else None
+        detail = message or top_level_error or google_status or "Unknown error"
+        if message and google_status and google_status not in message:
+            detail = f"{message} ({google_status})"
+        status_text = _safe_error_text(status)
+        status_suffix = f" (status {status_text})" if status_text and status_text != "0" else ""
+        err_msg = f"Image upload failed{status_suffix}: {detail}"
         log.error("%s", err_msg)
         raise ValueError(err_msg)
 

flow-agent/tests/test_extension_flow_urls.py

@@ -0,0 +1,68 @@
+import json
+from pathlib import Path
+
+
+EXTENSION_DIR = Path(__file__).resolve().parents[2] / "flow-extension"
+
+
+def test_manifest_allows_new_and_legacy_flow_pages():
+    manifest = json.loads((EXTENSION_DIR / "manifest.json").read_text(encoding="utf-8"))
+
+    assert "https://flow.google.com/*" in manifest["host_permissions"]
+    assert "https://flow.google.com/*" in manifest["content_scripts"][0]["matches"]
+    assert "https://flow.google.com/*" in manifest["web_accessible_resources"][0]["matches"]
+    assert "https://labs.google/fx/tools/flow*" in manifest["content_scripts"][0]["matches"]
+
+
+def test_background_uses_precise_flow_page_eligibility_and_shared_tab_patterns():
+    source = (EXTENSION_DIR / "background.js").read_text(encoding="utf-8")
+
+    assert "'https://flow.google.com/*'" in source
+    assert "parsed.hostname === 'flow.google.com'" in source
+    assert "parsed.hostname !== 'labs.google'" in source
+    assert "return createdTab;" in source
+    assert "url: '*://labs.google/*'" not in source
+    assert source.count("url: FLOW_TAB_URLS") >= 3
+
+
+def test_background_opens_captcha_capable_project_pages():
+    source = (EXTENSION_DIR / "background.js").read_text(encoding="utf-8")
+
+    # labs.google/fx/tools/flow redirects to the flow.google.com home page, which
+    # never loads reCAPTCHA; only /project/<id> pages do.
+    assert "const FLOW_URL = 'https://flow.google.com/';" in source
+    assert "function isFlowProjectUrl(url)" in source
+    assert "/^\\/project\\/[^/]+/" in source
+    assert "https://flow.google.com/project/${encodeURIComponent(projectId)}" in source
+    assert "tabs.filter((t) => isFlowProjectUrl(t.url))" in source
+    assert "solveCaptcha(id, captchaAction, projectId)" in source
+
+
+def test_background_refreshes_token_through_labs_handoff():
+    source = (EXTENSION_DIR / "background.js").read_text(encoding="utf-8")
+
+    # The ya29 bearer is only observable during the labs.google -> flow.google.com
+    # redirect; reloading a flow.google.com tab does not re-capture it.
+    assert "const TOKEN_URL = 'https://labs.google/fx/tools/flow';" in source
+    assert "async function refreshTokenViaLabs()" in source
+    assert source.count("await refreshTokenViaLabs()") >= 2
+    assert "chrome.tabs.reload(tabs[0].id)" not in source
+
+
+def test_extension_verifies_captcha_bridge_before_using_a_tab():
+    background = (EXTENSION_DIR / "background.js").read_text(encoding="utf-8")
+    content = (EXTENSION_DIR / "content.js").read_text(encoding="utf-8")
+    injected = (EXTENSION_DIR / "injected.js").read_text(encoding="utf-8")
+
+    # A tab matching a Flow URL is not enough: its bridge must answer a ping,
+    # otherwise every request burns the content-script timeout.
+    assert "async function bridgeAlive(tabId)" in background
+    assert "if (!(await bridgeAlive(tab.id))) continue;" in background
+    assert "type: 'PING_BRIDGE'" in background
+    assert "msg.type !== 'PING_BRIDGE'" in content
+    assert "'FLOW_AGENT_PING'" in injected and "'FLOW_AGENT_PONG'" in injected
+    # GET_CAPTCHA is re-dispatched until injected.js answers; injected.js dedups.
+    assert "setInterval(dispatch, 500)" in content
+    assert "_captchaInFlight" in injected
+    assert "grecaptcha execute timeout" in injected
+

flow-agent/tests/test_media_storage.py

@@ -2,6 +2,7 @@
 import importlib
 import json
 import os
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -216,6 +217,61 @@ async def api_request(self, endpoint, body, **kwargs):
     assert media_store.get_for_file(image_path, project_id="project-a")["media_id"] == "fresh-id"
 
 
+@pytest.mark.parametrize(
+    ("response", "expected"),
+    [
+        (
+            {"status": 403, "error": "CAPTCHA_FAILED"},
+            "Image upload failed (status 403): CAPTCHA_FAILED",
+        ),
+        ({"error": "Request timed out"}, "Image upload failed: Request timed out"),
+        (
+            {"status": 500, "error": {"debug": {"internal": "sensitive-context"}}},
+            "Image upload failed (status 500): Unknown error",
+        ),
+        (
+            {
+                "status": 400,
+                "data": {
+                    "error": {
+                        "message": "The image could not be processed",
+                        "status": "INVALID_ARGUMENT",
+                    }
+                },
+            },
+            "Image upload failed (status 400): The image could not be processed (INVALID_ARGUMENT)",
+        ),
+    ],
+)
+def test_upload_image_preserves_safe_actionable_errors(
+    monkeypatch, tmp_path, response, expected
+):
+    _configure_store(monkeypatch, tmp_path)
+    image_path = tmp_path / "reference.png"
+    image_path.write_bytes(PNG_BYTES)
+
+    class Bridge:
+        async def api_request(self, endpoint, body, **kwargs):
+            return response
+
+    with pytest.raises(ValueError, match=re.escape(expected)):
+        asyncio.run(upload_image(Bridge(), str(image_path), "project-a"))
+
+
+def test_upload_image_still_accepts_successful_media_response(monkeypatch, tmp_path):
+    _configure_store(monkeypatch, tmp_path)
+    image_path = tmp_path / "reference.png"
+    image_path.write_bytes(PNG_BYTES)
+
+    class Bridge:
+        async def api_request(self, endpoint, body, **kwargs):
+            return {"status": 200, "data": {"media": {"name": "uploaded-media-id"}}}
+
+    media_id = asyncio.run(upload_image(Bridge(), str(image_path), "project-a"))
+
+    assert media_id == "uploaded-media-id"
+
+
 def test_upload_is_reused_as_image_to_video_reference_without_duplicates(monkeypatch, tmp_path):
     history_path = _configure_store(monkeypatch, tmp_path)
     image_path = tmp_path / "start-frame.png"

flow-extension/README.md

@@ -19,4 +19,12 @@ Chrome bridge for [kodelyx/flow-agent](https://github.com/kodelyx/flow-agent). I
 4. Open <https://labs.google/fx/tools/flow>, sign in, and keep the tab open.
 5. Click the extension icon to open Flow Agent in Chrome's side panel.
 
+## Flow site compatibility
+
+The extension recognizes both the current `https://flow.google.com/` site and
+the legacy `https://labs.google/fx/tools/flow` route. This is partial issue #10
+compatibility: authentication and CAPTCHA behavior, plus the existing REST and
+upload calls on the new site, have not been verified end to end and may still
+require follow-up changes.
+
 Main documentation: [Flow Agent README](../README.md)