From cda35db088def9bfa656872d40908adc289b4daa Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:34:45 -0700 Subject: [PATCH 01/37] Let model tests outlast the Studio bridge deadline and stop them when the browser leaves --- src/firefly_weave/studio/service.py | 104 +++++++++------ tests/unit/test_studio_bridge_deadlines.py | 147 +++++++++++++++++++++ 2 files changed, 210 insertions(+), 41 deletions(-) create mode 100644 tests/unit/test_studio_bridge_deadlines.py diff --git a/src/firefly_weave/studio/service.py b/src/firefly_weave/studio/service.py index 0116a265..429af7c3 100644 --- a/src/firefly_weave/studio/service.py +++ b/src/firefly_weave/studio/service.py @@ -23,9 +23,10 @@ import re import secrets import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from pathlib import Path +from types import MappingProxyType from typing import TYPE_CHECKING, Any from urllib.parse import urlsplit from uuid import UUID @@ -39,6 +40,7 @@ from firefly_weave import __version__ from firefly_weave.contracts.connector_descriptors import ADAPTER from firefly_weave.contracts.surface import OPERATIONS +from firefly_weave.operations.ephemeral import until_disconnect from firefly_weave.sdk.auth import AuthError from firefly_weave.sdk.credentials import CredentialError @@ -49,6 +51,11 @@ BODY_LIMIT = 2 * 1024 * 1024 RESPONSE_LIMIT = 8 * 1024 * 1024 COOKIE = "weave_studio_session" +# Seconds the bridge waits for the platform. A model test calls the model itself, and a local model +# that loads for the first time can take two minutes; the platform stops that test after 130 seconds, +# so the bridge waits a little longer for its answer and cancels it if the browser leaves first. +BRIDGE_SECONDS = 35.0 +LONG_OPERATIONS: Mapping[str, float] = MappingProxyType({"ai_connections.test": 140.0}) STUDIO_FAMILIES = frozenset( { "identity", @@ -252,6 +259,10 @@ def pair(self, code: str) -> Response: return response def check_scope(self, path: str, method: str) -> int: + return self.match(path, method)[0] + + def match(self, path: str, method: str) -> tuple[int, str | None]: + """The bridge status for a path and the operation it names; None when no operation matches.""" profile = self.options.profile assert profile is not None for operation in OPERATIONS.values(): @@ -273,13 +284,13 @@ def check_scope(self, path: str, method: str) -> int: ("environment", profile.environment_id), ): if name in values and (selected is None or values[name] != str(selected)): - return 403 + return 403, operation.id # Variable segments are resource IDs, the finite catalog kind or a bounded adapter # name. A segment that fits another operation's template keeps matching (for example # /connector-descriptors also fits the {collection} template). if all(self.segment(name, value) for name, value in values.items()): - return 200 - return 404 + return 200, operation.id + return 404, None @staticmethod def segment(name: str, value: str) -> bool: @@ -300,7 +311,7 @@ async def bridge(self, request: Request, path: str) -> Response: return problem(409, "WV-STUDIO-OFFLINE", "Connect to a platform in Settings to use connected operations") if any(part in path for part in ("%", "\\", "?", "#", "..")): return problem(404, "WV-STUDIO-ROUTE", "Unknown platform operation") - status = self.check_scope(path, request.method) + status, operation = self.match(path, request.method) if status != 200: return problem(status, "WV-STUDIO-ROUTE", "Operation is outside the selected profile or API surface") try: @@ -329,44 +340,55 @@ async def bridge(self, request: Request, path: str) -> Response: headers[name] = request.headers[name] # Do not inherit upstream cookie authentication across browser operations. self.client.cookies.clear() - async with asyncio.timeout(35): - async with self.client.stream( - request.method, - profile.base_url.rstrip("/") + path, - params=request.query_params, - headers=headers, - content=await request.body(), - ) as upstream: - if upstream.is_redirect: - return problem(502, "WV-STUDIO-REDIRECT", "The platform returned an unexpected redirect") - if upstream.headers.get("content-encoding", "identity").strip().lower() != "identity": - return problem(502, "WV-STUDIO-ENCODING", "Platform response must use identity encoding") - body = bytearray() - async for chunk in upstream.aiter_bytes(): - body.extend(chunk) - if len(body) > RESPONSE_LIMIT: + long = LONG_OPERATIONS.get(operation or "") + seconds = long if long is not None else BRIDGE_SECONDS + content = await request.body() + + async def forward() -> Response: + async with asyncio.timeout(seconds): + async with self.client.stream( + request.method, + profile.base_url.rstrip("/") + path, + params=request.query_params, + headers=headers, + content=content, + timeout=seconds if long is not None else httpx.USE_CLIENT_DEFAULT, + ) as upstream: + if upstream.is_redirect: + return problem(502, "WV-STUDIO-REDIRECT", "The platform returned an unexpected redirect") + if upstream.headers.get("content-encoding", "identity").strip().lower() != "identity": + return problem(502, "WV-STUDIO-ENCODING", "Platform response must use identity encoding") + body = bytearray() + async for chunk in upstream.aiter_bytes(): + body.extend(chunk) + if len(body) > RESPONSE_LIMIT: + return problem( + 502, + "WV-STUDIO-RESPONSE-LIMIT", + "Platform response exceeds Studio's limit; narrow the query", + ) + if ( + self.connection_generation != generation + or self.options.profile is not profile + or self.options.token_provider is not provider + ): return problem( - 502, - "WV-STUDIO-RESPONSE-LIMIT", - "Platform response exceeds Studio's limit; narrow the query", + 503, + "WV-STUDIO-CONNECTION-CHANGED", + "Connection changed; discard the previous result. " + "A submitted command may have completed in its original scope", ) - if ( - self.connection_generation != generation - or self.options.profile is not profile - or self.options.token_provider is not provider - ): - return problem( - 503, - "WV-STUDIO-CONNECTION-CHANGED", - "Connection changed; discard the previous result. " - "A submitted command may have completed in its original scope", - ) - safe_headers = { - key: value - for key, value in upstream.headers.items() - if key.lower() in {"content-type", "etag", "retry-after"} - } - return Response(bytes(body), status_code=upstream.status_code, headers=safe_headers) + safe_headers = { + key: value + for key, value in upstream.headers.items() + if key.lower() in {"content-type", "etag", "retry-after"} + } + return Response(bytes(body), status_code=upstream.status_code, headers=safe_headers) + + if long is not None: + # A person can cancel a long model test; the platform call stops when the browser leaves. + return await until_disconnect(forward(), request.receive) + return await forward() except (AuthError, CredentialError): return problem(401, "WV-AUTH-REQUIRED", "Sign in again with weave auth login using the selected profile") except (httpx.HTTPError, TimeoutError): diff --git a/tests/unit/test_studio_bridge_deadlines.py b/tests/unit/test_studio_bridge_deadlines.py new file mode 100644 index 00000000..4ffc63e8 --- /dev/null +++ b/tests/unit/test_studio_bridge_deadlines.py @@ -0,0 +1,147 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""A model test may take longer than other platform calls, and stops when its browser leaves.""" + +import asyncio +from pathlib import Path +from types import MappingProxyType + +import httpx +import pytest +from starlette.requests import Request +from starlette.testclient import TestClient + +from firefly_weave.studio import service as studio_service +from firefly_weave.studio.host import make_studio_app +from firefly_weave.studio.service import StudioOptions, StudioProfile, StudioService + +ORIGIN = "http://127.0.0.1:8766" +CODE = "terminal-only-pairing-code" +TENANT = "00000000-0000-0000-0000-000000000001" +PROJECT_ID = "00000000-0000-0000-0000-000000000002" +ENVIRONMENT_ID = "00000000-0000-0000-0000-000000000003" +REVISION = "00000000-0000-0000-0000-0000000000aa" +ENVIRONMENT = f"/api/v1/tenants/{TENANT}/projects/{PROJECT_ID}/environments/{ENVIRONMENT_ID}" +MODEL_TEST = f"{ENVIRONMENT}/ai/connections/{REVISION}/test" +READINESS = f"{ENVIRONMENT}/ai/readiness" +PROFILE = StudioProfile( + name="Dev", base_url="https://api.example", tenant_id=TENANT, project_id=PROJECT_ID, environment_id=ENVIRONMENT_ID +) + + +def assets(tmp_path: Path) -> Path: + folder = tmp_path / "assets" + folder.mkdir() + (folder / "index.html").write_text("Weave Studio") + return folder + + +def paired(tmp_path: Path, upstream) -> TestClient: + browser = TestClient( + make_studio_app( + StudioOptions( + origin=ORIGIN, + assets=assets(tmp_path), + pairing_code=CODE, + profile=PROFILE, + token_provider=lambda: "host-token", + transport=httpx.MockTransport(upstream), + ) + ), + base_url=ORIGIN, + ) + response = browser.post("/studio/session", json={"code": CODE}, headers={"Origin": ORIGIN}) + assert response.status_code == 200 + browser.headers.update({"Origin": ORIGIN, "X-Weave-CSRF": response.json()["csrfToken"]}) + return browser + + +def test_the_bridge_names_the_operation_a_path_matches(tmp_path): + service = StudioService(StudioOptions(origin=ORIGIN, assets=assets(tmp_path), profile=PROFILE)) + assert service.match(MODEL_TEST, "POST") == (200, "ai_connections.test") + assert service.match(READINESS, "GET") == (200, "ai_readiness.read") + other = MODEL_TEST.replace(ENVIRONMENT_ID, "00000000-0000-0000-0000-000000000009") + assert service.match(other, "POST") == (403, "ai_connections.test") + assert service.match("/api/v1/unknown", "GET") == (404, None) + assert service.check_scope(MODEL_TEST, "POST") == 200 + + +def test_a_model_test_may_outlast_the_ordinary_deadline(tmp_path, monkeypatch): + monkeypatch.setattr(studio_service, "BRIDGE_SECONDS", 0.2) + monkeypatch.setattr(studio_service, "LONG_OPERATIONS", MappingProxyType({"ai_connections.test": 3.0})) + + async def upstream(request: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.5) + if request.url.path.endswith("/test"): + return httpx.Response(200, json={"ok": True, "code": "ok", "latency_ms": 500, "model": "qwen3:4b"}) + return httpx.Response(200, json={"items": []}) + + with paired(tmp_path, upstream) as browser: + tested = browser.post("/studio/api" + MODEL_TEST, json={"model": "qwen3:4b", "probe_tools": True}) + assert tested.status_code == 200, tested.text + assert tested.json()["model"] == "qwen3:4b" + slow = browser.get("/studio/api" + READINESS) + assert slow.status_code == 502 + assert slow.json()["code"] == "WV-STUDIO-UPSTREAM" + + +async def test_a_model_test_stops_when_the_browser_leaves(tmp_path): + started, cancelled = asyncio.Event(), asyncio.Event() + + async def upstream(request: httpx.Request) -> httpx.Response: + started.set() + try: + await asyncio.sleep(60) + except asyncio.CancelledError: + cancelled.set() + raise + return httpx.Response(200, json={}) + + service = StudioService( + StudioOptions( + origin=ORIGIN, + assets=assets(tmp_path), + profile=PROFILE, + token_provider=lambda: "host-token", + transport=httpx.MockTransport(upstream), + ) + ) + messages = [{"type": "http.request", "body": b'{"model":"qwen3:4b","probe_tools":true}', "more_body": False}] + + async def receive(): + if messages: + return messages.pop(0) + await started.wait() + return {"type": "http.disconnect"} + + scope = { + "type": "http", + "method": "POST", + "path": "/studio/api" + MODEL_TEST, + "raw_path": ("/studio/api" + MODEL_TEST).encode(), + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + "scheme": "http", + "server": ("127.0.0.1", 8766), + "root_path": "", + } + try: + with pytest.raises(asyncio.CancelledError): + await service.bridge(Request(scope, receive), MODEL_TEST) + assert cancelled.is_set() + finally: + await service.close() From 231e69319dc13e19523a0f180e24c1c3de41af87 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:48:52 -0700 Subject: [PATCH 02/37] Give the local owner every setup role and add weave platform user grant --- src/firefly_weave/cli/platform.py | 68 ++++++++-- src/firefly_weave/sdk/platform.py | 153 +++++++++++++++++++++- tests/acceptance/journeys.toml | 2 +- tests/acceptance/test_j0_bring_up.py | 10 +- tests/unit/cli/test_platform_docker.py | 26 ++++ tests/unit/cli/test_platform_sign_in.py | 160 ++++++++++++++++++++++++ tests/unit/test_acceptance_journeys.py | 4 +- 7 files changed, 409 insertions(+), 14 deletions(-) diff --git a/src/firefly_weave/cli/platform.py b/src/firefly_weave/cli/platform.py index 22fcbbbd..431103c9 100644 --- a/src/firefly_weave/cli/platform.py +++ b/src/firefly_weave/cli/platform.py @@ -23,7 +23,7 @@ import sys from collections.abc import Callable from pathlib import Path -from typing import Any +from typing import Any, NoReturn import click @@ -223,10 +223,26 @@ def level(scope: dict[str, str]) -> str: click.echo(" " + value["next"][1]) -@platform.command() +def _usage(message: str, output: str) -> NoReturn: + # The CLI hides click's own usage messages because they may echo option values; this fixed text has none. + emit_result(error_result("WV-CLI-USAGE", message), output) + raise SystemExit(EXIT_USAGE) + + +def _show_grants(value: dict[str, Any]) -> None: + def level(scope: dict[str, str]) -> str: + return "environment" if "environment_id" in scope else "project" if "project_id" in scope else "tenant" + + click.echo( + f"Granted to {value['username']}: " + + ", ".join(f"{item['role']} (demo {level(item['scope'])})" for item in value["grants"]) + ) + click.echo("Sign in again in Studio or the CLI to use the new roles.") + + +@platform.group(invoke_without_command=True) @click.option( "--username", - required=True, help="New sign-in name: 3 to 64 lowercase letters and digits, optionally joined by '.', '_', or '-'.", ) @click.option( @@ -239,8 +255,8 @@ def level(scope: dict[str, str]) -> str: "One of: " + ", ".join(lifecycle.PERSON_ROLES) + ".", ) @click.option("--output", type=click.Choice(["text", "json"]), default="text") -@click.pass_obj -def user(directory: Path, username: str, roles: tuple[str, ...], output: str) -> None: +@click.pass_context +def user(context: click.Context, username: str | None, roles: tuple[str, ...], output: str) -> None: """Create a person who can sign in to this local platform (development only). Creates an account in the owned Keycloak with a generated password, a @@ -251,8 +267,45 @@ def user(directory: Path, username: str, roles: tuple[str, ...], output: str) -> Default roles are the smallest set to author, publish, activate, run, inspect runs, and work on human tasks: developer on the demo project, and deployer, operator, viewer, and task_participant on the demo environment. + Use `user grant` to add roles to an account later. + """ + if context.invoked_subcommand is not None: + if username is not None or roles: + _usage( + "Put --username and --role after the subcommand: " + "weave platform user grant --username NAME --role ROLE.", + output, + ) + return + if username is None: + _usage("Missing option '--username'. Use --help for supported arguments.", output) + directory: Path = context.obj + name: str = username + _call(lambda: lifecycle.user(directory, name, roles), output, _show_person) + + +@user.command("grant") +@click.option("--username", required=True, help="An account this platform created with `user` or `up`.") +@click.option( + "--role", + "roles", + multiple=True, + required=True, + type=click.Choice(lifecycle.PERSON_ROLES), + metavar="ROLE", + help="Role to add in the demo workspace; repeat for several roles. One of: " + + ", ".join(lifecycle.PERSON_ROLES) + + ".", +) +@click.option("--output", type=click.Choice(["text", "json"]), default="text") +@click.pass_obj +def user_grant(directory: Path, username: str, roles: tuple[str, ...], output: str) -> None: + """Add roles in the demo workspace to an existing local account (development only). + + Grants are added, never removed, and are scoped like the roles `user` gives. + The account must have been created by this platform's `user` or `up`. """ - _call(lambda: lifecycle.user(directory, username, roles), output, _show_person) + _call(lambda: lifecycle.grant(directory, username, roles), output, _show_grants) @platform.group() @@ -656,7 +709,8 @@ def ai_models_remove(directory: Path, provider: str, model: str, output: str) -> "roles", multiple=True, type=click.Choice(lifecycle.PERSON_ROLES), - help="Role for the initial account; repeat for several roles. Requires --username.", + help="Role for the initial account; repeat for several roles. Requires --username. Without --role the " + "account gets the local owner roles: " + ", ".join(lifecycle.LOCAL_OWNER_ROLES) + ".", ) @click.option( "--allow-private-origin", diff --git a/src/firefly_weave/sdk/platform.py b/src/firefly_weave/sdk/platform.py index e96ecf7e..416dacf5 100644 --- a/src/firefly_weave/sdk/platform.py +++ b/src/firefly_weave/sdk/platform.py @@ -107,6 +107,26 @@ "email_manager", "execution_manager", ) +# The account `up --username` creates gets every role a local owner needs to set up the demo +# workspace from Studio: AI connections, Weave AI, executions, workers and deployments. Other +# accounts keep the defaults above; `user grant` adds roles to any account this platform created. +LOCAL_OWNER_ROLES = ( + "developer", + "deployer", + "operator", + "viewer", + "task_participant", + "tenant_admin", + "lumi_manager", + "execution_manager", + "worker_operator", + "deployment_reader", + "deployment_planner", + "deployment_approver", + "deployment_operator", +) +_PEOPLE = "people.json" +_PEOPLE_FORMAT = "weave/local-people-v1" class PlatformError(ValueError): @@ -1514,8 +1534,8 @@ def _check_username(username: str) -> None: ) -def _person_roles(roles: Sequence[str]) -> tuple[str, ...]: - selected = tuple(roles) or DEFAULT_PERSON_ROLES +def _person_roles(roles: Sequence[str], default: Sequence[str] = DEFAULT_PERSON_ROLES) -> tuple[str, ...]: + selected = tuple(roles) or tuple(default) if len(set(selected)) != len(selected) or any(role not in PERSON_ROLES for role in selected): raise PlatformError("Choose each role at most once from: " + ", ".join(PERSON_ROLES) + ".") return selected @@ -1533,6 +1553,29 @@ def _person_grants(roles: tuple[str, ...], scope: dict[str, str]) -> list[dict[s ] +def _people(directory: Path) -> dict[str, Any]: + path = directory / _PEOPLE + if not path.exists(): + return {"format": _PEOPLE_FORMAT, "people": {}} + value = strict_json(read_file(path, 1048576, private=True)) + if ( + not isinstance(value, dict) + or value.get("format") != _PEOPLE_FORMAT + or not isinstance(value.get("people"), dict) + ): + raise PlatformError("The private record of local accounts is not readable; it was not changed.") + return value + + +def _remember_person(directory: Path, username: str, principal: str, subject: str) -> None: + """Record which Weave person a local sign-in belongs to, so later role grants find it.""" + with _exclusive(directory, ".people.lock", "Another platform user command is active."): + path = directory / _PEOPLE + value = _people(directory) + value["people"][username] = {"principal_id": principal, "subject": subject} + _write(path, value, replace=path.exists()) + + def _exists(username: str) -> str: return ( f"A local sign-in account named {username} already exists. " @@ -1776,6 +1819,7 @@ def user( import asyncio principal, subject = asyncio.run(_enroll(state, username, password, grants, host_token, admin_secret, transport)) + _remember_person(directory, username, principal, subject) return { "ok": True, "username": username, @@ -1792,6 +1836,109 @@ def user( } +async def _grant_roles( + state: dict[str, Any], + username: str, + person: dict[str, Any], + grants: list[dict[str, Any]], + host_token: str, + admin_secret: str, + transport: httpx.AsyncBaseTransport | None, +) -> None: + import httpx + + from firefly_weave.contracts.access import Scope + from firefly_weave.contracts.members import MemberGrantRequest, PrincipalIdentityRequest + from firefly_weave.sdk.client import WeaveClient + from firefly_weave.sdk.errors import WeaveError + from firefly_weave.sdk.members import MembersClient + + keycloak = f"http://localhost:{state['ports']['keycloak']}" + users = keycloak + "/admin/realms/weave/users" + tenant = Scope(tenant_id=UUID(grants[0]["scope"]["tenant_id"])) + async with ( + httpx.AsyncClient(timeout=10, trust_env=False, follow_redirects=False, transport=transport) as identity, + WeaveClient(_summary(state)["api_url"], lambda: host_token, tenant, timeout=15, transport=transport) as client, + ): + admin = await _keycloak_admin(identity, keycloak, admin_secret) + if await _keycloak_user(identity, users, admin, username) != person.get("subject"): + raise PlatformError( + f"The local sign-in account {username} is not the one this platform recorded; no role was granted." + ) + members = MembersClient(client) + principal = UUID(str(person["principal_id"])) + try: + # Linking the recorded sign-in again changes nothing, and is refused if it belongs to another person. + await members.link_identity( + principal, + PrincipalIdentityRequest( + provider_id=_PROVIDER_ID, issuer=keycloak + "/realms/weave", subject=str(person["subject"]) + ), + ) + for item in grants: + scope = { + key: UUID(item["scope"][key]) for key in ("project_id", "environment_id") if key in item["scope"] + } + await members.grant( + MemberGrantRequest.model_validate({"principal_id": principal, "role": item["role"], **scope}) + ) + except WeaveError as error: + raise PlatformError( + f"The local API refused the role grant ({error.code}). Roles granted before the refusal stay in place." + ) from None + + +def grant( + directory: Path, + username: str, + roles: Sequence[str], + *, + transport: httpx.AsyncBaseTransport | None = None, +) -> dict[str, Any]: + """Grant more roles in the demo workspace to a person that `user` or `up` created (development only).""" + _check_username(username) + if not roles: + raise PlatformError("Choose at least one role with --role.") + selected = _person_roles(roles) + if not (directory / "platform.json").exists(): + raise PlatformError("No local platform exists here yet. Create it first: " + _command(directory, "setup")) + state = _load(directory) + if not (directory / "first-run.json").exists(): + raise PlatformError("Create the demo workspace first: " + _command(directory, "demo")) + person = _people(directory)["people"].get(username) + if not isinstance(person, dict): + raise PlatformError( + f"This platform has no record of {username}. Accounts created before this version are not recorded: " + "assign roles in Studio under Settings › People and access, or create a new account with " + + _command(directory, "user --username NAME") + ) + grants = _person_grants(selected, _demo_receipt(directory)["scope"]) + try: + import httpx # noqa: F401 + except ModuleNotFoundError: + raise PlatformError("Granting roles requires the client extra: install firefly-weave[client].") from None + _check_engine(state) + _verify_runtime(state) + api = _summary(state)["api_url"] + if not _probe(api + "/health/ready"): + raise PlatformError("Start the API in another terminal (" + _command(directory, "start") + "), then retry.") + admin_secret = _env_file(directory / "identity.env").get("WEAVE_KC_ADMIN_SECRET") + if not admin_secret: + raise PlatformError("Private identity configuration is incomplete; retain it for recovery.") + _token(state) + host_token = strict_json(read_file(directory / "host-token.json", 65536, private=True))["access_token"] + import asyncio + + asyncio.run(_grant_roles(state, username, person, grants, host_token, admin_secret, transport)) + return { + "ok": True, + "username": username, + "principal_id": str(person["principal_id"]), + "grants": grants, + "development_only": True, + } + + def up( directory: Path, source: Path, @@ -1808,7 +1955,7 @@ def up( if roles and username is None: raise PlatformError("Account roles require --username.") - selected = _person_roles(roles) + selected = _person_roles(roles, LOCAL_OWNER_ROLES) if username is not None: _check_username(username) origins = platform_origins.requested(private_origins) diff --git a/tests/acceptance/journeys.toml b/tests/acceptance/journeys.toml index da851f35..6140af9c 100644 --- a/tests/acceptance/journeys.toml +++ b/tests/acceptance/journeys.toml @@ -52,7 +52,7 @@ "language-docs" = false "ai-gateway" = true "ai-setup" = false -"ai-models-screen" = false +"ai-models-screen" = true "ai-task" = false "weave-ai-ollama" = false "agents" = false diff --git a/tests/acceptance/test_j0_bring_up.py b/tests/acceptance/test_j0_bring_up.py index ad256739..90189220 100644 --- a/tests/acceptance/test_j0_bring_up.py +++ b/tests/acceptance/test_j0_bring_up.py @@ -29,7 +29,7 @@ import pytest from firefly_weave import __version__ -from firefly_weave.sdk.platform import PERSON_ROLES +from firefly_weave.sdk.platform import LOCAL_OWNER_ROLES, PERSON_ROLES pytestmark = pytest.mark.acceptance @@ -67,6 +67,10 @@ def ready(api_url: str, timeout: float = 60) -> bool: return False +def _expect_roles(grants: list[dict[str, object]], roles: tuple[str, ...]) -> None: + assert sorted({str(grant["role"]) for grant in grants}) == sorted(roles) + + @pytest.mark.journey_step("J0.1") def test_j0_01_version(run): assert run.require(run.weave("--version")).stdout == f"Firefly Weave {__version__}\n" @@ -101,6 +105,10 @@ def test_j0_03_up_with_the_fixture_origin(run): for purpose in ("event-delivery", "http-connector") ], } + run.check( + "J0.3/owner-roles", + lambda: _expect_roles(value["account"]["grants"], LOCAL_OWNER_ROLES), + ) @pytest.mark.journey_step("J0.4") diff --git a/tests/unit/cli/test_platform_docker.py b/tests/unit/cli/test_platform_docker.py index f2315fcb..981e5337 100644 --- a/tests/unit/cli/test_platform_docker.py +++ b/tests/unit/cli/test_platform_docker.py @@ -221,6 +221,32 @@ def setup(*args, **kwargs): assert calls == ["setup", "start", "user", "start"] +def test_up_gives_its_new_account_the_local_owner_roles(tmp_path, monkeypatch): + directory = tmp_path / "owner" + state = { + "mode": "docker", + "context": "owned", + "directory": str(directory), + "stage": "ready", + "id": "d" * 24, + "ports": {"container_api": 55201}, + } + seen = [] + + def setup(*args, **kwargs): + directory.mkdir(mode=0o700) + + monkeypatch.setattr(platform, "setup", setup) + monkeypatch.setattr(platform, "_load", lambda p: state) + monkeypatch.setattr(platform, "start", lambda *a: None) + monkeypatch.setattr(platform, "demo", lambda *a: {"receipt": {"status": "succeeded"}}) + monkeypatch.setattr(platform, "user", lambda d, u, roles: seen.append(tuple(roles)) or {"username": u}) + platform.up(directory, tmp_path, "owned", username="owner") + assert seen == [platform.LOCAL_OWNER_ROLES] + receipt = json.loads((directory / "up-user.json").read_text()) + assert receipt["roles"] == list(platform.LOCAL_OWNER_ROLES) + + def test_uncertain_account_creation_blocks_replay(owned, monkeypatch): directory, state = owned platform._write(directory / "up-user.json", {"stage": "attempted", "username": "developer"}) diff --git a/tests/unit/cli/test_platform_sign_in.py b/tests/unit/cli/test_platform_sign_in.py index ffefbb43..b80b7ad0 100644 --- a/tests/unit/cli/test_platform_sign_in.py +++ b/tests/unit/cli/test_platform_sign_in.py @@ -710,3 +710,163 @@ def test_cli_help_documents_development_only_default_roles(): assert "development only" in text.lower() for role in ("developer", "deployer", "operator", "viewer", "task_participant"): assert role in text + + +def test_local_owner_roles_cover_every_setup_screen_and_stay_person_roles(): + assert set(platform.LOCAL_OWNER_ROLES) <= set(platform.PERSON_ROLES) + assert set(platform.DEFAULT_PERSON_ROLES) < set(platform.LOCAL_OWNER_ROLES) + for role in ("tenant_admin", "lumi_manager", "execution_manager", "worker_operator", "deployment_operator"): + assert role in platform.LOCAL_OWNER_ROLES + assert not set(platform.LOCAL_OWNER_ROLES) & {"platform_admin", "worker", "deployment_runner"} + + +def test_user_records_the_account_for_later_grants(demo_workspace): + directory, _, _ = demo_workspace + value = platform.user(directory, "ada.lovelace", transport=httpx.MockTransport(Identity())) + path = directory / "people.json" + assert path.stat().st_mode & 0o777 == 0o600 + record = json.loads(path.read_text()) + assert record == { + "format": "weave/local-people-v1", + "people": {"ada.lovelace": {"principal_id": PRINCIPAL, "subject": SUBJECT}}, + } + assert value["password"] not in path.read_text() + + +def _recorded(directory, username="ada.lovelace"): + platform._write( + directory / "people.json", + {"format": "weave/local-people-v1", "people": {username: {"principal_id": PRINCIPAL, "subject": SUBJECT}}}, + ) + + +def test_grant_adds_roles_to_a_recorded_account_after_proving_its_sign_in(demo_workspace): + directory, _, _ = demo_workspace + _recorded(directory) + identity = Identity() + identity.users["ada.lovelace"] = SUBJECT + value = platform.grant( + directory, "ada.lovelace", ("tenant_admin", "lumi_manager"), transport=httpx.MockTransport(identity) + ) + assert identity.paths() == [ + ("POST", "/realms/master/protocol/openid-connect/token"), + ("GET", "/admin/realms/weave/users"), + ("POST", f"/api/v1/admin/principals/{PRINCIPAL}/identity-links"), + *[("POST", f"/api/v1/tenants/{TENANT}/members")] * 2, + ] + link = json.loads(identity.requests[2].content) + assert link == {"provider_id": "local-keycloak", "issuer": KEYCLOAK + "/realms/weave", "subject": SUBJECT} + grants = [json.loads(request.content) for request in identity.requests[3:]] + assert [(g["role"], g["project_id"], g["environment_id"]) for g in grants] == [ + ("tenant_admin", None, None), + ("lumi_manager", PROJECT, ENVIRONMENT), + ] + assert value == { + "ok": True, + "username": "ada.lovelace", + "principal_id": PRINCIPAL, + "grants": [ + {"role": "tenant_admin", "scope": {"tenant_id": TENANT}}, + { + "role": "lumi_manager", + "scope": {"tenant_id": TENANT, "project_id": PROJECT, "environment_id": ENVIRONMENT}, + }, + ], + "development_only": True, + } + + +def test_grant_refuses_an_unrecorded_account_before_any_request(demo_workspace): + directory, _, _ = demo_workspace + + def refuse(request): + raise AssertionError("no request may be sent") + + with pytest.raises(platform.PlatformError, match="no record of grace"): + platform.grant(directory, "grace", ("viewer",), transport=httpx.MockTransport(refuse)) + + +def test_grant_refuses_a_replaced_sign_in_account(demo_workspace): + directory, _, _ = demo_workspace + _recorded(directory) + identity = Identity() + identity.users["ada.lovelace"] = "66666666-6666-4666-8666-666666666666" + with pytest.raises(platform.PlatformError, match="not the one this platform recorded"): + platform.grant(directory, "ada.lovelace", ("viewer",), transport=httpx.MockTransport(identity)) + assert ("POST", f"/api/v1/tenants/{TENANT}/members") not in identity.paths() + + +def test_grant_refuses_when_the_sign_in_belongs_to_another_principal(demo_workspace): + directory, _, _ = demo_workspace + _recorded(directory) + identity = Identity(link_status=409) + identity.users["ada.lovelace"] = SUBJECT + with pytest.raises(platform.PlatformError, match="refused the role grant"): + platform.grant(directory, "ada.lovelace", ("viewer",), transport=httpx.MockTransport(identity)) + assert ("POST", f"/api/v1/tenants/{TENANT}/members") not in identity.paths() + + +@pytest.mark.parametrize( + ("roles", "message"), + [((), "at least one role"), (("platform_admin",), "Choose each role"), (("viewer", "viewer"), "Choose each role")], +) +def test_grant_checks_roles_before_any_request(demo_workspace, roles, message): + directory, _, _ = demo_workspace + _recorded(directory) + with pytest.raises(platform.PlatformError, match=message): + platform.grant( + directory, "ada.lovelace", roles, transport=httpx.MockTransport(lambda request: pytest.fail("sent")) + ) + + +def test_cli_user_grant_prints_the_grants(demo_workspace, monkeypatch): + directory, _, _ = demo_workspace + _recorded(directory) + identity = Identity() + identity.users["ada.lovelace"] = SUBJECT + monkeypatch.setattr(platform, "grant", functools.partial(platform.grant, transport=httpx.MockTransport(identity))) + result = CliRunner().invoke( + cli, + [ + "platform", + "--directory", + str(directory), + "user", + "grant", + "--username", + "ada.lovelace", + "--role", + "tenant_admin", + ], + ) + assert result.exit_code == 0, result.output + assert "Granted to ada.lovelace: tenant_admin (demo tenant)" in result.output + + +def test_cli_user_options_belong_after_the_grant_subcommand(demo_workspace): + directory, _, _ = demo_workspace + result = CliRunner().invoke( + cli, + [ + "platform", + "--directory", + str(directory), + "user", + "--username", + "ada", + "grant", + "--username", + "ada", + "--role", + "viewer", + ], + ) + assert result.exit_code == 2 + assert "after the subcommand" in result.output + + +def test_cli_user_without_a_username_or_subcommand_is_a_usage_error(demo_workspace): + directory, _, _ = demo_workspace + result = CliRunner().invoke(cli, ["platform", "--directory", str(directory), "user"]) + assert result.exit_code == 2 + assert "--username" in result.output diff --git a/tests/unit/test_acceptance_journeys.py b/tests/unit/test_acceptance_journeys.py index 5780c377..6a3660db 100644 --- a/tests/unit/test_acceptance_journeys.py +++ b/tests/unit/test_acceptance_journeys.py @@ -129,12 +129,12 @@ def test_the_committed_map_is_valid(): enablement = journeys.Enablement.load(ROOT / "tests/acceptance/journeys.toml") assert enablement.capabilities["acceptance-foundations"] is True landed = [name for name, value in enablement.capabilities.items() if value] - assert landed == ["text-operators", "ai-gateway", "acceptance-foundations"] + assert landed == ["text-operators", "ai-gateway", "ai-models-screen", "acceptance-foundations"] assert enablement.journey_steps("J0") == tuple(f"J0.{n}" for n in range(1, 15)) assert enablement.profiles["pr"] == ("J0", "J1", "J2", "J3", "J5", "J8", "J9") assert enablement.criteria["J3"] == ("SC1", "SC3", "SC7") assert enablement.applies("J0.12", "ai") and not enablement.applies("J0.12", "pr") - assert enablement.missing("J0.3/owner-roles") == ("ai-models-screen|compose-operations",) + assert enablement.missing("J0.3/owner-roles") == () def test_versions_toml_pins_match_the_files_that_use_them(): From 31ba4db5ccdcdac9c04aba525b3e435cc8f7ff69 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:48:52 -0700 Subject: [PATCH 03/37] Add cancellable Studio requests and a typed client for AI setup --- studio/src/app/api.ts | 31 +- .../integrations/ai-provider-connection.ts | 20 ++ studio/src/app/integrations/ai-setup-api.ts | 273 ++++++++++++++++++ studio/tests/ai-provider-connection.test.ts | 28 ++ studio/tests/api.test.ts | 58 ++++ 5 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 studio/src/app/integrations/ai-setup-api.ts diff --git a/studio/src/app/api.ts b/studio/src/app/api.ts index 93558e93..bc627b99 100644 --- a/studio/src/app/api.ts +++ b/studio/src/app/api.ts @@ -88,6 +88,30 @@ const replayable = (status: number, code: string, retryAfter: string | null) => (status === 503 && code === "WV-COMPATIBILITY" && !(Number(retryAfter) > longestPause)); +/** A signal that aborts when `signal` does, or after `timeout` milliseconds. */ +function cancellable(signal: AbortSignal, timeout: number): AbortSignal { + const controller = new AbortController(); + const follow = () => controller.abort(signal.reason); + const timer = setTimeout( + () => + controller.abort( + new DOMException("The request took too long.", "TimeoutError"), + ), + timeout, + ); + controller.signal.addEventListener( + "abort", + () => { + clearTimeout(timer); + signal.removeEventListener("abort", follow); + }, + { once: true }, + ); + if (signal.aborted) follow(); + else signal.addEventListener("abort", follow, { once: true }); + return controller.signal; +} + export class StudioApi { session: Session = { paired: false, @@ -103,6 +127,8 @@ export class StudioApi { /** * Calls the paired local host. `timeout` (milliseconds) bounds the wait; * platform checks that reach a remote server or identity provider pass more. + * `signal` lets the caller cancel (a model test); a canceled request is never + * sent again. */ async request( path: string, @@ -110,6 +136,7 @@ export class StudioApi { body?: unknown, headers: Record = {}, timeout = 15000, + signal?: AbortSignal, ): Promise { if (!path.startsWith("/studio/") || path.startsWith("//")) throw Error("Studio requests must use the same-origin host boundary."); @@ -126,7 +153,9 @@ export class StudioApi { credentials: "same-origin", headers: h, body: body === undefined ? undefined : JSON.stringify(body), - signal: AbortSignal.timeout(timeout), + signal: signal + ? cancellable(signal, timeout) + : AbortSignal.timeout(timeout), }); const value: unknown = await response .json() diff --git a/studio/src/app/integrations/ai-provider-connection.ts b/studio/src/app/integrations/ai-provider-connection.ts index 3305a14f..50a7a93f 100644 --- a/studio/src/app/integrations/ai-provider-connection.ts +++ b/studio/src/app/integrations/ai-provider-connection.ts @@ -24,6 +24,26 @@ export interface AiConnection { revision: number; connector: string; config: { provider: string; endpoint: string; apiVersion?: string }; + /** The secret handle by slot; never a secret value. */ + secretRef?: Record; +} +/** The secret handles of a connection revision, whichever spelling the platform used. */ +export function secretRefs(connection: AiConnection): Record { + const value = + connection.secretRef ?? + (connection as unknown as { secret_refs?: Record }) + .secret_refs; + return value && typeof value === "object" ? value : {}; +} +/** The newest revision of each connection name, sorted by name. */ +export function latestByName(connections: AiConnection[]): AiConnection[] { + const latest = new Map(); + for (const connection of connections) { + const known = latest.get(connection.name); + if (!known || connection.revision > known.revision) + latest.set(connection.name, connection); + } + return [...latest.values()].sort((a, b) => a.name.localeCompare(b.name)); } export interface AiConnectionDraft { name: string; diff --git a/studio/src/app/integrations/ai-setup-api.ts b/studio/src/app/integrations/ai-setup-api.ts new file mode 100644 index 00000000..0fcfe47e --- /dev/null +++ b/studio/src/app/integrations/ai-setup-api.ts @@ -0,0 +1,273 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// Calls to the platform's AI setup operations. The types mirror the platform's +// payloads and keep their snake_case. The platform checks every permission; +// Studio only shows what a person may do. +import { ApiError, type StudioApi } from "../api"; +import { + aiConnections, + aiConnector, + aiSetupRows, + latestByName, + type AiConnection, +} from "./ai-provider-connection"; + +export type AiProvider = + | "openai-chat" + | "openai-responses" + | "azure-chat" + | "azure-responses" + | "anthropic"; +export interface AiEndpoint { + id: string; + label: string; + url: string; + providers: AiProvider[]; + compat: "ollama" | null; + credential: "none" | "required"; + models: "served" | "listed"; + context_tokens: number | null; + development_only: boolean; +} +export interface AiEndpoints { + policy: "loaded" | "absent"; + endpoints: AiEndpoint[]; +} +export interface AiModel { + name: string; + approved: boolean; + available: boolean; + tools: "yes" | "no" | "unknown"; + context_tokens?: number; + size_bytes?: number; + family?: string; +} +export interface AiModels { + approval: "served" | "listed" | "unknown"; + discovery: string; + discovered_at: string | null; + models: AiModel[]; +} +export interface AiTestResult { + ok: boolean; + code: string; + latency_ms: number | null; + model: string; + tool_calling: "supported" | "unsupported" | "unknown"; + context_tokens?: number; +} +export type ReadinessId = + | "worker_installed" + | "actions_published" + | "worker_online" + | "connection" + | "key_available" + | "connection_authorized" + | "model_responds" + | "weave_ai"; +export type ReadinessStatus = "done" | "action" | "not_needed" | "unknown"; +export interface ReadinessFix { + kind: "operation" | "command" | "navigate" | "ask"; + label: string; + operation?: + | "ai_setup.publish" + | "ai_setup.grant" + | "ai_connections.test" + | "lumi.configuration.write" + | null; + command?: string | null; + route?: "settings/ai" | "operate/workers" | null; +} +export interface ReadinessItem { + id: ReadinessId; + status: ReadinessStatus; + detail: string; + fix?: ReadinessFix; +} +export interface Readiness { + items: ReadinessItem[]; +} +export interface BindableConnection { + connection_id: string; + revision_id: string; + name: string; + connector: string; + label: string; + enabled: boolean; + provider?: string; + endpoint_origin?: string; +} +interface BindablePage { + items: BindableConnection[]; + next_cursor: string | null; +} +export interface ConnectionRequestBody { + name: string; + connector_version_id: string; + config: { + provider: AiProvider; + endpoint: string; + secretSlot: "apiKey"; + apiVersion?: string; + }; + secretRef: { apiKey: string }; + allowed_destinations: string[]; +} +export interface LumiConfiguration { + enabled: boolean; + connection_revision_id: string; + profile: Record; + revision: number; +} +export interface LumiStatus { + configured: boolean; + provider?: string | null; + model?: string | null; +} + +/** A model test: the platform stops it after 130 seconds; Studio waits a little longer. */ +export const TEST_TIMEOUT_MS = 150_000; +/** Model discovery: the AI gateway gives a server 15 seconds. */ +export const DISCOVERY_TIMEOUT_MS = 30_000; + +export class AiSetupApi { + constructor(private readonly api: StudioApi) {} + private get environment() { + return this.api.environment; + } + endpoints() { + return this.api.request(`${this.environment}/ai/endpoints`); + } + models(revision: string, refresh = false, signal?: AbortSignal) { + return this.api.request( + `${this.environment}/ai/models?connection=${encodeURIComponent(revision)}&refresh=${refresh}`, + "GET", + undefined, + {}, + DISCOVERY_TIMEOUT_MS, + signal, + ); + } + readiness() { + return this.api.request(`${this.environment}/ai/readiness`); + } + test(revision: string, model: string, signal?: AbortSignal) { + return this.api.request( + `${this.environment}/ai/connections/${encodeURIComponent(revision)}/test`, + "POST", + { model, probe_tools: true }, + {}, + TEST_TIMEOUT_MS, + signal, + ); + } + publish() { + return this.api.request>( + `${this.environment}/ai/setup/publish`, + "POST", + undefined, + {}, + 60_000, + ); + } + grant(revision: string) { + return this.api.request<{ granted: boolean; not_needed?: boolean }>( + `${this.environment}/ai/setup/grant`, + "POST", + { connection_revision_id: revision }, + ); + } + /** The newest revision of each AI connection (needs connection.manage). */ + async connections(): Promise { + return latestByName( + aiConnections(await aiSetupRows(this.api, "connections", true)), + ); + } + /** AI connections this person may choose (needs connection.bind). */ + async bindable(): Promise { + const items: BindableConnection[] = []; + let cursor: string | null = null; + for (let page = 0; page < 20; page++) { + const query: string = `connector=${encodeURIComponent(aiConnector)}&limit=100${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`; + const result: BindablePage = await this.api.request( + `${this.environment}/bindable-connections?${query}`, + ); + items.push(...result.items); + if (!result.next_cursor) return items; + cursor = result.next_cursor; + } + throw Error( + "Too many connections to list here. Ask an administrator to narrow the environment's configuration.", + ); + } + /** The published AI connector version, or "" before the AI actions are published. */ + async connectorId(): Promise { + const rows = await aiSetupRows(this.api, "connectors", false); + return String( + rows.find( + (row) => + !row["unavailable"] && + !row["retired"] && + `${row["name"]}@${row["version"]}` === aiConnector, + )?.["id"] ?? "", + ); + } + /** Creates a connection revision; a retry with the same key never adds a second one. */ + create(body: ConnectionRequestBody, key: string) { + return this.api.mutate( + `${this.environment}/connections`, + "POST", + body, + undefined, + key, + ); + } + async lumiConfiguration(): Promise { + try { + return await this.api.request( + `${this.environment}/lumi/configuration`, + ); + } catch (error) { + if (error instanceof ApiError && error.status === 404) return null; + throw error; + } + } + saveLumi(body: Record, revision: number | null) { + return this.api.request( + `${this.environment}/lumi/configuration`, + "PUT", + body, + revision === null ? {} : { "If-Match": `"${revision}"` }, + ); + } + lumiStatus() { + return this.api.request(`${this.environment}/lumi/status`); + } + /** The fixed reply schema every Weave AI configuration carries. */ + async lumiReplySchema(): Promise { + const schema = await this.api.request>( + "/studio/contracts/lumi-configuration", + ); + const defs = (schema["$defs"] ?? {}) as Record; + const reply = + defs["LumiProfile"]?.["properties"]?.["outputSchema"]?.["const"]; + if (!reply) + throw Error("The host did not provide the fixed Weave AI reply schema."); + return reply; + } +} diff --git a/studio/tests/ai-provider-connection.test.ts b/studio/tests/ai-provider-connection.test.ts index ed30d50c..b4aada82 100644 --- a/studio/tests/ai-provider-connection.test.ts +++ b/studio/tests/ai-provider-connection.test.ts @@ -20,6 +20,8 @@ import { aiConnectionRequest, aiConnections, aiConnector, + latestByName, + secretRefs, } from "../src/app/integrations/ai-provider-connection"; const draft = { name: "azure-models", @@ -108,3 +110,29 @@ describe("AI provider connection authority", () => { ).toEqual([valid]); }); }); +describe("AI connection lists", () => { + const base = { + connector: aiConnector, + config: { provider: "openai-chat", endpoint: "http://ollama:11434/v1" }, + }; + it("keeps the latest revision of each name, newest first by name", () => { + const rows = [ + { ...base, id: "a1", name: "ollama-local", revision: 1 }, + { ...base, id: "b1", name: "openai", revision: 1 }, + { ...base, id: "a2", name: "ollama-local", revision: 2 }, + ]; + expect(latestByName(rows as never).map((row) => row.id)).toEqual([ + "a2", + "b1", + ]); + }); + it("reads secret handles under either wire spelling", () => { + expect( + secretRefs({ secretRef: { apiKey: "no-credential" } } as never), + ).toEqual({ apiKey: "no-credential" }); + expect(secretRefs({ secret_refs: { apiKey: "k" } } as never)).toEqual({ + apiKey: "k", + }); + expect(secretRefs({} as never)).toEqual({}); + }); +}); diff --git a/studio/tests/api.test.ts b/studio/tests/api.test.ts index 705384c8..f04a5197 100644 --- a/studio/tests/api.test.ts +++ b/studio/tests/api.test.ts @@ -259,3 +259,61 @@ describe("capacity rejections", () => { expect(fetch).toHaveBeenCalledTimes(1); }); }); +describe("cancellable requests", () => { + it("stops a request when its caller cancels it", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + (_path: string, options: RequestInit) => + new Promise((_resolve, reject) => + options.signal!.addEventListener("abort", () => + reject(options.signal!.reason), + ), + ), + ), + ); + const api = new StudioApi(); + const controller = new AbortController(); + const pending = api.request( + "/studio/api/x", + "POST", + {}, + {}, + 150000, + controller.signal, + ); + controller.abort(new DOMException("The test was canceled.", "AbortError")); + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + }); + it("still stops a cancellable request at its deadline", async () => { + vi.useFakeTimers(); + try { + vi.stubGlobal( + "fetch", + vi.fn( + (_path: string, options: RequestInit) => + new Promise((_resolve, reject) => + options.signal!.addEventListener("abort", () => + reject(options.signal!.reason), + ), + ), + ), + ); + const pending = new StudioApi().request( + "/studio/api/x", + "GET", + undefined, + {}, + 100, + new AbortController().signal, + ); + const outcome = expect(pending).rejects.toMatchObject({ + name: "TimeoutError", + }); + await vi.advanceTimersByTimeAsync(150); + await outcome; + } finally { + vi.useRealTimers(); + } + }); +}); From 64f8c47c849cb036a21d6f3a2fb92d01adf8945a Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:50:27 -0700 Subject: [PATCH 04/37] Add the rules for Settings AI models with their tests --- .../src/app/integrations/ai-models-model.ts | 737 ++++++++++++++++++ studio/tests/ai-models-model.test.ts | 613 +++++++++++++++ 2 files changed, 1350 insertions(+) create mode 100644 studio/src/app/integrations/ai-models-model.ts create mode 100644 studio/tests/ai-models-model.test.ts diff --git a/studio/src/app/integrations/ai-models-model.ts b/studio/src/app/integrations/ai-models-model.ts new file mode 100644 index 00000000..6ef5f6ad --- /dev/null +++ b/studio/src/app/integrations/ai-models-model.ts @@ -0,0 +1,737 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// The rules Settings › AI models applies, kept out of the components so they +// can be tested alone: which endpoints a provider offers, how a connection +// request is built, which model to suggest, what a test result means and how +// the readiness checklist reads. The platform checks all of it again. +import { ApiError } from "../api"; +import { describeError } from "../errors"; +import type { SelectOption } from "../forms/ui/select"; +import { plainServerMessage } from "./connection-copy"; +import { secretRefs, type AiConnection } from "./ai-provider-connection"; +import type { + AiEndpoint, + AiModel, + AiProvider, + AiTestResult, + BindableConnection, + ConnectionRequestBody, + LumiConfiguration, + Readiness, + ReadinessFix, + ReadinessId, + ReadinessStatus, +} from "./ai-setup-api"; + +/** The reserved handle of connections that use no credential. */ +export const NO_CREDENTIAL = "no-credential"; +export const SUGGESTED_MODEL = "qwen3:4b"; +export const PLAIN_HTTP_REFUSED = + "This address is not approved for plain HTTP. A platform operator must add it to the private-origin policy."; +export const GATEWAY_MISSING = + "Tests need the AI gateway. Run `weave platform ai enable`, or ask an operator to deploy it."; +export const RATE_LIMITED = + "Too many AI connection tests. Wait a minute, then test again."; +const NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/; + +export type ProviderCard = "ollama" | "openai" | "anthropic" | "azure"; +export interface CardInfo { + id: ProviderCard; + title: string; + kind: string; + icon: "model" | "cloud"; + providers: readonly AiProvider[]; +} +export const PROVIDER_CARDS: readonly CardInfo[] = [ + { + id: "ollama", + title: "Ollama", + kind: "Local models", + icon: "model", + providers: ["openai-chat"], + }, + { + id: "openai", + title: "OpenAI", + kind: "Cloud", + icon: "cloud", + providers: ["openai-responses", "openai-chat"], + }, + { + id: "anthropic", + title: "Anthropic", + kind: "Cloud", + icon: "cloud", + providers: ["anthropic"], + }, + { + id: "azure", + title: "Azure OpenAI", + kind: "Cloud", + icon: "cloud", + providers: ["azure-responses", "azure-chat"], + }, +]; +export const cardInfo = (card: ProviderCard): CardInfo => + PROVIDER_CARDS.find((item) => item.id === card)!; +export const API_STYLES: Readonly< + Record<"openai" | "azure", readonly { value: AiProvider; label: string }[]> +> = { + openai: [ + { value: "openai-responses", label: "Responses" }, + { value: "openai-chat", label: "Chat Completions" }, + ], + azure: [ + { value: "azure-responses", label: "Responses" }, + { value: "azure-chat", label: "Chat Completions" }, + ], +}; +const STANDARD: Partial> = { + openai: "https://api.openai.com/v1", + anthropic: "https://api.anthropic.com", +}; +const DEFAULT_NAME: Record = { + ollama: "ollama-local", + openai: "openai", + anthropic: "anthropic", + azure: "azure-openai", +}; +const DEFAULT_HANDLE: Record = { + ollama: "", + openai: "openai-api-key", + anthropic: "anthropic-api-key", + azure: "azure-openai-api-key", +}; + +/** The address as the platform stores it: origin plus path, without a lone trailing slash. */ +export function normalizedEndpoint(value: string): string { + const text = value.trim(); + const url = new URL(text); + return ( + url.origin + + (url.pathname === "/" && !text.endsWith("/") ? "" : url.pathname) + ); +} +export function originOf(value: string): string { + try { + return new URL(value).origin; + } catch { + return value; + } +} +export function endpointFor( + url: string, + endpoints: readonly AiEndpoint[], +): AiEndpoint | undefined { + try { + const wanted = normalizedEndpoint(url); + return endpoints.find( + (endpoint) => normalizedEndpoint(endpoint.url) === wanted, + ); + } catch { + return undefined; + } +} +export function presetsFor( + card: ProviderCard, + endpoints: readonly AiEndpoint[], +): AiEndpoint[] { + const providers = cardInfo(card).providers; + return endpoints.filter((endpoint) => + card === "ollama" + ? endpoint.compat === "ollama" + : endpoint.compat === null && + endpoint.providers.some((provider) => providers.includes(provider)), + ); +} +export const keylessConnection = (connection: AiConnection): boolean => + secretRefs(connection)["apiKey"] === NO_CREDENTIAL; +export function cardOf( + connection: AiConnection, + endpoints: readonly AiEndpoint[], +): ProviderCard { + if ( + keylessConnection(connection) || + endpointFor(connection.config.endpoint, endpoints)?.compat === "ollama" + ) + return "ollama"; + const provider = connection.config.provider; + return provider === "anthropic" + ? "anthropic" + : provider.startsWith("azure-") + ? "azure" + : "openai"; +} +export function cardOfBindable( + item: BindableConnection, + endpoints: readonly AiEndpoint[], +): ProviderCard { + if ( + endpoints.some( + (endpoint) => + endpoint.compat === "ollama" && + originOf(endpoint.url) === item.endpoint_origin, + ) + ) + return "ollama"; + const provider = item.provider ?? ""; + return provider === "anthropic" + ? "anthropic" + : provider.startsWith("azure-") + ? "azure" + : "openai"; +} + +export interface SheetDraft { + card: ProviderCard; + name: string; + endpoint: string; + /** The address was typed, not chosen from the approved endpoints. */ + custom: boolean; + provider: AiProvider; + apiVersion: string; + handle: string; +} +export function draftFor( + card: ProviderCard, + endpoints: readonly AiEndpoint[], + existing?: AiConnection | null, +): SheetDraft { + const presets = presetsFor(card, endpoints); + if (existing) { + const handle = secretRefs(existing)["apiKey"] ?? ""; + return { + card, + name: existing.name, + endpoint: existing.config.endpoint, + custom: !endpointFor(existing.config.endpoint, presets), + provider: existing.config.provider as AiProvider, + apiVersion: existing.config.apiVersion ?? "", + handle: handle === NO_CREDENTIAL ? "" : handle, + }; + } + const preferred = + presets.find((endpoint) => endpoint.url === STANDARD[card]) ?? presets[0]; + return { + card, + name: DEFAULT_NAME[card], + endpoint: preferred?.url ?? STANDARD[card] ?? "", + custom: !preferred, + provider: cardInfo(card).providers[0], + apiVersion: "", + handle: DEFAULT_HANDLE[card], + }; +} +/** What makes two drafts the same request. */ +export const draftKey = (draft: SheetDraft): string => + JSON.stringify([ + draft.card, + draft.name.trim(), + draft.endpoint.trim(), + draft.provider, + draft.apiVersion.trim(), + draft.handle.trim(), + ]); +/** The chosen endpoint is an approved preset that sends no credential. */ +export function keyless( + draft: SheetDraft, + endpoints: readonly AiEndpoint[], +): boolean { + if (draft.custom) return false; + return ( + endpointFor(draft.endpoint, presetsFor(draft.card, endpoints)) + ?.credential === "none" + ); +} + +export type SheetField = + | "name" + | "endpoint" + | "apiVersion" + | "handle" + | "model" + | "form"; +export class FieldError extends Error { + constructor( + readonly field: SheetField, + message: string, + ) { + super(message); + } +} +/** The request for a new connection revision; throws FieldError for the field to fix. */ +export function connectionRequest( + draft: SheetDraft, + connectorId: string, + endpoints: readonly AiEndpoint[], + policy: "loaded" | "absent", +): ConnectionRequestBody { + const name = draft.name.trim(); + if (!NAME.test(name)) + throw new FieldError( + "name", + "Letters, numbers, dots, dashes or underscores.", + ); + let url: URL; + try { + url = new URL(draft.endpoint.trim()); + } catch { + throw new FieldError("endpoint", "Enter the model endpoint's address."); + } + if (url.username || url.password || url.search || url.hash) + throw new FieldError( + "endpoint", + "Use an address without credentials, query parameters or fragments.", + ); + const preset = draft.custom + ? undefined + : endpointFor(draft.endpoint, presetsFor(draft.card, endpoints)); + if (url.protocol === "http:") { + if (!preset) throw new FieldError("endpoint", PLAIN_HTTP_REFUSED); + } else if (url.protocol !== "https:") + throw new FieldError("endpoint", "Use an https:// address."); + else if (policy === "loaded" && draft.card !== "ollama" && !preset) + throw new FieldError( + "endpoint", + "Choose an approved endpoint. A platform operator approves model endpoints.", + ); + if (draft.card === "azure" && !draft.apiVersion.trim()) + throw new FieldError("apiVersion", "Azure requires an API version."); + const noKey = preset?.credential === "none"; + const handle = draft.handle.trim(); + if (!noKey && handle === NO_CREDENTIAL) + throw new FieldError( + "handle", + "no-credential is reserved for approved local endpoints without a key.", + ); + if (!noKey && !NAME.test(handle)) + throw new FieldError( + "handle", + "Enter the secret handle your operator approved, never the key itself.", + ); + if (!connectorId) + throw new FieldError( + "form", + "Publish the AI actions first, from the readiness checklist.", + ); + return { + name, + connector_version_id: connectorId, + config: { + provider: draft.provider, + endpoint: normalizedEndpoint(draft.endpoint), + secretSlot: "apiKey", + ...(draft.card === "azure" + ? { apiVersion: draft.apiVersion.trim() } + : {}), + }, + secretRef: { apiKey: noKey ? NO_CREDENTIAL : handle }, + allowed_destinations: [url.origin], + }; +} + +export interface SheetProblem { + field: SheetField; + message: string; + code?: string; +} +const record = (value: unknown): Record => + value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +function fieldFor(pointer: string): SheetField { + if (pointer === "/name") return "name"; + if ( + pointer.startsWith("/config/endpoint") || + pointer.startsWith("/allowed_destinations") + ) + return "endpoint"; + if (pointer.startsWith("/config/apiVersion")) return "apiVersion"; + if (pointer.startsWith("/secretRef")) return "handle"; + return "form"; +} +/** Field problems from a 422 WV-CONNECTION answer, each on the field its pointer names. */ +export function sheetProblems(detail: unknown): SheetProblem[] { + const body = record(detail); + const items = Array.isArray(body["diagnostics"]) + ? body["diagnostics"] + : record(body["result"])["diagnostics"]; + if (!Array.isArray(items)) return []; + return items.slice(0, 20).map((value) => { + const item = record(value); + const message = item["message"]; + const code = String(item["code"] ?? ""); + return { + field: fieldFor(typeof item["path"] === "string" ? item["path"] : ""), + message: + typeof message === "string" && message.trim() && message.length <= 240 + ? plainServerMessage(message) + : "The platform rejected this setting.", + code: /^WV-CONNECTION-[A-Z]+$/.test(code) ? code : undefined, + }; + }); +} + +export function formatSize(bytes?: number): string { + if (bytes === undefined) return ""; + return bytes >= 1e9 + ? `${(bytes / 1e9).toFixed(1)} GB` + : `${Math.max(1, Math.round(bytes / 1e6))} MB`; +} +export function modelFacts(model: AiModel): string { + if (!model.approved) return "Not approved"; + if (!model.available) return "Not installed on this server"; + return [ + formatSize(model.size_bytes), + model.tools === "yes" ? "tools" : model.tools === "no" ? "no tools" : "", + model.context_tokens + ? `${Math.round(model.context_tokens / 1024)}k context` + : "", + ] + .filter(Boolean) + .join(" · "); +} +export function modelOptions(models: readonly AiModel[]): SelectOption[] { + return models.map((model) => ({ + value: model.name, + label: model.name, + description: modelFacts(model), + disabled: !model.approved || !model.available, + })); +} +export function defaultModel(models: readonly AiModel[], current = ""): string { + const usable = models.filter((model) => model.approved && model.available); + return ( + ( + usable.find((model) => model.name === current) ?? + usable.find((model) => model.name === SUGGESTED_MODEL) ?? + usable.find((model) => model.tools === "yes") ?? + usable[0] + )?.name ?? "" + ); +} + +/** The platform's copy for each model error code (contracts/ai.py); a test keeps them equal. */ +export const AI_COPY: Readonly> = { + LLM_POLICY: + "This model is not approved for this endpoint. Ask an operator to approve it.", + LLM_CONNECTION: + "The AI connection does not match this step. Choose the connection again.", + LLM_MODEL_NOT_FOUND: + "{model} is not installed on {endpoint}. Pull it with `ollama pull {model}`.", + LLM_NO_TOOL_SUPPORT: + "{model} cannot call tools. Choose a tool-capable model, such as qwen3:4b.", + LLM_UNREACHABLE: + "Weave could not reach the model endpoint. Check that Ollama is running.", + LLM_TIMEOUT: + "The model did not answer within {n} seconds. Local models can be slow on first load.", + LLM_CONTEXT_LIMIT: + "The prompt is too long for this model's context ({n} tokens).", + LLM_AUTH: "The provider refused the credential.", + LLM_RATE_LIMITED: "The provider is rate limiting requests.", + LLM_CAPACITY: "Weave is busy; the step will retry.", + LLM_OUTPUT: "The model's answer did not match the result fields.", + LLM_LIMIT: "The step reached its token or request limit.", + LLM_INPUT: "The AI step input is not valid.", + LLM_OPTIONS: "This provider does not accept these model options.", + LLM_REASONING: "The reasoning pattern did not reach a final answer.", + LLM_PROVIDER: "The model provider returned an unexpected error.", +}; +export function aiMessage( + code: string, + values: { model?: string; endpoint?: string; n?: number }, +): string { + return (AI_COPY[code] ?? AI_COPY["LLM_PROVIDER"]) + .replaceAll("{model}", values.model ?? "The model") + .replaceAll("{endpoint}", values.endpoint ?? "the endpoint") + .replaceAll("{n}", (values.n ?? 0).toLocaleString("en-US")); +} + +export type TestState = + | { kind: "idle" } + | { kind: "running"; model: string } + | { kind: "canceled" } + | { kind: "done"; result: AiTestResult; origin: string } + | { + kind: "refused"; + message: string; + code: string; + status: number; + origin: string; + }; +export interface TestView { + tone: "neutral" | "success" | "warning" | "danger"; + text: string; + details: [string, string][]; +} +/** The gateway's own test budget, which a timed-out test reports. */ +const TEST_SECONDS = 120; +export function testView(state: TestState, connection: string): TestView { + switch (state.kind) { + case "idle": + return { tone: "neutral", text: "Not tested", details: [] }; + case "canceled": + return { tone: "neutral", text: "Test canceled.", details: [] }; + case "running": + return { + tone: "neutral", + text: `Calling ${state.model} on ${connection}…`, + details: [], + }; + case "refused": + return { + tone: "danger", + text: state.message, + details: [ + ["Code", state.code], + ["Endpoint", state.origin], + ["HTTP status", `${Math.floor(state.status / 100)}xx`], + ], + }; + case "done": { + const result = state.result; + const seconds = + result.latency_ms === null + ? "" + : ` in ${(result.latency_ms / 1000).toFixed(1)} s`; + const details: [string, string][] = [ + ["Code", result.code], + ["Endpoint", state.origin], + [ + "Latency", + result.latency_ms === null ? "Unknown" : `${result.latency_ms} ms`, + ], + ]; + if (!result.ok) + return { + tone: "danger", + text: aiMessage(result.code, { + model: result.model, + endpoint: state.origin, + n: + result.code === "LLM_TIMEOUT" + ? TEST_SECONDS + : (result.context_tokens ?? 0), + }), + details, + }; + if (result.tool_calling === "unsupported") + return { + tone: "warning", + text: `Connected. ${result.model} answered${seconds} but cannot call tools. Use it for AI tasks, not AI agents.`, + details, + }; + return { + tone: "success", + text: + result.tool_calling === "supported" + ? `Connected. ${result.model} answered${seconds} and can call tools.` + : `Connected. ${result.model} answered${seconds}.`, + details, + }; + } + } +} +/** A test the platform refused before calling the model. Never shows the response body. */ +export function refusal(error: unknown, origin: string): TestState { + const status = error instanceof ApiError ? error.status : 0; + const code = error instanceof ApiError ? error.code : ""; + const message = + code === "WV-AI-GATEWAY-MISSING" + ? GATEWAY_MISSING + : code === "WV-AI-RATE-LIMITED" + ? RATE_LIMITED + : describeError(error).message; + return { + kind: "refused", + message, + code: code || "WV-STUDIO", + status, + origin, + }; +} + +export const READINESS_LABELS: Readonly> = { + worker_installed: "Agentic worker installed", + actions_published: "AI actions published", + worker_online: "Worker online", + connection: "AI connection", + key_available: "Key available", + connection_authorized: "Connection authorized", + model_responds: "Model responds", + weave_ai: "Weave AI", +}; +const DETAILS: Readonly> = { + worker_installed: "Installed", + worker_missing: "Not installed", + actions_published: "Published", + actions_missing: "Not published yet", + catalog_mismatch: "Published versions differ from this platform's", + worker_online: "Online", + worker_offline: "Offline", + connection_available: "Set up", + connection_missing: "No AI connection yet", + key_grant_available: "Available", + key_missing: "Key not found", + no_credential: "Not needed", + connection_authorized: "Authorized", + authorization_missing: "Not authorized yet", + test_passed: "Answered in the last 24 hours", + test_failed: "The last test failed", + test_required: "Not tested yet", + test_stale: "Last tested more than a day ago", + weave_ai_ready: "Ready", + weave_ai_missing: "Not set up", + weave_ai_disabled: "Turned off", + gateway_missing: "Needs the AI gateway", + record_unavailable: "Couldn't check", + limit_exceeded: "Too many records to check", +}; +const ICONS: Readonly> = { + done: "ok", + action: "warning", + not_needed: "off", + unknown: "help", +}; +export interface ReadinessRow { + id: ReadinessId; + label: string; + status: ReadinessStatus; + tone: "success" | "warning" | "neutral"; + icon: string; + detail: string; + fix?: ReadinessFix; +} +export function readinessRows( + result: Readiness, + roleLabel: (role: string) => string, + lastTest?: { model: string; latency_ms: number | null }, +): ReadinessRow[] { + return result.items.map((item) => ({ + id: item.id, + label: READINESS_LABELS[item.id], + status: item.status, + tone: + item.status === "done" + ? "success" + : item.status === "action" + ? "warning" + : "neutral", + icon: ICONS[item.status], + detail: item.detail.startsWith("requires_") + ? `Requires ${roleLabel(item.detail.slice("requires_".length))}` + : item.id === "model_responds" && item.status === "done" && lastTest + ? `${lastTest.model}${lastTest.latency_ms === null ? "" : ` · ${(lastTest.latency_ms / 1000).toFixed(1)} s`}` + : (DETAILS[item.detail] ?? "Couldn't check"), + fix: item.fix, + })); +} +export function allReady( + result: Readiness, + except: readonly ReadinessId[] = [], +): boolean { + return result.items.every( + (item) => + except.includes(item.id) || + item.status === "done" || + item.status === "not_needed", + ); +} + +export type FixAction = + | { kind: "button"; label: string } + | { kind: "copy"; label: string; command: string } + | { kind: "link"; label: string } + | { kind: "text"; label: string }; +const NEEDS: Readonly> = { + "ai_setup.publish": "definition.publish", + "ai_setup.grant": "connection.manage", + "ai_connections.test": "connection.manage", + "lumi.configuration.write": "lumi.manage", +}; +/** What a person can do about an item; null when the fix is not theirs to run. */ +export function fixAction( + fix: ReadinessFix | undefined, + can: (capability: string) => boolean, +): FixAction | null { + if (!fix) return null; + if (fix.kind === "operation") + return fix.operation && can(NEEDS[fix.operation]) + ? { kind: "button", label: fix.label } + : null; + if (fix.kind === "command" && fix.command) + return { kind: "copy", label: fix.label, command: fix.command }; + if (fix.kind === "navigate") return { kind: "link", label: fix.label }; + // "Requires {role}" repeats the row's detail, which names the role in words. + return fix.label.startsWith("Requires ") + ? null + : { kind: "text", label: fix.label }; +} + +export interface WeaveAiDraft { + enabled: boolean; + connection: string; + model: string; +} +/** Limits for a new Weave AI configuration; a saved one keeps its own. */ +export const WEAVE_AI_DEFAULTS: Readonly> = { + options: { max_tokens: 1024 }, + reasoning: { pattern: "none", maxSteps: 6 }, + maxCalls: 8, + timeoutSeconds: 180, +}; +export function weaveAiBody( + draft: WeaveAiDraft, + connection: AiConnection, + current: LumiConfiguration | null, + replySchema: unknown, +): Record { + const model = draft.model.trim(); + if (!model) throw new FieldError("model", "Choose a model."); + const { + provider: _provider, + model: _model, + outputSchema: _schema, + ...kept + } = (current?.profile ?? WEAVE_AI_DEFAULTS) as Record; + return { + enabled: draft.enabled, + connection_revision_id: connection.id, + profile: { + ...kept, + provider: connection.config.provider, + model, + outputSchema: replySchema, + }, + }; +} +export function weaveAiChoices( + connections: readonly AiConnection[], +): SelectOption[] { + return connections.map((connection) => { + const noKey = keylessConnection(connection); + return { + value: connection.id, + label: `${connection.name} · revision ${connection.revision}`, + description: noKey + ? "Weave AI can't use a connection without a key yet" + : connection.config.endpoint, + disabled: noKey, + }; + }); +} diff --git a/studio/tests/ai-models-model.test.ts b/studio/tests/ai-models-model.test.ts new file mode 100644 index 00000000..67c27de8 --- /dev/null +++ b/studio/tests/ai-models-model.test.ts @@ -0,0 +1,613 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +import { execFileSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; +import { ApiError } from "../src/app/api"; +import type { AiConnection } from "../src/app/integrations/ai-provider-connection"; +import type { + AiEndpoint, + AiModel, + Readiness, +} from "../src/app/integrations/ai-setup-api"; +import { + AI_COPY, + FieldError, + PLAIN_HTTP_REFUSED, + aiMessage, + allReady, + cardOf, + connectionRequest, + defaultModel, + draftFor, + fixAction, + keyless, + modelOptions, + presetsFor, + readinessRows, + refusal, + sheetProblems, + testView, + weaveAiBody, + weaveAiChoices, +} from "../src/app/integrations/ai-models-model"; +import { python, pythonAvailable } from "./python-path"; + +const ollama: AiEndpoint = { + id: "ollama-local", + label: "Ollama (Weave-managed)", + url: "http://ollama:11434/v1", + providers: ["openai-chat"], + compat: "ollama", + credential: "none", + models: "served", + context_tokens: 8192, + development_only: true, +}; +const openai: AiEndpoint = { + id: "openai", + label: "OpenAI", + url: "https://api.openai.com/v1", + providers: ["openai-chat", "openai-responses"], + compat: null, + credential: "required", + models: "listed", + context_tokens: 128000, + development_only: false, +}; +const endpoints = [ollama, openai]; +const keylessConnection: AiConnection = { + id: "r1", + name: "ollama-local", + revision: 1, + connector: "weave-agentic-provider@1.0.0", + config: { provider: "openai-chat", endpoint: "http://ollama:11434/v1" }, + secretRef: { apiKey: "no-credential" }, +}; +const cloudConnection: AiConnection = { + id: "r2", + name: "openai", + revision: 3, + connector: "weave-agentic-provider@1.0.0", + config: { + provider: "openai-responses", + endpoint: "https://api.openai.com/v1", + }, + secretRef: { apiKey: "openai-api-key" }, +}; +const qwen: AiModel = { + name: "qwen3:4b", + approved: true, + available: true, + tools: "yes", + context_tokens: 8192, + size_bytes: 2_500_000_000, +}; +const gemma: AiModel = { + name: "gemma3:270m", + approved: true, + available: true, + tools: "no", + context_tokens: 8192, + size_bytes: 291_000_000, +}; + +describe("provider presets and connection requests", () => { + it("offers only Ollama endpoints on the Ollama card and cloud endpoints elsewhere", () => { + expect(presetsFor("ollama", endpoints)).toEqual([ollama]); + expect(presetsFor("openai", endpoints)).toEqual([openai]); + expect(presetsFor("anthropic", endpoints)).toEqual([]); + }); + it("builds a keyless Ollama connection from an approved preset", () => { + const draft = draftFor("ollama", endpoints); + expect(keyless(draft, endpoints)).toBe(true); + expect(connectionRequest(draft, "connector", endpoints, "loaded")).toEqual({ + name: "ollama-local", + connector_version_id: "connector", + config: { + provider: "openai-chat", + endpoint: "http://ollama:11434/v1", + secretSlot: "apiKey", + }, + secretRef: { apiKey: "no-credential" }, + allowed_destinations: ["http://ollama:11434"], + }); + }); + it.each([ + "http://10.0.0.5:11434/v1", + "http://localhost:11434/v1", + "http://ollama.example.com/v1", + ])("refuses plain HTTP that no preset approves: %s", (endpoint) => { + const draft = { ...draftFor("ollama", endpoints), custom: true, endpoint }; + expect(() => + connectionRequest(draft, "connector", endpoints, "loaded"), + ).toThrow(PLAIN_HTTP_REFUSED); + }); + it("asks for a secret handle on credentialed endpoints and never sends the reserved one", () => { + const draft = draftFor("openai", endpoints); + expect(draft).toMatchObject({ + name: "openai", + provider: "openai-responses", + endpoint: "https://api.openai.com/v1", + handle: "openai-api-key", + }); + expect( + connectionRequest(draft, "connector", endpoints, "loaded").secretRef, + ).toEqual({ apiKey: "openai-api-key" }); + for (const handle of ["no-credential", "sk live key", ""]) { + const error = (() => { + try { + connectionRequest( + { ...draft, handle }, + "connector", + endpoints, + "loaded", + ); + } catch (caught) { + return caught; + } + })(); + expect(error).toBeInstanceOf(FieldError); + expect((error as FieldError).field).toBe("handle"); + } + }); + it("accepts only approved cloud endpoints while a policy is loaded", () => { + const draft = { + ...draftFor("openai", endpoints), + custom: true, + endpoint: "https://proxy.example.com/v1", + }; + expect(() => + connectionRequest(draft, "connector", endpoints, "loaded"), + ).toThrow("approved endpoint"); + expect( + connectionRequest(draft, "connector", endpoints, "absent").config + .endpoint, + ).toBe("https://proxy.example.com/v1"); + }); + it.each([ + "https://user:password@model.example", + "https://model.example/?api-key=secret", + "https://model.example/#secret", + "ftp://model.example", + ])("refuses an endpoint carrying anything but an address: %s", (endpoint) => { + const draft = { ...draftFor("openai", []), custom: true, endpoint }; + expect(() => connectionRequest(draft, "c", [], "absent")).toThrow( + FieldError, + ); + }); + it("requires Azure's API version and keeps it only for Azure", () => { + const azure = { + ...draftFor("azure", []), + endpoint: "https://team.openai.azure.com", + custom: true, + }; + expect(() => connectionRequest(azure, "c", [], "absent")).toThrow( + "API version", + ); + expect( + connectionRequest( + { ...azure, apiVersion: "2025-04-01-preview" }, + "c", + [], + "absent", + ).config, + ).toMatchObject({ + provider: "azure-responses", + apiVersion: "2025-04-01-preview", + }); + }); + it("needs the published AI connector before saving", () => { + expect(() => + connectionRequest(draftFor("ollama", endpoints), "", endpoints, "loaded"), + ).toThrow("Publish"); + }); + it("puts each server problem on its field", () => { + const detail = { + code: "WV-CONNECTION", + result: { + diagnostics: [ + { + path: "/config/endpoint", + code: "WV-CONNECTION-CONFIG", + message: "Plain HTTP needs a private-origin entry.", + }, + { + path: "/secretRef/apiKey", + code: "WV-CONNECTION-SECRET", + message: "The secret handle is not available.", + }, + { + path: "/name", + code: "WV-CONNECTION-CONFIG", + message: "Choose another name.", + }, + ], + }, + }; + expect(sheetProblems(detail).map((p) => p.field)).toEqual([ + "endpoint", + "handle", + "name", + ]); + }); + it("knows which card a saved connection belongs to", () => { + expect(cardOf(keylessConnection, endpoints)).toBe("ollama"); + expect(cardOf(cloudConnection, endpoints)).toBe("openai"); + expect( + cardOf( + { + ...cloudConnection, + config: { + provider: "anthropic", + endpoint: "https://api.anthropic.com", + }, + }, + endpoints, + ), + ).toBe("anthropic"); + }); +}); + +describe("models", () => { + it("suggests qwen3:4b, then a tool-capable model, and never one that is not approved or installed", () => { + expect(defaultModel([gemma, qwen])).toBe("qwen3:4b"); + expect(defaultModel([gemma, { ...qwen, name: "llama3.2:3b" }])).toBe( + "llama3.2:3b", + ); + expect(defaultModel([{ ...qwen, approved: false }, gemma])).toBe( + "gemma3:270m", + ); + expect(defaultModel([{ ...qwen, available: false }])).toBe(""); + expect(defaultModel([gemma, qwen], "gemma3:270m")).toBe("gemma3:270m"); + }); + it("describes each model with size, tools and context, and disables the rest", () => { + expect(modelOptions([qwen, { ...gemma, approved: false }])).toEqual([ + { + value: "qwen3:4b", + label: "qwen3:4b", + description: "2.5 GB · tools · 8k context", + disabled: false, + }, + { + value: "gemma3:270m", + label: "gemma3:270m", + description: "Not approved", + disabled: true, + }, + ]); + }); +}); + +describe("test states", () => { + const origin = "http://ollama:11434"; + it("reads a tool-capable success, a no-tools warning and a failure", () => { + expect( + testView( + { + kind: "done", + origin, + result: { + ok: true, + code: "ok", + latency_ms: 2400, + model: "qwen3:4b", + tool_calling: "supported", + }, + }, + "ollama-local", + ).text, + ).toBe("Connected. qwen3:4b answered in 2.4 s and can call tools."); + const warning = testView( + { + kind: "done", + origin, + result: { + ok: true, + code: "ok", + latency_ms: 800, + model: "gemma3:270m", + tool_calling: "unsupported", + }, + }, + "ollama-local", + ); + expect(warning).toMatchObject({ + tone: "warning", + text: "Connected. gemma3:270m answered in 0.8 s but cannot call tools. Use it for AI tasks, not AI agents.", + }); + const failed = testView( + { + kind: "done", + origin, + result: { + ok: false, + code: "LLM_MODEL_NOT_FOUND", + latency_ms: 30, + model: "qwen3:4b", + tool_calling: "unknown", + }, + }, + "ollama-local", + ); + expect(failed.text).toBe( + "qwen3:4b is not installed on http://ollama:11434. Pull it with `ollama pull qwen3:4b`.", + ); + expect(failed.details).toEqual([ + ["Code", "LLM_MODEL_NOT_FOUND"], + ["Endpoint", "http://ollama:11434"], + ["Latency", "30 ms"], + ]); + expect( + testView({ kind: "running", model: "qwen3:4b" }, "ollama-local").text, + ).toBe("Calling qwen3:4b on ollama-local…"); + expect(testView({ kind: "idle" }, "x").text).toBe("Not tested"); + }); + it("explains a missing gateway and the test limit without response bodies", () => { + const missing = refusal( + new ApiError(503, { + code: "WV-AI-GATEWAY-MISSING", + message: "secret detail from a body", + }), + origin, + ); + expect(missing).toMatchObject({ + kind: "refused", + message: + "Tests need the AI gateway. Run `weave platform ai enable`, or ask an operator to deploy it.", + }); + expect(JSON.stringify(testView(missing, "x"))).not.toContain( + "secret detail", + ); + expect(testView(missing, "x").details).toContainEqual([ + "HTTP status", + "5xx", + ]); + expect( + refusal(new ApiError(429, { code: "WV-AI-RATE-LIMITED" }), origin), + ).toMatchObject({ + message: "Too many AI connection tests. Wait a minute, then test again.", + }); + }); +}); + +describe.skipIf(!pythonAvailable())("error copy matches the platform", () => { + it("has the same text and placeholders as the server's table", () => { + const output = execFileSync( + python, + [ + "-c", + "import json; from firefly_weave.contracts.ai import AI_ERRORS, ai_message; " + + "print(json.dumps({'copy': {c: e.message for c, e in AI_ERRORS.items()}, " + + "'filled': {c: ai_message(c, model='qwen3:4b', endpoint='http://ollama:11434', n=8192) for c in AI_ERRORS}}))", + ], + { cwd: "..", encoding: "utf8" }, + ); + const server = JSON.parse(output) as { + copy: Record; + filled: Record; + }; + expect(AI_COPY).toEqual(server.copy); + for (const [code, text] of Object.entries(server.filled)) + expect( + aiMessage(code, { + model: "qwen3:4b", + endpoint: "http://ollama:11434", + n: 8192, + }), + ).toBe(text); + }); +}); + +describe("readiness", () => { + const item = (id: string, status: string, detail: string, fix?: object) => ({ + id, + status, + detail, + ...(fix ? { fix } : {}), + }); + const local = { + items: [ + item("worker_installed", "done", "worker_installed"), + item("actions_published", "action", "actions_missing", { + kind: "operation", + label: "Publish", + operation: "ai_setup.publish", + }), + item("worker_online", "done", "worker_online"), + item("connection", "done", "connection_available"), + item("key_available", "not_needed", "no_credential"), + item("connection_authorized", "not_needed", "no_credential"), + item("model_responds", "action", "test_required", { + kind: "operation", + label: "Test connection", + operation: "ai_connections.test", + }), + item("weave_ai", "unknown", "requires_lumi_user", { + kind: "ask", + label: "Requires lumi_user", + }), + ], + } as Readiness; + it("labels each item, its state and who may fix it", () => { + const rows = readinessRows(local, (role) => + role === "lumi_user" ? "Weave AI user" : role, + ); + expect(rows.map((row) => [row.label, row.detail, row.icon])).toEqual([ + ["Agentic worker installed", "Installed", "ok"], + ["AI actions published", "Not published yet", "warning"], + ["Worker online", "Online", "ok"], + ["AI connection", "Set up", "ok"], + ["Key available", "Not needed", "off"], + ["Connection authorized", "Not needed", "off"], + ["Model responds", "Not tested yet", "warning"], + ["Weave AI", "Requires Weave AI user", "help"], + ]); + expect( + readinessRows( + { + items: local.items.map((i) => + i.id === "model_responds" + ? { ...i, status: "done", detail: "test_passed" } + : i, + ), + } as Readiness, + (role) => role, + { model: "qwen3:4b", latency_ms: 2400 }, + )[6].detail, + ).toBe("qwen3:4b · 2.4 s"); + }); + it("is ready only when every item is done or not needed", () => { + expect(allReady(local)).toBe(false); + const done = { + items: local.items.map((i) => ({ + ...i, + status: i.status === "not_needed" ? "not_needed" : "done", + })), + } as Readiness; + expect(allReady(done)).toBe(true); + expect( + allReady( + { + items: done.items.map((i) => + i.id === "weave_ai" ? { ...i, status: "action" } : i, + ), + } as Readiness, + ["weave_ai"], + ), + ).toBe(true); + }); + it("offers an operation only to people who may run it", () => { + const publish = { + kind: "operation", + label: "Publish", + operation: "ai_setup.publish", + } as const; + expect(fixAction(publish, () => true)).toEqual({ + kind: "button", + label: "Publish", + }); + expect(fixAction(publish, (c) => c !== "definition.publish")).toBeNull(); + expect( + fixAction( + { kind: "operation", label: "Authorize", operation: "ai_setup.grant" }, + (c) => c !== "connection.manage", + ), + ).toBeNull(); + expect( + fixAction( + { + kind: "command", + label: "Ask an operator to enable AI", + command: "weave platform ai enable --ollama auto", + }, + () => false, + ), + ).toEqual({ + kind: "copy", + label: "Ask an operator to enable AI", + command: "weave platform ai enable --ollama auto", + }); + expect( + fixAction({ kind: "ask", label: "Requires lumi_user" }, () => false), + ).toBeNull(); + expect( + fixAction( + { kind: "ask", label: "Ask an operator to set the connection secret" }, + () => false, + ), + ).toEqual({ + kind: "text", + label: "Ask an operator to set the connection secret", + }); + }); +}); + +describe("Weave AI", () => { + it("keeps the saved profile's limits and takes provider and model from the choice", () => { + const body = weaveAiBody( + { enabled: true, connection: "r2", model: "gpt-5-mini" }, + cloudConnection, + { + enabled: false, + connection_revision_id: "old", + revision: 4, + profile: { + provider: "anthropic", + model: "x", + options: { max_tokens: 2048 }, + reasoning: { pattern: "none", maxSteps: 6 }, + maxCalls: 4, + timeoutSeconds: 90, + outputSchema: { old: true }, + }, + }, + { type: "object" }, + ); + expect(body).toEqual({ + enabled: true, + connection_revision_id: "r2", + profile: { + options: { max_tokens: 2048 }, + reasoning: { pattern: "none", maxSteps: 6 }, + maxCalls: 4, + timeoutSeconds: 90, + provider: "openai-responses", + model: "gpt-5-mini", + outputSchema: { type: "object" }, + }, + }); + }); + it("uses 1,024 output tokens and three minutes for a new configuration", () => { + expect( + weaveAiBody( + { enabled: true, connection: "r2", model: "gpt-5-mini" }, + cloudConnection, + null, + {}, + ).profile, + ).toMatchObject({ + options: { max_tokens: 1024 }, + maxCalls: 8, + timeoutSeconds: 180, + }); + expect(() => + weaveAiBody( + { enabled: true, connection: "r2", model: " " }, + cloudConnection, + null, + {}, + ), + ).toThrow("Choose a model"); + }); + it("lists connections without a key as not usable for Weave AI yet", () => { + expect(weaveAiChoices([keylessConnection, cloudConnection])).toEqual([ + { + value: "r1", + label: "ollama-local · revision 1", + description: "Weave AI can't use a connection without a key yet", + disabled: true, + }, + { + value: "r2", + label: "openai · revision 3", + description: "https://api.openai.com/v1", + disabled: false, + }, + ]); + }); +}); From 56615519003163668ff6ad067dfb5790bb693f6b Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:57:30 -0700 Subject: [PATCH 05/37] Add the AI models tab to Settings --- studio/src/app/settings/ai-models-page.ts | 42 ++++++++++++ studio/src/app/settings/settings-page.ts | 72 +++++++++----------- studio/src/app/settings/settings-routes.ts | 27 +++++--- studio/tests/browser/settings-routes.spec.ts | 34 ++++++++- studio/tests/settings-routes.test.ts | 35 ++++++---- 5 files changed, 146 insertions(+), 64 deletions(-) create mode 100644 studio/src/app/settings/ai-models-page.ts diff --git a/studio/src/app/settings/ai-models-page.ts b/studio/src/app/settings/ai-models-page.ts new file mode 100644 index 00000000..f53c5bd0 --- /dev/null +++ b/studio/src/app/settings/ai-models-page.ts @@ -0,0 +1,42 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// Settings › AI models: one place to connect models, check that AI steps can +// run, and set up Weave AI. The shell owns the AI connection sheet; this page +// asks it to open and reloads when an AI connection changes. +import { ChangeDetectionStrategy, Component, input } from "@angular/core"; +import type { Profile, StudioApi } from "../api"; + +/** What the page reads from the shell. */ +export interface AiModelsHost { + readonly api: StudioApi; + readonly profile: Profile | null; + can(capability: string, resource?: string): boolean; +} + +@Component({ + selector: "weave-ai-models-page", + standalone: true, + changeDetection: ChangeDetectionStrategy.Eager, + template: `

+ Connect a model once. Workflows and Weave AI use the connections you set up + here. +

`, +}) +export class AiModelsPage { + host = input.required(); +} diff --git a/studio/src/app/settings/settings-page.ts b/studio/src/app/settings/settings-page.ts index 49b22610..4e679bd7 100644 --- a/studio/src/app/settings/settings-page.ts +++ b/studio/src/app/settings/settings-page.ts @@ -16,8 +16,9 @@ Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ // Settings: Platforms (saved platforms, how Studio starts, keyboard controls), -// People and access for administrators, and Preferences, each tab at its own -// address (/settings/). The shell (App) +// People and access for administrators, AI models once a platform is +// connected, and Preferences, each tab at its own address (/settings/). +// The shell (App) // owns the state and the commands; this page loads lazily (@defer). import { ChangeDetectionStrategy, @@ -32,6 +33,7 @@ import { ModalSheet, sheetWhen } from "../modal-sheet"; import { PlatformList } from "../platform-list"; import { RowMenu, type RowMenuItem } from "../row-menu"; import { statusLabel } from "../status-labels"; +import { AiModelsPage } from "./ai-models-page"; import { settingsPath, settingsTab, @@ -55,7 +57,7 @@ interface AccountRow { selector: "weave-settings-page", changeDetection: ChangeDetectionStrategy.Eager, standalone: true, - imports: [Icon, PlatformList, RowMenu, ModalSheet], + imports: [Icon, PlatformList, RowMenu, ModalSheet, AiModelsPage], styleUrl: "./settings-page.css", template: `@let h = host();
@@ -66,40 +68,6 @@ interface AccountRow {

- @if (h.profile) { -
-

AI models

-

- Provider connections are shared environment resources. Weave AI uses - its own model settings; workflow AI profiles stay in each workflow. -

- @if (h.can("connection.manage")) { - - } - @if (h.can("lumi.manage")) { - - } - @if (!h.can("lumi.manage") || !h.can("connection.manage")) { -

- Ask an administrator for Weave AI manager and connection.manage - grants in this environment to set up Weave AI and its provider - connection. -

- } -

- For a workflow model, add an AI task and open Configure workflow AI - profiles. A platform operator installs the Agentic worker and Weave AI - gateway and provisions approved secret handles. -

-
- } @let current = tab();
}
+ } @else if (current === "ai") { +
+ @defer (on immediate) { + + } @placeholder { +

Loading AI models…

+ } +
} @else {
tab.id); - const index = order.indexOf(this.tab()); + // Arrows move from the focused tab, which may differ from the open one. + const focused = (event.target as HTMLElement | null)?.id?.replace( + /^settings-tab-/, + "", + ); + const from = order.includes(focused as SettingsTab) + ? (focused as SettingsTab) + : this.tab(); + const index = order.indexOf(from); const next = event.key === "Home" ? 0 diff --git a/studio/src/app/settings/settings-routes.ts b/studio/src/app/settings/settings-routes.ts index 32475d5c..1b8bd71e 100644 --- a/studio/src/app/settings/settings-routes.ts +++ b/studio/src/app/settings/settings-routes.ts @@ -15,34 +15,43 @@ limitations under the License. Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ -// Settings has one address per tab: /settings/platforms, /settings/people -// and /settings/preferences; /settings opens Platforms. People and access -// shows only to people who can manage members or grants. +// Settings has one address per tab: /settings/platforms, /settings/people, +// /settings/ai and /settings/preferences; /settings opens Platforms. People +// and access shows only to people who can manage members or grants; AI models +// shows once a platform is connected. import type { Routes } from "@angular/router"; -export type SettingsTab = "platforms" | "people" | "preferences"; +export type SettingsTab = "platforms" | "people" | "ai" | "preferences"; export const SETTINGS_TABS: readonly { id: SettingsTab; label: string }[] = [ { id: "platforms", label: "Platforms" }, { id: "people", label: "People and access" }, + { id: "ai", label: "AI models" }, { id: "preferences", label: "Preferences" }, ]; +/** Which optional tabs this person sees. */ +export interface SettingsAccess { + people: boolean; + ai: boolean; +} export const settingsPath = (tab: SettingsTab): string => `/settings/${tab}`; /** The tabs this person sees, in order. */ -export function settingsTabs(canSeePeople: boolean) { - return SETTINGS_TABS.filter((tab) => tab.id !== "people" || canSeePeople); +export function settingsTabs(access: SettingsAccess) { + return SETTINGS_TABS.filter( + (tab) => + (tab.id !== "people" || access.people) && (tab.id !== "ai" || access.ai), + ); } /** The tab an address opens; an unknown or hidden tab opens Platforms. */ export function settingsTab( pathname: string, - canSeePeople: boolean, + access: SettingsAccess, ): SettingsTab { const segment = pathname.split("/")[2] ?? ""; return ( - settingsTabs(canSeePeople).find((tab) => tab.id === segment)?.id ?? - "platforms" + settingsTabs(access).find((tab) => tab.id === segment)?.id ?? "platforms" ); } diff --git a/studio/tests/browser/settings-routes.spec.ts b/studio/tests/browser/settings-routes.spec.ts index 8da247b9..28a42fd0 100644 --- a/studio/tests/browser/settings-routes.spec.ts +++ b/studio/tests/browser/settings-routes.spec.ts @@ -16,7 +16,7 @@ Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ import { expect, test } from "@playwright/test"; -import { offline } from "./support"; +import { allCapabilities, connected, offline } from "./support"; test("Settings tabs have their own addresses, and Preferences turns on the new editor", async ({ page, @@ -67,3 +67,35 @@ test("/settings opens Platforms, and the tabs follow the arrow keys", async ({ await expect(page.getByRole("tab", { name: "Platforms" })).toBeFocused(); await expect(page).toHaveURL(/\/settings\/platforms$/); }); + +test("AI models has its own address once a platform is connected", async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await connected(page, { capabilities: allCapabilities }); + await page.goto("/settings/ai"); + const tab = page.getByRole("tab", { name: "AI models" }); + await expect(tab).toHaveAttribute("aria-selected", "true"); + await expect(page.getByRole("tabpanel", { name: "AI models" })).toContainText( + "Connect a model once. Workflows and Weave AI use the connections you set up here.", + ); + await expect( + page.getByRole("button", { name: "Weave AI settings" }), + ).toHaveCount(0); + await page.getByRole("tab", { name: "Platforms" }).focus(); + await page.keyboard.press("ArrowRight"); + await expect(page.getByRole("tab", { name: "AI models" })).toBeFocused(); + await expect(page).toHaveURL(/\/settings\/ai$/); +}); + +test("AI models is not offered while working locally", async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await offline(page); + await page.goto("/settings/ai"); + await expect(page).toHaveURL(/\/settings\/ai$/); + await expect(page.getByRole("tab", { name: "AI models" })).toHaveCount(0); + await expect(page.getByRole("tab", { name: "Platforms" })).toHaveAttribute( + "aria-selected", + "true", + ); +}); diff --git a/studio/tests/settings-routes.test.ts b/studio/tests/settings-routes.test.ts index 60903adc..6b61b640 100644 --- a/studio/tests/settings-routes.test.ts +++ b/studio/tests/settings-routes.test.ts @@ -24,46 +24,53 @@ import { settingsTabs, } from "../src/app/settings/settings-routes"; +const everything = { people: true, ai: true }; +const nothing = { people: false, ai: false }; describe("Settings addresses", () => { it("gives each tab its own address", () => { expect(SETTINGS_TABS.map((tab) => settingsPath(tab.id))).toEqual([ "/settings/platforms", "/settings/people", + "/settings/ai", "/settings/preferences", ]); }); - it("opens the tab an address names", () => { - expect(settingsTab("/settings/preferences", false)).toBe("preferences"); - expect(settingsTab("/settings/people", true)).toBe("people"); - expect(settingsTab("/settings/platforms", true)).toBe("platforms"); + expect(settingsTab("/settings/preferences", nothing)).toBe("preferences"); + expect(settingsTab("/settings/people", everything)).toBe("people"); + expect(settingsTab("/settings/ai", everything)).toBe("ai"); + expect(settingsTab("/settings/platforms", everything)).toBe("platforms"); }); - it("opens Platforms for /settings, an unknown tab or a tab the person can't see", () => { - expect(settingsTab("/settings", true)).toBe("platforms"); - expect(settingsTab("/settings/ai", true)).toBe("platforms"); - expect(settingsTab("/settings/people", false)).toBe("platforms"); + expect(settingsTab("/settings", everything)).toBe("platforms"); + expect(settingsTab("/settings/administration", everything)).toBe( + "platforms", + ); + expect(settingsTab("/settings/people", nothing)).toBe("platforms"); + expect(settingsTab("/settings/ai", nothing)).toBe("platforms"); }); - - it("shows People and access only to people who can manage it", () => { - expect(settingsTabs(false).map((tab) => tab.label)).toEqual([ + it("shows People and access to managers and AI models once a platform is connected", () => { + expect(settingsTabs(nothing).map((tab) => tab.label)).toEqual([ "Platforms", "Preferences", ]); - expect(settingsTabs(true).map((tab) => tab.label)).toEqual([ + expect( + settingsTabs({ people: false, ai: true }).map((tab) => tab.label), + ).toEqual(["Platforms", "AI models", "Preferences"]); + expect(settingsTabs(everything).map((tab) => tab.label)).toEqual([ "Platforms", "People and access", + "AI models", "Preferences", ]); }); - it("routes /settings to Platforms and each tab to the shell", () => { const paths = routes.map((route) => route.path); expect(routes.find((route) => route.path === "settings")).toMatchObject({ redirectTo: "settings/platforms", pathMatch: "full", }); - for (const tab of ["platforms", "people", "preferences"]) + for (const tab of ["platforms", "people", "ai", "preferences"]) expect(paths).toContain(`settings/${tab}`); expect(paths.indexOf("settings/preferences")).toBeLessThan( paths.indexOf("**"), From aed73084f37660ae11b2812bc69b43e88542915f Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:12:27 -0700 Subject: [PATCH 06/37] Check the AI models tab name where Settings now defines it --- studio/tests/weave-ai-naming.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/studio/tests/weave-ai-naming.test.ts b/studio/tests/weave-ai-naming.test.ts index 2afb31ec..9475dc56 100644 --- a/studio/tests/weave-ai-naming.test.ts +++ b/studio/tests/weave-ai-naming.test.ts @@ -55,9 +55,10 @@ describe("Weave AI naming", () => { expect(panel).toContain('heading="Ask Weave AI"'); expect(panel).toContain('closeLabel="Close Weave AI"'); expect(panel).toContain('aria-label="Message to Weave AI"'); - const settings = read("app/settings/settings-page.ts"); - expect(settings).toContain("

AI models

"); - expect(settings).not.toContain("AI setup"); + expect(read("app/settings/settings-routes.ts")).toContain( + '{ id: "ai", label: "AI models" }', + ); + expect(read("app/settings/settings-page.ts")).not.toContain("AI setup"); }); it("keeps the lumi identifiers the API, roles and storage use", () => { From 8f3adc6c5714a8fd44f80cdbc4306ddfd3a1b975 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:12:27 -0700 Subject: [PATCH 07/37] Add the AI connection side sheet with discovery and connection tests --- studio/src/app/app.html | 12 + studio/src/app/app.ts | 29 +- studio/src/app/dialog.ts | 9 +- .../app/integrations/ai-connection-sheet.css | 121 +++ .../app/integrations/ai-connection-sheet.ts | 712 ++++++++++++++ studio/src/app/modal-sheet.ts | 1 + studio/src/app/settings/ai-models-page.ts | 22 +- studio/src/styles.css | 19 + studio/tests/browser/ai-models-po.ts | 356 +++++++ studio/tests/browser/ai-setup.spec.ts | 921 ++++-------------- studio/tests/browser/ai-task-profile.spec.ts | 331 +++++++ studio/tests/ndv-sample-account.test.ts | 2 + 12 files changed, 1816 insertions(+), 719 deletions(-) create mode 100644 studio/src/app/integrations/ai-connection-sheet.css create mode 100644 studio/src/app/integrations/ai-connection-sheet.ts create mode 100644 studio/tests/browser/ai-models-po.ts create mode 100644 studio/tests/browser/ai-task-profile.spec.ts diff --git a/studio/src/app/app.html b/studio/src/app/app.html index b81ac1cf..60e9e829 100644 --- a/studio/src/app/app.html +++ b/studio/src/app/app.html @@ -622,6 +622,18 @@

Server response

@defer (when lumiOpen) { } + @if (aiSheet && profile) { + @defer (on immediate) { + + } + }
diff --git a/studio/src/app/app.ts b/studio/src/app/app.ts index d3b212df..cb4757bb 100644 --- a/studio/src/app/app.ts +++ b/studio/src/app/app.ts @@ -151,6 +151,11 @@ import { DesignerView } from "./designer/designer-view"; // Integration and template UI loads lazily (@defer): only types are imported // from these modules besides the component classes listed in `imports`. import { IntegrationDialogs } from "./integrations/integration-dialogs"; +import { + AiConnectionSheet, + type AiSheetRequest, + type AiTestRecord, +} from "./integrations/ai-connection-sheet"; import { TemplateGallery } from "./templates/template-gallery"; import { sheetWhen } from "./modal-sheet"; import { ToastHost, ToastService, type ToastAction } from "./toast"; @@ -356,6 +361,7 @@ const sideEffects: Record = { StepPicker, StartRunDialog, IntegrationDialogs, + AiConnectionSheet, TemplateGallery, ToastHost, DesignerView, @@ -4767,8 +4773,24 @@ export class App implements CanvasHost { this.cdr.markForCheck(); } /** Opens "Connect to an API", prefilled from the builder's hand-off for this slot. */ - openAiConnectionDialog() { - this.connectionDialog = { kind: "ai", fromBuilder: null }; + /** The AI connection side sheet, open over any view. */ + aiSheet: AiSheetRequest | null = null; + /** Grows whenever an AI connection or its test changes; AI screens reload on it. */ + aiSetupGeneration = 0; + /** Test results of this Studio session, by connection revision; never stored. */ + readonly aiTests = new Map(); + /** Opens the AI connection side sheet. */ + openAiConnectionDialog(request: AiSheetRequest = {}) { + this.aiSheet = request; + this.cdr.markForCheck(); + } + aiConnectionSaved() { + this.aiSetupGeneration++; + if (this.view === "connections") void this.refresh(); + } + aiConnectionTested(record: AiTestRecord) { + this.aiTests.set(record.revision, record); + this.aiSetupGeneration++; } openConnectionDialog(slot = "") { const use = this.lastUse; @@ -7038,6 +7060,9 @@ export class App implements CanvasHost { } /** Forgets everything read from the platform; the open workflow is kept. */ private clearPlatformState() { + this.aiSheet = null; + this.aiTests.clear(); + this.aiSetupGeneration++; this.stepDataGeneration++; this.importGeneration++; this.listSequence++; diff --git a/studio/src/app/dialog.ts b/studio/src/app/dialog.ts index 87491d98..73c694e6 100644 --- a/studio/src/app/dialog.ts +++ b/studio/src/app/dialog.ts @@ -66,10 +66,15 @@ export class Autofocus implements AfterViewInit { selector: "weave-modal", standalone: true, imports: [Icon], - template: `
+ template: `
+
- Connect OpenAI - -
`, +
+

Providers

+
+
    + @for (card of cards; track card.id) { + @let total = count(card.id); + @let action = cardAction(card.id); +
  • +

    + {{ card.title }} +

    +

    {{ card.kind }}

    +

    + @if (total) { + Connected · {{ total }} + } @else { + Not set up + } +

    + @if (action) { + + } +
    • + } +
+

+ Connections +

+ @if (listError) { + + } @else if (!canManage() && !canBind()) { +

Seeing AI connections requires the Deployer role.

+ } @else { +
    + @for (item of visible; track item.connection.id) { +
  • +
    + {{ + item.connection.name + }} + + {{ providerLabel(item) }} + {{ + item.connection.config.endpoint + }} + @if (developmentOnly(item)) { + Development only + } + + @if (!item.readOnly) { + {{ testText(item) }} + } +
    +
    + @if (item.readOnly) { + + } @else { + + + } +
    +
    • + } @empty { +
  • + {{ + listed + ? "No AI connections yet. Connect a provider above." + : "Loading…" + }} +
    • + } +
+ } + +
+

{{ announcement }}

`, }) -export class AiModelsPage { +export class AiModelsPage implements DoCheck, OnDestroy { host = input.required(); + readonly cards = PROVIDER_CARDS; + private readonly cdr = inject(ChangeDetectorRef); + private readonly element = inject>(ElementRef); + /** The AI setup generation and the person's access the page last loaded for. */ + private loadedFor = ""; + private loads = 0; + private alive = true; + private setup: AiSetupApi | null = null; + readiness: Readiness | null = null; + readinessError = ""; + checking = false; + endpoints: AiEndpoint[] = []; + connections: AiConnection[] = []; + bindable: BindableConnection[] | null = null; + listed = false; + listError = ""; + running = ""; + failures: Record = {}; + announcement = ""; + + ngDoCheck() { + // Capabilities arrive after the page may first render; what the page may + // list depends on them, so a change in access reloads it too. + const key = `${this.host().aiSetupGeneration}:${this.canManage()}:${this.canBind()}`; + if (key === this.loadedFor) return; + this.loadedFor = key; + void this.load(); + } + ngOnDestroy() { + this.alive = false; + } + canManage() { + return this.host().can("connection.manage"); + } + canBind() { + return this.host().can("connection.bind"); + } + async load() { + const load = ++this.loads; + this.setup = new AiSetupApi(this.host().api); + await Promise.all([this.recheck(load), this.loadConnections(load)]); + } + async recheck(load = this.loads) { + const setup = this.setup; + if (!setup) return; + this.checking = true; + try { + const readiness = await setup.readiness(); + if (this.alive && load === this.loads) { + this.readiness = readiness; + this.readinessError = ""; + } + } catch (error) { + if (this.alive && load === this.loads) + this.readinessError = + error instanceof ApiError && error.status === 403 + ? "Checking AI readiness requires the Viewer role." + : describeError(error).message; + } finally { + if (this.alive) { + this.checking = false; + this.cdr.markForCheck(); + } + } + } + private async loadConnections(load: number) { + const setup = this.setup!; + try { + const endpoints = await setup + .endpoints() + .catch(() => ({ policy: "absent" as const, endpoints: [] })); + const connections = this.canManage() ? await setup.connections() : []; + const bindable = + !this.canManage() && this.canBind() ? await setup.bindable() : null; + if (!this.alive || load !== this.loads) return; + this.endpoints = endpoints.endpoints ?? []; + this.connections = connections; + this.bindable = bindable; + this.listError = ""; + this.listed = true; + } catch (error) { + if (this.alive && load === this.loads) + this.listError = describeError(error).message; + } finally { + if (this.alive) this.cdr.markForCheck(); + } + } + get visible(): Row[] { + if (this.bindable) + return this.bindable.map((item) => ({ + connection: fromBindable(item), + card: cardOfBindable(item, this.endpoints), + readOnly: true, + })); + return this.connections.map((connection) => ({ + connection, + card: cardOf(connection, this.endpoints), + readOnly: false, + })); + } + get ready() { + return !!this.readiness && allReady(this.readiness); + } + get rows(): ReadinessRow[] { + if (!this.readiness) return []; + const latest = [...this.host().aiTests.values()] + .filter( + (record) => + record.result.ok && + this.connections.some((c) => c.id === record.revision), + ) + .sort((a, b) => b.at - a.at)[0]; + return readinessRows( + this.readiness, + (role) => this.host().roleLabel(role), + latest?.result, + ); + } + get actions() { + const host = this.host(); + return this.rows.map((row) => + fixAction(row.fix, (capability) => host.can(capability)), + ); + } + count(card: ProviderCard) { + return this.visible.filter((row) => row.card === card).length; + } + cardAction(card: ProviderCard): string { + if (this.canManage()) return this.count(card) ? "Manage" : "Connect"; + return this.count(card) ? "View" : ""; + } + providerLabel(row: Row) { + const title = cardInfo(row.card).title; + const provider = row.connection.config.provider; + return row.card === "openai" || row.card === "azure" + ? `${title} · ${provider.endsWith("-chat") ? "Chat Completions" : "Responses"}` + : title; + } + developmentOnly(row: Row) { + const origin = row.connection.config.endpoint; + return ( + endpointFor(origin, this.endpoints)?.development_only ?? + this.endpoints.some( + (endpoint) => + endpoint.development_only && endpoint.url.startsWith(origin + "/"), + ) + ); + } + testText(row: Row) { + const record = this.host().aiTests.get(row.connection.id); + if (!record) return "Not tested in this session"; + const result = record.result; + if (!result.ok) return `Last test failed · ${result.code}`; + return `${result.model}${result.latency_ms === null ? "" : ` · ${(result.latency_ms / 1000).toFixed(1)} s`}`; + } + testTone(row: Row) { + const record = this.host().aiTests.get(row.connection.id); + return record ? (record.result.ok ? "success" : "danger") : null; + } + openCard(card: ProviderCard) { + const first = this.visible.find((row) => row.card === card); + this.host().openAiConnectionDialog({ + card, + connection: first?.connection ?? null, + }); + } + openRow(row: Row, focus?: "model") { + this.host().openAiConnectionDialog({ + card: row.card, + connection: row.connection, + focus, + }); + } + menu(row: Row): RowMenuItem[] { + return [ + { label: "Edit connection", run: () => this.openRow(row) }, + { + label: "Copy connection ID", + run: () => void this.copy(row.connection.id), + }, + ]; + } + async act(row: ReadinessRow) { + const fix = row.fix; + const host = this.host(); + const setup = this.setup; + if (!fix || !setup || this.running) return; + if (fix.kind === "navigate") { + if (fix.route === "operate/workers") await host.navigate("workers"); + else + this.focus( + row.id === "weave_ai" ? "#weave-ai-title" : "#ai-providers-title", + ); + return; + } + if (fix.kind !== "operation") return; + if (fix.operation === "ai_connections.test") { + const target = + this.connections.find( + (connection) => !host.aiTests.get(connection.id)?.result.ok, + ) ?? this.connections[0]; + if (target) + host.openAiConnectionDialog({ + card: cardOf(target, this.endpoints), + connection: target, + focus: "model", + }); + return; + } + if (fix.operation === "lumi.configuration.write") { + this.focus("#weave-ai-title"); + return; + } + this.running = row.id; + this.failures = {}; + try { + if (fix.operation === "ai_setup.publish") { + await setup.publish(); + this.announcement = "Published the AI actions."; + } else if (fix.operation === "ai_setup.grant") { + const targets = this.connections.filter( + (connection) => !keylessConnection(connection), + ); + for (const connection of targets) await setup.grant(connection.id); + this.announcement = + targets.length === 1 + ? `Authorized ${targets[0].name}.` + : `Authorized ${targets.length} AI connections.`; + } + host.notify(this.announcement); + await this.recheck(); + } catch (error) { + const plain = describeError(error); + if (this.alive) + this.failures = { + [row.id]: plain.code + ? `${plain.message} (${plain.code})` + : plain.message, + }; + } finally { + if (this.alive) { + this.running = ""; + this.cdr.markForCheck(); + } + } + } + async copy(text: string) { + try { + await navigator.clipboard.writeText(text); + this.announcement = "Copied."; + } catch { + this.announcement = "Copy isn't available here. Select the text instead."; + } + this.cdr.markForCheck(); + } + private focus(selector: string) { + this.element.nativeElement.querySelector(selector)?.focus(); + } } diff --git a/studio/src/app/settings/ai-readiness-list.ts b/studio/src/app/settings/ai-readiness-list.ts new file mode 100644 index 00000000..e967eaef --- /dev/null +++ b/studio/src/app/settings/ai-readiness-list.ts @@ -0,0 +1,161 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// The AI readiness checklist: one row per fact the platform checked, with the +// fix the person may run. The page runs the fixes; this list only shows them. +import { + ChangeDetectionStrategy, + Component, + input, + output, +} from "@angular/core"; +import { Icon } from "../icon"; +import type { FixAction, ReadinessRow } from "../integrations/ai-models-model"; + +@Component({ + selector: "weave-ai-readiness-list", + standalone: true, + changeDetection: ChangeDetectionStrategy.Eager, + imports: [Icon], + template: `
    + @for (row of rows(); track row.id; let index = $index) { + @let action = actions()[index]; +
  • + + {{ row.label }} · {{ row.detail }} + @if (action) { + @if (action.kind === "button") { + + } @else if (action.kind === "link") { + + } @else if (action.kind === "copy") { + + {{ action.command }} + + + } @else { + {{ action.label }} + } + } + @if (failures()[row.id]; as failure) { + + } +
    • + } +
`, + styles: ` + .readiness-list { + list-style: none; + margin: 0; + padding: 0; + display: grid; + } + .readiness-row { + display: grid; + grid-template-columns: 16px minmax(0, 1fr) auto; + align-items: center; + column-gap: var(--space-2); + row-gap: var(--space-1); + min-height: var(--control-sm); + padding: var(--space-1) 0; + border-bottom: 1px solid var(--line); + } + .readiness-icon[data-tone="success"] { + color: var(--success-ink); + } + .readiness-icon[data-tone="warning"] { + color: var(--warning-ink); + } + .readiness-icon[data-tone="neutral"] { + color: var(--muted); + } + .readiness-text { + min-width: 0; + overflow-wrap: anywhere; + } + .readiness-detail, + .readiness-ask { + color: var(--muted); + } + .readiness-ask { + font: var(--type-small); + } + .readiness-command, + .readiness-error { + grid-column: 2 / -1; + margin: 0; + } + .readiness-command { + display: flex; + align-items: center; + gap: var(--space-2); + min-width: 0; + } + .readiness-command code { + flex: 1 1 auto; + min-width: 0; + overflow-wrap: anywhere; + font: var(--type-mono); + background: var(--sunken); + padding: var(--space-1) var(--space-2); + border-radius: var(--radius-xs); + } + @media (max-width: 767px), (pointer: coarse) { + .readiness-row { + min-height: var(--control-touch); + } + } + `, +}) +export class AiReadinessList { + rows = input.required(); + actions = input.required<(FixAction | null)[]>(); + /** The item whose fix is running, or "". */ + running = input(""); + failures = input>({}); + act = output(); + copy = output(); +} diff --git a/studio/tests/browser/ai-models-po.ts b/studio/tests/browser/ai-models-po.ts index 29d9ac2c..85f30291 100644 --- a/studio/tests/browser/ai-models-po.ts +++ b/studio/tests/browser/ai-models-po.ts @@ -227,13 +227,13 @@ export async function mockAiPlatform( }); await page.route(`${ENV}/ai/setup/publish`, (r) => { state.publishes++; - state.readiness = readiness({ - actions_published: { - status: "done", - detail: "actions_published", - fix: undefined, - }, - }); + state.readiness = { + items: state.readiness.items.map((item) => + item.id === "actions_published" + ? { id: item.id, status: "done", detail: "actions_published" } + : item, + ), + }; return json(r, { connector: {}, actions: [{}] }); }); await page.route(`${ENV}/ai/setup/grant`, (r) => { diff --git a/studio/tests/browser/ai-setup.spec.ts b/studio/tests/browser/ai-setup.spec.ts index f842f436..8f64b4e8 100644 --- a/studio/tests/browser/ai-setup.spec.ts +++ b/studio/tests/browser/ai-setup.spec.ts @@ -26,9 +26,12 @@ import { AI_CAPABILITIES, CONNECTOR_ID, OLLAMA_REVISION, + OPENAI_REVISION, mockAiPlatform, ollamaConnection, openAiModels, + openaiConnection, + readiness, sheet, type AiPlatform, } from "./ai-models-po"; @@ -364,3 +367,181 @@ test("Home offers Connect a model to people who manage connections", async ({ await page.getByRole("button", { name: "Connect a model" }).click(); await expect(page).toHaveURL(/\/settings\/ai$/); }); + +for (const size of [ + { width: 1440, height: 900 }, + { width: 600, height: 500 }, +]) + test.describe(`AI models page at ${size.width}x${size.height}`, () => { + test.use({ viewport: size }); + + test("shows providers, connections and the checklist", async ({ page }) => { + await start(page, { connections: [ollamaConnection, openaiConnection] }); + const cards = page.locator(".provider-card"); + await expect(cards).toHaveCount(4); + await expect(cards.nth(0)).toContainText("Ollama"); + await expect(cards.nth(0)).toContainText("Connected · 1"); + await expect(cards.nth(2)).toContainText("Not set up"); + await expect( + page.getByRole("button", { name: "Manage Ollama" }), + ).toBeVisible(); + await expect( + page.getByRole("button", { name: "Connect Anthropic" }), + ).toBeVisible(); + const row = page + .locator(".connection-row") + .filter({ hasText: "ollama-local" }); + await expect(row).toContainText("http://ollama:11434/v1"); + await expect(row).toContainText("Development only"); + await expect(row).toContainText("Not tested in this session"); + const items = page.locator(".readiness-row"); + await expect(items).toHaveCount(8); + await expect(items.nth(4)).toContainText("Key available · Not needed"); + await expect(items.nth(7)).toContainText("Weave AI · Not set up"); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + ).toBe(true); + if (size.width < 1024) { + const readinessBox = await page.locator(".ai-readiness").boundingBox(); + const providersBox = await page.locator(".ai-providers").boundingBox(); + expect(readinessBox!.y + readinessBox!.height).toBeLessThanOrEqual( + providersBox!.y, + ); + } + }); + + test("Publish and Authorize run once and the checklist rechecks", async ({ + page, + }) => { + const platform = await start(page, { + connections: [ollamaConnection, openaiConnection], + readiness: readiness({ + actions_published: { + status: "action", + detail: "actions_missing", + fix: { + kind: "operation", + label: "Publish", + operation: "ai_setup.publish", + }, + }, + key_available: { status: "done", detail: "key_grant_available" }, + connection_authorized: { + status: "action", + detail: "authorization_missing", + fix: { + kind: "operation", + label: "Authorize", + operation: "ai_setup.grant", + }, + }, + }), + }); + const publish = page.getByRole("button", { + name: "Publish", + exact: true, + }); + await publish.dblclick(); + await expect(page.locator(".readiness-row").nth(1)).toContainText( + "AI actions published · Published", + ); + expect(platform.publishes).toBe(1); + await page + .getByRole("button", { name: "Authorize", exact: true }) + .click(); + await expect(page.locator(".readiness-row").nth(5)).toContainText( + "Connection authorized · Authorized", + ); + expect(platform.grants).toEqual([ + { connection_revision_id: OPENAI_REVISION }, + ]); + }); + }); + +test("a viewer sees the checklist without setup controls", async ({ page }) => { + await start(page, { + capabilities: ["catalog.read", "status.read"], + connections: [ollamaConnection], + readiness: readiness({ + connection: { + status: "unknown", + detail: "requires_deployer", + fix: { kind: "ask", label: "Requires deployer" }, + }, + key_available: { + status: "unknown", + detail: "requires_tenant_admin", + fix: { kind: "ask", label: "Requires tenant_admin" }, + }, + connection_authorized: { + status: "unknown", + detail: "requires_tenant_admin", + fix: { kind: "ask", label: "Requires tenant_admin" }, + }, + model_responds: { + status: "unknown", + detail: "requires_tenant_admin", + fix: { kind: "ask", label: "Requires tenant_admin" }, + }, + weave_ai: { + status: "unknown", + detail: "requires_lumi_user", + fix: { kind: "ask", label: "Requires lumi_user" }, + }, + }), + }); + await expect(page.locator(".readiness-row").nth(4)).toContainText( + "Key available · Requires Tenant administrator", + ); + await expect(page.locator(".readiness-row").nth(7)).toContainText( + "Weave AI · Requires Weave AI user", + ); + expect(await page.locator(".ai-readiness").textContent()).not.toContain( + "tenant_admin", + ); + await expect(page.getByRole("button", { name: /^Connect / })).toHaveCount(0); + await expect(page.locator(".ai-providers")).toContainText( + "Seeing AI connections requires the Deployer role.", + ); +}); + +test("the operator command is copied and the workers link opens Operate", async ({ + page, + context, +}) => { + await context.grantPermissions(["clipboard-read", "clipboard-write"]); + await start(page, { + readiness: readiness({ + worker_installed: { + status: "action", + detail: "worker_missing", + fix: { + kind: "command", + label: "Ask an operator to enable AI", + command: "weave platform ai enable --ollama auto", + }, + }, + worker_online: { + status: "action", + detail: "worker_offline", + fix: { + kind: "navigate", + label: "Open workers", + route: "operate/workers", + }, + }, + }), + }); + await page + .getByRole("button", { + name: "Copy weave platform ai enable --ollama auto", + }) + .click(); + expect(await page.evaluate(() => navigator.clipboard.readText())).toBe( + "weave platform ai enable --ollama auto", + ); + await page.getByRole("button", { name: "Open workers" }).click(); + await expect(page).toHaveURL(/\/operate\/workers$/); +}); From 4cfda678e92af490c6021557c3f94e09916703b2 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:55:34 -0700 Subject: [PATCH 15/37] Move Weave AI settings to Settings AI models and remove the old AI connection form --- studio/src/app/app.ts | 1 - .../ai-provider-connection-form.ts | 531 ------------------ .../integrations/ai-provider-connection.ts | 78 --- studio/src/app/lumi/lumi-panel.ts | 460 +-------------- studio/src/app/settings/ai-models-page.ts | 14 +- studio/src/app/settings/weave-ai-card.ts | 322 +++++++++++ studio/tests/ai-provider-connection.test.ts | 74 +-- studio/tests/browser/ai-setup.spec.ts | 117 ++++ studio/tests/browser/ai-task-profile.spec.ts | 162 +++--- studio/tests/browser/weave-ai.spec.ts | 199 +------ 10 files changed, 550 insertions(+), 1408 deletions(-) delete mode 100644 studio/src/app/integrations/ai-provider-connection-form.ts create mode 100644 studio/src/app/settings/weave-ai-card.ts diff --git a/studio/src/app/app.ts b/studio/src/app/app.ts index f6c5556d..c6fbe481 100644 --- a/studio/src/app/app.ts +++ b/studio/src/app/app.ts @@ -416,7 +416,6 @@ export class App implements CanvasHost { ? (this.operationsView()?.lumiAttachments ?? []) : []; } - lumiSettingsRequested = false; platformMenuOpen = false; platformBusy = ""; // Fences platform checks and status reads against later platform changes. diff --git a/studio/src/app/integrations/ai-provider-connection-form.ts b/studio/src/app/integrations/ai-provider-connection-form.ts deleted file mode 100644 index 97470f76..00000000 --- a/studio/src/app/integrations/ai-provider-connection-form.ts +++ /dev/null @@ -1,531 +0,0 @@ -/* -Copyright 2026 Firefly Software Foundation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Author: Firefly Software Foundation -SPDX-License-Identifier: Apache-2.0 -*/ -import { - ChangeDetectionStrategy, - ChangeDetectorRef, - Component, - ElementRef, - Injector, - afterNextRender, - OnDestroy, - OnInit, - inject, - input, - output, -} from "@angular/core"; -import { ApiError, type StudioApi } from "../api"; -import { describeError } from "../errors"; -import { Select } from "../forms/ui/select"; -import { - aiConnector, - aiConnectionRequest, - aiProviderEndpoint, - aiProviders, - aiSetupRows, - type AiConnection, - type AiConnectionDraft, -} from "./ai-provider-connection"; - -@Component({ - selector: "weave-ai-provider-connection-form", - standalone: true, - changeDetection: ChangeDetectionStrategy.Eager, - imports: [Select], - template: ` - @if (!canManage()) { -

- Ask an administrator for connection.manage in this environment to create - a provider connection. -

- } @else if (done) { -

Connection created

-

{{ done.name }} · revision {{ done.revision }}

-

Saved · Not tested

-

- The configuration was validated. Provider access has not been tested. No - model request was sent. A platform operator must authorize the worker - release to use this connection before a workflow can run; Weave AI uses - its separately configured gateway. -

- - } @else { -
    - @for (label of stages; track label; let index = $index) { -
  1. - - {{ label }} -
    2. - } -
- @if (loading) { -

Checking the published AI connector…

- } @else if (!connectorId) { -

- Publish {{ connector }} in this project first. A platform operator - installs the Agentic worker and provisions the scoped API key handle. -

- - } -
-
- @if (step === 0) { -

Choose a provider

-

- Name this connection so authors can recognize it. Choose models or - Azure deployments later in workflow AI profiles or Weave AI - settings. -

- - - } @else if (step === 1) { -

Provider access

-

- Choose the credential handle approved for {{ providerLabel }} in - this environment. -

- @if (draft.provider.startsWith("azure-")) { - -

- Enter the base endpoint for your Azure resource. -

- -

- Use the API version approved for your deployment. -

- } @else { -

Standard endpoint: {{ standardEndpoint }}

-
- Advanced: custom endpoint - -

- Override only for an approved proxy or compatible service. The - worker or Weave AI gateway must allow the exact endpoint. -

-
- } - -

- Enter the handle supplied by your platform operator. The operator - stores and grants the secret for this environment. Never paste the - API key. -

- } @else { -

Review your connection

-

- Check these details before creating an immutable connection - revision. -

-
-
-
Connection name
-
{{ draft.name.trim() }}
-
-
-
Provider
-
{{ providerLabel }}
-
-
-
Endpoint
-
{{ reviewedEndpoint }}
-
- @if (draft.provider.startsWith("azure-")) { -
-
Azure API version
-
{{ draft.apiVersion.trim() }}
-
- } -
-
API key secret handle
-
{{ draft.handle.trim() }}
-
-
-

- The platform will validate the configuration. No model request or - connectivity test is sent. Your operator still needs to authorize - the worker release or configure Weave AI's gateway before use. -

- } -
- @if (step > 0) { - - } - -
-
-
- } - @if (error) { - - } - `, - styles: ` - :host { - display: grid; - gap: 16px; - min-width: 0; - } - .progress { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 8px; - list-style: none; - padding: 0 0 16px; - margin: 0; - border-bottom: 1px solid var(--line); - } - .progress li { - display: flex; - align-items: center; - gap: 6px; - color: var(--muted); - font-size: 13px; - } - .progress li[aria-current] { - color: var(--text); - font-weight: 600; - } - .number { - display: grid; - place-items: center; - width: 24px; - height: 24px; - flex: 0 0 24px; - border: 1px solid var(--field-border); - border-radius: 50%; - } - [aria-current] .number { - background: var(--accent); - border-color: var(--accent); - color: var(--on-accent); - } - .complete .number { - background: var(--selected); - color: var(--text); - } - fieldset { - display: grid; - gap: 14px; - border: 0; - padding: 0; - margin: 0; - min-width: 0; - } - .wizard-title { - margin: 0; - font-size: 17px; - line-height: 1.4; - } - label { - display: grid; - gap: 6px; - } - details[open] { - display: grid; - gap: 10px; - } - summary { - cursor: pointer; - padding: 8px 0; - } - input { - min-width: 0; - width: 100%; - box-sizing: border-box; - } - p { - margin: 0; - overflow-wrap: anywhere; - } - .actions { - display: flex; - justify-content: flex-end; - flex-wrap: wrap; - gap: 8px; - border-top: 1px solid var(--line); - margin-top: 4px; - padding-top: 16px; - } - .actions button { - min-height: 40px; - } - .actions button:first-child:not(.primary) { - margin-right: auto; - } - .review { - margin: 0; - display: grid; - gap: 12px; - min-width: 0; - } - .review div { - display: grid; - grid-template-columns: minmax(100px, 1fr) minmax(0, 1.5fr); - gap: 12px; - } - dt { - color: var(--muted); - } - dd { - margin: 0; - overflow-wrap: anywhere; - font-weight: 500; - } - .review-note { - border-left: 3px solid var(--accent); - padding: 10px 12px; - background: var(--raised); - } - @media (max-width: 420px) { - .progress li { - flex-direction: column; - align-items: flex-start; - } - .review div { - grid-template-columns: minmax(0, 1fr); - gap: 3px; - } - } - `, -}) -export class AiProviderConnectionForm implements OnInit, OnDestroy { - api = input.required(); - canManage = input.required(); - created = output(); - finished = output(); - providers = aiProviders; - connector = aiConnector; - draft: AiConnectionDraft = { - name: "", - provider: "", - endpoint: "", - apiVersion: "", - handle: "", - }; - readonly stages = ["Provider", "Access", "Review"]; - step = 0; - get providerLabel() { - return ( - this.providers.find((provider) => provider.value === this.draft.provider) - ?.label ?? "your provider" - ); - } - get standardEndpoint() { - return aiProviderEndpoint(this.draft.provider); - } - get reviewedEndpoint() { - return aiConnectionRequest(this.draft, this.connectorId).config.endpoint; - } - chooseProvider(provider: string) { - if (provider === this.draft.provider) return; - this.draft.provider = provider; - this.draft.endpoint = aiProviderEndpoint(provider); - this.draft.apiVersion = ""; - } - connectorId = ""; - loading = false; - busy = false; - error = ""; - done: AiConnection | null = null; - private alive = true; - private cdr = inject(ChangeDetectorRef); - private host = inject>(ElementRef); - private injector = inject(Injector); - private focus(selector = ".wizard-title") { - const step = this.step; - afterNextRender( - () => { - if (this.alive && this.step === step) - this.host.nativeElement.querySelector(selector)?.focus(); - }, - { injector: this.injector }, - ); - } - back() { - if (this.busy || this.loading || this.step === 0) return; - this.step--; - this.error = ""; - this.focus(); - } - private attempt: { - body: ReturnType; - key: string; - } | null = null; - ngOnInit() { - if (this.canManage()) void this.load(); - } - ngOnDestroy() { - this.alive = false; - } - value(event: Event) { - return (event.target as HTMLInputElement).value; - } - async load() { - this.loading = true; - this.error = ""; - try { - const rows = await aiSetupRows(this.api(), "connectors", false); - if (!this.alive) return; - this.connectorId = String( - rows.find( - (row) => - !row["unavailable"] && - !row["retired"] && - `${row["name"]}@${row["version"]}` === aiConnector, - )?.["id"] ?? "", - ); - } catch (error) { - if (this.alive) - this.error = - error instanceof ApiError && error.status === 403 - ? "Reading the published provider connector requires catalog.read in this project. Ask an administrator for that grant as well as connection.manage in this environment." - : describeError(error).message; - } finally { - if (this.alive) { - this.loading = false; - this.cdr.markForCheck(); - } - } - } - async submit(event: Event) { - event.preventDefault(); - if (this.busy || this.loading || !this.canManage() || !this.connectorId) - return; - this.error = ""; - try { - if (this.step === 0) { - if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(this.draft.name.trim())) - throw Error( - "Enter a connection name using letters, numbers, dots, dashes or underscores.", - ); - if ( - !this.providers.some( - (provider) => provider.value === this.draft.provider, - ) - ) - throw Error("Choose a provider."); - this.step = 1; - this.focus(); - return; - } - const body = aiConnectionRequest(this.draft, this.connectorId); - if (this.step === 1) { - this.step = 2; - this.focus(); - return; - } - if ( - !this.attempt || - JSON.stringify(this.attempt.body) !== JSON.stringify(body) - ) - this.attempt = { body, key: crypto.randomUUID() }; - this.busy = true; - const result = await this.api().mutate( - `${this.api().environment}/connections`, - "POST", - this.attempt.body, - undefined, - this.attempt.key, - ); - if (!this.alive) return; - this.done = result; - this.created.emit(result); - this.focus(); - } catch (error) { - if (this.alive) { - this.error = describeError(error).message; - this.focus(".error"); - if ( - error instanceof ApiError && - error.code === "WV-IDEMPOTENCY-CONFLICT" - ) - this.attempt = null; - } - } finally { - if (this.alive) { - this.busy = false; - this.cdr.markForCheck(); - } - } - } -} diff --git a/studio/src/app/integrations/ai-provider-connection.ts b/studio/src/app/integrations/ai-provider-connection.ts index 50a7a93f..2ad993cb 100644 --- a/studio/src/app/integrations/ai-provider-connection.ts +++ b/studio/src/app/integrations/ai-provider-connection.ts @@ -45,84 +45,6 @@ export function latestByName(connections: AiConnection[]): AiConnection[] { } return [...latest.values()].sort((a, b) => a.name.localeCompare(b.name)); } -export interface AiConnectionDraft { - name: string; - provider: string; - endpoint: string; - apiVersion: string; - handle: string; -} -export const aiProviders = [ - { value: "openai-responses", label: "OpenAI Responses" }, - { value: "openai-chat", label: "OpenAI Chat" }, - { value: "azure-responses", label: "Azure OpenAI Responses" }, - { value: "azure-chat", label: "Azure OpenAI Chat" }, - { value: "anthropic", label: "Anthropic" }, -]; -export function aiProviderEndpoint(provider: string): string { - if (provider === "openai-chat" || provider === "openai-responses") - return "https://api.openai.com/v1"; - return provider === "anthropic" ? "https://api.anthropic.com" : ""; -} -export function aiConnectionRequest( - draft: AiConnectionDraft, - connector: string, -) { - if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(draft.name.trim())) - throw Error( - "Enter a connection name using letters, numbers, dots, dashes or underscores.", - ); - if (!aiProviders.some((p) => p.value === draft.provider)) - throw Error("Choose a provider."); - let endpoint: URL; - const endpointValue = - draft.endpoint.trim() || aiProviderEndpoint(draft.provider); - try { - endpoint = new URL(endpointValue); - } catch { - throw Error("Enter the provider's trusted HTTPS endpoint."); - } - if ( - endpoint.protocol !== "https:" || - endpoint.username || - endpoint.password || - endpoint.search || - endpoint.hash - ) - throw Error( - "Use an HTTPS endpoint without credentials, query parameters or fragments.", - ); - if (draft.provider.startsWith("azure-") && !draft.apiVersion.trim()) - throw Error( - "Azure requires an API version supplied by your platform operator.", - ); - if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(draft.handle.trim())) - throw Error( - "Enter an operator-approved API key secret handle, never the key itself.", - ); - if (!connector) - throw Error( - `${aiConnector} must be published in this project before creating this connection.`, - ); - return { - name: draft.name.trim(), - connector_version_id: connector, - config: { - provider: draft.provider, - endpoint: - endpoint.origin + - (endpoint.pathname === "/" && !endpointValue.endsWith("/") - ? "" - : endpoint.pathname), - secretSlot: "apiKey", - ...(draft.provider.startsWith("azure-") - ? { apiVersion: draft.apiVersion.trim() } - : {}), - }, - secretRef: { apiKey: draft.handle.trim() }, - allowed_destinations: [endpoint.origin], - }; -} /** Bounded lists fail visibly rather than hiding an older pinned revision. */ export async function aiSetupRows( api: StudioApi, diff --git a/studio/src/app/lumi/lumi-panel.ts b/studio/src/app/lumi/lumi-panel.ts index c7768e33..3c305105 100644 --- a/studio/src/app/lumi/lumi-panel.ts +++ b/studio/src/app/lumi/lumi-panel.ts @@ -25,20 +25,8 @@ import { inject, } from "@angular/core"; import type { App } from "../app"; -import { ApiError } from "../api"; import { parse } from "yaml"; -import { Select } from "../forms/ui/select"; -import { AiProviderConnectionForm } from "../integrations/ai-provider-connection-form"; -import { - aiConnections, - aiSetupRows, - type AiConnection, -} from "../integrations/ai-provider-connection"; import { Modal } from "../dialog"; -import { AiProfileEditor } from "../integrations/ai-profile-editor"; -import { AiSetupWizard } from "../integrations/ai-setup-wizard"; -import type { Schema } from "../task-schema"; -import { missingData } from "../forms/core/form-model"; import { describeError } from "../errors"; import { exportFile } from "../export-file"; import { @@ -58,13 +46,7 @@ const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; selector: "weave-lumi-panel", standalone: true, changeDetection: ChangeDetectionStrategy.Eager, - imports: [ - Modal, - AiProfileEditor, - AiSetupWizard, - Select, - AiProviderConnectionForm, - ], + imports: [Modal], template: ` @if (host.lumiOpen) {
-
+
@if (status?.configured) { {{ status.provider }} · {{ status.model }} } - @if (host.profile && host.can("lumi.manage")) { - }
@@ -103,155 +77,16 @@ const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;

Ask an administrator for access to Weave AI in this environment.

- } @else if (!settings && !status?.configured && !loading) { + } @else if (!status?.configured && !loading) {

- Weave AI is unavailable in this environment. An administrator - configures its model and connection; a platform operator enables the - Weave AI gateway. Saving model settings alone does not deploy the - gateway. + Weave AI is not set up in this environment. A Weave AI manager sets + it up in Settings › AI models.

} @if (loading) {

Loading Weave AI…

} - @if (settings) { -
-

- Choose Weave AI's model and connection for this environment. - Workflow AI profiles have their own settings. -

- @if (!host.can("connection.manage")) { -

- To select or save a provider connection, ask an administrator - for connection.manage in this environment as well as Weave AI - manager. -

- } - @if (configSchema) { - -
- -
-
-

- Choose Weave AI's provider connection. Credentials stay with - the platform operator. -

- -

- Weave AI pins this exact revision. Choose a connection with - the same provider as the profile; changing a workflow - profile will not change these settings. -

- @if (configConnection && !selectedConnection) { -

- Choose an available connection that matches this provider. -

- } - @if (host.can("connection.manage")) { - @if (!connectionOptions.length) { -

- No matching provider connections yet. Create one with an - operator-approved secret handle, or change the provider. -

- } -
- -
- @if (newConnection) { - - } - } -
-
-
- @for (row of settingsSummary; track row.label) { -
-
{{ row.label }}
-
{{ row.value }}
-
- } -
-

- Saving applies these settings to Weave AI in the current - environment. It does not change workflow AI profiles or send - a model request. The platform operator must enable the Weave - AI gateway and authorize this connection before you can ask - Weave AI. -

-
-
- @if (configError) { - - } -
-
- -
- - } @else if (configBusy) { -

Loading Weave AI settings…

- } - @if (!configSchema && configError) { - - } -
- -
-
- } @else if (review; as item) { + @if (review; as item) {

{{ item.title }}

@@ -442,35 +277,6 @@ const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; [hidden] { display: none !important; } - .settings-connection { - display: grid; - gap: 12px; - min-width: 0; - } - .settings-summary { - margin: 0; - border-top: 1px solid var(--line); - } - .settings-summary div { - display: grid; - grid-template-columns: minmax(100px, 1fr) minmax(0, 2fr); - gap: 16px; - padding: 12px 0; - border-bottom: 1px solid var(--line); - } - .settings-summary dt { - color: var(--muted); - } - .settings-summary dd { - margin: 0; - overflow-wrap: anywhere; - } - @media (max-width: 480px) { - .settings-summary div { - grid-template-columns: 1fr; - gap: 4px; - } - } .lumi-conversation { display: grid; gap: 16px; @@ -494,7 +300,6 @@ const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; max-width: 100%; white-space: normal; } - .lumi-settings, .lumi-review, fieldset { display: grid; @@ -533,8 +338,6 @@ const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; }) export class LumiPanel implements DoCheck { @Input({ required: true }) host!: App; - readonly validateAiProfile = (profile: Record) => - this.host.api.validateLlmProfile({ ...profile, outputSchema: {} }); private cdr = inject(ChangeDetectorRef); private element = inject>(ElementRef); private keepFocus() { @@ -549,7 +352,6 @@ export class LumiPanel implements DoCheck { back() { this.keepFocus(); this.review = null; - this.settings = false; } conversation = new LumiConversation(); message = ""; @@ -562,58 +364,6 @@ export class LumiPanel implements DoCheck { null; review: ProposalReview | null = null; validating = false; - settings = false; - settingsStep = 0; - readonly settingsHeadings = [ - "Choose Weave AI's model", - "Choose a provider connection", - "Review Weave AI settings", - ]; - get settingsSummary() { - const profile = object(this.config["profile"]); - const connection = this.connections.find( - (c) => c.id === this.configConnection, - ); - return [ - { - label: "Assistant", - value: this.config["enabled"] ? "Enabled" : "Disabled", - }, - { label: "Provider", value: profile["provider"] }, - { label: "Model or deployment", value: profile["model"] }, - { - label: "Connection", - value: connection - ? `${connection.name} · revision ${connection.revision}` - : "Unavailable", - }, - { - label: "Endpoint", - value: connection?.config.endpoint ?? "Unavailable", - }, - { - label: "Reasoning", - value: object(profile["reasoning"])["pattern"] ?? "none", - }, - { label: "Maximum calls", value: profile["maxCalls"] }, - { label: "Timeout", value: `${profile["timeoutSeconds"]} seconds` }, - { - label: "Maximum output tokens", - value: object(profile["options"])["max_tokens"] ?? "Provider default", - }, - ]; - } - configBusy = false; - configError = ""; - configSchema: Schema | null = null; - configInitial: Record = {}; - config: Record = {}; - configValid = true; - configRevision: number | null = null; - configConnection = ""; - connections: AiConnection[] = []; - newConnection = false; - private replySchema: unknown; private wasOpen = false; ngDoCheck() { const key = JSON.stringify([ @@ -631,10 +381,6 @@ export class LumiPanel implements DoCheck { } if (this.host.lumiOpen && !this.wasOpen) void this.loadStatus(); this.wasOpen = this.host.lumiOpen; - if (this.host.lumiOpen && this.host.lumiSettingsRequested) { - this.host.lumiSettingsRequested = false; - void this.openSettings(); - } } private reset() { this.message = ""; @@ -643,23 +389,12 @@ export class LumiPanel implements DoCheck { this.review = null; this.status = null; this.busy = false; - this.settings = false; - this.configSchema = null; - this.config = {}; - this.configInitial = {}; - this.configRevision = null; - this.configConnection = ""; - this.connections = []; - this.newConnection = false; this.error = ""; this.loading = false; - this.configBusy = false; this.validating = false; - this.configError = ""; - this.replySchema = undefined; } clear() { - if (this.busy || this.configBusy) return; + if (this.busy) return; this.conversation.clear(); this.message = ""; this.review = null; @@ -881,179 +616,4 @@ export class LumiPanel implements DoCheck { item.source, ); } - get connectionOptions() { - const provider = object(this.config["profile"])["provider"]; - return this.connections - .filter((c) => c.config.provider === provider) - .map((c) => ({ - value: c.id, - label: `${c.name} · revision ${c.revision}`, - description: c.config.endpoint, - })); - } - get selectedConnection() { - return this.connectionOptions.some( - (option) => option.value === this.configConnection, - ); - } - async loadConnections() { - if (!this.host.can("connection.manage")) return; - const generation = this.conversation.generation; - try { - const rows = await aiSetupRows(this.host.api, "connections", true); - if (generation === this.conversation.generation) - this.connections = aiConnections(rows); - } catch (error) { - if (generation === this.conversation.generation) - this.configError = describeError(error).message; - } finally { - this.cdr.markForCheck(); - } - } - connectionCreated(connection: AiConnection) { - this.connections = [ - ...this.connections.filter((c) => c.id !== connection.id), - connection, - ]; - this.configConnection = connection.id; - } - get modelComplete() { - return ( - !!this.configSchema && - this.configValid && - !missingData(this.configSchema, this.config).length - ); - } - get configComplete() { - return ( - this.host.can("connection.manage") && - this.selectedConnection && - this.modelComplete - ); - } - async openSettings() { - if (this.configBusy || !this.host.profile || !this.host.can("lumi.manage")) - return; - this.keepFocus(); - this.settings = true; - this.settingsStep = 0; - this.newConnection = false; - this.configSchema = null; - this.keepFocus(); - this.configBusy = true; - this.configError = ""; - const generation = this.conversation.generation; - try { - const schema = await this.host.api.request>( - "/studio/contracts/lumi-configuration", - ); - const defs = object(schema["$defs"]); - const profile = object(defs["LumiProfile"]); - this.replySchema = object(object(profile["properties"])["outputSchema"])[ - "const" - ]; - if (!this.replySchema) - throw Error( - "The host did not provide the fixed Weave AI reply schema.", - ); - const properties = { ...object(profile["properties"]) }; - delete properties["outputSchema"]; - const { connection_revision_id: _, ...configProperties } = object( - schema["properties"], - ); - const editable = { - ...schema, - properties: configProperties, - required: (schema["required"] as string[]).filter( - (key) => key !== "connection_revision_id", - ), - $defs: { - ...defs, - LumiProfile: { - ...profile, - properties, - required: (profile["required"] as string[]).filter( - (key) => key !== "outputSchema", - ), - }, - }, - }; - let configuration: Record = { - enabled: true, - profile: { - options: { max_tokens: 2048 }, - reasoning: { pattern: "none", maxSteps: 6 }, - maxCalls: 8, - timeoutSeconds: 120, - }, - }; - try { - configuration = await this.host.api.request>( - `${this.host.api.environment}/lumi/configuration`, - ); - } catch (error) { - if (!(error instanceof ApiError && error.status === 404)) throw error; - } - if (generation !== this.conversation.generation) return; - this.configRevision = configuration["revision"] ?? null; - const { revision, connection_revision_id, ...data } = configuration; - this.configConnection = connection_revision_id ?? ""; - const cleanProfile = { ...object(data["profile"]) }; - delete cleanProfile["outputSchema"]; - this.configInitial = { ...data, profile: cleanProfile }; - this.config = structuredClone(this.configInitial); - this.configSchema = editable as Schema; - await this.loadConnections(); - } catch (error) { - if (generation === this.conversation.generation) - this.configError = describeError(error).message; - } finally { - if (generation === this.conversation.generation) { - this.configBusy = false; - this.cdr.markForCheck(); - } - } - } - async saveSettings() { - if ( - this.configBusy || - this.settingsStep !== 2 || - !this.configComplete || - !this.host.can("lumi.manage") - ) - return; - this.configBusy = true; - this.configError = ""; - const generation = this.conversation.generation; - try { - const body = { - ...this.config, - connection_revision_id: this.configConnection, - profile: { - ...object(this.config["profile"]), - outputSchema: this.replySchema, - }, - }; - const result = await this.host.api.request>( - `${this.host.api.environment}/lumi/configuration`, - "PUT", - body, - this.configRevision === null - ? {} - : { "If-Match": `"${this.configRevision}"` }, - ); - if (generation !== this.conversation.generation) return; - this.configRevision = result["revision"]; - this.settings = false; - await this.loadStatus(); - } catch (error) { - if (generation === this.conversation.generation) - this.configError = describeError(error).message; - } finally { - if (generation === this.conversation.generation) { - this.configBusy = false; - this.cdr.markForCheck(); - } - } - } } diff --git a/studio/src/app/settings/ai-models-page.ts b/studio/src/app/settings/ai-models-page.ts index 11378351..c5c06f34 100644 --- a/studio/src/app/settings/ai-models-page.ts +++ b/studio/src/app/settings/ai-models-page.ts @@ -58,6 +58,7 @@ import { type Readiness, } from "../integrations/ai-setup-api"; import { AiReadinessList } from "./ai-readiness-list"; +import { WeaveAiCard } from "./weave-ai-card"; /** What the page reads from and asks of the shell. */ export interface AiModelsHost { @@ -92,7 +93,7 @@ const fromBindable = (item: BindableConnection): AiConnection => ({ selector: "weave-ai-models-page", standalone: true, changeDetection: ChangeDetectionStrategy.Eager, - imports: [Icon, RowMenu, AiReadinessList], + imports: [Icon, RowMenu, AiReadinessList, WeaveAiCard], styleUrl: "./ai-models-page.css", template: `

Connect a model once. Workflows and Weave AI use the connections you set @@ -241,6 +242,17 @@ const fromBindable = (item: BindableConnection): AiConnection => ({ }

+

{{ announcement }}

`, }) diff --git a/studio/src/app/settings/weave-ai-card.ts b/studio/src/app/settings/weave-ai-card.ts new file mode 100644 index 00000000..1af70ec8 --- /dev/null +++ b/studio/src/app/settings/weave-ai-card.ts @@ -0,0 +1,322 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// Weave AI's settings in Settings › AI models: on or off, the AI connection +// and the model. Its limits (output tokens, calls, time) keep their saved +// values; a new configuration starts from Weave AI's defaults. +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + OnDestroy, + DoCheck, + inject, + input, + output, +} from "@angular/core"; +import { ApiError, type StudioApi } from "../api"; +import { describeError } from "../errors"; +import { Select } from "../forms/ui/select"; +import type { AiConnection } from "../integrations/ai-provider-connection"; +import { + FieldError, + defaultModel, + keylessConnection, + modelOptions, + weaveAiBody, + weaveAiChoices, + type WeaveAiDraft, +} from "../integrations/ai-models-model"; +import { + AiSetupApi, + type AiModels, + type LumiConfiguration, + type LumiStatus, +} from "../integrations/ai-setup-api"; + +@Component({ + selector: "weave-ai-card", + standalone: true, + changeDetection: ChangeDetectionStrategy.Eager, + imports: [Select], + template: `
+

Weave AI

+ @if (status) { + {{ + status.configured ? "Set up · " + status.model : "Not set up" + }} + } +
+

+ Weave AI answers questions about your workflows and runs, and drafts + changes for you to review. +

+ @if (!manager()) { +

+ {{ + canUse() + ? "Configuring Weave AI requires the Weave AI manager role." + : "Using Weave AI requires the Weave AI user role." + }} +

+ } @else if (!canChoose()) { +

+ Choosing Weave AI's connection also requires the Tenant administrator + role. +

+ } @else if (loading) { +

Loading Weave AI settings…

+ } @else { +
+ +
+ + +
+ @if (!usable.length) { +

+ Weave AI needs an AI connection with a secret handle. Connections + without a key can't be used for Weave AI yet. +

+ } + @if (error) { + + } +
+ +
+
+ } +

{{ announcement }}

`, + styles: ` + :host { + display: block; + } + .panel-heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-2); + min-height: var(--control-md); + margin-bottom: var(--space-3); + } + h2 { + margin: 0; + font: var(--type-title-sm); + } + .card-intro, + .empty { + margin: 0 0 var(--space-4); + color: var(--muted); + } + .weave-ai-form { + display: grid; + gap: var(--space-4); + } + .weave-ai-fields { + display: grid; + gap: var(--space-4); + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + } + .card-actions { + display: flex; + justify-content: flex-end; + } + `, +}) +export class WeaveAiCard implements DoCheck, OnDestroy { + api = input.required(); + /** lumi.manage */ + manager = input(false); + /** lumi.manage and connection.manage: saving reads the chosen connection. */ + canChoose = input(false); + /** lumi.use */ + canUse = input(false); + connections = input([]); + saved = output(); + private readonly cdr = inject(ChangeDetectorRef); + private setup!: AiSetupApi; + private replySchema: unknown = null; + private alive = true; + loading = true; + busy = false; + error = ""; + announcement = ""; + status: LumiStatus | null = null; + current: LumiConfiguration | null = null; + models: AiModels | null = null; + draft: WeaveAiDraft = { enabled: true, connection: "", model: "" }; + + get choices() { + return weaveAiChoices(this.connections()); + } + get usable() { + return this.connections().filter( + (connection) => !keylessConnection(connection), + ); + } + get modelChoices() { + return modelOptions(this.models?.models ?? []); + } + checked(event: Event) { + return (event.target as HTMLInputElement).checked; + } + /** The access the card last loaded for; capabilities can arrive after it renders. */ + private loadedFor = ""; + private loads = 0; + ngDoCheck() { + const key = `${this.canUse()}:${this.manager()}:${this.canChoose()}`; + if (key === this.loadedFor) return; + this.loadedFor = key; + void this.load(); + } + private async load() { + const load = ++this.loads; + this.setup = new AiSetupApi(this.api()); + this.loading = true; + try { + if (this.canUse() || this.manager()) + this.status = await this.setup.lumiStatus().catch(() => null); + if (this.canChoose()) { + const [current, reply] = await Promise.all([ + this.setup.lumiConfiguration(), + this.setup.lumiReplySchema(), + ]); + if (!this.alive || load !== this.loads) return; + this.current = current; + this.replySchema = reply; + this.draft = { + enabled: current?.enabled ?? true, + connection: current?.connection_revision_id ?? "", + model: String(current?.profile?.["model"] ?? ""), + }; + if (this.draft.connection) await this.loadModels(this.draft.connection); + } + } catch (error) { + if (this.alive && load === this.loads) + this.error = describeError(error).message; + } finally { + if (this.alive && load === this.loads) { + this.loading = false; + this.cdr.markForCheck(); + } + } + } + ngOnDestroy() { + this.alive = false; + } + async chooseConnection(id: string) { + if (id === this.draft.connection) return; + this.draft = { ...this.draft, connection: id, model: "" }; + await this.loadModels(id); + } + private async loadModels(id: string) { + try { + const models = await this.setup.models(id); + if (!this.alive || this.draft.connection !== id) return; + this.models = models; + this.draft = { + ...this.draft, + model: defaultModel(models.models, this.draft.model), + }; + } catch (error) { + if (this.alive) this.error = describeError(error).message; + } finally { + if (this.alive) this.cdr.markForCheck(); + } + } + async save() { + const connection = this.connections().find( + (item) => item.id === this.draft.connection, + ); + if (this.busy || !connection) return; + this.error = ""; + let body: Record; + try { + body = weaveAiBody( + this.draft, + connection, + this.current, + this.replySchema, + ); + } catch (error) { + if (error instanceof FieldError) { + this.error = error.message; + return; + } + throw error; + } + this.busy = true; + try { + this.current = await this.setup.saveLumi( + body, + this.current?.revision ?? null, + ); + if (!this.alive) return; + this.status = await this.setup.lumiStatus().catch(() => this.status); + this.announcement = "Saved Weave AI settings."; + this.saved.emit(); + } catch (error) { + if (!this.alive) return; + this.error = + error instanceof ApiError && error.code === "WV-LUMI-REVISION" + ? "Weave AI settings changed since you opened them. Reload the page to see them, then save again." + : describeError(error).message; + } finally { + if (this.alive) { + this.busy = false; + this.cdr.markForCheck(); + } + } + } +} diff --git a/studio/tests/ai-provider-connection.test.ts b/studio/tests/ai-provider-connection.test.ts index 0f4a5e7f..ff6cc6fd 100644 --- a/studio/tests/ai-provider-connection.test.ts +++ b/studio/tests/ai-provider-connection.test.ts @@ -17,84 +17,12 @@ SPDX-License-Identifier: Apache-2.0 */ import { describe, expect, it } from "vitest"; import { - aiConnectionRequest, aiConnections, aiConnector, latestByName, secretRefs, } from "../src/app/integrations/ai-provider-connection"; -const draft = { - name: "azure-models", - provider: "azure-responses", - endpoint: "https://approved.openai.azure.com/", - apiVersion: "2025-04-01-preview", - handle: "model-key", -}; -describe("AI provider connection authority", () => { - it.each([ - ["openai-chat", "https://api.openai.com/v1"], - ["openai-responses", "https://api.openai.com/v1"], - ["anthropic", "https://api.anthropic.com"], - ])("uses the standard endpoint for %s", (provider, endpoint) => { - const body = aiConnectionRequest( - { ...draft, provider, endpoint: "" }, - "connector", - ); - expect(body.config.endpoint).toBe(endpoint); - expect(body.allowed_destinations).toEqual([new URL(endpoint).origin]); - }); - it("requires Azure's resource endpoint", () => { - expect(() => - aiConnectionRequest({ ...draft, endpoint: "" }, "connector"), - ).toThrow("HTTPS endpoint"); - }); - it("canonicalizes the saved endpoint and destination together without removing the base path", () => { - const body = aiConnectionRequest( - { ...draft, endpoint: "https://APPROVED.openai.azure.com:443/custom/" }, - "connector", - ); - expect(body.config.endpoint).toBe( - "https://approved.openai.azure.com/custom/", - ); - expect(body.allowed_destinations).toEqual([ - "https://approved.openai.azure.com", - ]); - }); - it.each([ - "http://model.invalid", - "https://user:password@model.invalid", - "https://model.invalid/?api-key=secret", - "https://model.invalid/#secret", - ])( - "rejects an endpoint carrying authority outside HTTPS origin: %s", - (endpoint) => { - expect(() => - aiConnectionRequest({ ...draft, endpoint }, "connector"), - ).toThrow(); - }, - ); - it("requires the explicit Azure API version and operator handle", () => { - expect(() => - aiConnectionRequest({ ...draft, apiVersion: "" }, "connector"), - ).toThrow("Azure requires"); - expect(() => - aiConnectionRequest({ ...draft, handle: "key with spaces" }, "connector"), - ).toThrow("secret handle"); - }); - it("keeps provider/model separate and derives only the approved origin", () => { - const body = aiConnectionRequest(draft, "connector"); - expect(body.allowed_destinations).toEqual([ - "https://approved.openai.azure.com", - ]); - expect(body.config).not.toHaveProperty("model"); - expect(body.secretRef).toEqual({ apiKey: "model-key" }); - expect( - aiConnectionRequest( - { ...draft, provider: "openai-responses" }, - "connector", - ).config, - ).not.toHaveProperty("apiVersion"); - }); +describe("AI connection revisions", () => { it("excludes unavailable and unrelated connection revisions", () => { const valid = { id: "ai", diff --git a/studio/tests/browser/ai-setup.spec.ts b/studio/tests/browser/ai-setup.spec.ts index 8f64b4e8..a7bb0d09 100644 --- a/studio/tests/browser/ai-setup.spec.ts +++ b/studio/tests/browser/ai-setup.spec.ts @@ -545,3 +545,120 @@ test("the operator command is copied and the workers link opens Operate", async await page.getByRole("button", { name: "Open workers" }).click(); await expect(page).toHaveURL(/\/operate\/workers$/); }); + +test("saves Weave AI on a cloud connection with a suggested model, keeping its limits", async ({ + page, +}) => { + const platform = await start(page, { + connections: [ollamaConnection, openaiConnection], + models: [ + { + name: "gpt-5-mini", + approved: true, + available: true, + tools: "yes", + context_tokens: 128000, + }, + ], + }); + const card = page.locator("weave-ai-card"); + await expect(card.getByRole("heading", { name: "Weave AI" })).toBeVisible(); + await card.getByLabel("Connection", { exact: true }).click(); + await expect( + page.getByRole("option", { name: /ollama-local/ }), + ).toHaveAttribute("aria-disabled", "true"); + await page.getByRole("option", { name: /openai · revision 1/ }).click(); + await expect(card.getByLabel("Model", { exact: true })).toHaveValue( + "gpt-5-mini", + ); + await card.getByRole("button", { name: "Save Weave AI settings" }).click(); + await expect.poll(() => platform.lumiWrites.length).toBe(1); + expect(platform.lumiWrites[0]).toEqual({ + ifMatch: null, + body: { + enabled: true, + connection_revision_id: OPENAI_REVISION, + profile: { + options: { max_tokens: 1024 }, + reasoning: { pattern: "none", maxSteps: 6 }, + maxCalls: 8, + timeoutSeconds: 180, + provider: "openai-responses", + model: "gpt-5-mini", + outputSchema: { type: "object" }, + }, + }, + }); + await expect.poll(() => platform.readinessReads).toBeGreaterThan(1); +}); + +test("explains that connections without a key can't be used for Weave AI yet", async ({ + page, +}) => { + await start(page, { connections: [ollamaConnection] }); + await expect(page.locator("weave-ai-card")).toContainText( + "Weave AI needs an AI connection with a secret handle. Connections without a key can't be used for Weave AI yet.", + ); +}); + +test("a Weave AI manager without connection management sees why the card is read-only", async ({ + page, +}) => { + await start(page, { + capabilities: ["catalog.read", "status.read", "lumi.use", "lumi.manage"], + }); + await expect(page.locator("weave-ai-card")).toContainText( + "Choosing Weave AI's connection also requires the Tenant administrator role.", + ); + await expect( + page.getByRole("button", { name: "Save Weave AI settings" }), + ).toHaveCount(0); +}); + +test("saving Weave AI on a configured connection keeps its limits and sends its revision", async ({ + page, +}) => { + const platform = await start(page, { + connections: [ollamaConnection, openaiConnection], + models: [ + { + name: "gpt-5-mini", + approved: true, + available: true, + tools: "yes", + context_tokens: 128000, + }, + ], + lumi: { + enabled: true, + connection_revision_id: OPENAI_REVISION, + profile: { + provider: "openai-responses", + model: "gpt-5-mini", + options: { max_tokens: 512 }, + reasoning: { pattern: "none", maxSteps: 6 }, + maxCalls: 4, + timeoutSeconds: 120, + outputSchema: { type: "object" }, + }, + revision: 7, + }, + }); + const card = page.locator("weave-ai-card"); + await expect(card.getByLabel("Model", { exact: true })).toHaveValue( + "gpt-5-mini", + ); + await card.getByRole("switch", { name: "Turn on Weave AI" }).uncheck(); + await card.getByRole("button", { name: "Save Weave AI settings" }).click(); + await expect.poll(() => platform.lumiWrites.length).toBe(1); + expect(platform.lumiWrites[0].ifMatch).toBe('"7"'); + expect(platform.lumiWrites[0].body).toMatchObject({ + enabled: false, + profile: { + options: { max_tokens: 512 }, + maxCalls: 4, + timeoutSeconds: 120, + model: "gpt-5-mini", + }, + }); +}); diff --git a/studio/tests/browser/ai-task-profile.spec.ts b/studio/tests/browser/ai-task-profile.spec.ts index 2a1e7274..f06d9697 100644 --- a/studio/tests/browser/ai-task-profile.spec.ts +++ b/studio/tests/browser/ai-task-profile.spec.ts @@ -171,57 +171,37 @@ for (const width of [1440, 600]) }); }); +/** Opens the workflow AI profile dialog of a new AI task. */ +async function openProfiles(page: Page) { + await newWorkflow(page); + await page + .locator(".palette") + .getByRole("button", { name: "AI task", exact: true }) + .click(); + await new DesignerPage(page).selectStep("ask-ai-1"); + const form = page.locator("weave-llm-inspector"); + await form + .getByRole("button", { + name: "Configure workflow AI profiles", + exact: true, + }) + .click(); + return form; +} + test("AI model basics and advanced options preserve edits, invalid drafts and explicit clears", async ({ page, }) => { - const capture = await setup(page); - capture.connections.push({ - id: connectionId, - name: "approved-ai", - revision: 1, - connector: "weave-agentic-provider@1.0.0", - config: { - provider: "openai-responses", - endpoint: "https://api.openai.com/v1", - }, - }); - const profile = { - provider: "openai-responses", - model: "approved-model", - options: { max_tokens: 1024, temperature: 0.5, top_p: 0.8 }, - reasoning: { pattern: "none", maxSteps: 7 }, - maxCalls: 9, - timeoutSeconds: 90, - outputSchema: lumiSchema.$defs.LumiProfile.properties.outputSchema.const, - }; - let saved = { - enabled: true, - connection_revision_id: connectionId, - profile, - revision: 1, - }; - await page.route("**/lumi/configuration", async (route) => { - if (route.request().method() === "PUT") { - const body = route.request().postDataJSON(); - capture.configurations.push(body); - saved = { ...body, revision: saved.revision + 1 }; - } - await route.fulfill({ json: saved }); - }); - await page.getByRole("button", { name: "Ask Weave AI", exact: true }).click(); - await page - .getByRole("dialog", { name: "Ask Weave AI", exact: true }) - .getByRole("button", { name: "Weave AI settings", exact: true }) - .click(); - const lumi = page.getByRole("dialog", { name: "Ask Weave AI", exact: true }); - await expect(lumi.getByLabel("Model", { exact: true })).toHaveValue( - "approved-model", - ); - await expect(lumi.getByLabel("Max tokens", { exact: true })).toHaveValue( - "1024", + await setup(page); + const form = await openProfiles(page); + await selectChoice( + form.getByLabel("Provider", { exact: true }), + "openai-responses", ); + await form.getByLabel("Model", { exact: true }).fill("approved-model"); + await form.getByLabel("Max tokens", { exact: true }).fill("1024"); await expect( - lumi.getByRole("spinbutton", { + form.getByRole("spinbutton", { name: "Temperature (optional)", exact: true, }), @@ -229,56 +209,50 @@ test("AI model basics and advanced options preserve edits, invalid drafts and ex await page.screenshot({ path: "../.superpowers/editor-ux/ai-profile-basic.png", }); - const advanced = lumi.getByText("Advanced model settings", { exact: true }); + const advanced = form.getByText("Advanced model settings", { exact: true }); await advanced.click(); await page.screenshot({ path: "../.superpowers/editor-ux/ai-profile-advanced.png", }); - await lumi + await form .getByRole("spinbutton", { name: "Temperature (optional)", exact: true }) .fill("3"); await advanced.click(); - await lumi.getByLabel("Model", { exact: true }).fill("updated-model"); + await form.getByLabel("Model", { exact: true }).fill("updated-model"); await expect( - lumi.getByRole("button", { name: "Continue to connection", exact: true }), + form.getByRole("button", { name: "Continue to connection", exact: true }), ).toBeDisabled(); await advanced.click(); await expect( - lumi.getByRole("spinbutton", { + form.getByRole("spinbutton", { name: "Temperature (optional)", exact: true, }), ).toHaveValue("3"); - await lumi + await form .getByRole("spinbutton", { name: "Temperature (optional)", exact: true }) .fill(""); await advanced.click(); - await lumi.getByLabel("Max tokens", { exact: true }).fill("2048"); - await lumi + await form.getByLabel("Max tokens", { exact: true }).fill("2048"); + await form .getByRole("button", { name: "Continue to connection", exact: true }) .click(); - await lumi + await form + .getByRole("button", { name: "Add AI connection slot", exact: true }) + .click(); + await form .getByRole("button", { name: "Review settings", exact: true }) .click(); - await lumi - .getByRole("button", { name: "Save Weave AI settings", exact: true }) + await form + .getByRole("button", { name: "Apply workflow AI settings", exact: true }) .click(); - await expect(lumi.getByLabel("Message to Weave AI")).toBeVisible(); - expect(capture.configurations[0].profile).toEqual({ - ...profile, + const profile = parse(await sourceText(page)).spec.llmProfiles.default; + expect(profile).toMatchObject({ + provider: "openai-responses", model: "updated-model", - options: { max_tokens: 2048, top_p: 0.8 }, + options: { max_tokens: 2048 }, }); - await lumi - .getByRole("button", { name: "Weave AI settings", exact: true }) - .click(); - await lumi.getByText("Advanced model settings", { exact: true }).click(); - await expect( - lumi.getByRole("spinbutton", { - name: "Temperature (optional)", - exact: true, - }), - ).toHaveValue(""); + expect(profile["options"]).not.toHaveProperty("temperature"); }); test("canonical model validation blocks pending and invalid settings and ignores stale replies", async ({ @@ -289,43 +263,45 @@ test("canonical model validation blocks pending and invalid settings and ignores await page.route("**/studio/contracts/llm-profile/validate", (route) => { requests.push(route); }); - await page.getByRole("button", { name: "Ask Weave AI", exact: true }).click(); - await page - .getByRole("dialog", { name: "Ask Weave AI", exact: true }) - .getByRole("button", { name: "Weave AI settings", exact: true }) - .click(); - const lumi = page.getByRole("dialog", { name: "Ask Weave AI", exact: true }); + const form = await openProfiles(page); await selectChoice( - lumi.getByLabel("Provider", { exact: true }), + form.getByLabel("Provider", { exact: true }), "openai-responses", ); - await lumi.getByLabel("Model", { exact: true }).fill("first-model"); - await expect.poll(() => requests.length).toBe(1); - const next = lumi.getByRole("button", { + await form.getByLabel("Max tokens", { exact: true }).fill("512"); + await form.getByLabel("Model", { exact: true }).fill("first-model"); + const first = requests.length; + await expect.poll(() => requests.length).toBeGreaterThan(0); + const next = form.getByRole("button", { name: "Continue to connection", exact: true, }); await expect(next).toBeDisabled(); - await lumi.getByLabel("Model", { exact: true }).fill("second-model"); - await expect.poll(() => requests.length).toBe(2); + await form.getByLabel("Model", { exact: true }).fill("second-model"); + await expect.poll(() => requests.length).toBeGreaterThan(Math.max(first, 1)); + const stale = requests.slice(0, -1); + const latest = requests[requests.length - 1]; const message = "A model call timeout cannot exceed the total AI step timeout"; - await requests[1].fulfill({ + await latest.fulfill({ json: { valid: false, issues: [{ path: [], message }] }, }); - await expect(lumi.getByRole("alert")).toContainText(message); - await requests[0].fulfill({ json: { valid: true, issues: [] } }); + await expect(form.getByRole("alert")).toContainText(message); + for (const route of stale) + await route.fulfill({ json: { valid: true, issues: [] } }); await expect(next).toBeDisabled(); - await expect(lumi.getByRole("alert")).toContainText(message); - await lumi.getByLabel("Model", { exact: true }).fill("corrected-model"); - await expect.poll(() => requests.length).toBe(3); - expect(requests[2].request().postDataJSON()).toMatchObject({ + await expect(form.getByRole("alert")).toContainText(message); + const before = requests.length; + await form.getByLabel("Model", { exact: true }).fill("corrected-model"); + await expect.poll(() => requests.length).toBe(before + 1); + const corrected = requests[before]; + expect(corrected.request().postDataJSON()).toMatchObject({ provider: "openai-responses", model: "corrected-model", outputSchema: {}, }); - expect(requests[2].request().postDataJSON()).not.toHaveProperty("profile"); - await requests[2].fulfill({ json: { valid: true, issues: [] } }); + expect(corrected.request().postDataJSON()).not.toHaveProperty("profile"); + await corrected.fulfill({ json: { valid: true, issues: [] } }); await expect(next).toBeEnabled(); - await expect(lumi.getByRole("alert")).toHaveCount(0); + await expect(form.getByRole("alert")).toHaveCount(0); }); diff --git a/studio/tests/browser/weave-ai.spec.ts b/studio/tests/browser/weave-ai.spec.ts index 11ec61a5..fd06c562 100644 --- a/studio/tests/browser/weave-ai.spec.ts +++ b/studio/tests/browser/weave-ai.spec.ts @@ -15,7 +15,6 @@ limitations under the License. Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ -import { execFileSync } from "node:child_process"; import { expect, test } from "@playwright/test"; import { allCapabilities, @@ -25,7 +24,6 @@ import { sourceText, } from "./support"; import { DesignerPage } from "./designer-po"; -import { python } from "../python-path"; const proposal = `apiVersion: weave/v1alpha1 kind: Workflow @@ -179,186 +177,25 @@ test("context defaults off, stale proposals cannot replace later edits, and relo await expect(panel).not.toContainText("Private conversation"); }); -for (const viewport of [ - { width: 1440, height: 900 }, - { width: 390, height: 700 }, -]) - test(`Weave AI settings wizard preserves drafts and saves reviewed configuration at ${viewport.width}px`, async ({ - page, - }, testInfo) => { - await page.setViewportSize(viewport); - const schema = JSON.parse( - execFileSync( - python, - [ - "-c", - "import json; from firefly_weave.contracts.lumi import LumiConfigurationRequest; print(json.dumps(LumiConfigurationRequest.model_json_schema(by_alias=True)))", - ], - { cwd: "..", encoding: "utf8" }, - ), - ); - const fixed = schema.$defs.LumiProfile.properties.outputSchema.const; - await connected(page, { - capabilities: [...allCapabilities, "lumi.use", "lumi.manage"], - }); - await page.route("**/lumi/status", (r) => - r.fulfill({ - json: { - configured: true, - provider: "openai-responses", - model: "before", - }, - }), - ); - await page.route("**/studio/contracts/lumi-configuration", (r) => - r.fulfill({ json: schema }), - ); - await page.route("**/studio/contracts/llm-profile/validate", (r) => - r.fulfill({ json: { valid: true, issues: [] } }), - ); - const original = { - enabled: true, - connection_revision_id: "11111111-1111-4111-8111-111111111111", - profile: { - provider: "openai-responses", - model: "before", - options: { max_tokens: 512 }, - reasoning: { pattern: "none", maxSteps: 6 }, - maxCalls: 8, - timeoutSeconds: 120, - outputSchema: fixed, - }, - revision: 7, - }; - await page.route("**/environments/development/connections?*", (r) => - r.fulfill({ - json: { - items: [ - { - id: original.connection_revision_id, - name: "lumi-provider", - revision: 1, - connector: "weave-agentic-provider@1.0.0", - config: { - provider: "openai-responses", - endpoint: "https://api.openai.com", - }, - }, - ], - next_cursor: null, - }, - }), - ); - let saved: any; - let etag: string | undefined; - let finishSave: () => void = () => {}; - const saving = new Promise((resolve) => { - finishSave = resolve; - }); - await page.route("**/lumi/configuration", async (r) => { - if (r.request().method() === "PUT") { - saved = r.request().postDataJSON(); - etag = r.request().headers()["if-match"]; - await saving; - return r.fulfill({ json: { ...saved, revision: 8 } }); - } - return r.fulfill({ json: original }); - }); - await newWorkflow(page); - await page - .getByRole("button", { name: "Ask Weave AI", exact: true }) - .click(); - const panel = page.getByRole("dialog", { - name: "Ask Weave AI", - exact: true, - }); - await panel - .getByRole("button", { name: "Weave AI settings", exact: true }) - .click(); - await expect( - panel.getByRole("button", { name: "New conversation", exact: true }), - ).toBeHidden(); - await panel.getByLabel("Model", { exact: true }).fill("configured-model"); - await panel.getByLabel("Max tokens", { exact: true }).fill("1024"); - await expect(panel.getByText("Output schema", { exact: true })).toHaveCount( - 0, - ); - await expect( - panel.getByRole("button", { - name: "Save Weave AI settings", - exact: true, - }), - ).toHaveCount(0); - await panel - .getByRole("button", { name: "Continue to connection", exact: true }) - .click(); - await expect( - panel.getByRole("heading", { - name: "Choose a provider connection", - exact: true, - }), - ).toBeFocused(); - expect(saved).toBeUndefined(); - await panel.getByRole("button", { name: "Back", exact: true }).click(); - await expect(panel.getByLabel("Model", { exact: true })).toHaveValue( - "configured-model", - ); - await expect(panel.getByLabel("Max tokens", { exact: true })).toHaveValue( - "1024", - ); - await panel - .getByRole("button", { name: "Continue to connection", exact: true }) - .click(); - await panel - .getByRole("button", { name: "Review settings", exact: true }) - .click(); - await expect( - panel.getByRole("heading", { - name: "Review Weave AI settings", - exact: true, - }), - ).toBeFocused(); - await expect(panel.locator(".settings-summary")).toContainText( - "configured-model", - ); - await expect(panel.locator(".settings-summary")).toContainText( - "https://api.openai.com", - ); - await expect(panel.locator(".settings-summary")).toContainText( - "lumi-provider", - ); - expect(saved).toBeUndefined(); - await panel.screenshot({ - path: testInfo.outputPath(`lumi-review-${viewport.width}.png`), - }); - expect(await panel.evaluate((el) => el.scrollWidth <= el.clientWidth)).toBe( - true, - ); - await panel - .getByRole("button", { name: "Save Weave AI settings", exact: true }) - .click(); - await expect( - panel.getByRole("button", { name: "Weave AI settings", exact: true }), - ).toBeHidden(); - await expect( - panel.getByRole("button", { name: "New conversation", exact: true }), - ).toBeHidden(); - await expect( - panel.getByRole("button", { name: "Reload settings", exact: true }), - ).toBeDisabled(); - finishSave(); - await expect(panel.getByLabel("Message to Weave AI")).toBeVisible(); - await expect( - panel.getByRole("button", { name: "Weave AI settings", exact: true }), - ).toBeVisible(); - await expect( - panel.getByRole("button", { name: "New conversation", exact: true }), - ).toBeEnabled(); - expect(etag).toBe('"7"'); - expect(saved.profile.model).toBe("configured-model"); - expect(saved.profile.options.max_tokens).toBe(1024); - expect(saved.profile.outputSchema).toEqual(fixed); +test("Set up Weave AI opens Settings › AI models", async ({ page }) => { + await connected(page, { + capabilities: [...allCapabilities, "lumi.use", "lumi.manage"], }); + await page.route("**/lumi/status", (r) => + r.fulfill({ json: { configured: false } }), + ); + await newWorkflow(page); + await page.getByRole("button", { name: "Ask Weave AI", exact: true }).click(); + const panel = page.getByRole("dialog", { name: "Ask Weave AI", exact: true }); + await expect(panel).toContainText( + "Weave AI is not set up in this environment.", + ); + await panel + .getByRole("button", { name: "Set up Weave AI", exact: true }) + .click(); + await expect(panel).toHaveCount(0); + await expect(page).toHaveURL(/\/settings\/ai$/); +}); test("non-workflow proposals are validated before saving a reviewed file", async ({ page, From 283ac8a344cc11c07c9a21cc6037e70e2573f1e4 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:02:59 -0700 Subject: [PATCH 16/37] Keep keyboard focus in the AI connection sheet while a test runs and after a save --- studio/src/app/dialog.ts | 32 ++++++++ .../app/integrations/ai-connection-sheet.ts | 36 ++++++++- studio/tests/browser/ai-setup.spec.ts | 73 +++++++++++++++++++ 3 files changed, 139 insertions(+), 2 deletions(-) diff --git a/studio/src/app/dialog.ts b/studio/src/app/dialog.ts index 73c694e6..9e8c586f 100644 --- a/studio/src/app/dialog.ts +++ b/studio/src/app/dialog.ts @@ -81,6 +81,7 @@ export class Autofocus implements AfterViewInit { [attr.aria-labelledby]="titleId" [attr.aria-describedby]="describedBy() || null" (keydown)="keydown($event)" + (focusout)="keepFocus($event)" >

{{ heading() }}

@@ -122,7 +123,37 @@ export class Modal implements AfterViewInit, OnDestroy { resize: ResizeObserver; mutation: MutationObserver; } | null = null; + /** + * Keys pressed while focus fell to the page (a focused control was removed) + * still reach the topmost dialog, so Tab and Escape keep working. + */ + private readonly strayKeys = (event: KeyboardEvent) => { + if (!this.lostFocus() || (event.key !== "Tab" && event.key !== "Escape")) + return; + if (event.key === "Tab") { + event.preventDefault(); + this.panel.focus({ preventScroll: true }); + } else this.keydown(event); + }; + /** Focus left for nowhere (body) while this is the topmost dialog. */ + private lostFocus() { + const panel = this.panel; + const active = document.activeElement; + return ( + !!panel?.isConnected && + (!active || active === document.body) && + [...document.querySelectorAll(".modal-panel")].at(-1) === panel + ); + } + /** A focused control that is disabled or removed hands focus back to the dialog. */ + keepFocus(event: FocusEvent) { + if (event.relatedTarget) return; + queueMicrotask(() => { + if (this.lostFocus()) this.panel.focus({ preventScroll: true }); + }); + } ngAfterViewInit() { + document.addEventListener("keydown", this.strayKeys); this.watchFooter(); queueMicrotask(() => { const panel = this.panel; @@ -170,6 +201,7 @@ export class Modal implements AfterViewInit, OnDestroy { this.footerWatch = { resize, mutation }; } ngOnDestroy() { + document.removeEventListener("keydown", this.strayKeys); this.footerWatch?.resize.disconnect(); this.footerWatch?.mutation.disconnect(); const opener = this.opener; diff --git a/studio/src/app/integrations/ai-connection-sheet.ts b/studio/src/app/integrations/ai-connection-sheet.ts index a56ab97f..528c5e91 100644 --- a/studio/src/app/integrations/ai-connection-sheet.ts +++ b/studio/src/app/integrations/ai-connection-sheet.ts @@ -398,7 +398,12 @@ export interface AiTestRecord { }
@if (canManage()) { - } @else if (action.kind === "copy") { - {{ action.command }} + + @for (word of words(action.command); track $index) {{{ word }}{{ $last ? "" : " " }}}
diff --git a/studio/src/app/settings/weave-ai-card.ts b/studio/src/app/settings/weave-ai-card.ts index 1af70ec8..5e39bc8a 100644 --- a/studio/src/app/settings/weave-ai-card.ts +++ b/studio/src/app/settings/weave-ai-card.ts @@ -101,22 +101,27 @@ import { >
+ @if (connectionNote; as note) { +

{{ note }}

+ } @if (!usable.length) {

Weave AI needs an AI connection with a secret handle. Connections @@ -130,7 +135,7 @@ import { @@ -172,6 +177,12 @@ import { display: flex; justify-content: flex-end; } + /* The dropdowns match the buttons: 36 px, or 44 px for touch. */ + @media (max-width: 767px), (pointer: coarse) { + .weave-ai-fields weave-select { + --select-height: var(--control-touch); + } + } `, }) export class WeaveAiCard implements DoCheck, OnDestroy { @@ -183,6 +194,8 @@ export class WeaveAiCard implements DoCheck, OnDestroy { /** lumi.use */ canUse = input(false); connections = input([]); + /** The shell's AI setup generation: the card reloads when it changes. */ + generation = input(0); saved = output(); private readonly cdr = inject(ChangeDetectorRef); private setup!: AiSetupApi; @@ -196,6 +209,39 @@ export class WeaveAiCard implements DoCheck, OnDestroy { current: LumiConfiguration | null = null; models: AiModels | null = null; draft: WeaveAiDraft = { enabled: true, connection: "", model: "" }; + /** The saved connection revision, read when the list may not hold it. */ + pinned: AiConnection | null = null; + /** The saved connection revision could not be read. */ + pinnedMissing = false; + + /** + * The connection Save uses: the chosen one, or for a saved revision that + * is no longer the newest, the newest revision of that connection. + */ + get chosen(): AiConnection | null { + const id = this.draft.connection; + if (!id) return null; + const listed = this.connections().find((item) => item.id === id); + if (listed) return listed; + const pinned = this.pinned; + if (!pinned || pinned.id !== id) return null; + return ( + this.connections().find( + (item) => item.name === pinned.name && item.id !== pinned.id, + ) ?? null + ); + } + /** Why the saved connection is not the one shown, when it isn't. */ + get connectionNote(): string { + const id = this.draft.connection; + if (!id || this.connections().some((item) => item.id === id)) return ""; + const chosen = this.chosen; + if (chosen && this.pinned) + return `Weave AI uses revision ${this.pinned.revision} of ${this.pinned.name}; saving moves it to revision ${chosen.revision}.`; + if (this.pinnedMissing || (this.pinned && this.connections().length)) + return "Weave AI's connection is no longer available. Choose a connection, then save."; + return ""; + } get choices() { return weaveAiChoices(this.connections()); @@ -215,7 +261,7 @@ export class WeaveAiCard implements DoCheck, OnDestroy { private loadedFor = ""; private loads = 0; ngDoCheck() { - const key = `${this.canUse()}:${this.manager()}:${this.canChoose()}`; + const key = `${this.generation()}:${this.canUse()}:${this.manager()}:${this.canChoose()}`; if (key === this.loadedFor) return; this.loadedFor = key; void this.load(); @@ -232,8 +278,14 @@ export class WeaveAiCard implements DoCheck, OnDestroy { this.setup.lumiConfiguration(), this.setup.lumiReplySchema(), ]); + const pinnedId = current?.connection_revision_id ?? ""; + const pinned = pinnedId + ? await this.setup.connection(pinnedId).catch(() => null) + : null; if (!this.alive || load !== this.loads) return; this.current = current; + this.pinned = pinned; + this.pinnedMissing = !!pinnedId && !pinned; this.replySchema = reply; this.draft = { enabled: current?.enabled ?? true, @@ -276,10 +328,12 @@ export class WeaveAiCard implements DoCheck, OnDestroy { } } async save() { - const connection = this.connections().find( - (item) => item.id === this.draft.connection, - ); - if (this.busy || !connection) return; + if (this.busy) return; + const connection = this.chosen; + if (!connection) { + this.error = "Choose a connection for Weave AI."; + return; + } this.error = ""; let body: Record; try { diff --git a/studio/tests/browser/ai-models-layout.spec.ts b/studio/tests/browser/ai-models-layout.spec.ts index a6551d10..82dede8a 100644 --- a/studio/tests/browser/ai-models-layout.spec.ts +++ b/studio/tests/browser/ai-models-layout.spec.ts @@ -260,8 +260,35 @@ for (const size of sizes) "a connection action", ); } + // Weave AI's dropdowns are exactly as high as the buttons beside them; + // side by side they share a top, stacked they share a left edge. + const card = page.locator("weave-ai-card"); + // The bordered field, not the text box inside it. + const dropdowns = await boxes(card.locator("weave-select .choice-field")); + expect(dropdowns).toHaveLength(2); + expectAligned( + [d.control, ...dropdowns.map((dropdown) => dropdown.height)], + "Weave AI dropdown heights", + 0.5, + ); + if (Math.abs(dropdowns[0].y - dropdowns[1].y) <= 1) + expectAligned( + dropdowns.map((dropdown) => dropdown.y), + "Weave AI dropdown tops", + ); + else + expectAligned( + dropdowns.map((dropdown) => dropdown.x), + "Weave AI dropdown left edges", + ); + // A dropdown's chevron sits inside its 44 px field, which opens the + // same list: the field is the target, measured above. if (d.touch) - await minimumHeight(root.locator("button"), 44, "a touch target"); + await minimumHeight( + root.locator("button:not(.choice-field button)"), + 44, + "a touch target", + ); }); test("the side sheet fills its edge and lines up its fields and footer", async ({ diff --git a/studio/tests/browser/ai-models-po.ts b/studio/tests/browser/ai-models-po.ts index 035da4b0..048fedc3 100644 --- a/studio/tests/browser/ai-models-po.ts +++ b/studio/tests/browser/ai-models-po.ts @@ -259,18 +259,24 @@ export async function mockAiPlatform( }); const scripted = state.createAnswers.shift(); if (scripted) return json(r, scripted.body, scripted.status); + // Saving a name again adds its next revision; earlier ones stay readable. + const earlier = state.connections.filter((c) => c["name"] === body["name"]); + const revision = + Math.max(0, ...earlier.map((c) => Number(c["revision"]))) + 1; + const first = + body["name"] === "ollama-local" ? OLLAMA_REVISION : OPENAI_REVISION; const created = { ...body, - id: body["name"] === "ollama-local" ? OLLAMA_REVISION : OPENAI_REVISION, - revision: 1, + id: + revision === 1 + ? first + : `${first.slice(0, -2)}${String(revision).padStart(2, "0")}`, + revision, connector: "weave-agentic-provider@1.0.0", connector_digest: "digest", adapter: "agentic", }; - state.connections = [ - ...state.connections.filter((c) => c["name"] !== body["name"]), - created, - ]; + state.connections = [...state.connections, created]; return json(r, created, 201); }); await page.route(`${ENV}/connections/*`, (r) => { diff --git a/studio/tests/browser/ai-setup.spec.ts b/studio/tests/browser/ai-setup.spec.ts index 86d1a417..863bef5d 100644 --- a/studio/tests/browser/ai-setup.spec.ts +++ b/studio/tests/browser/ai-setup.spec.ts @@ -663,6 +663,93 @@ test("saving Weave AI on a configured connection keeps its limits and sends its }); }); +const GPT = { + name: "gpt-5-mini", + approved: true, + available: true, + tools: "yes", + context_tokens: 128000, +}; +const pinnedOpenAi = { + enabled: true, + connection_revision_id: OPENAI_REVISION, + profile: { + provider: "openai-responses", + model: "gpt-5-mini", + outputSchema: { type: "object" }, + }, + revision: 7, +}; + +test("after its connection gets a new revision, Weave AI says so and saves on the new one", async ({ + page, +}) => { + const platform = await start(page, { + connections: [ollamaConnection, openaiConnection], + models: [GPT], + lumi: pinnedOpenAi, + }); + const card = page.locator("weave-ai-card"); + await expect(card.getByLabel("Model", { exact: true })).toHaveValue( + "gpt-5-mini", + ); + // Edit the connection Weave AI uses: saving adds revision 2, and the card + // reads Weave AI's settings again. + let reads = 0; + page.on("request", (request) => { + if ( + request.method() === "GET" && + request.url().endsWith("/lumi/configuration") + ) + reads++; + }); + await page.getByRole("button", { name: "Test openai" }).click(); + const form = sheet(page, "openai"); + await form.getByLabel("Secret handle").fill("openai-team-key"); + await form.getByRole("button", { name: "Save as a new revision" }).click(); + await expect(form.getByRole("status").last()).toContainText("revision 2"); + await page.keyboard.press("Escape"); + await expect(form).toHaveCount(0); + await expect.poll(() => reads).toBeGreaterThan(0); + + await expect(card).toContainText( + "Weave AI uses revision 1 of openai; saving moves it to revision 2.", + ); + await expect(card.getByLabel("Connection", { exact: true })).toHaveValue( + "openai · revision 2", + ); + await card.getByRole("switch", { name: "Turn on Weave AI" }).uncheck(); + await card.getByRole("button", { name: "Save Weave AI settings" }).click(); + await expect.poll(() => platform.lumiWrites.length).toBe(1); + const second = platform.connections.find( + (c) => c["name"] === "openai" && c["revision"] === 2, + )!; + expect(platform.lumiWrites[0].ifMatch).toBe('"7"'); + expect(platform.lumiWrites[0].body).toMatchObject({ + enabled: false, + connection_revision_id: second["id"], + profile: { model: "gpt-5-mini" }, + }); +}); + +test("when Weave AI's connection can't be found, the card asks for one before saving", async ({ + page, +}) => { + const platform = await start(page, { + connections: [ollamaConnection], + models: [GPT], + lumi: pinnedOpenAi, + }); + const card = page.locator("weave-ai-card"); + await expect(card).toContainText( + "Weave AI's connection is no longer available. Choose a connection, then save.", + ); + await expect( + card.getByRole("button", { name: "Save Weave AI settings" }), + ).toBeDisabled(); + expect(platform.lumiWrites).toEqual([]); +}); + /** The accessible name of the focused element, when it sits in the AI connection sheet. */ const focusInSheet = (page: Page) => page.evaluate(() => { From 2444baf17ce542fbd0becd9d2ff4f688ab82acae Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 00:05:46 -0700 Subject: [PATCH 30/37] Show the secret set command with HANDLE in place of anything that is not a handle --- studio/src/app/integrations/ai-connection-sheet.ts | 6 ++---- studio/tests/browser/ai-setup.spec.ts | 9 +++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/studio/src/app/integrations/ai-connection-sheet.ts b/studio/src/app/integrations/ai-connection-sheet.ts index 5ab1a304..acef0fe8 100644 --- a/studio/src/app/integrations/ai-connection-sheet.ts +++ b/studio/src/app/integrations/ai-connection-sheet.ts @@ -276,10 +276,8 @@ export interface AiTestRecord { on the platform:

- weave platform secret set --handle - {{ draft.handle.trim() || "HANDLE" }} + + {{ secretCommand }}