Add bring-your-own OpenAI and Jev API keys

.env.example

@@ -5,7 +5,8 @@ OPENAI_API_KEY=
 
 # chat-latest follows ChatGPT Instant. gpt-* uses OpenAI; claude-* uses Anthropic.
 # Model access/availability must be verified with your provider.
-# Blank or unsupported prefixes and missing selected-provider keys fail at startup.
+# Blank or unsupported model prefixes fail at startup.
+# Keys are optional for BYOK-only hosting; visitors enter both keys in the app.
 LLM_MODEL=chat-latest
 
 # Jev selects the renderer and visualization before each new user turn.

README.md

@@ -143,14 +143,23 @@ Prerequisites: Node.js 22+, pnpm 9+, Python 3.12+, and [uv](https://docs.astral.
 git clone https://github.com/CopilotKit/OpenIntelligentUI.git
 cd OpenIntelligentUI
 make setup
-# Set OPENAI_API_KEY and TYPESAFE_API_KEY in apps/agent/.env
+# Optional: set shared OPENAI_API_KEY and TYPESAFE_API_KEY in apps/agent/.env
+# Or use API keys in the chat header after starting the app
 make dev
 ```
 
 Open the [app](http://localhost:3000). The [agent health endpoint](http://localhost:8123/health) confirms the agent service is running. See [Getting started](docs/getting-started.md) for configuration and verification.
 
 The default model is `chat-latest`, OpenAI’s documented alias for the latest ChatGPT Instant model, requiring `OPENAI_API_KEY`. Jev (`jev-latest`, requiring `TYPESAFE_API_KEY`) selects the renderer and visualization for each user turn. Basic tables use A2UI; charts, diagrams, calculators, and maps use Open Generative UI. Provider failures are surfaced instead of silently substituting another router or model. See [Visualization routing](docs/visualization-routing.md).
 
+## Use your own API keys
+
+Open **API keys** in the chat header, enter an OpenAI key and a Jev key, then choose **Test connection** and **Save keys**. The connection test makes a small request to each provider. Your keys use `chat-latest` for answers and `jev-latest` for visualization routing, with usage billed to your provider accounts.
+
+Keys stay in browser memory until refresh or **Clear keys**; the app does not save them in browser storage. Requests send them through the application server to their respective providers. They are kept out of chat state and checkpoints, and hosted LangSmith tracing is disabled for requests using your keys. Only use this feature on a deployment whose operator you trust. Saving or clearing keys starts a new chat; **New chat** keeps your keys for the current session.
+
+The agent can run without shared provider keys. Visitors must then supply both keys before chatting. Configured server credentials remain available when visitors have not supplied their own.
+
 ## How it works
 
 The Python Deep Agent uses a task-first system prompt and focused skills. CopilotKit carries the agent stream to the Next.js app. Native components handle supported structured tasks; `generateSandboxedUi` streams custom HTML, CSS, and JavaScript into an isolated iframe.

apps/agent/main.py

@@ -8,13 +8,20 @@
 
 from dotenv import load_dotenv
 from fastapi import FastAPI
-from copilotkit import CopilotKitMiddleware, LangGraphAGUIAgent
+from fastapi.responses import JSONResponse
+from copilotkit import CopilotKitMiddleware
 from ag_ui_langgraph import add_langgraph_fastapi_endpoint
 from deepagents import create_deep_agent
 
 from src.anthropic_compat import ConsecutiveSystemMessagesMiddleware
 from src.bounded_memory_saver import BoundedMemorySaver
-from src.model import build_model
+from src.model import build_model, RequestModelMiddleware, CredentialScopedModel
+from src.credentials import (
+    CredentialsMiddleware,
+    CredentialScopedAgent,
+    current_credentials,
+    validate_credentials,
+)
 from src.visualization_router import JevVisualizationMiddleware
 from src.skill_backend import SKILL_SOURCES, build_agent_backend
 from src.query import query_data
@@ -27,9 +34,10 @@
 load_dotenv()
 
 agent = create_deep_agent(
-    model=build_model(),
+    model=CredentialScopedModel(fallback=build_model(allow_unconfigured=True)),
     tools=[query_data, get_trip_stop_images, plan_visualization, *todo_tools, generate_form],
     middleware=[
+        RequestModelMiddleware(),
         CopilotKitMiddleware(),
         JevVisualizationMiddleware(),
         ConsecutiveSystemMessagesMiddleware(),
@@ -42,6 +50,15 @@
 )
 
 app = FastAPI()
+app.add_middleware(CredentialsMiddleware)
+
+
+@app.post("/credentials/validate")
+async def validate_provider_credentials():
+    credentials = current_credentials.get()
+    if credentials is None:
+        return JSONResponse({"ok": False, "code": "invalid_keys"}, status_code=400)
+    return await validate_credentials(credentials)
 
 
 @app.get("/health")
@@ -51,7 +68,7 @@ def health():
 
 add_langgraph_fastapi_endpoint(
     app=app,
-    agent=LangGraphAGUIAgent(
+    agent=CredentialScopedAgent(
         name="sample_agent",
         description="Open Generative UI by CopilotKit — answers you can interact with",
         graph=agent,

apps/agent/src/credentials.py

@@ -0,0 +1,129 @@
+"""Ephemeral provider credentials, scoped to the complete ASGI stream lifetime."""
+
+import asyncio
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+import hashlib
+import hmac
+import json
+import os
+
+import httpx
+from copilotkit import LangGraphAGUIAgent
+from langsmith import tracing_context
+from starlette.responses import JSONResponse
+
+
+@dataclass(frozen=True)
+class Credentials:
+    openai: str = field(repr=False)
+    jev: str = field(repr=False)
+
+
+current_credentials: ContextVar[Credentials | None] = ContextVar('provider_credentials', default=None)
+SENSITIVE_HEADERS = {b'x-openai-api-key', b'x-jev-api-key'}
+
+
+def parse_credentials(headers):
+    values = {}
+    for name, value in headers:
+        name = name.lower()
+        if name in SENSITIVE_HEADERS:
+            if name in values:
+                raise ValueError('invalid_keys')
+            # Reject controls before trimming: HTTP whitespace is spaces only.
+            if any(c < 32 or c > 126 for c in value):
+                raise ValueError('invalid_keys')
+            value = value.strip(b' ')
+            if not value or len(value) > 4096 or any(c <= 32 for c in value):
+                raise ValueError('invalid_keys')
+            values[name] = value.decode('ascii')
+    if not values:
+        return None
+    if len(values) != 2:
+        raise ValueError('invalid_keys')
+    return Credentials(values[b'x-openai-api-key'], values[b'x-jev-api-key'])
+
+
+def server_credentials_available():
+    model = os.environ.get('LLM_MODEL', 'chat-latest').strip()
+    key = 'ANTHROPIC_API_KEY' if model.startswith('claude-') else 'OPENAI_API_KEY'
+    return bool(os.environ.get(key, '').strip() and os.environ.get('TYPESAFE_API_KEY', '').strip())
+
+
+class CredentialsMiddleware:
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        if scope['type'] != 'http':
+            return await self.app(scope, receive, send)
+        try:
+            credentials = parse_credentials(scope.get('headers', []))
+        except ValueError:
+            return await JSONResponse({'ok': False, 'code': 'invalid_keys'}, status_code=400)(scope, receive, send)
+        if scope['path'] == '/' and not credentials and not server_credentials_available():
+            return await JSONResponse({'ok': False, 'code': 'credentials_required'}, status_code=401)(scope, receive, send)
+        clean_scope = {**scope, 'headers': [(k, v) for k, v in scope.get('headers', []) if k.lower() not in SENSITIVE_HEADERS]}
+        token = current_credentials.set(credentials)
+        try:
+            # BYOK provider calls and graph runs must never enter hosted traces.
+            with tracing_context(enabled=False if credentials else None):
+                await self.app(clean_scope, receive, send)
+        finally:
+            current_credentials.reset(token)
+
+
+def private_thread_id(thread_id):
+    credentials = current_credentials.get()
+    if credentials:
+        key = hashlib.sha256(json.dumps([credentials.openai, credentials.jev]).encode()).digest()
+        namespace = 'byok'
+    else:
+        key = b'open-intelligent-ui-server-threads-v1'
+        namespace = 'server'
+    return namespace + ':' + hmac.new(key, thread_id.encode(), hashlib.sha256).hexdigest()
+
+
+class CredentialScopedAgent(LangGraphAGUIAgent):
+    async def run(self, input):
+        original = input.thread_id
+        scoped = input.model_copy(update={'thread_id': private_thread_id(original)})
+        async for event in super().run(scoped):
+            # The wire protocol retains the client ID; only graph storage sees the private ID.
+            if hasattr(event, 'thread_id'):
+                event = event.model_copy(update={'thread_id': original})
+            yield event
+
+
+async def validate_credentials(credentials):
+    """Exercise both APIs; never forward their response bodies or exceptions."""
+    from src.visualization_router import JEV_URL, request_options
+
+    async def check(provider, url, options):
+        try:
+            async with httpx.AsyncClient(timeout=12, follow_redirects=False, trust_env=False) as client:
+                response = await client.post(url, **options)
+            if response.is_success:
+                return None
+            if response.status_code in {400, 401, 403, 404}:
+                return provider + '_invalid'
+        except httpx.HTTPError:
+            pass
+        return 'provider_unavailable'
+
+    token = current_credentials.set(credentials)
+    try:
+        results = await asyncio.gather(
+            check('openai', 'https://api.openai.com/v1/chat/completions', {
+                'headers': {'Authorization': f'Bearer {credentials.openai}'},
+                'json': {'model': 'chat-latest', 'messages': [{'role': 'user', 'content': 'Reply OK.'}], 'max_completion_tokens': 8},
+            }),
+            check('jev', JEV_URL, request_options([{'role': 'human', 'content': 'Hello'}])),
+        )
+    finally:
+        current_credentials.reset(token)
+    error = next((result for result in results if result), None)
+    if error:
+        return JSONResponse({'ok': False, 'code': error}, status_code=503 if error == 'provider_unavailable' else 401)
+    return JSONResponse({'ok': True})

apps/agent/src/model.py

@@ -1,7 +1,12 @@
 """Chat model factory for the agent."""
 
 import os
+from contextlib import contextmanager
 
+from langchain.agents.middleware import AgentMiddleware
+from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
+from langchain_core.messages import AIMessageChunk
+from pydantic import Field
 from langchain_anthropic import ChatAnthropic
 from langchain_core.language_models.chat_models import BaseChatModel
 
@@ -14,7 +19,7 @@
 MAX_TOKENS = 64000
 
 
-def build_model() -> BaseChatModel:
+def build_model(*, allow_unconfigured: bool = False) -> BaseChatModel:
     model_name = os.environ.get("LLM_MODEL", DEFAULT_MODEL).strip()
     is_openai = model_name == "chat-latest" or model_name.startswith("gpt-")
     if not is_openai and not model_name.startswith("claude-"):
@@ -24,6 +29,8 @@ def build_model() -> BaseChatModel:
         )
     key_name = "OPENAI_API_KEY" if is_openai else "ANTHROPIC_API_KEY"
     if not os.environ.get(key_name, "").strip():
+        if allow_unconfigured:
+            return UnconfiguredModel()
         raise ValueError(
             f"{key_name} is required for the selected LLM_MODEL provider. "
             "Set it in the agent environment and restart the agent."
@@ -37,3 +44,136 @@ def build_model() -> BaseChatModel:
         model=model_name,
         max_tokens=MAX_TOKENS,
     )
+
+
+class UnconfiguredModel(BaseChatModel):
+    """Compile a BYOK-only graph without constructing a provider with fake credentials."""
+
+    @property
+    def _llm_type(self):
+        return "unconfigured"
+
+    def _generate(self, *args, **kwargs):
+        raise ValueError("Provider credentials are required. Add OpenAI and Jev keys.")
+
+    def bind_tools(self, tools, **kwargs):
+        return self
+
+
+def request_model(fallback):
+    from src.credentials import current_credentials
+    credentials = current_credentials.get()
+    if credentials is None:
+        return fallback
+    from langchain_openai import ChatOpenAI
+    from openai import AsyncOpenAI, OpenAI, DefaultAsyncHttpxClient, DefaultHttpxClient
+
+    # SDK None defaults inherit server organization/project settings. Explicit
+    # clients with empty scopes and isolated transports prevent that inheritance,
+    # including server HTTP(S)_PROXY and OPENAI_PROXY settings.
+    options = {
+        "api_key": credentials.openai,
+        "base_url": "https://api.openai.com/v1",
+        "organization": "",
+        "project": "",
+    }
+    sync_client = OpenAI(**options, http_client=DefaultHttpxClient(trust_env=False))
+    async_client = AsyncOpenAI(
+        **options, http_client=DefaultAsyncHttpxClient(trust_env=False)
+    )
+    # After construction, None omits these headers without rereading env vars.
+    for client in (sync_client, async_client):
+        client.organization = None
+        client.project = None
+        client.admin_api_key = None
+        client.webhook_secret = None
+        # The SDK merges OPENAI_CUSTOM_HEADERS even with default_headers={}.
+        # Clear that merge before any request so host Authorization cannot win.
+        client._custom_headers = {}
+    model = ChatOpenAI(
+        model=DEFAULT_MODEL,
+        api_key=credentials.openai,
+        base_url=options["base_url"],
+        openai_proxy="",
+        root_client=sync_client,
+        client=sync_client.chat.completions,
+        root_async_client=async_client,
+        async_client=async_client.chat.completions,
+    )
+    # LangChain's metadata field separately inherits organization environment
+    # values even when prebuilt SDK clients are supplied. Clear the inert field.
+    model.openai_organization = None
+    return model
+
+
+@contextmanager
+def provider_error_boundary():
+    from src.credentials import current_credentials
+    try:
+        yield
+    except Exception:
+        if current_credentials.get() is None:
+            raise
+        raise ValueError("OpenAI request failed. Check your key and model access, or retry.") from None
+
+
+class RequestModelMiddleware(AgentMiddleware):
+    def wrap_model_call(self, request, handler):
+        with provider_error_boundary():
+            return handler(request.override(model=request_model(request.model)))
+
+    async def awrap_model_call(self, request, handler):
+        with provider_error_boundary():
+            return await handler(request.override(model=request_model(request.model)))
+
+
+class CredentialScopedModel(BaseChatModel):
+    """Route DeepAgents' internal summary/subagent model calls through BYOK too."""
+
+    fallback: BaseChatModel = Field(exclude=True)
+
+    @property
+    def _llm_type(self):
+        return "credential-scoped"
+
+    def bind_tools(self, tools, **kwargs):
+        return self.bind(_credential_tools=tools, _credential_tool_options=kwargs)
+
+    def _target(self, kwargs):
+        model = request_model(self.fallback)
+        options = dict(kwargs)
+        tools = options.pop("_credential_tools", None)
+        tool_options = options.pop("_credential_tool_options", {})
+        if tools is not None:
+            model = model.bind_tools(tools, **tool_options)
+        return model, options
+
+    def _generate(self, messages, stop=None, run_manager=None, **kwargs):
+        with provider_error_boundary():
+            model, options = self._target(kwargs)
+            message = model.invoke(messages, stop=stop, **options)
+            return ChatResult(generations=[ChatGeneration(message=message)])
+
+    async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs):
+        with provider_error_boundary():
+            model, options = self._target(kwargs)
+            message = await model.ainvoke(messages, stop=stop, **options)
+            return ChatResult(generations=[ChatGeneration(message=message)])
+
+    def _stream(self, messages, stop=None, run_manager=None, **kwargs):
+        with provider_error_boundary():
+            model, options = self._target(kwargs)
+            for chunk in model.stream(messages, stop=stop, **options):
+                yield ChatGenerationChunk(message=as_chunk(chunk))
+
+    async def _astream(self, messages, stop=None, run_manager=None, **kwargs):
+        with provider_error_boundary():
+            model, options = self._target(kwargs)
+            async for chunk in model.astream(messages, stop=stop, **options):
+                yield ChatGenerationChunk(message=as_chunk(chunk))
+
+
+def as_chunk(message):
+    if isinstance(message, AIMessageChunk):
+        return message
+    return AIMessageChunk(**message.model_dump(exclude={"type"}))