diff --git a/README.md b/README.md index 039a1e42..7b2a179c 100644 --- a/README.md +++ b/README.md @@ -101,24 +101,24 @@ import { Factuality } from "autoevals"; ## Using other AI providers -When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI proxy](https://www.braintrust.dev/docs/guides/proxy). +When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI-compatible API. If `OPENAI_BASE_URL` is not set, it will look for a `BRAINTRUST_AI_GATEWAY_URL` environment variable and then default to the [Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway). -If you choose to use the proxy, you'll also get: +When you use the Braintrust Gateway, you'll also get: - Simplified access to many AI providers - Reduced costs with automatic request caching - Increased observability when you enable logging to Braintrust -The proxy is free to use, even if you don't have a Braintrust account. +The Braintrust-hosted Gateway is free to use while it is in beta. -If you have a Braintrust account, you can optionally set the `BRAINTRUST_API_KEY` environment variable instead of `OPENAI_API_KEY` to unlock additional features like logging and monitoring. You can also route requests to [supported AI providers and models](https://www.braintrust.dev/docs/guides/proxy#supported-models) or custom models you have configured in Braintrust. +Set the `BRAINTRUST_API_KEY` environment variable to authenticate Gateway requests. You can also route requests to supported AI providers and models or custom models you have configured in Braintrust.
### Python ```python -# NOTE: ensure BRAINTRUST_API_KEY is set in your environment and OPENAI_API_KEY is not set +# NOTE: ensure BRAINTRUST_API_KEY is set in your environment from autoevals.llm import * # Create an LLM-based evaluator using the Claude 3.5 Sonnet model from Anthropic @@ -139,7 +139,7 @@ print(f"Factuality metadata: {result.metadata['rationale']}") ### TypeScript ```typescript -// NOTE: ensure BRAINTRUST_API_KEY is set in your environment and OPENAI_API_KEY is not set +// NOTE: ensure BRAINTRUST_API_KEY is set in your environment import { Factuality } from "autoevals"; (async () => { diff --git a/js/oai.test.ts b/js/oai.test.ts index 20979a54..5adf9ba7 100644 --- a/js/oai.test.ts +++ b/js/oai.test.ts @@ -26,10 +26,14 @@ beforeAll(() => { let OPENAI_API_KEY: string | undefined; let OPENAI_BASE_URL: string | undefined; +let BRAINTRUST_API_KEY: string | undefined; +let BRAINTRUST_AI_GATEWAY_URL: string | undefined; beforeEach(() => { OPENAI_API_KEY = process.env.OPENAI_API_KEY; OPENAI_BASE_URL = process.env.OPENAI_BASE_URL; + BRAINTRUST_API_KEY = process.env.BRAINTRUST_API_KEY; + BRAINTRUST_AI_GATEWAY_URL = process.env.BRAINTRUST_AI_GATEWAY_URL; }); afterEach(() => { @@ -37,6 +41,8 @@ afterEach(() => { process.env.OPENAI_API_KEY = OPENAI_API_KEY; process.env.OPENAI_BASE_URL = OPENAI_BASE_URL; + process.env.BRAINTRUST_API_KEY = BRAINTRUST_API_KEY; + process.env.BRAINTRUST_AI_GATEWAY_URL = BRAINTRUST_AI_GATEWAY_URL; // Reset init state init({ client: undefined, defaultModel: undefined }); @@ -120,13 +126,14 @@ describe("OAI", () => { ); }); - test("calls proxy if everything unset", async () => { + test("calls gateway if everything unset", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + delete process.env.BRAINTRUST_AI_GATEWAY_URL; + process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( - http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => { - debugger; + http.post("https://gateway.braintrust.dev/chat/completions", () => { return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE); }), ); @@ -137,7 +144,28 @@ describe("OAI", () => { messages: [{ role: "user", content: "Hello" }], }); - debugger; + expect(response.choices[0].message.content).toBe( + "Hello, I am a mock response!", + ); + }); + + test("uses configured Braintrust Gateway URL", async () => { + delete process.env.OPENAI_API_KEY; + delete process.env.OPENAI_BASE_URL; + process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; + process.env.BRAINTRUST_AI_GATEWAY_URL = " https://gateway.example.com "; + + server.use( + http.post("https://gateway.example.com/chat/completions", () => { + return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE); + }), + ); + + const client = buildOpenAIClient({}); + const response = await client.chat.completions.create({ + model: "gpt-4", + messages: [{ role: "user", content: "Hello" }], + }); expect(response.choices[0].message.content).toBe( "Hello, I am a mock response!", @@ -147,9 +175,11 @@ describe("OAI", () => { test("default wraps", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + delete process.env.BRAINTRUST_AI_GATEWAY_URL; + process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( - http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => { + http.post("https://gateway.braintrust.dev/chat/completions", () => { return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE); }), ); @@ -173,9 +203,11 @@ describe("OAI", () => { test("wraps once", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + delete process.env.BRAINTRUST_AI_GATEWAY_URL; + process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( - http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => { + http.post("https://gateway.braintrust.dev/chat/completions", () => { return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE); }), ); diff --git a/js/oai.ts b/js/oai.ts index 15157180..6da1dee0 100644 --- a/js/oai.ts +++ b/js/oai.ts @@ -88,7 +88,18 @@ export function extractOpenAIArgs>( }; } -const PROXY_URL = "https://api.braintrust.dev/v1/proxy"; +const DEFAULT_GATEWAY_URL = "https://gateway.braintrust.dev"; + +const getGatewayURL = (): string => + (process.env.BRAINTRUST_AI_GATEWAY_URL ?? "").trim() || DEFAULT_GATEWAY_URL; + +const isGatewayBaseURL = (baseURL: string): boolean => { + const normalizedBaseURL = baseURL.replace(/\/+$/, ""); + return ( + normalizedBaseURL === DEFAULT_GATEWAY_URL || + normalizedBaseURL === getGatewayURL().replace(/\/+$/, "") + ); +}; const resolveOpenAIClient = (options: OpenAIAuth): OpenAI => { const { @@ -121,13 +132,18 @@ const resolveOpenAIClient = (options: OpenAIAuth): OpenAI => { }); } + const baseURL = + openAiBaseUrl || process.env.OPENAI_BASE_URL || getGatewayURL(); + const apiKey = + openAiApiKey || + (isGatewayBaseURL(baseURL) + ? process.env.BRAINTRUST_API_KEY || process.env.OPENAI_API_KEY + : process.env.OPENAI_API_KEY || process.env.BRAINTRUST_API_KEY); + return new OpenAI({ - apiKey: - openAiApiKey || - process.env.OPENAI_API_KEY || - process.env.BRAINTRUST_API_KEY, + apiKey, organization: openAiOrganizationId, - baseURL: openAiBaseUrl || process.env.OPENAI_BASE_URL || PROXY_URL, + baseURL, defaultHeaders: openAiDefaultHeaders, dangerouslyAllowBrowser: openAiDangerouslyAllowBrowser, }); @@ -179,7 +195,7 @@ export interface InitOptions { /** * An OpenAI-compatible client to use for all evaluations. * This can be an OpenAI client, or any client that implements the OpenAI API - * (e.g., configured to use the Braintrust proxy with Anthropic, Gemini, etc.) + * (e.g., configured to use the Braintrust Gateway with Anthropic, Gemini, etc.) */ client?: OpenAI; /** @@ -192,7 +208,7 @@ export interface InitOptions { * default models for different evaluation types. Only the specified models * are updated; others remain unchanged. * - * When using non-OpenAI providers via the Braintrust proxy, set this to + * When using non-OpenAI providers via the Braintrust Gateway, set this to * the appropriate model string (e.g., "claude-3-5-sonnet-20241022"). * * @example @@ -243,14 +259,14 @@ export interface InitOptions { * init({ client: new OpenAI() }); * * @example - * // Using with Anthropic via Braintrust proxy + * // Using with Anthropic via Braintrust Gateway * import { init } from "autoevals"; * import { OpenAI } from "openai"; * * init({ * client: new OpenAI({ * apiKey: process.env.BRAINTRUST_API_KEY, - * baseURL: "https://api.braintrust.dev/v1/proxy", + * baseURL: process.env.BRAINTRUST_AI_GATEWAY_URL || "https://gateway.braintrust.dev", * }), * defaultModel: { * completion: "claude-3-5-sonnet-20241022", diff --git a/py/autoevals/__init__.py b/py/autoevals/__init__.py index 10bda72b..0a5841ad 100644 --- a/py/autoevals/__init__.py +++ b/py/autoevals/__init__.py @@ -15,7 +15,7 @@ - Both sync and async evaluation support - Configurable scoring parameters - Detailed feedback through metadata -- Integration with OpenAI and other LLM providers through Braintrust AI Proxy +- Integration with OpenAI and other LLM providers through the Braintrust Gateway **Client setup**: @@ -43,10 +43,10 @@ evaluator = ClosedQA(client=client) ``` -**Multi-provider support via the Braintrust AI Proxy**: +**Multi-provider support via the Braintrust Gateway**: -Autoevals supports multiple LLM providers (Anthropic, Azure, etc.) through the Braintrust AI Proxy. -Configure your client to use the proxy and set the default model: +Autoevals supports multiple LLM providers (Anthropic, Azure, etc.) through the Braintrust Gateway. +Configure your client to use the Gateway and set the default model: ```python import os @@ -54,9 +54,9 @@ from autoevals import init from autoevals.llm import Factuality -# Configure client to use Braintrust AI Proxy with Claude +# Configure client to use the Braintrust Gateway with Claude client = AsyncOpenAI( - base_url="https://api.braintrust.dev/v1/proxy", + base_url=os.getenv("BRAINTRUST_AI_GATEWAY_URL") or "https://gateway.braintrust.dev", api_key=os.getenv("BRAINTRUST_API_KEY"), ) diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index 2ccb50f0..7a20e299 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -9,7 +9,16 @@ from dataclasses import dataclass from typing import Any, Optional, Protocol, TypedDict, TypeVar, Union, cast, runtime_checkable -PROXY_URL = "https://api.braintrust.dev/v1/proxy" +GATEWAY_URL = "https://gateway.braintrust.dev" + + +def _gateway_url() -> str: + return os.environ.get("BRAINTRUST_AI_GATEWAY_URL", "").strip() or GATEWAY_URL + + +def _is_gateway_url(base_url: str) -> bool: + normalized_base_url = base_url.rstrip("/") + return normalized_base_url == GATEWAY_URL or normalized_base_url == _gateway_url().rstrip("/") class DefaultModelConfig(TypedDict, total=False): @@ -195,7 +204,9 @@ def __post_init__(self): if not has_customization and not isinstance(self.openai, NamedWrapper): self.openai = wrap_openai(self.openai) - self._is_wrapped = isinstance(self.openai, NamedWrapper) + self._is_wrapped = isinstance(self.openai, NamedWrapper) or ( + not has_customization and wrap_openai.__module__.startswith("braintrust.") + ) openai_module = get_openai_module() @@ -431,7 +442,7 @@ def init( models for different evaluation types. Only the specified models are updated; others remain unchanged. - When using non-OpenAI providers via the Braintrust proxy, set this to the + When using non-OpenAI providers via the Braintrust Gateway, set this to the appropriate model string (e.g., "claude-3-5-sonnet-20241022"). Example: @@ -448,7 +459,7 @@ def init( init( client=OpenAI( api_key=os.environ["BRAINTRUST_API_KEY"], - base_url="https://api.braintrust.dev/v1/proxy", + base_url=os.getenv("BRAINTRUST_AI_GATEWAY_URL") or "https://gateway.braintrust.dev", ), default_model={ "completion": "claude-3-5-sonnet-20241022", @@ -520,7 +531,8 @@ def prepare_openai( Deprecated: Use the `client` argument and set the `openai`. base_url (str, optional): Base URL for API requests. If not provided, will - use OPENAI_BASE_URL from environment or fall back to PROXY_URL. + use OPENAI_BASE_URL from environment or fall back to BRAINTRUST_AI_GATEWAY_URL + or GATEWAY_URL. Deprecated: Use the `client` argument and set the `openai`. Returns: @@ -559,11 +571,14 @@ def prepare_openai( ) warned_deprecated_api_key_base_url = True + if base_url is None: + base_url = os.environ.get("OPENAI_BASE_URL") or _gateway_url() # prepare the default openai sdk, if not provided if api_key is None: - api_key = os.environ.get("OPENAI_API_KEY") or os.environ.get("BRAINTRUST_API_KEY") - if base_url is None: - base_url = os.environ.get("OPENAI_BASE_URL", PROXY_URL) + if _is_gateway_url(base_url): + api_key = os.environ.get("BRAINTRUST_API_KEY") or os.environ.get("OPENAI_API_KEY") + else: + api_key = os.environ.get("OPENAI_API_KEY") or os.environ.get("BRAINTRUST_API_KEY") if hasattr(openai_module, "OpenAI"): openai_module = cast(OpenAIV1Module, openai_module) diff --git a/py/autoevals/test_oai.py b/py/autoevals/test_oai.py index b3454dd7..1f3447d5 100644 --- a/py/autoevals/test_oai.py +++ b/py/autoevals/test_oai.py @@ -7,6 +7,7 @@ from autoevals import init # type: ignore[import] from autoevals.oai import ( # type: ignore[import] + GATEWAY_URL, LLMClient, OpenAIV0Module, OpenAIV1Module, @@ -28,6 +29,7 @@ def reset_env_and_client(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.setenv("OPENAI_API_KEY", "test-key") monkeypatch.setenv("OPENAI_BASE_URL", "http://test-url") + monkeypatch.delenv("BRAINTRUST_AI_GATEWAY_URL", raising=False) monkeypatch.setattr("autoevals.oai._named_wrapper", None) monkeypatch.setattr("autoevals.oai._wrap_openai", None) monkeypatch.setattr("autoevals.oai._openai_module", None) @@ -89,6 +91,32 @@ def test_prepare_openai_defaults(): assert openai_obj.base_url == "http://test-url" +def test_prepare_openai_defaults_to_gateway(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("BRAINTRUST_AI_GATEWAY_URL", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + monkeypatch.setenv("BRAINTRUST_API_KEY", "braintrust-key") + + prepared_client = prepare_openai() + + openai_obj = unwrap_named_wrapper(prepared_client.openai) + assert openai_obj.api_key == "braintrust-key" + assert str(openai_obj.base_url).rstrip("/") == GATEWAY_URL + + +def test_prepare_openai_uses_configured_gateway_url(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.setenv("BRAINTRUST_AI_GATEWAY_URL", " https://gateway.example.com ") + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + monkeypatch.setenv("BRAINTRUST_API_KEY", "braintrust-key") + + prepared_client = prepare_openai() + + openai_obj = unwrap_named_wrapper(prepared_client.openai) + assert openai_obj.api_key == "braintrust-key" + assert str(openai_obj.base_url).rstrip("/") == "https://gateway.example.com" + + def test_prepare_openai_with_plain_openai(): client = openai.OpenAI(api_key="api-key", base_url="http://test") prepared_client = prepare_openai(client=client)