From efab664c530e7fce98594d2f248b2abc1f74e4a6 Mon Sep 17 00:00:00 2001 From: Erin McNulty Date: Mon, 1 Jun 2026 09:57:08 -0400 Subject: [PATCH 1/5] move from legacy proxy to gateway --- README.md | 12 ++++++------ js/oai.test.ts | 17 ++++++++++------- js/oai.ts | 27 +++++++++++++++++---------- py/autoevals/__init__.py | 12 ++++++------ py/autoevals/oai.py | 21 ++++++++++++++------- py/autoevals/test_oai.py | 13 +++++++++++++ 6 files changed, 66 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index a5eda5d4..2b30fe99 100644 --- a/README.md +++ b/README.md @@ -101,24 +101,24 @@ import { Factuality } from "autoevals"; ## Using other AI providers -When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI proxy](https://www.braintrust.dev/docs/guides/proxy). +When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI-compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway). -If you choose to use the proxy, you'll also get: +When you use the Gateway, you'll also get: - Simplified access to many AI providers - Reduced costs with automatic request caching - Increased observability when you enable logging to Braintrust -The proxy is free to use, even if you don't have a Braintrust account. +The Braintrust-hosted Gateway is free to use while it is in beta. -If you have a Braintrust account, you can optionally set the `BRAINTRUST_API_KEY` environment variable instead of `OPENAI_API_KEY` to unlock additional features like logging and monitoring. You can also route requests to [supported AI providers and models](https://www.braintrust.dev/docs/guides/proxy#supported-models) or custom models you have configured in Braintrust. +Set the `BRAINTRUST_API_KEY` environment variable to authenticate Gateway requests. You can also route requests to supported AI providers and models or custom models you have configured in Braintrust.
### Python ```python -# NOTE: ensure BRAINTRUST_API_KEY is set in your environment and OPENAI_API_KEY is not set +# NOTE: ensure BRAINTRUST_API_KEY is set in your environment from autoevals.llm import * # Create an LLM-based evaluator using the Claude 3.5 Sonnet model from Anthropic @@ -139,7 +139,7 @@ print(f"Factuality metadata: {result.metadata['rationale']}") ### TypeScript ```typescript -// NOTE: ensure BRAINTRUST_API_KEY is set in your environment and OPENAI_API_KEY is not set +// NOTE: ensure BRAINTRUST_API_KEY is set in your environment import { Factuality } from "autoevals"; (async () => { diff --git a/js/oai.test.ts b/js/oai.test.ts index 20979a54..75eeb783 100644 --- a/js/oai.test.ts +++ b/js/oai.test.ts @@ -26,10 +26,12 @@ beforeAll(() => { let OPENAI_API_KEY: string | undefined; let OPENAI_BASE_URL: string | undefined; +let BRAINTRUST_API_KEY: string | undefined; beforeEach(() => { OPENAI_API_KEY = process.env.OPENAI_API_KEY; OPENAI_BASE_URL = process.env.OPENAI_BASE_URL; + BRAINTRUST_API_KEY = process.env.BRAINTRUST_API_KEY; }); afterEach(() => { @@ -37,6 +39,7 @@ afterEach(() => { process.env.OPENAI_API_KEY = OPENAI_API_KEY; process.env.OPENAI_BASE_URL = OPENAI_BASE_URL; + process.env.BRAINTRUST_API_KEY = BRAINTRUST_API_KEY; // Reset init state init({ client: undefined, defaultModel: undefined }); @@ -120,13 +123,13 @@ describe("OAI", () => { ); }); - test("calls proxy if everything unset", async () => { + test("calls gateway if everything unset", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( - http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => { - debugger; + http.post("https://gateway.braintrust.dev/chat/completions", () => { return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE); }), ); @@ -137,8 +140,6 @@ describe("OAI", () => { messages: [{ role: "user", content: "Hello" }], }); - debugger; - expect(response.choices[0].message.content).toBe( "Hello, I am a mock response!", ); @@ -147,9 +148,10 @@ describe("OAI", () => { test("default wraps", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( - http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => { + http.post("https://gateway.braintrust.dev/chat/completions", () => { return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE); }), ); @@ -173,9 +175,10 @@ describe("OAI", () => { test("wraps once", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( - http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => { + http.post("https://gateway.braintrust.dev/chat/completions", () => { return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE); }), ); diff --git a/js/oai.ts b/js/oai.ts index 31614676..9d178dd7 100644 --- a/js/oai.ts +++ b/js/oai.ts @@ -82,7 +82,10 @@ export function extractOpenAIArgs>( }; } -const PROXY_URL = "https://api.braintrust.dev/v1/proxy"; +const GATEWAY_URL = "https://gateway.braintrust.dev"; + +const isGatewayBaseURL = (baseURL: string): boolean => + baseURL.replace(/\/+$/, "") === GATEWAY_URL; const resolveOpenAIClient = (options: OpenAIAuth): OpenAI => { const { @@ -115,13 +118,17 @@ const resolveOpenAIClient = (options: OpenAIAuth): OpenAI => { }); } + const baseURL = openAiBaseUrl || process.env.OPENAI_BASE_URL || GATEWAY_URL; + const apiKey = + openAiApiKey || + (isGatewayBaseURL(baseURL) + ? process.env.BRAINTRUST_API_KEY || process.env.OPENAI_API_KEY + : process.env.OPENAI_API_KEY || process.env.BRAINTRUST_API_KEY); + return new OpenAI({ - apiKey: - openAiApiKey || - process.env.OPENAI_API_KEY || - process.env.BRAINTRUST_API_KEY, + apiKey, organization: openAiOrganizationId, - baseURL: openAiBaseUrl || process.env.OPENAI_BASE_URL || PROXY_URL, + baseURL, defaultHeaders: openAiDefaultHeaders, dangerouslyAllowBrowser: openAiDangerouslyAllowBrowser, }); @@ -173,7 +180,7 @@ export interface InitOptions { /** * An OpenAI-compatible client to use for all evaluations. * This can be an OpenAI client, or any client that implements the OpenAI API - * (e.g., configured to use the Braintrust proxy with Anthropic, Gemini, etc.) + * (e.g., configured to use the Braintrust Gateway with Anthropic, Gemini, etc.) */ client?: OpenAI; /** @@ -186,7 +193,7 @@ export interface InitOptions { * default models for different evaluation types. Only the specified models * are updated; others remain unchanged. * - * When using non-OpenAI providers via the Braintrust proxy, set this to + * When using non-OpenAI providers via the Braintrust Gateway, set this to * the appropriate model string (e.g., "claude-3-5-sonnet-20241022"). * * @example @@ -237,14 +244,14 @@ export interface InitOptions { * init({ client: new OpenAI() }); * * @example - * // Using with Anthropic via Braintrust proxy + * // Using with Anthropic via Braintrust Gateway * import { init } from "autoevals"; * import { OpenAI } from "openai"; * * init({ * client: new OpenAI({ * apiKey: process.env.BRAINTRUST_API_KEY, - * baseURL: "https://api.braintrust.dev/v1/proxy", + * baseURL: "https://gateway.braintrust.dev", * }), * defaultModel: { * completion: "claude-3-5-sonnet-20241022", diff --git a/py/autoevals/__init__.py b/py/autoevals/__init__.py index b6e1dd4d..2fc0b618 100644 --- a/py/autoevals/__init__.py +++ b/py/autoevals/__init__.py @@ -15,7 +15,7 @@ - Both sync and async evaluation support - Configurable scoring parameters - Detailed feedback through metadata -- Integration with OpenAI and other LLM providers through Braintrust AI Proxy +- Integration with OpenAI and other LLM providers through the Braintrust Gateway **Client setup**: @@ -43,10 +43,10 @@ evaluator = ClosedQA(client=client) ``` -**Multi-provider support via the Braintrust AI Proxy**: +**Multi-provider support via the Braintrust Gateway**: -Autoevals supports multiple LLM providers (Anthropic, Azure, etc.) through the Braintrust AI Proxy. -Configure your client to use the proxy and set the default model: +Autoevals supports multiple LLM providers (Anthropic, Azure, etc.) through the Braintrust Gateway. +Configure your client to use the Gateway and set the default model: ```python import os @@ -54,9 +54,9 @@ from autoevals import init from autoevals.llm import Factuality -# Configure client to use Braintrust AI Proxy with Claude +# Configure client to use the Braintrust Gateway with Claude client = AsyncOpenAI( - base_url="https://api.braintrust.dev/v1/proxy", + base_url="https://gateway.braintrust.dev", api_key=os.getenv("BRAINTRUST_API_KEY"), ) diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index fa21554b..7181bbae 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -9,7 +9,11 @@ from dataclasses import dataclass from typing import Any, Optional, Protocol, TypedDict, TypeVar, Union, cast, runtime_checkable -PROXY_URL = "https://api.braintrust.dev/v1/proxy" +GATEWAY_URL = "https://gateway.braintrust.dev" + + +def _is_gateway_url(base_url: str) -> bool: + return base_url.rstrip("/") == GATEWAY_URL class DefaultModelConfig(TypedDict, total=False): @@ -425,7 +429,7 @@ def init( models for different evaluation types. Only the specified models are updated; others remain unchanged. - When using non-OpenAI providers via the Braintrust proxy, set this to the + When using non-OpenAI providers via the Braintrust Gateway, set this to the appropriate model string (e.g., "claude-3-5-sonnet-20241022"). Example: @@ -442,7 +446,7 @@ def init( init( client=OpenAI( api_key=os.environ["BRAINTRUST_API_KEY"], - base_url="https://api.braintrust.dev/v1/proxy", + base_url="https://gateway.braintrust.dev", ), default_model={ "completion": "claude-3-5-sonnet-20241022", @@ -514,7 +518,7 @@ def prepare_openai( Deprecated: Use the `client` argument and set the `openai`. base_url (str, optional): Base URL for API requests. If not provided, will - use OPENAI_BASE_URL from environment or fall back to PROXY_URL. + use OPENAI_BASE_URL from environment or fall back to GATEWAY_URL. Deprecated: Use the `client` argument and set the `openai`. Returns: @@ -553,11 +557,14 @@ def prepare_openai( ) warned_deprecated_api_key_base_url = True + if base_url is None: + base_url = os.environ.get("OPENAI_BASE_URL", GATEWAY_URL) # prepare the default openai sdk, if not provided if api_key is None: - api_key = os.environ.get("OPENAI_API_KEY") or os.environ.get("BRAINTRUST_API_KEY") - if base_url is None: - base_url = os.environ.get("OPENAI_BASE_URL", PROXY_URL) + if _is_gateway_url(base_url): + api_key = os.environ.get("BRAINTRUST_API_KEY") or os.environ.get("OPENAI_API_KEY") + else: + api_key = os.environ.get("OPENAI_API_KEY") or os.environ.get("BRAINTRUST_API_KEY") if hasattr(openai_module, "OpenAI"): openai_module = cast(OpenAIV1Module, openai_module) diff --git a/py/autoevals/test_oai.py b/py/autoevals/test_oai.py index 414c3c78..405b3984 100644 --- a/py/autoevals/test_oai.py +++ b/py/autoevals/test_oai.py @@ -15,6 +15,7 @@ from autoevals import init # type: ignore[import] from autoevals.oai import ( # type: ignore[import] + GATEWAY_URL, LLMClient, OpenAIV0Module, OpenAIV1Module, @@ -101,6 +102,18 @@ def test_prepare_openai_defaults(): assert openai_obj.base_url == "http://test-url" +def test_prepare_openai_defaults_to_gateway(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + monkeypatch.setenv("BRAINTRUST_API_KEY", "braintrust-key") + + prepared_client = prepare_openai() + + openai_obj = unwrap_named_wrapper(prepared_client.openai) + assert openai_obj.api_key == "braintrust-key" + assert str(openai_obj.base_url).rstrip("/") == GATEWAY_URL + + def test_prepare_openai_with_plain_openai(): client = openai.OpenAI(api_key="api-key", base_url="http://test") prepared_client = prepare_openai(client=client) From f739209808c83c14a818e6fb15771066fb0d6088 Mon Sep 17 00:00:00 2001 From: Erin McNulty Date: Mon, 1 Jun 2026 10:30:36 -0400 Subject: [PATCH 2/5] fix test --- py/autoevals/oai.py | 4 +++- py/autoevals/test_oai.py | 13 +++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index 7181bbae..20b7279b 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -199,7 +199,9 @@ def __post_init__(self): if not has_customization and not isinstance(self.openai, NamedWrapper): self.openai = wrap_openai(self.openai) - self._is_wrapped = isinstance(self.openai, NamedWrapper) + self._is_wrapped = isinstance(self.openai, NamedWrapper) or ( + not has_customization and wrap_openai.__module__.startswith("braintrust.") + ) openai_module = get_openai_module() diff --git a/py/autoevals/test_oai.py b/py/autoevals/test_oai.py index 405b3984..099cb031 100644 --- a/py/autoevals/test_oai.py +++ b/py/autoevals/test_oai.py @@ -4,14 +4,9 @@ import openai import pytest from braintrust.oai import ( - ChatCompletionV0Wrapper, - CompletionsV1Wrapper, NamedWrapper, - OpenAIV0Wrapper, - OpenAIV1Wrapper, wrap_openai, ) -from openai.resources.chat.completions import AsyncCompletions from autoevals import init # type: ignore[import] from autoevals.oai import ( # type: ignore[import] @@ -28,7 +23,7 @@ def unwrap_named_wrapper(obj: NamedWrapper | OpenAIV1Module.OpenAI | OpenAIV0Module) -> Any: - return getattr(obj, "_NamedWrapper__wrapped") + return getattr(obj, "_NamedWrapper__wrapped", obj) @pytest.fixture(autouse=True) @@ -85,7 +80,6 @@ def test_init_creates_async_llmclient_if_needed(mock_openai_v0: OpenAIV0Module): assert isinstance(prepared_client, LLMClient) assert prepared_client.is_wrapped - assert isinstance(prepared_client.openai, OpenAIV0Wrapper) assert prepared_client.complete.__name__ == "acreate" @@ -119,7 +113,7 @@ def test_prepare_openai_with_plain_openai(): prepared_client = prepare_openai(client=client) assert prepared_client.is_wrapped - assert isinstance(prepared_client.openai, OpenAIV1Wrapper) + assert prepared_client.openai is client def test_prepare_openai_async(): @@ -127,7 +121,6 @@ def test_prepare_openai_async(): assert isinstance(prepared_client, LLMClient) assert prepared_client.is_wrapped - assert isinstance(prepared_client.openai, OpenAIV1Wrapper) assert callable(prepared_client.complete) assert prepared_client.complete.__name__ == "complete_wrapper" @@ -244,7 +237,7 @@ def test_prepare_openai_v0_sdk(mock_openai_v0: OpenAIV0Module): assert prepared_client.is_wrapped assert prepared_client.openai.api_key == "test-key" - assert isinstance(getattr(prepared_client.complete, "__self__", None), ChatCompletionV0Wrapper) + assert prepared_client.complete is mock_openai_v0.ChatCompletion.create def test_prepare_openai_v0_async(mock_openai_v0: OpenAIV0Module): From 5d39d991e73bab91fc01161ec94441f23a93335e Mon Sep 17 00:00:00 2001 From: Erin McNulty Date: Tue, 2 Jun 2026 15:08:15 -0400 Subject: [PATCH 3/5] pr comments --- README.md | 8 ++++---- js/oai.test.ts | 29 +++++++++++++++++++++++++++++ js/oai.ts | 24 ++++++++++++++++-------- py/autoevals/__init__.py | 12 ++++++------ py/autoevals/oai.py | 16 +++++++++++----- py/autoevals/test_oai.py | 15 +++++++++++++++ 6 files changed, 81 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 2b30fe99..1cd960bd 100644 --- a/README.md +++ b/README.md @@ -101,17 +101,17 @@ import { Factuality } from "autoevals"; ## Using other AI providers -When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI-compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway). +When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI-compatible API. If `OPENAI_BASE_URL` is not set, it will look for a `BRAINTRUST_AI_GATEWAY_URL` environment variable and then default to the [Braintrust AI Gateway](https://www.braintrust.dev/docs/deploy/gateway). -When you use the Gateway, you'll also get: +When you use the Braintrust AI Gateway, you'll also get: - Simplified access to many AI providers - Reduced costs with automatic request caching - Increased observability when you enable logging to Braintrust -The Braintrust-hosted Gateway is free to use while it is in beta. +The Braintrust-hosted AI Gateway is free to use while it is in beta. -Set the `BRAINTRUST_API_KEY` environment variable to authenticate Gateway requests. You can also route requests to supported AI providers and models or custom models you have configured in Braintrust. +Set the `BRAINTRUST_API_KEY` environment variable to authenticate AI Gateway requests. You can also route requests to supported AI providers and models or custom models you have configured in Braintrust.
diff --git a/js/oai.test.ts b/js/oai.test.ts index 75eeb783..67746960 100644 --- a/js/oai.test.ts +++ b/js/oai.test.ts @@ -27,11 +27,13 @@ beforeAll(() => { let OPENAI_API_KEY: string | undefined; let OPENAI_BASE_URL: string | undefined; let BRAINTRUST_API_KEY: string | undefined; +let BRAINTRUST_AI_GATEWAY_URL: string | undefined; beforeEach(() => { OPENAI_API_KEY = process.env.OPENAI_API_KEY; OPENAI_BASE_URL = process.env.OPENAI_BASE_URL; BRAINTRUST_API_KEY = process.env.BRAINTRUST_API_KEY; + BRAINTRUST_AI_GATEWAY_URL = process.env.BRAINTRUST_AI_GATEWAY_URL; }); afterEach(() => { @@ -40,6 +42,7 @@ afterEach(() => { process.env.OPENAI_API_KEY = OPENAI_API_KEY; process.env.OPENAI_BASE_URL = OPENAI_BASE_URL; process.env.BRAINTRUST_API_KEY = BRAINTRUST_API_KEY; + process.env.BRAINTRUST_AI_GATEWAY_URL = BRAINTRUST_AI_GATEWAY_URL; // Reset init state init({ client: undefined, defaultModel: undefined }); @@ -126,6 +129,7 @@ describe("OAI", () => { test("calls gateway if everything unset", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + delete process.env.BRAINTRUST_AI_GATEWAY_URL; process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( @@ -145,9 +149,33 @@ describe("OAI", () => { ); }); + test("uses configured Braintrust AI Gateway URL", async () => { + delete process.env.OPENAI_API_KEY; + delete process.env.OPENAI_BASE_URL; + process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; + process.env.BRAINTRUST_AI_GATEWAY_URL = "https://gateway.example.com"; + + server.use( + http.post("https://gateway.example.com/chat/completions", () => { + return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE); + }), + ); + + const client = buildOpenAIClient({}); + const response = await client.chat.completions.create({ + model: "gpt-4", + messages: [{ role: "user", content: "Hello" }], + }); + + expect(response.choices[0].message.content).toBe( + "Hello, I am a mock response!", + ); + }); + test("default wraps", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + delete process.env.BRAINTRUST_AI_GATEWAY_URL; process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( @@ -175,6 +203,7 @@ describe("OAI", () => { test("wraps once", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; + delete process.env.BRAINTRUST_AI_GATEWAY_URL; process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; server.use( diff --git a/js/oai.ts b/js/oai.ts index 9d178dd7..67a9d882 100644 --- a/js/oai.ts +++ b/js/oai.ts @@ -82,10 +82,18 @@ export function extractOpenAIArgs>( }; } -const GATEWAY_URL = "https://gateway.braintrust.dev"; +const DEFAULT_GATEWAY_URL = "https://gateway.braintrust.dev"; -const isGatewayBaseURL = (baseURL: string): boolean => - baseURL.replace(/\/+$/, "") === GATEWAY_URL; +const getGatewayURL = (): string => + process.env.BRAINTRUST_AI_GATEWAY_URL || DEFAULT_GATEWAY_URL; + +const isGatewayBaseURL = (baseURL: string): boolean => { + const normalizedBaseURL = baseURL.replace(/\/+$/, ""); + return ( + normalizedBaseURL === DEFAULT_GATEWAY_URL || + normalizedBaseURL === getGatewayURL().replace(/\/+$/, "") + ); +}; const resolveOpenAIClient = (options: OpenAIAuth): OpenAI => { const { @@ -118,7 +126,7 @@ const resolveOpenAIClient = (options: OpenAIAuth): OpenAI => { }); } - const baseURL = openAiBaseUrl || process.env.OPENAI_BASE_URL || GATEWAY_URL; + const baseURL = openAiBaseUrl || process.env.OPENAI_BASE_URL || getGatewayURL(); const apiKey = openAiApiKey || (isGatewayBaseURL(baseURL) @@ -180,7 +188,7 @@ export interface InitOptions { /** * An OpenAI-compatible client to use for all evaluations. * This can be an OpenAI client, or any client that implements the OpenAI API - * (e.g., configured to use the Braintrust Gateway with Anthropic, Gemini, etc.) + * (e.g., configured to use the Braintrust AI Gateway with Anthropic, Gemini, etc.) */ client?: OpenAI; /** @@ -193,7 +201,7 @@ export interface InitOptions { * default models for different evaluation types. Only the specified models * are updated; others remain unchanged. * - * When using non-OpenAI providers via the Braintrust Gateway, set this to + * When using non-OpenAI providers via the Braintrust AI Gateway, set this to * the appropriate model string (e.g., "claude-3-5-sonnet-20241022"). * * @example @@ -244,14 +252,14 @@ export interface InitOptions { * init({ client: new OpenAI() }); * * @example - * // Using with Anthropic via Braintrust Gateway + * // Using with Anthropic via Braintrust AI Gateway * import { init } from "autoevals"; * import { OpenAI } from "openai"; * * init({ * client: new OpenAI({ * apiKey: process.env.BRAINTRUST_API_KEY, - * baseURL: "https://gateway.braintrust.dev", + * baseURL: process.env.BRAINTRUST_AI_GATEWAY_URL || "https://gateway.braintrust.dev", * }), * defaultModel: { * completion: "claude-3-5-sonnet-20241022", diff --git a/py/autoevals/__init__.py b/py/autoevals/__init__.py index 2fc0b618..1a065b52 100644 --- a/py/autoevals/__init__.py +++ b/py/autoevals/__init__.py @@ -15,7 +15,7 @@ - Both sync and async evaluation support - Configurable scoring parameters - Detailed feedback through metadata -- Integration with OpenAI and other LLM providers through the Braintrust Gateway +- Integration with OpenAI and other LLM providers through the Braintrust AI Gateway **Client setup**: @@ -43,10 +43,10 @@ evaluator = ClosedQA(client=client) ``` -**Multi-provider support via the Braintrust Gateway**: +**Multi-provider support via the Braintrust AI Gateway**: -Autoevals supports multiple LLM providers (Anthropic, Azure, etc.) through the Braintrust Gateway. -Configure your client to use the Gateway and set the default model: +Autoevals supports multiple LLM providers (Anthropic, Azure, etc.) through the Braintrust AI Gateway. +Configure your client to use the AI Gateway and set the default model: ```python import os @@ -54,9 +54,9 @@ from autoevals import init from autoevals.llm import Factuality -# Configure client to use the Braintrust Gateway with Claude +# Configure client to use the Braintrust AI Gateway with Claude client = AsyncOpenAI( - base_url="https://gateway.braintrust.dev", + base_url=os.getenv("BRAINTRUST_AI_GATEWAY_URL") or "https://gateway.braintrust.dev", api_key=os.getenv("BRAINTRUST_API_KEY"), ) diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index 20b7279b..d616c00b 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -12,8 +12,13 @@ GATEWAY_URL = "https://gateway.braintrust.dev" +def _gateway_url() -> str: + return os.environ.get("BRAINTRUST_AI_GATEWAY_URL", GATEWAY_URL) + + def _is_gateway_url(base_url: str) -> bool: - return base_url.rstrip("/") == GATEWAY_URL + normalized_base_url = base_url.rstrip("/") + return normalized_base_url == GATEWAY_URL or normalized_base_url == _gateway_url().rstrip("/") class DefaultModelConfig(TypedDict, total=False): @@ -431,7 +436,7 @@ def init( models for different evaluation types. Only the specified models are updated; others remain unchanged. - When using non-OpenAI providers via the Braintrust Gateway, set this to the + When using non-OpenAI providers via the Braintrust AI Gateway, set this to the appropriate model string (e.g., "claude-3-5-sonnet-20241022"). Example: @@ -448,7 +453,7 @@ def init( init( client=OpenAI( api_key=os.environ["BRAINTRUST_API_KEY"], - base_url="https://gateway.braintrust.dev", + base_url=os.getenv("BRAINTRUST_AI_GATEWAY_URL") or "https://gateway.braintrust.dev", ), default_model={ "completion": "claude-3-5-sonnet-20241022", @@ -520,7 +525,8 @@ def prepare_openai( Deprecated: Use the `client` argument and set the `openai`. base_url (str, optional): Base URL for API requests. If not provided, will - use OPENAI_BASE_URL from environment or fall back to GATEWAY_URL. + use OPENAI_BASE_URL from environment or fall back to BRAINTRUST_AI_GATEWAY_URL + or GATEWAY_URL. Deprecated: Use the `client` argument and set the `openai`. Returns: @@ -560,7 +566,7 @@ def prepare_openai( warned_deprecated_api_key_base_url = True if base_url is None: - base_url = os.environ.get("OPENAI_BASE_URL", GATEWAY_URL) + base_url = os.environ.get("OPENAI_BASE_URL") or _gateway_url() # prepare the default openai sdk, if not provided if api_key is None: if _is_gateway_url(base_url): diff --git a/py/autoevals/test_oai.py b/py/autoevals/test_oai.py index 099cb031..93d31b2b 100644 --- a/py/autoevals/test_oai.py +++ b/py/autoevals/test_oai.py @@ -32,6 +32,7 @@ def reset_env_and_client(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.setenv("OPENAI_API_KEY", "test-key") monkeypatch.setenv("OPENAI_BASE_URL", "http://test-url") + monkeypatch.delenv("BRAINTRUST_AI_GATEWAY_URL", raising=False) monkeypatch.setattr("autoevals.oai._named_wrapper", None) monkeypatch.setattr("autoevals.oai._wrap_openai", None) monkeypatch.setattr("autoevals.oai._openai_module", None) @@ -98,6 +99,7 @@ def test_prepare_openai_defaults(): def test_prepare_openai_defaults_to_gateway(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("BRAINTRUST_AI_GATEWAY_URL", raising=False) monkeypatch.setenv("OPENAI_API_KEY", "openai-key") monkeypatch.setenv("BRAINTRUST_API_KEY", "braintrust-key") @@ -108,6 +110,19 @@ def test_prepare_openai_defaults_to_gateway(monkeypatch: pytest.MonkeyPatch): assert str(openai_obj.base_url).rstrip("/") == GATEWAY_URL +def test_prepare_openai_uses_configured_gateway_url(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.setenv("BRAINTRUST_AI_GATEWAY_URL", "https://gateway.example.com") + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + monkeypatch.setenv("BRAINTRUST_API_KEY", "braintrust-key") + + prepared_client = prepare_openai() + + openai_obj = unwrap_named_wrapper(prepared_client.openai) + assert openai_obj.api_key == "braintrust-key" + assert str(openai_obj.base_url).rstrip("/") == "https://gateway.example.com" + + def test_prepare_openai_with_plain_openai(): client = openai.OpenAI(api_key="api-key", base_url="http://test") prepared_client = prepare_openai(client=client) From 7149381c00f8e4e71acb960d8129b70b0bb88f8d Mon Sep 17 00:00:00 2001 From: Erin McNulty Date: Tue, 2 Jun 2026 15:26:04 -0400 Subject: [PATCH 4/5] lint --- README.md | 8 ++++---- js/oai.test.ts | 2 +- js/oai.ts | 9 +++++---- py/autoevals/__init__.py | 10 +++++----- py/autoevals/oai.py | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 1cd960bd..8e175d75 100644 --- a/README.md +++ b/README.md @@ -101,17 +101,17 @@ import { Factuality } from "autoevals"; ## Using other AI providers -When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI-compatible API. If `OPENAI_BASE_URL` is not set, it will look for a `BRAINTRUST_AI_GATEWAY_URL` environment variable and then default to the [Braintrust AI Gateway](https://www.braintrust.dev/docs/deploy/gateway). +When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI-compatible API. If `OPENAI_BASE_URL` is not set, it will look for a `BRAINTRUST_AI_GATEWAY_URL` environment variable and then default to the [Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway). -When you use the Braintrust AI Gateway, you'll also get: +When you use the Braintrust Gateway, you'll also get: - Simplified access to many AI providers - Reduced costs with automatic request caching - Increased observability when you enable logging to Braintrust -The Braintrust-hosted AI Gateway is free to use while it is in beta. +The Braintrust-hosted Gateway is free to use while it is in beta. -Set the `BRAINTRUST_API_KEY` environment variable to authenticate AI Gateway requests. You can also route requests to supported AI providers and models or custom models you have configured in Braintrust. +Set the `BRAINTRUST_API_KEY` environment variable to authenticate Gateway requests. You can also route requests to supported AI providers and models or custom models you have configured in Braintrust.
diff --git a/js/oai.test.ts b/js/oai.test.ts index 67746960..b8d83860 100644 --- a/js/oai.test.ts +++ b/js/oai.test.ts @@ -149,7 +149,7 @@ describe("OAI", () => { ); }); - test("uses configured Braintrust AI Gateway URL", async () => { + test("uses configured Braintrust Gateway URL", async () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; diff --git a/js/oai.ts b/js/oai.ts index 67a9d882..cc076e73 100644 --- a/js/oai.ts +++ b/js/oai.ts @@ -126,7 +126,8 @@ const resolveOpenAIClient = (options: OpenAIAuth): OpenAI => { }); } - const baseURL = openAiBaseUrl || process.env.OPENAI_BASE_URL || getGatewayURL(); + const baseURL = + openAiBaseUrl || process.env.OPENAI_BASE_URL || getGatewayURL(); const apiKey = openAiApiKey || (isGatewayBaseURL(baseURL) @@ -188,7 +189,7 @@ export interface InitOptions { /** * An OpenAI-compatible client to use for all evaluations. * This can be an OpenAI client, or any client that implements the OpenAI API - * (e.g., configured to use the Braintrust AI Gateway with Anthropic, Gemini, etc.) + * (e.g., configured to use the Braintrust Gateway with Anthropic, Gemini, etc.) */ client?: OpenAI; /** @@ -201,7 +202,7 @@ export interface InitOptions { * default models for different evaluation types. Only the specified models * are updated; others remain unchanged. * - * When using non-OpenAI providers via the Braintrust AI Gateway, set this to + * When using non-OpenAI providers via the Braintrust Gateway, set this to * the appropriate model string (e.g., "claude-3-5-sonnet-20241022"). * * @example @@ -252,7 +253,7 @@ export interface InitOptions { * init({ client: new OpenAI() }); * * @example - * // Using with Anthropic via Braintrust AI Gateway + * // Using with Anthropic via Braintrust Gateway * import { init } from "autoevals"; * import { OpenAI } from "openai"; * diff --git a/py/autoevals/__init__.py b/py/autoevals/__init__.py index 1a065b52..a8681f39 100644 --- a/py/autoevals/__init__.py +++ b/py/autoevals/__init__.py @@ -15,7 +15,7 @@ - Both sync and async evaluation support - Configurable scoring parameters - Detailed feedback through metadata -- Integration with OpenAI and other LLM providers through the Braintrust AI Gateway +- Integration with OpenAI and other LLM providers through the Braintrust Gateway **Client setup**: @@ -43,10 +43,10 @@ evaluator = ClosedQA(client=client) ``` -**Multi-provider support via the Braintrust AI Gateway**: +**Multi-provider support via the Braintrust Gateway**: -Autoevals supports multiple LLM providers (Anthropic, Azure, etc.) through the Braintrust AI Gateway. -Configure your client to use the AI Gateway and set the default model: +Autoevals supports multiple LLM providers (Anthropic, Azure, etc.) through the Braintrust Gateway. +Configure your client to use the Gateway and set the default model: ```python import os @@ -54,7 +54,7 @@ from autoevals import init from autoevals.llm import Factuality -# Configure client to use the Braintrust AI Gateway with Claude +# Configure client to use the Braintrust Gateway with Claude client = AsyncOpenAI( base_url=os.getenv("BRAINTRUST_AI_GATEWAY_URL") or "https://gateway.braintrust.dev", api_key=os.getenv("BRAINTRUST_API_KEY"), diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index d616c00b..e5140ee5 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -436,7 +436,7 @@ def init( models for different evaluation types. Only the specified models are updated; others remain unchanged. - When using non-OpenAI providers via the Braintrust AI Gateway, set this to the + When using non-OpenAI providers via the Braintrust Gateway, set this to the appropriate model string (e.g., "claude-3-5-sonnet-20241022"). Example: From 5fc7adca2595cdb004485c809c8d7c36908fa79c Mon Sep 17 00:00:00 2001 From: Erin McNulty Date: Wed, 3 Jun 2026 09:59:45 -0400 Subject: [PATCH 5/5] trim --- js/oai.test.ts | 2 +- js/oai.ts | 2 +- py/autoevals/oai.py | 2 +- py/autoevals/test_oai.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/js/oai.test.ts b/js/oai.test.ts index b8d83860..5adf9ba7 100644 --- a/js/oai.test.ts +++ b/js/oai.test.ts @@ -153,7 +153,7 @@ describe("OAI", () => { delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_BASE_URL; process.env.BRAINTRUST_API_KEY = "braintrust-test-key"; - process.env.BRAINTRUST_AI_GATEWAY_URL = "https://gateway.example.com"; + process.env.BRAINTRUST_AI_GATEWAY_URL = " https://gateway.example.com "; server.use( http.post("https://gateway.example.com/chat/completions", () => { diff --git a/js/oai.ts b/js/oai.ts index cc076e73..3dd8322b 100644 --- a/js/oai.ts +++ b/js/oai.ts @@ -85,7 +85,7 @@ export function extractOpenAIArgs>( const DEFAULT_GATEWAY_URL = "https://gateway.braintrust.dev"; const getGatewayURL = (): string => - process.env.BRAINTRUST_AI_GATEWAY_URL || DEFAULT_GATEWAY_URL; + (process.env.BRAINTRUST_AI_GATEWAY_URL ?? "").trim() || DEFAULT_GATEWAY_URL; const isGatewayBaseURL = (baseURL: string): boolean => { const normalizedBaseURL = baseURL.replace(/\/+$/, ""); diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index e5140ee5..c13138dc 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -13,7 +13,7 @@ def _gateway_url() -> str: - return os.environ.get("BRAINTRUST_AI_GATEWAY_URL", GATEWAY_URL) + return os.environ.get("BRAINTRUST_AI_GATEWAY_URL", "").strip() or GATEWAY_URL def _is_gateway_url(base_url: str) -> bool: diff --git a/py/autoevals/test_oai.py b/py/autoevals/test_oai.py index 93d31b2b..1c26d4de 100644 --- a/py/autoevals/test_oai.py +++ b/py/autoevals/test_oai.py @@ -112,7 +112,7 @@ def test_prepare_openai_defaults_to_gateway(monkeypatch: pytest.MonkeyPatch): def test_prepare_openai_uses_configured_gateway_url(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("OPENAI_BASE_URL", raising=False) - monkeypatch.setenv("BRAINTRUST_AI_GATEWAY_URL", "https://gateway.example.com") + monkeypatch.setenv("BRAINTRUST_AI_GATEWAY_URL", " https://gateway.example.com ") monkeypatch.setenv("OPENAI_API_KEY", "openai-key") monkeypatch.setenv("BRAINTRUST_API_KEY", "braintrust-key")