diff --git a/openapi/spec.json b/openapi/spec.json index efa8482..c8aa922 100644 --- a/openapi/spec.json +++ b/openapi/spec.json @@ -94,11 +94,6 @@ "format": "uuid", "description": "SpanIframe id" }, - "EvalStatusPageIdParam": { - "type": "string", - "format": "uuid", - "description": "EvalStatusPage id" - }, "FunctionIdParam": { "type": "string", "format": "uuid", @@ -193,10 +188,6 @@ "type": "string", "description": "Name of the span_iframe to search for" }, - "EvalStatusPageName": { - "type": "string", - "description": "Name of the eval_status_page to search for" - }, "FunctionName": { "type": "string", "description": "Name of the function to search for" @@ -1694,6 +1685,24 @@ "nullable": true, "description": "Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified." }, + "internal_metadata": { + "type": "object", + "nullable": true, + "properties": { + "dataset_filter": { + "type": "object", + "nullable": true, + "additionalProperties": { + "nullable": true + }, + "description": "BTQL filter payload used to evaluate a subset of a linked dataset." + } + }, + "additionalProperties": { + "nullable": true + }, + "description": "Braintrust-controlled metadata about the experiment." + }, "parameters_id": { "type": "string", "nullable": true, @@ -1778,6 +1787,24 @@ "nullable": true, "description": "Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified." }, + "internal_metadata": { + "type": "object", + "nullable": true, + "properties": { + "dataset_filter": { + "type": "object", + "nullable": true, + "additionalProperties": { + "nullable": true + }, + "description": "BTQL filter payload used to evaluate a subset of a linked dataset." + } + }, + "additionalProperties": { + "nullable": true + }, + "description": "Braintrust-controlled metadata about the experiment." + }, "parameters_id": { "type": "string", "nullable": true, @@ -1853,6 +1880,24 @@ "nullable": true, "description": "Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified." }, + "internal_metadata": { + "type": "object", + "nullable": true, + "properties": { + "dataset_filter": { + "type": "object", + "nullable": true, + "additionalProperties": { + "nullable": true + }, + "description": "BTQL filter payload used to evaluate a subset of a linked dataset." + } + }, + "additionalProperties": { + "nullable": true + }, + "description": "Braintrust-controlled metadata about the experiment." + }, "parameters_id": { "type": "string", "nullable": true, @@ -5022,6 +5067,64 @@ "id" ] }, + "SpanScope": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "span" + ] + } + }, + "required": [ + "type" + ], + "description": "Process individual spans" + }, + "TraceScope": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "trace" + ] + }, + "idle_seconds": { + "type": "number", + "description": "Consider trace complete after this many seconds of inactivity (default: 30)" + } + }, + "required": [ + "type" + ], + "description": "Process entire traces (all spans sharing the same root_span_id)" + }, + "GroupScope": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "group" + ] + }, + "group_by": { + "type": "string", + "description": "Field path to group by, e.g. metadata.session_id" + }, + "idle_seconds": { + "type": "number", + "description": "Optional: trigger after this many seconds of inactivity" + } + }, + "required": [ + "type", + "group_by" + ], + "description": "Process spans/traces grouped by a field (e.g., session_id)" + }, "RetentionObjectType": { "type": "string", "enum": [ @@ -5101,64 +5204,6 @@ "function" ] }, - "SpanScope": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "span" - ] - } - }, - "required": [ - "type" - ], - "description": "Process individual spans" - }, - "TraceScope": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "trace" - ] - }, - "idle_seconds": { - "type": "number", - "description": "Consider trace complete after this many seconds of inactivity (default: 30)" - } - }, - "required": [ - "type" - ], - "description": "Process entire traces (all spans sharing the same root_span_id)" - }, - "GroupScope": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "group" - ] - }, - "group_by": { - "type": "string", - "description": "Field path to group by, e.g. metadata.session_id" - }, - "idle_seconds": { - "type": "number", - "description": "Optional: trigger after this many seconds of inactivity" - } - }, - "required": [ - "type", - "group_by" - ], - "description": "Process spans/traces grouped by a field (e.g., session_id)" - }, "TopicAutomationDataScope": { "anyOf": [ { @@ -5547,6 +5592,23 @@ ], "description": "The definition of what to export" }, + "scope": { + "anyOf": [ + { + "$ref": "#/components/schemas/SpanScope" + }, + { + "$ref": "#/components/schemas/TraceScope" + }, + { + "$ref": "#/components/schemas/GroupScope" + }, + { + "type": "null" + } + ], + "description": "Execution scope for export automation. Defaults to span-level execution." + }, "export_path": { "type": "string", "description": "The path to export the results to. It should include the storage protocol and prefix, e.g. s3://bucket-name/path/to/export" @@ -5590,6 +5652,25 @@ "role_arn", "external_id" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gcp_service_account" + ] + }, + "service_account_email": { + "type": "string", + "description": "The GCP service account email to impersonate" + } + }, + "required": [ + "type", + "service_account_email" + ] } ] }, @@ -5887,6 +5968,23 @@ ], "description": "The definition of what to export" }, + "scope": { + "anyOf": [ + { + "$ref": "#/components/schemas/SpanScope" + }, + { + "$ref": "#/components/schemas/TraceScope" + }, + { + "$ref": "#/components/schemas/GroupScope" + }, + { + "type": "null" + } + ], + "description": "Execution scope for export automation. Defaults to span-level execution." + }, "export_path": { "type": "string", "description": "The path to export the results to. It should include the storage protocol and prefix, e.g. s3://bucket-name/path/to/export" @@ -5930,6 +6028,25 @@ "role_arn", "external_id" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gcp_service_account" + ] + }, + "service_account_email": { + "type": "string", + "description": "The GCP service account email to impersonate" + } + }, + "required": [ + "type", + "service_account_email" + ] } ] }, @@ -6222,6 +6339,23 @@ ], "description": "The definition of what to export" }, + "scope": { + "anyOf": [ + { + "$ref": "#/components/schemas/SpanScope" + }, + { + "$ref": "#/components/schemas/TraceScope" + }, + { + "$ref": "#/components/schemas/GroupScope" + }, + { + "type": "null" + } + ], + "description": "Execution scope for export automation. Defaults to span-level execution." + }, "export_path": { "type": "string", "description": "The path to export the results to. It should include the storage protocol and prefix, e.g. s3://bucket-name/path/to/export" @@ -6265,6 +6399,25 @@ "role_arn", "external_id" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gcp_service_account" + ] + }, + "service_account_email": { + "type": "string", + "description": "The GCP service account email to impersonate" + } + }, + "required": [ + "type", + "service_account_email" + ] } ] }, @@ -6911,193 +7064,7 @@ } } }, - "EvalStatusPageTheme": { - "type": "string", - "enum": [ - "light", - "dark" - ], - "description": "The theme for the page" - }, - "EvalStatusPageConfig": { - "type": "object", - "properties": { - "score_columns": { - "type": "array", - "nullable": true, - "items": { - "type": "string" - }, - "description": "The score columns to display on the page" - }, - "metric_columns": { - "type": "array", - "nullable": true, - "items": { - "type": "string" - }, - "description": "The metric columns to display on the page" - }, - "grouping_field": { - "type": "string", - "nullable": true, - "description": "The metadata field to use for grouping experiments (model)" - }, - "filter": { - "type": "string", - "nullable": true, - "description": "BTQL filter to apply to experiment data" - }, - "sort_by": { - "type": "string", - "nullable": true, - "description": "Field to sort results by (format: 'score:' or 'metric:')" - }, - "sort_order": { - "type": "string", - "nullable": true, - "enum": [ - "asc", - "desc", - null - ], - "description": "Sort order (ascending or descending)" - }, - "api_key": { - "type": "string", - "nullable": true, - "description": "The API key used for fetching experiment data" - } - }, - "description": "Configuration for what data to display" - }, - "EvalStatusPage": { - "type": "object", - "properties": { - "id": { - "type": "string", - "format": "uuid", - "description": "Unique identifier for the eval status page" - }, - "project_id": { - "type": "string", - "format": "uuid", - "description": "Unique identifier for the project that the eval status page belongs under" - }, - "user_id": { - "type": "string", - "nullable": true, - "format": "uuid", - "description": "Identifies the user who created the eval status page" - }, - "created": { - "type": "string", - "nullable": true, - "format": "date-time", - "description": "Date of eval status page creation" - }, - "deleted_at": { - "type": "string", - "nullable": true, - "format": "date-time", - "description": "Date of eval status page deletion, or null if the eval status page is still active" - }, - "name": { - "type": "string", - "description": "Name of the eval status page" - }, - "description": { - "type": "string", - "nullable": true, - "description": "Textual description of the eval status page" - }, - "logo_url": { - "type": "string", - "nullable": true, - "description": "URL of the logo to display on the page" - }, - "theme": { - "$ref": "#/components/schemas/EvalStatusPageTheme" - }, - "config": { - "$ref": "#/components/schemas/EvalStatusPageConfig" - } - }, - "required": [ - "id", - "project_id", - "name", - "theme", - "config" - ], - "description": "A public eval status page that displays aggregate experiment results" - }, - "CreateEvalStatusPage": { - "type": "object", - "properties": { - "project_id": { - "type": "string", - "format": "uuid", - "description": "Unique identifier for the project that the eval status page belongs under" - }, - "name": { - "type": "string", - "description": "Name of the eval status page" - }, - "description": { - "type": "string", - "nullable": true, - "description": "Textual description of the eval status page" - }, - "logo_url": { - "type": "string", - "nullable": true, - "description": "URL of the logo to display on the page" - }, - "theme": { - "$ref": "#/components/schemas/EvalStatusPageTheme" - }, - "config": { - "$ref": "#/components/schemas/EvalStatusPageConfig" - } - }, - "required": [ - "project_id", - "name", - "theme", - "config" - ], - "description": "A public eval status page that displays aggregate experiment results" - }, - "PatchEvalStatusPage": { - "type": "object", - "properties": { - "name": { - "type": "string", - "nullable": true, - "description": "Name of the eval status page" - }, - "description": { - "type": "string", - "nullable": true, - "description": "Textual description of the eval status page" - }, - "logo_url": { - "type": "string", - "nullable": true, - "description": "URL of the logo to display on the page" - }, - "theme": { - "$ref": "#/components/schemas/EvalStatusPageTheme", - "nullable": true - }, - "config": { - "$ref": "#/components/schemas/EvalStatusPageConfig", - "nullable": true - } - } - }, - "CodeBundle": { + "CodeBundle": { "type": "object", "properties": { "runtime_context": { @@ -9041,6 +9008,10 @@ "type": "string", "nullable": true }, + "topicMapReportKey": { + "type": "string", + "nullable": true + }, "chartHeight": { "type": "number", "nullable": true @@ -9191,9 +9162,14 @@ "enum": [ "traces", "spans", + "topics", null ] }, + "cluster": { + "type": "string", + "nullable": true + }, "freezeColumns": { "type": "boolean", "nullable": true @@ -10912,6 +10888,16 @@ "dataset_id": { "type": "string" }, + "dataset_version": { + "type": "string", + "nullable": true, + "description": "The version of the dataset to evaluate" + }, + "dataset_environment": { + "type": "string", + "nullable": true, + "description": "The environment tag that resolves to the dataset version to evaluate" + }, "_internal_btql": { "type": "object", "nullable": true, @@ -10935,6 +10921,16 @@ "dataset_name": { "type": "string" }, + "dataset_version": { + "type": "string", + "nullable": true, + "description": "The version of the dataset to evaluate" + }, + "dataset_environment": { + "type": "string", + "nullable": true, + "description": "The environment tag that resolves to the dataset version to evaluate" + }, "_internal_btql": { "type": "object", "nullable": true, @@ -11226,15 +11222,6 @@ "name": "span_iframe_id", "in": "path" }, - "EvalStatusPageIdParam": { - "schema": { - "$ref": "#/components/schemas/EvalStatusPageIdParam" - }, - "required": true, - "description": "EvalStatusPage id", - "name": "eval_status_page_id", - "in": "path" - }, "FunctionIdParam": { "schema": { "$ref": "#/components/schemas/FunctionIdParam" @@ -11435,16 +11422,6 @@ "in": "query", "allowReserved": true }, - "EvalStatusPageName": { - "schema": { - "$ref": "#/components/schemas/EvalStatusPageName" - }, - "required": false, - "description": "Name of the eval_status_page to search for", - "name": "eval_status_page_name", - "in": "query", - "allowReserved": true - }, "FunctionName": { "schema": { "$ref": "#/components/schemas/FunctionName" @@ -23999,858 +23976,6 @@ } } }, - "/v1/eval_status_page": { - "post": { - "tags": [ - "EvalStatusPages" - ], - "security": [ - { - "bearerAuth": [] - }, - {} - ], - "operationId": "postEvalStatusPage", - "description": "Create a new eval_status_page. If there is an existing eval_status_page with the same name as the one specified in the request, will return the existing eval_status_page unmodified", - "summary": "Create eval_status_page", - "requestBody": { - "description": "Any desired information about the new eval_status_page object", - "required": false, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateEvalStatusPage" - } - } - } - }, - "responses": { - "200": { - "description": "Returns the new eval_status_page object", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvalStatusPage" - } - } - } - }, - "400": { - "description": "The request was unacceptable, often due to missing a required parameter", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "401": { - "description": "No valid API key provided", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "403": { - "description": "The API key doesn’t have permissions to perform the request", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "429": { - "description": "Too many requests hit the API too quickly. We recommend an exponential backoff of your requests", - "headers": { - "Retry-After": { - "schema": { - "type": "string" - } - } - }, - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "500": { - "description": "Something went wrong on Braintrust's end. (These are rare.)", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - } - } - }, - "put": { - "tags": [ - "EvalStatusPages" - ], - "security": [ - { - "bearerAuth": [] - }, - {} - ], - "operationId": "putEvalStatusPage", - "description": "Create or replace eval_status_page. If there is an existing eval_status_page with the same name as the one specified in the request, will replace the existing eval_status_page with the provided fields", - "summary": "Create or replace eval_status_page", - "requestBody": { - "description": "Any desired information about the new eval_status_page object", - "required": false, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateEvalStatusPage" - } - } - } - }, - "responses": { - "200": { - "description": "Returns the new eval_status_page object", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvalStatusPage" - } - } - } - }, - "400": { - "description": "The request was unacceptable, often due to missing a required parameter", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "401": { - "description": "No valid API key provided", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "403": { - "description": "The API key doesn’t have permissions to perform the request", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "429": { - "description": "Too many requests hit the API too quickly. We recommend an exponential backoff of your requests", - "headers": { - "Retry-After": { - "schema": { - "type": "string" - } - } - }, - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "500": { - "description": "Something went wrong on Braintrust's end. (These are rare.)", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - } - } - }, - "get": { - "operationId": "getEvalStatusPage", - "tags": [ - "EvalStatusPages" - ], - "description": "List out all eval_status_pages. The eval_status_pages are sorted by creation date, with the most recently-created eval_status_pages coming first", - "summary": "List eval_status_pages", - "security": [ - { - "bearerAuth": [] - }, - {} - ], - "parameters": [ - { - "$ref": "#/components/parameters/AppLimitParam" - }, - { - "$ref": "#/components/parameters/StartingAfter" - }, - { - "$ref": "#/components/parameters/EndingBefore" - }, - { - "$ref": "#/components/parameters/Ids" - }, - { - "$ref": "#/components/parameters/EvalStatusPageName" - }, - { - "$ref": "#/components/parameters/OrgName" - } - ], - "responses": { - "200": { - "description": "Returns a list of eval_status_page objects", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "objects": { - "type": "array", - "items": { - "$ref": "#/components/schemas/EvalStatusPage" - }, - "description": "A list of eval_status_page objects" - } - }, - "required": [ - "objects" - ], - "additionalProperties": false - } - } - } - }, - "400": { - "description": "The request was unacceptable, often due to missing a required parameter", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "401": { - "description": "No valid API key provided", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "403": { - "description": "The API key doesn’t have permissions to perform the request", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "429": { - "description": "Too many requests hit the API too quickly. We recommend an exponential backoff of your requests", - "headers": { - "Retry-After": { - "schema": { - "type": "string" - } - } - }, - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "500": { - "description": "Something went wrong on Braintrust's end. (These are rare.)", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - } - } - }, - "options": { - "operationId": "optionsEvalStatusPage", - "description": "Enable CORS", - "summary": "Enable CORS (`/v1/eval_status_page`)", - "security": [], - "tags": [ - "CORS" - ], - "responses": { - "200": { - "description": "Response for CORS method", - "headers": { - "Access-Control-Allow-Credentials": { - "schema": { - "type": "string" - } - }, - "Access-Control-Allow-Headers": { - "schema": { - "type": "string" - } - }, - "Access-Control-Allow-Methods": { - "schema": { - "type": "string" - } - }, - "Access-Control-Allow-Origin": { - "schema": { - "type": "string" - } - }, - "Access-Control-Max-Age": { - "schema": { - "type": "string" - } - } - }, - "content": {} - }, - "400": { - "description": "The request was unacceptable, often due to missing a required parameter", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - } - } - } - }, - "/v1/eval_status_page/{eval_status_page_id}": { - "get": { - "operationId": "getEvalStatusPageId", - "tags": [ - "EvalStatusPages" - ], - "description": "Get a eval_status_page object by its id", - "summary": "Get eval_status_page", - "security": [ - { - "bearerAuth": [] - }, - {} - ], - "parameters": [ - { - "$ref": "#/components/parameters/EvalStatusPageIdParam" - } - ], - "responses": { - "200": { - "description": "Returns the eval_status_page object", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvalStatusPage" - } - } - } - }, - "400": { - "description": "The request was unacceptable, often due to missing a required parameter", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "401": { - "description": "No valid API key provided", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "403": { - "description": "The API key doesn’t have permissions to perform the request", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "429": { - "description": "Too many requests hit the API too quickly. We recommend an exponential backoff of your requests", - "headers": { - "Retry-After": { - "schema": { - "type": "string" - } - } - }, - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "500": { - "description": "Something went wrong on Braintrust's end. (These are rare.)", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - } - } - }, - "patch": { - "operationId": "patchEvalStatusPageId", - "tags": [ - "EvalStatusPages" - ], - "description": "Partially update a eval_status_page object. Specify the fields to update in the payload. Any object-type fields will be deep-merged with existing content. Currently we do not support removing fields or setting them to null.", - "summary": "Partially update eval_status_page", - "security": [ - { - "bearerAuth": [] - }, - {} - ], - "parameters": [ - { - "$ref": "#/components/parameters/EvalStatusPageIdParam" - } - ], - "requestBody": { - "description": "Fields to update", - "required": false, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PatchEvalStatusPage" - } - } - } - }, - "responses": { - "200": { - "description": "Returns the eval_status_page object", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvalStatusPage" - } - } - } - }, - "400": { - "description": "The request was unacceptable, often due to missing a required parameter", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "401": { - "description": "No valid API key provided", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "403": { - "description": "The API key doesn’t have permissions to perform the request", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "429": { - "description": "Too many requests hit the API too quickly. We recommend an exponential backoff of your requests", - "headers": { - "Retry-After": { - "schema": { - "type": "string" - } - } - }, - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "500": { - "description": "Something went wrong on Braintrust's end. (These are rare.)", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - } - } - }, - "delete": { - "operationId": "deleteEvalStatusPageId", - "tags": [ - "EvalStatusPages" - ], - "description": "Delete a eval_status_page object by its id", - "summary": "Delete eval_status_page", - "security": [ - { - "bearerAuth": [] - }, - {} - ], - "parameters": [ - { - "$ref": "#/components/parameters/EvalStatusPageIdParam" - } - ], - "responses": { - "200": { - "description": "Returns the deleted eval_status_page object", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvalStatusPage" - } - } - } - }, - "400": { - "description": "The request was unacceptable, often due to missing a required parameter", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "401": { - "description": "No valid API key provided", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "403": { - "description": "The API key doesn’t have permissions to perform the request", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "429": { - "description": "Too many requests hit the API too quickly. We recommend an exponential backoff of your requests", - "headers": { - "Retry-After": { - "schema": { - "type": "string" - } - } - }, - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - }, - "500": { - "description": "Something went wrong on Braintrust's end. (These are rare.)", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - } - } - }, - "options": { - "operationId": "optionsEvalStatusPageId", - "description": "Enable CORS", - "summary": "Enable CORS (`/v1/eval_status_page/{eval_status_page_id}`)", - "security": [], - "tags": [ - "CORS" - ], - "parameters": [ - { - "$ref": "#/components/parameters/EvalStatusPageIdParam" - } - ], - "responses": { - "200": { - "description": "Response for CORS method", - "headers": { - "Access-Control-Allow-Credentials": { - "schema": { - "type": "string" - } - }, - "Access-Control-Allow-Headers": { - "schema": { - "type": "string" - } - }, - "Access-Control-Allow-Methods": { - "schema": { - "type": "string" - } - }, - "Access-Control-Allow-Origin": { - "schema": { - "type": "string" - } - }, - "Access-Control-Max-Age": { - "schema": { - "type": "string" - } - } - }, - "content": {} - }, - "400": { - "description": "The request was unacceptable, often due to missing a required parameter", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - }, - "application/json": { - "schema": { - "nullable": true - } - } - } - } - } - } - }, "/v1/function": { "post": { "tags": [ diff --git a/openapi/spec.yaml b/openapi/spec.yaml index 1ed3bb0..d2255ae 100644 --- a/openapi/spec.yaml +++ b/openapi/spec.yaml @@ -80,10 +80,6 @@ components: type: string format: uuid description: SpanIframe id - EvalStatusPageIdParam: - type: string - format: uuid - description: EvalStatusPage id FunctionIdParam: type: string format: uuid @@ -157,9 +153,6 @@ components: SpanIframeName: type: string description: Name of the span_iframe to search for - EvalStatusPageName: - type: string - description: Name of the eval_status_page to search for FunctionName: type: string description: Name of the function to search for @@ -1632,6 +1625,19 @@ components: description: Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. + internal_metadata: + type: object + nullable: true + properties: + dataset_filter: + type: object + nullable: true + additionalProperties: + nullable: true + description: BTQL filter payload used to evaluate a subset of a linked dataset. + additionalProperties: + nullable: true + description: Braintrust-controlled metadata about the experiment. parameters_id: type: string nullable: true @@ -1705,6 +1711,19 @@ components: description: Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. + internal_metadata: + type: object + nullable: true + properties: + dataset_filter: + type: object + nullable: true + additionalProperties: + nullable: true + description: BTQL filter payload used to evaluate a subset of a linked dataset. + additionalProperties: + nullable: true + description: Braintrust-controlled metadata about the experiment. parameters_id: type: string nullable: true @@ -1773,6 +1792,19 @@ components: description: Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. + internal_metadata: + type: object + nullable: true + properties: + dataset_filter: + type: object + nullable: true + additionalProperties: + nullable: true + description: BTQL filter payload used to evaluate a subset of a linked dataset. + additionalProperties: + nullable: true + description: Braintrust-controlled metadata about the experiment. parameters_id: type: string nullable: true @@ -4540,6 +4572,47 @@ components: available required: - id + SpanScope: + type: object + properties: + type: + type: string + enum: + - span + required: + - type + description: Process individual spans + TraceScope: + type: object + properties: + type: + type: string + enum: + - trace + idle_seconds: + type: number + description: "Consider trace complete after this many seconds of inactivity + (default: 30)" + required: + - type + description: Process entire traces (all spans sharing the same root_span_id) + GroupScope: + type: object + properties: + type: + type: string + enum: + - group + group_by: + type: string + description: Field path to group by, e.g. metadata.session_id + idle_seconds: + type: number + description: "Optional: trigger after this many seconds of inactivity" + required: + - type + - group_by + description: Process spans/traces grouped by a field (e.g., session_id) RetentionObjectType: type: string enum: @@ -4592,47 +4665,6 @@ components: the current span matches. required: - function - SpanScope: - type: object - properties: - type: - type: string - enum: - - span - required: - - type - description: Process individual spans - TraceScope: - type: object - properties: - type: - type: string - enum: - - trace - idle_seconds: - type: number - description: "Consider trace complete after this many seconds of inactivity - (default: 30)" - required: - - type - description: Process entire traces (all spans sharing the same root_span_id) - GroupScope: - type: object - properties: - type: - type: string - enum: - - group - group_by: - type: string - description: Field path to group by, e.g. metadata.session_id - idle_seconds: - type: number - description: "Optional: trigger after this many seconds of inactivity" - required: - - type - - group_by - description: Process spans/traces grouped by a field (e.g., session_id) TopicAutomationDataScope: anyOf: - type: object @@ -4888,6 +4920,14 @@ components: - type - btql_query description: The definition of what to export + scope: + anyOf: + - $ref: "#/components/schemas/SpanScope" + - $ref: "#/components/schemas/TraceScope" + - $ref: "#/components/schemas/GroupScope" + - type: "null" + description: Execution scope for export automation. Defaults to span-level + execution. export_path: type: string description: The path to export the results to. It should include the storage @@ -4923,6 +4963,18 @@ components: - type - role_arn - external_id + - type: object + properties: + type: + type: string + enum: + - gcp_service_account + service_account_email: + type: string + description: The GCP service account email to impersonate + required: + - type + - service_account_email batch_size: type: number nullable: true @@ -5121,6 +5173,14 @@ components: - type - btql_query description: The definition of what to export + scope: + anyOf: + - $ref: "#/components/schemas/SpanScope" + - $ref: "#/components/schemas/TraceScope" + - $ref: "#/components/schemas/GroupScope" + - type: "null" + description: Execution scope for export automation. Defaults to span-level + execution. export_path: type: string description: The path to export the results to. It should include the storage @@ -5156,6 +5216,18 @@ components: - type - role_arn - external_id + - type: object + properties: + type: + type: string + enum: + - gcp_service_account + service_account_email: + type: string + description: The GCP service account email to impersonate + required: + - type + - service_account_email batch_size: type: number nullable: true @@ -5349,6 +5421,14 @@ components: - type - btql_query description: The definition of what to export + scope: + anyOf: + - $ref: "#/components/schemas/SpanScope" + - $ref: "#/components/schemas/TraceScope" + - $ref: "#/components/schemas/GroupScope" + - type: "null" + description: Execution scope for export automation. Defaults to span-level + execution. export_path: type: string description: The path to export the results to. It should include the storage @@ -5384,6 +5464,18 @@ components: - type - role_arn - external_id + - type: object + properties: + type: + type: string + enum: + - gcp_service_account + service_account_email: + type: string + description: The GCP service account email to impersonate + required: + - type + - service_account_email batch_size: type: number nullable: true @@ -5855,153 +5947,6 @@ components: type: string nullable: true description: Textual description of the span iframe - EvalStatusPageTheme: - type: string - enum: - - light - - dark - description: The theme for the page - EvalStatusPageConfig: - type: object - properties: - score_columns: - type: array - nullable: true - items: - type: string - description: The score columns to display on the page - metric_columns: - type: array - nullable: true - items: - type: string - description: The metric columns to display on the page - grouping_field: - type: string - nullable: true - description: The metadata field to use for grouping experiments (model) - filter: - type: string - nullable: true - description: BTQL filter to apply to experiment data - sort_by: - type: string - nullable: true - description: "Field to sort results by (format: 'score:' or - 'metric:')" - sort_order: - type: string - nullable: true - enum: - - asc - - desc - - null - description: Sort order (ascending or descending) - api_key: - type: string - nullable: true - description: The API key used for fetching experiment data - description: Configuration for what data to display - EvalStatusPage: - type: object - properties: - id: - type: string - format: uuid - description: Unique identifier for the eval status page - project_id: - type: string - format: uuid - description: Unique identifier for the project that the eval status page belongs - under - user_id: - type: string - nullable: true - format: uuid - description: Identifies the user who created the eval status page - created: - type: string - nullable: true - format: date-time - description: Date of eval status page creation - deleted_at: - type: string - nullable: true - format: date-time - description: Date of eval status page deletion, or null if the eval status page - is still active - name: - type: string - description: Name of the eval status page - description: - type: string - nullable: true - description: Textual description of the eval status page - logo_url: - type: string - nullable: true - description: URL of the logo to display on the page - theme: - $ref: "#/components/schemas/EvalStatusPageTheme" - config: - $ref: "#/components/schemas/EvalStatusPageConfig" - required: - - id - - project_id - - name - - theme - - config - description: A public eval status page that displays aggregate experiment results - CreateEvalStatusPage: - type: object - properties: - project_id: - type: string - format: uuid - description: Unique identifier for the project that the eval status page belongs - under - name: - type: string - description: Name of the eval status page - description: - type: string - nullable: true - description: Textual description of the eval status page - logo_url: - type: string - nullable: true - description: URL of the logo to display on the page - theme: - $ref: "#/components/schemas/EvalStatusPageTheme" - config: - $ref: "#/components/schemas/EvalStatusPageConfig" - required: - - project_id - - name - - theme - - config - description: A public eval status page that displays aggregate experiment results - PatchEvalStatusPage: - type: object - properties: - name: - type: string - nullable: true - description: Name of the eval status page - description: - type: string - nullable: true - description: Textual description of the eval status page - logo_url: - type: string - nullable: true - description: URL of the logo to display on the page - theme: - $ref: "#/components/schemas/EvalStatusPageTheme" - nullable: true - config: - $ref: "#/components/schemas/EvalStatusPageConfig" - nullable: true CodeBundle: type: object properties: @@ -7346,6 +7291,9 @@ components: layout: type: string nullable: true + topicMapReportKey: + type: string + nullable: true chartHeight: type: number nullable: true @@ -7451,7 +7399,11 @@ components: enum: - traces - spans + - topics - null + cluster: + type: string + nullable: true freezeColumns: type: boolean nullable: true @@ -8762,6 +8714,15 @@ components: properties: dataset_id: type: string + dataset_version: + type: string + nullable: true + description: The version of the dataset to evaluate + dataset_environment: + type: string + nullable: true + description: The environment tag that resolves to the dataset version to + evaluate _internal_btql: type: object nullable: true @@ -8777,6 +8738,15 @@ components: type: string dataset_name: type: string + dataset_version: + type: string + nullable: true + description: The version of the dataset to evaluate + dataset_environment: + type: string + nullable: true + description: The environment tag that resolves to the dataset version to + evaluate _internal_btql: type: object nullable: true @@ -9000,13 +8970,6 @@ components: description: SpanIframe id name: span_iframe_id in: path - EvalStatusPageIdParam: - schema: - $ref: "#/components/schemas/EvalStatusPageIdParam" - required: true - description: EvalStatusPage id - name: eval_status_page_id - in: path FunctionIdParam: schema: $ref: "#/components/schemas/FunctionIdParam" @@ -9165,14 +9128,6 @@ components: name: span_iframe_name in: query allowReserved: true - EvalStatusPageName: - schema: - $ref: "#/components/schemas/EvalStatusPageName" - required: false - description: Name of the eval_status_page to search for - name: eval_status_page_name - in: query - allowReserved: true FunctionName: schema: $ref: "#/components/schemas/FunctionName" @@ -17096,538 +17051,6 @@ paths: application/json: schema: nullable: true - /v1/eval_status_page: - post: - tags: - - EvalStatusPages - security: - - bearerAuth: [] - - {} - operationId: postEvalStatusPage - description: Create a new eval_status_page. If there is an existing - eval_status_page with the same name as the one specified in the request, - will return the existing eval_status_page unmodified - summary: Create eval_status_page - requestBody: - description: Any desired information about the new eval_status_page object - required: false - content: - application/json: - schema: - $ref: "#/components/schemas/CreateEvalStatusPage" - responses: - "200": - description: Returns the new eval_status_page object - content: - application/json: - schema: - $ref: "#/components/schemas/EvalStatusPage" - "400": - description: The request was unacceptable, often due to missing a required - parameter - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "401": - description: No valid API key provided - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "403": - description: The API key doesn’t have permissions to perform the request - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "429": - description: Too many requests hit the API too quickly. We recommend an - exponential backoff of your requests - headers: - Retry-After: - schema: - type: string - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "500": - description: Something went wrong on Braintrust's end. (These are rare.) - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - put: - tags: - - EvalStatusPages - security: - - bearerAuth: [] - - {} - operationId: putEvalStatusPage - description: Create or replace eval_status_page. If there is an existing - eval_status_page with the same name as the one specified in the request, - will replace the existing eval_status_page with the provided fields - summary: Create or replace eval_status_page - requestBody: - description: Any desired information about the new eval_status_page object - required: false - content: - application/json: - schema: - $ref: "#/components/schemas/CreateEvalStatusPage" - responses: - "200": - description: Returns the new eval_status_page object - content: - application/json: - schema: - $ref: "#/components/schemas/EvalStatusPage" - "400": - description: The request was unacceptable, often due to missing a required - parameter - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "401": - description: No valid API key provided - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "403": - description: The API key doesn’t have permissions to perform the request - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "429": - description: Too many requests hit the API too quickly. We recommend an - exponential backoff of your requests - headers: - Retry-After: - schema: - type: string - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "500": - description: Something went wrong on Braintrust's end. (These are rare.) - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - get: - operationId: getEvalStatusPage - tags: - - EvalStatusPages - description: List out all eval_status_pages. The eval_status_pages are sorted by - creation date, with the most recently-created eval_status_pages coming - first - summary: List eval_status_pages - security: - - bearerAuth: [] - - {} - parameters: - - $ref: "#/components/parameters/AppLimitParam" - - $ref: "#/components/parameters/StartingAfter" - - $ref: "#/components/parameters/EndingBefore" - - $ref: "#/components/parameters/Ids" - - $ref: "#/components/parameters/EvalStatusPageName" - - $ref: "#/components/parameters/OrgName" - responses: - "200": - description: Returns a list of eval_status_page objects - content: - application/json: - schema: - type: object - properties: - objects: - type: array - items: - $ref: "#/components/schemas/EvalStatusPage" - description: A list of eval_status_page objects - required: - - objects - additionalProperties: false - "400": - description: The request was unacceptable, often due to missing a required - parameter - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "401": - description: No valid API key provided - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "403": - description: The API key doesn’t have permissions to perform the request - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "429": - description: Too many requests hit the API too quickly. We recommend an - exponential backoff of your requests - headers: - Retry-After: - schema: - type: string - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "500": - description: Something went wrong on Braintrust's end. (These are rare.) - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - options: - operationId: optionsEvalStatusPage - description: Enable CORS - summary: Enable CORS (`/v1/eval_status_page`) - security: [] - tags: - - CORS - responses: - "200": - description: Response for CORS method - headers: - Access-Control-Allow-Credentials: - schema: - type: string - Access-Control-Allow-Headers: - schema: - type: string - Access-Control-Allow-Methods: - schema: - type: string - Access-Control-Allow-Origin: - schema: - type: string - Access-Control-Max-Age: - schema: - type: string - content: {} - "400": - description: The request was unacceptable, often due to missing a required - parameter - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - /v1/eval_status_page/{eval_status_page_id}: - get: - operationId: getEvalStatusPageId - tags: - - EvalStatusPages - description: Get a eval_status_page object by its id - summary: Get eval_status_page - security: - - bearerAuth: [] - - {} - parameters: - - $ref: "#/components/parameters/EvalStatusPageIdParam" - responses: - "200": - description: Returns the eval_status_page object - content: - application/json: - schema: - $ref: "#/components/schemas/EvalStatusPage" - "400": - description: The request was unacceptable, often due to missing a required - parameter - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "401": - description: No valid API key provided - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "403": - description: The API key doesn’t have permissions to perform the request - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "429": - description: Too many requests hit the API too quickly. We recommend an - exponential backoff of your requests - headers: - Retry-After: - schema: - type: string - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "500": - description: Something went wrong on Braintrust's end. (These are rare.) - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - patch: - operationId: patchEvalStatusPageId - tags: - - EvalStatusPages - description: Partially update a eval_status_page object. Specify the fields to - update in the payload. Any object-type fields will be deep-merged with - existing content. Currently we do not support removing fields or setting - them to null. - summary: Partially update eval_status_page - security: - - bearerAuth: [] - - {} - parameters: - - $ref: "#/components/parameters/EvalStatusPageIdParam" - requestBody: - description: Fields to update - required: false - content: - application/json: - schema: - $ref: "#/components/schemas/PatchEvalStatusPage" - responses: - "200": - description: Returns the eval_status_page object - content: - application/json: - schema: - $ref: "#/components/schemas/EvalStatusPage" - "400": - description: The request was unacceptable, often due to missing a required - parameter - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "401": - description: No valid API key provided - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "403": - description: The API key doesn’t have permissions to perform the request - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "429": - description: Too many requests hit the API too quickly. We recommend an - exponential backoff of your requests - headers: - Retry-After: - schema: - type: string - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "500": - description: Something went wrong on Braintrust's end. (These are rare.) - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - delete: - operationId: deleteEvalStatusPageId - tags: - - EvalStatusPages - description: Delete a eval_status_page object by its id - summary: Delete eval_status_page - security: - - bearerAuth: [] - - {} - parameters: - - $ref: "#/components/parameters/EvalStatusPageIdParam" - responses: - "200": - description: Returns the deleted eval_status_page object - content: - application/json: - schema: - $ref: "#/components/schemas/EvalStatusPage" - "400": - description: The request was unacceptable, often due to missing a required - parameter - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "401": - description: No valid API key provided - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "403": - description: The API key doesn’t have permissions to perform the request - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "429": - description: Too many requests hit the API too quickly. We recommend an - exponential backoff of your requests - headers: - Retry-After: - schema: - type: string - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - "500": - description: Something went wrong on Braintrust's end. (These are rare.) - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true - options: - operationId: optionsEvalStatusPageId - description: Enable CORS - summary: Enable CORS (`/v1/eval_status_page/{eval_status_page_id}`) - security: [] - tags: - - CORS - parameters: - - $ref: "#/components/parameters/EvalStatusPageIdParam" - responses: - "200": - description: Response for CORS method - headers: - Access-Control-Allow-Credentials: - schema: - type: string - Access-Control-Allow-Headers: - schema: - type: string - Access-Control-Allow-Methods: - schema: - type: string - Access-Control-Allow-Origin: - schema: - type: string - Access-Control-Max-Age: - schema: - type: string - content: {} - "400": - description: The request was unacceptable, often due to missing a required - parameter - content: - text/plain: - schema: - type: string - application/json: - schema: - nullable: true /v1/function: post: tags: