From 06051b8cca3f5fa4c0fa3d2403c66ab865036c66 Mon Sep 17 00:00:00 2001 From: thesujai Date: Tue, 15 Sep 2026 14:52:14 +0530 Subject: [PATCH 1/4] feat: integrate Sentry for error tracking and reporting in TrueFoundry --- .changeset/trueforge-sentry-p1.md | 6 + .../src/core/sandbox/Sandbox.ts | 4 + packages/trueforge/.env.example | 12 ++ packages/trueforge/package.json | 1 + packages/trueforge/src/config.ts | 12 ++ packages/trueforge/src/controller-main.ts | 6 +- packages/trueforge/src/controller.ts | 7 +- .../trueforge/src/controller/Controller.ts | 5 + .../src/controller/scheduleDispatch.ts | 9 ++ packages/trueforge/src/main.ts | 15 ++- .../trueforge/src/runtime/sessionResources.ts | 6 + .../src/sentry/captureCriticalException.ts | 23 ++++ packages/trueforge/src/sentry/index.ts | 15 +++ packages/trueforge/src/sentry/initSentry.ts | 68 ++++++++++ .../src/truefoundry/TrueFoundryAgentStore.ts | 17 ++- .../TrueFoundryServiceFoundryServerClient.ts | 15 ++- packages/trueforge/src/truefoundry/errors.ts | 7 +- .../src/truefoundry/initTrueFoundrySentry.ts | 83 ++++++++++++ pnpm-lock.yaml | 119 ++++++++++++++++++ 19 files changed, 420 insertions(+), 10 deletions(-) create mode 100644 .changeset/trueforge-sentry-p1.md create mode 100644 packages/trueforge/src/sentry/captureCriticalException.ts create mode 100644 packages/trueforge/src/sentry/index.ts create mode 100644 packages/trueforge/src/sentry/initSentry.ts create mode 100644 packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts diff --git a/.changeset/trueforge-sentry-p1.md b/.changeset/trueforge-sentry-p1.md new file mode 100644 index 000000000..ce53759d4 --- /dev/null +++ b/.changeset/trueforge-sentry-p1.md @@ -0,0 +1,6 @@ +--- +"@truefoundry/trueforge": minor +"@truefoundry/trueforge-core": patch +--- + +Add Sentry for P1 critical flows: TrueFoundry auth-server or SENTRY_DSN init, and agent-team captures for controller, dual-write, SFY hard failures, missing external_id, and sandbox init. diff --git a/packages/trueforge-core/src/core/sandbox/Sandbox.ts b/packages/trueforge-core/src/core/sandbox/Sandbox.ts index 47eac9169..82cbfba9b 100644 --- a/packages/trueforge-core/src/core/sandbox/Sandbox.ts +++ b/packages/trueforge-core/src/core/sandbox/Sandbox.ts @@ -101,6 +101,7 @@ export interface SandboxOptions { mcpConnectTimeoutMs: number; tracing: AgentTracing; logger: Logger; + onInitFailure?: ((error: unknown) => void) | undefined; } export const SANDBOX_EXEC_TOOL_NAME = 'exec'; @@ -210,6 +211,7 @@ export class Sandbox extends LocalToolMCP { private readonly logger: Logger; // Pre-resolved credential-store file content (null = clear / no git auth). private readonly resolvedGitCredentialsContent: string | null; + private readonly onInitFailure: ((error: unknown) => void) | undefined; private codeModeDispatcher: CodeModeDispatcher | undefined; private codeModeTransport: CodeModeTransport | undefined; /** Cached from transport.getClientInstall after sandbox init (when Code Mode is configured). */ @@ -234,6 +236,7 @@ export class Sandbox extends LocalToolMCP { this.requestTimeoutSeconds = Math.ceil(mcpBoundTimeoutMs / 1000) + NATS_REQUEST_TIMEOUT_BUFFER_SECONDS; this.logger = options.logger.child({ module: 'Sandbox' }); this.resolvedGitCredentialsContent = options.resolvedGitCredentialsContent ?? null; + this.onInitFailure = options.onInitFailure; if (this.existingSandboxId) { this.existingSandboxInfo = { sandbox_id: this.existingSandboxId }; @@ -528,6 +531,7 @@ export class Sandbox extends LocalToolMCP { ({ sandboxInfo, sandboxCreated } = await this.ensureReadySandbox()); } catch (e) { this.logger.error('Sandbox initialization failed', extractErrorLogFields(e)); + this.onInitFailure?.(e); const message = e instanceof Error ? e.message : 'Sandbox initialization failed'; const fallback = this.existingSandboxInfo; return toolResultResponse({ diff --git a/packages/trueforge/.env.example b/packages/trueforge/.env.example index 43161d6ab..431ae5823 100644 --- a/packages/trueforge/.env.example +++ b/packages/trueforge/.env.example @@ -248,6 +248,18 @@ POSTGRES_PORT=5432 # TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS=10000 ## Max ms for ServiceFoundry agent create/update/delete calls. Default 3000. # TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_AGENT_TIMEOUT_MS=3000 +## Auth server for Sentry DSN lookup when SENTRY_ENABLED=true in TrueFoundry mode. +# TRUEFOUNDRY_AUTH_SERVER_URL= +## Optional tenantName query param for the auth-server Sentry DSN lookup. +# TRUEFOUNDRY_TENANT_NAME= + +## Sentry error reporting (off by default). No-op when NODE_ENV is development/test/local. +# SENTRY_ENABLED=false +## Required when SENTRY_ENABLED=true and not in TrueFoundry mode. +# SENTRY_DSN= +## Extra tags applied on Sentry init (JSON object of strings), e.g. {"priority":"p1","team":"agent-team"}. +# SENTRY_ADDITIONAL_TAGS= + ## Optional per-tenant allowlist of model provider account names (JSON object). ## Tenants omitted are unfiltered. Example: {"internal":["openai-main","anthropic-main"]} # TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS={"internal":["openai-main"]} diff --git a/packages/trueforge/package.json b/packages/trueforge/package.json index 15896617d..a2c464801 100644 --- a/packages/trueforge/package.json +++ b/packages/trueforge/package.json @@ -66,6 +66,7 @@ "@hono/swagger-ui": "^0.2.2", "@hono/zod-openapi": "^1.6.1", "@modelcontextprotocol/sdk": "^1.30.0", + "@sentry/node": "^10.74.0", "@truefoundry/trueforge-core": "workspace:*", "@truefoundry/trueforge-sdk": "workspace:*", "better-sqlite3": "^13.0.3", diff --git a/packages/trueforge/src/config.ts b/packages/trueforge/src/config.ts index 6c89a6fea..7004a7a8a 100644 --- a/packages/trueforge/src/config.ts +++ b/packages/trueforge/src/config.ts @@ -807,6 +807,8 @@ export interface SharedServerConfiguration { OUTBOUND_URL_ALLOWED_HOSTS: string[]; /** Hosts always blocked. Env: `OUTBOUND_URL_BLOCKED_HOSTS` (JSON string array). Empty = none. */ OUTBOUND_URL_BLOCKED_HOSTS: string[]; + SENTRY_ENABLED: boolean; + SENTRY_DSN: string | undefined; } export type StandaloneServerConfiguration = SharedServerConfiguration & { @@ -970,6 +972,8 @@ export type DistributedServerConfiguration = SharedServerConfiguration & { * Unset / empty → web search tools are not registered. Env: `TRUEFOUNDRY_WEB_SEARCH_PROVIDER`. */ TRUEFOUNDRY_WEB_SEARCH_PROVIDER: TrueFoundryWebSearchProviderEnv | undefined; + TRUEFOUNDRY_AUTH_SERVER_URL: string | undefined; + TRUEFOUNDRY_TENANT_NAME: string | undefined; }; export type ServerConfiguration = StandaloneServerConfiguration | DistributedServerConfiguration; @@ -1103,6 +1107,12 @@ const shared: SharedServerConfiguration = { envKey: 'OUTBOUND_URL_BLOCKED_HOSTS', raw: getEnv('OUTBOUND_URL_BLOCKED_HOSTS'), }), + SENTRY_ENABLED: parseBoolean({ + envKey: 'SENTRY_ENABLED', + raw: getEnv('SENTRY_ENABLED'), + defaultValue: false, + }), + SENTRY_DSN: getEnv('SENTRY_DSN', { required: false }), }; const configuration: ServerConfiguration = standalone @@ -1202,6 +1212,8 @@ const configuration: ServerConfiguration = standalone TRUEFOUNDRY_WEB_SEARCH_PROVIDER: parseTrueFoundryWebSearchProvider( getEnv('TRUEFOUNDRY_WEB_SEARCH_PROVIDER', { required: false }), ), + TRUEFOUNDRY_AUTH_SERVER_URL: getEnv('TRUEFOUNDRY_AUTH_SERVER_URL', { required: false }), + TRUEFOUNDRY_TENANT_NAME: getEnv('TRUEFOUNDRY_TENANT_NAME', { required: false }), }; export function isOidcConfigured( diff --git a/packages/trueforge/src/controller-main.ts b/packages/trueforge/src/controller-main.ts index 1e4e7b26a..833c4067a 100644 --- a/packages/trueforge/src/controller-main.ts +++ b/packages/trueforge/src/controller-main.ts @@ -18,6 +18,7 @@ import { createDb } from './db/postgres/client'; import { PostgresScheduleStore } from './db/postgres/schedule-store/PostgresScheduleStore'; import { createControllerLogger } from './logger'; import { PACKAGE_VERSION } from './packageVersion'; +import { captureCriticalException, exitAfterFlushSentry, initSentry } from './sentry'; try { const logger = createControllerLogger({ @@ -26,6 +27,8 @@ try { version: PACKAGE_VERSION, }); + await initSentry(configuration, logger, { tags: { component: 'controller' } }); + if (configuration.STANDALONE) { // Not an error: in standalone the server process owns the controller in-process, so a // dedicated controller has nothing to do. Exit cleanly (e.g. `pnpm standalone:dev` also @@ -57,5 +60,6 @@ try { }); } catch (error) { console.error('Failed to start controller:', error instanceof Error ? error.message : error); - process.exit(1); + captureCriticalException(error, { tags: { module: 'controller', operation: 'boot' } }); + await exitAfterFlushSentry(1); } diff --git a/packages/trueforge/src/controller.ts b/packages/trueforge/src/controller.ts index 61c13c8ee..7c99aa467 100644 --- a/packages/trueforge/src/controller.ts +++ b/packages/trueforge/src/controller.ts @@ -3,6 +3,7 @@ import { Controller } from './controller/Controller'; import { scheduleDispatchLoop } from './controller/scheduleDispatch'; import type { IScheduleStore } from './db/scheduleStore'; import type { WithTransaction } from './db/transaction'; +import { captureCriticalException, exitAfterFlushSentry } from './sentry'; /** * Controller whose schedule loop hands runs to the server over HTTP @@ -52,7 +53,11 @@ export function runController(params: { // Passes only hold short transactions, so the deadline should never elapse. setTimeout(() => { logger.warn(`Controller drain timed out after ${String(gracefulTimeoutSeconds)}s, exiting`); - process.exit(1); + captureCriticalException(new Error('Controller drain timed out'), { + tags: { module: 'controller', operation: 'drain' }, + extra: { gracefulTimeoutSeconds }, + }); + void exitAfterFlushSentry(1); }, gracefulTimeoutSeconds * 1000).unref(); await controller.stop(); diff --git a/packages/trueforge/src/controller/Controller.ts b/packages/trueforge/src/controller/Controller.ts index 04b5de6bc..9b445552c 100644 --- a/packages/trueforge/src/controller/Controller.ts +++ b/packages/trueforge/src/controller/Controller.ts @@ -9,6 +9,7 @@ * process. Loops are written assuming this. */ import type { Logger } from 'winston'; +import { captureCriticalException } from '../sentry'; /** Reason passed to {@link AbortController.abort} when {@link Controller.stop} runs. */ export const CONTROLLER_STOPPED = 'controller-stopped'; @@ -120,6 +121,10 @@ export class Controller { return; } this.#logger.error('Control loop pass failed', { loop: loop.name, error }); + captureCriticalException(error, { + tags: { module: 'controller', operation: 'tick' }, + extra: { loop: loop.name }, + }); } })(); diff --git a/packages/trueforge/src/controller/scheduleDispatch.ts b/packages/trueforge/src/controller/scheduleDispatch.ts index 178c0886d..351fc9026 100644 --- a/packages/trueforge/src/controller/scheduleDispatch.ts +++ b/packages/trueforge/src/controller/scheduleDispatch.ts @@ -13,6 +13,7 @@ import type { WithTransaction } from '../db/transaction'; import { createTlsFetch, normalizeTlsUrl } from '../http/tls'; import { nextTriggerAfter } from '../runtime/cron'; import { InvalidCronError, type ScheduleRunStatus } from '../schemas/schedule'; +import { captureCriticalException } from '../sentry'; import type { ControlLoop } from './Controller'; /** @@ -303,6 +304,10 @@ export async function dispatchScheduledRuns(params: { run_id: run.id, error, }); + captureCriticalException(error, { + tags: { module: 'scheduleDispatch', operation: 'handoff' }, + extra: { schedule_id: schedule.id, run_id: run.id }, + }); await finishScheduledRun({ store, run, @@ -329,6 +334,10 @@ export async function dispatchScheduledRuns(params: { run_id: run.id, error, }); + captureCriticalException(error, { + tags: { module: 'scheduleDispatch', operation: 'processRun' }, + extra: { schedule_id: run.schedule_id, run_id: run.id }, + }); } } diff --git a/packages/trueforge/src/main.ts b/packages/trueforge/src/main.ts index 50a85f796..5efb23560 100644 --- a/packages/trueforge/src/main.ts +++ b/packages/trueforge/src/main.ts @@ -93,6 +93,7 @@ import { PACKAGE_VERSION } from './packageVersion'; import { ActiveTurnRegistry } from './runtime/activeTurns'; import { EventSubscriptionRegistry } from './runtime/event-subscription'; import type { ConnectedRedis } from './runtime/redis'; +import { captureCriticalException, exitAfterFlushSentry, initSentry } from './sentry'; import { printStandaloneStartupBanner } from './startupBanner'; import { InlineMcpServerStore } from './truefoundry/InlineMcpServerStore'; import { parseInlineMcpServers, parseInlineSkills, X_TFG_MCP, X_TFG_SKILLS } from './truefoundry/inlineResources'; @@ -675,6 +676,8 @@ try { version: PACKAGE_VERSION, }); + await initSentry(configuration, logger, { tags: { component: 'server' } }); + if (configuration.STANDALONE) { printStandaloneStartupBanner({ version: PACKAGE_VERSION, color: shouldColorize() }); await prepareCodeModeSocketParent({ path: configuration.CODE_MODE_SOCKET_PARENT, logger }); @@ -777,7 +780,8 @@ try { server.on('error', (error: unknown) => { console.error('Failed to start server:', error instanceof Error ? error.message : error); - process.exit(1); + captureCriticalException(error, { tags: { module: 'main', operation: 'listen' } }); + void exitAfterFlushSentry(1); }); // Graceful drain is the safe default for built and direct execution. @@ -794,7 +798,11 @@ try { // Arm at the start of each shutdown; unref so this timer alone cannot keep the process alive. setTimeout(() => { logger.warn(`Drain timed out after ${String(configuration.GRACEFUL_TIMEOUT_SECONDS)}s, exiting`); - process.exit(1); + captureCriticalException(new Error('Server drain timed out'), { + tags: { module: 'main', operation: 'drain' }, + extra: { gracefulTimeoutSeconds: configuration.GRACEFUL_TIMEOUT_SECONDS }, + }); + void exitAfterFlushSentry(1); }, configuration.GRACEFUL_TIMEOUT_SECONDS * 1000).unref(); const closed = new Promise(resolve => { @@ -844,5 +852,6 @@ try { } } catch (error) { console.error('Failed to start server:', error instanceof Error ? error.message : error); - process.exit(1); + captureCriticalException(error, { tags: { module: 'main', operation: 'startup' } }); + await exitAfterFlushSentry(1); } diff --git a/packages/trueforge/src/runtime/sessionResources.ts b/packages/trueforge/src/runtime/sessionResources.ts index f2670ba92..b6773d589 100644 --- a/packages/trueforge/src/runtime/sessionResources.ts +++ b/packages/trueforge/src/runtime/sessionResources.ts @@ -24,6 +24,7 @@ import { LocalSandboxProvider } from '../sandbox/local/provider/LocalSandboxProv import { getCachedLocalSandboxSupport, isLocalSandboxFallbackEnabled } from '../sandbox/localRuntime'; import { toSandboxProviderFromRecord } from '../sandbox/providerUtils'; import type { ReasoningEffort } from '../schemas/modelProvider'; +import { captureCriticalException } from '../sentry'; import { hasConfiguredWebSearchProvider } from '../websearch/providers'; export interface McpConnection { @@ -295,6 +296,11 @@ export function buildTurnSandbox(input: { skillMounter: new SkillMounter({ skills: input.skills ?? [] }), tracing: input.tracing, logger: input.logger, + onInitFailure: error => { + captureCriticalException(error, { + tags: { module: 'sandbox', operation: 'init' }, + }); + }, }); } diff --git a/packages/trueforge/src/sentry/captureCriticalException.ts b/packages/trueforge/src/sentry/captureCriticalException.ts new file mode 100644 index 000000000..580e01a6a --- /dev/null +++ b/packages/trueforge/src/sentry/captureCriticalException.ts @@ -0,0 +1,23 @@ +import * as Sentry from '@sentry/node'; +import { isTrueFoundryModeEnabled } from '../config'; + +export function captureCriticalException( + err: unknown, + options?: { tags?: Record; extra?: Record }, +): void { + Sentry.withScope(scope => { + scope.setTags({ + ...(isTrueFoundryModeEnabled() + ? { + priority: 'p1', + team: 'agent-team', + } + : {}), + ...options?.tags, + }); + if (options?.extra) { + scope.setExtras(options.extra); + } + Sentry.captureException(err); + }); +} diff --git a/packages/trueforge/src/sentry/index.ts b/packages/trueforge/src/sentry/index.ts new file mode 100644 index 000000000..f18754fef --- /dev/null +++ b/packages/trueforge/src/sentry/index.ts @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/node'; + +export { captureCriticalException } from './captureCriticalException'; +export { initSentry, type InitSentryOptions } from './initSentry'; + +export const SENTRY_FLUSH_TIMEOUT_MS = 2000; + +export async function flushSentry(timeoutMs: number = SENTRY_FLUSH_TIMEOUT_MS): Promise { + await Sentry.flush(timeoutMs); +} + +export async function exitAfterFlushSentry(exitCode: number): Promise { + await flushSentry(); + process.exit(exitCode); +} diff --git a/packages/trueforge/src/sentry/initSentry.ts b/packages/trueforge/src/sentry/initSentry.ts new file mode 100644 index 000000000..d2e524bc8 --- /dev/null +++ b/packages/trueforge/src/sentry/initSentry.ts @@ -0,0 +1,68 @@ +import * as Sentry from '@sentry/node'; +import type { Logger } from 'winston'; + +import { isTrueFoundryModeEnabled, type ServerConfiguration } from '../config'; +import { PACKAGE_VERSION } from '../packageVersion'; +import { initTrueFoundrySentry } from '../truefoundry/initTrueFoundrySentry'; + +function isLocalLikeEnv(nodeEnv: string | undefined): boolean { + return nodeEnv === 'development' || nodeEnv === 'test' || nodeEnv === 'local'; +} + +export interface InitSentryOptions { + tags?: Record; +} + +export async function initSentry( + config: ServerConfiguration, + logger: Pick, + options?: InitSentryOptions, +): Promise { + if (!config.SENTRY_ENABLED || isLocalLikeEnv(config.NODE_ENV)) { + logger.info('Sentry is not enabled (SENTRY_ENABLED=false or local-like NODE_ENV)'); + return; + } + + if (isTrueFoundryModeEnabled(config)) { + const authServerUrl = config.TRUEFOUNDRY_AUTH_SERVER_URL; + const apiKey = config.TRUEFOUNDRY_API_KEY; + if (authServerUrl === undefined || authServerUrl.trim() === '') { + logger.error('TRUEFOUNDRY_AUTH_SERVER_URL is required when SENTRY_ENABLED in TrueFoundry mode'); + return; + } + if (apiKey === undefined || apiKey.trim() === '') { + logger.error('TRUEFOUNDRY_API_KEY is required when SENTRY_ENABLED in TrueFoundry mode'); + return; + } + await initTrueFoundrySentry({ + config: { + TRUEFOUNDRY_AUTH_SERVER_URL: authServerUrl, + TRUEFOUNDRY_API_KEY: apiKey, + TRUEFOUNDRY_TENANT_NAME: config.TRUEFOUNDRY_TENANT_NAME, + TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS: config.TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS, + }, + logger, + version: PACKAGE_VERSION, + tags: options?.tags, + }); + return; + } + + const dsn = config.SENTRY_DSN; + if (dsn === undefined || dsn.trim() === '') { + logger.error('SENTRY_DSN is required when SENTRY_ENABLED outside TrueFoundry mode'); + return; + } + Sentry.init({ + dsn, + includeLocalVariables: false, + integrations: [], + }); + Sentry.getGlobalScope().setTag('TRUEFORGE_VERSION', PACKAGE_VERSION); + if (options?.tags) { + for (const [key, value] of Object.entries(options.tags)) { + Sentry.getGlobalScope().setTag(key, value); + } + } + logger.info('Sentry initialised'); +} diff --git a/packages/trueforge/src/truefoundry/TrueFoundryAgentStore.ts b/packages/trueforge/src/truefoundry/TrueFoundryAgentStore.ts index 2b96df378..ef9f48b58 100644 --- a/packages/trueforge/src/truefoundry/TrueFoundryAgentStore.ts +++ b/packages/trueforge/src/truefoundry/TrueFoundryAgentStore.ts @@ -16,6 +16,7 @@ import { import { PostgresAgentStore } from '../db/postgres/agent-store/PostgresAgentStore'; import type { Database } from '../db/postgres/types'; import { AGENT_DESCRIPTION_MAX_LENGTH } from '../schemas/agent'; +import { captureCriticalException } from '../sentry'; import { callerAccessToken, type ResolveAccessToken } from './accessToken'; import { TrueFoundryServiceFoundryServerClient, @@ -174,7 +175,14 @@ export class TrueFoundryAgentStore implements IAgentStore> failures.push(asError(cleanupError)); } if (failures.length > 1) { - throw new AggregateError(failures, 'createAgent failed and cleanup also failed', { cause: error }); + const aggregate = new AggregateError(failures, 'createAgent failed and cleanup also failed', { + cause: error, + }); + captureCriticalException(aggregate, { + tags: { module: 'TrueFoundryAgentStore', operation: 'dualWrite' }, + extra: { agent_id: created.id, tenant_id: input.tenant_id }, + }); + throw aggregate; } throw error; } @@ -226,11 +234,16 @@ export class TrueFoundryAgentStore implements IAgentStore> }), }); } catch (restoreError) { - throw new AggregateError( + const aggregate = new AggregateError( [asError(error), asError(restoreError)], 'updateAgent failed and ServiceFoundry restore also failed', { cause: restoreError }, ); + captureCriticalException(aggregate, { + tags: { module: 'TrueFoundryAgentStore', operation: 'dualWrite' }, + extra: { agent_id: input.id, tenant_id: input.tenant_id }, + }); + throw aggregate; } throw error; } diff --git a/packages/trueforge/src/truefoundry/TrueFoundryServiceFoundryServerClient.ts b/packages/trueforge/src/truefoundry/TrueFoundryServiceFoundryServerClient.ts index ab07a91b7..84a54c0a8 100644 --- a/packages/trueforge/src/truefoundry/TrueFoundryServiceFoundryServerClient.ts +++ b/packages/trueforge/src/truefoundry/TrueFoundryServiceFoundryServerClient.ts @@ -6,6 +6,7 @@ import type { Logger } from 'winston'; import { z } from 'zod'; import type { McpAuthStatus } from '../schemas/mcpServer'; +import { captureCriticalException } from '../sentry'; import { createInternalTlsDispatcher, normalizeInternalTlsUrl, type InternalTlsOptions } from './internalTls'; import { mapResolvedAgentSkillVersions, type ResolvedAgentSkillVersion } from './mapSfyAgentSkills'; import { parseSfyMcpAuthStatus, parseSfyMcpAuthorizeResult, type SfyMcpAuthSource } from './mapSfyMcpServers'; @@ -734,9 +735,14 @@ export class TrueFoundryServiceFoundryServerClient { } if (!response.ok) { const detail = await readServiceFoundryErrorMessage(response); - throw new HTTPException(424, { + const error = new HTTPException(424, { message: `TrueFoundry ServiceFoundry server request failed: ${detail ?? `HTTP ${String(response.status)}`}`, }); + captureCriticalException(error, { + tags: { module: 'TrueFoundryServiceFoundryServerClient', operation: 'requestJson' }, + extra: { url: input.url.href, method: input.method, status: response.status }, + }); + throw error; } if (response.status === 204) { return undefined; @@ -752,10 +758,15 @@ export class TrueFoundryServiceFoundryServerClient { url: input.url.href, ...extractErrorLogFields(error), }); - throw new HTTPException(424, { + const httpError = new HTTPException(424, { message: 'TrueFoundry ServiceFoundry server returned non-JSON', cause: error, }); + captureCriticalException(httpError, { + tags: { module: 'TrueFoundryServiceFoundryServerClient', operation: 'requestJson' }, + extra: { url: input.url.href, method: input.method }, + }); + throw httpError; } } } diff --git a/packages/trueforge/src/truefoundry/errors.ts b/packages/trueforge/src/truefoundry/errors.ts index 4a4e5fdde..a6eacb4bf 100644 --- a/packages/trueforge/src/truefoundry/errors.ts +++ b/packages/trueforge/src/truefoundry/errors.ts @@ -1,5 +1,6 @@ import { HTTPException } from 'hono/http-exception'; import type { AgentRecord } from '../db/agentStore'; +import { captureCriticalException } from '../sentry'; export const AGENT_EXTERNAL_ID_REQUIRED = 'Agent is missing a TrueFoundry external id'; export const TRUEFOUNDRY_MANAGED_STATUS = 424 as const; @@ -8,7 +9,11 @@ export const TRUEFOUNDRY_MANAGED_MESSAGE = 'This resource is managed by TrueFoun /** Require the remote identity needed for TrueFoundry agent operations. */ export function requireTrueFoundryAgentExternalId(agent: Pick): string { if (agent.external_id === null) { - throw new HTTPException(500, { message: AGENT_EXTERNAL_ID_REQUIRED }); + const error = new HTTPException(500, { message: AGENT_EXTERNAL_ID_REQUIRED }); + captureCriticalException(error, { + tags: { module: 'truefoundry', operation: 'requireExternalId' }, + }); + throw error; } return agent.external_id; } diff --git a/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts b/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts new file mode 100644 index 000000000..15c73452b --- /dev/null +++ b/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts @@ -0,0 +1,83 @@ +import type { NodeOptions } from '@sentry/node'; +import * as Sentry from '@sentry/node'; +import { fetch as undiciFetch } from 'undici'; +import type { Logger } from 'winston'; + +export const SENTRY_SERVICE_NAME = 'trueforge'; + +const AUTH_FETCH_TIMEOUT_MS = 10_000; + +export interface TrueFoundrySentryInitConfig { + TRUEFOUNDRY_AUTH_SERVER_URL: string; + TRUEFOUNDRY_API_KEY: string; + TRUEFOUNDRY_TENANT_NAME?: string | undefined; + TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS?: number | undefined; +} + +function isSentryNodeOptions(value: unknown): value is NodeOptions { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +async function fetchSentryAuth(config: TrueFoundrySentryInitConfig): Promise { + const url = `${config.TRUEFOUNDRY_AUTH_SERVER_URL}/api/v1/tenants/sentry-auth-data`; + const urlParamsObj: Record = { + serviceName: SENTRY_SERVICE_NAME, + }; + if (config.TRUEFOUNDRY_TENANT_NAME !== undefined && config.TRUEFOUNDRY_TENANT_NAME !== '') { + urlParamsObj['tenantName'] = config.TRUEFOUNDRY_TENANT_NAME; + } + const params = new URLSearchParams(urlParamsObj); + const timeoutMs = config.TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS ?? AUTH_FETCH_TIMEOUT_MS; + + try { + const res = await undiciFetch(`${url}?${params.toString()}`, { + headers: { Authorization: `Bearer ${config.TRUEFOUNDRY_API_KEY}` }, + signal: AbortSignal.timeout(timeoutMs), + }); + if (res.status !== 200) { + return undefined; + } + const data: unknown = await res.json(); + if (!isSentryNodeOptions(data)) { + return undefined; + } + return data; + } catch (err) { + if (err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError')) { + throw new Error(`Timed out fetching sentry auth data after ${String(timeoutMs)}ms`, { cause: err }); + } + throw err; + } +} + +export async function initTrueFoundrySentry(input: { + config: TrueFoundrySentryInitConfig; + logger: Pick; + version: string; + tags?: Record | undefined; +}): Promise { + const { config, logger, version, tags } = input; + try { + const sentryInitOptions = await fetchSentryAuth(config); + if (sentryInitOptions === undefined) { + logger.error('Failed to fetch sentry config. Skipping initialization'); + return false; + } + sentryInitOptions.includeLocalVariables = false; + sentryInitOptions.integrations = []; + Sentry.init(sentryInitOptions); + Sentry.getGlobalScope().setTag('TRUEFORGE_VERSION', version); + if (tags) { + for (const [key, value] of Object.entries(tags)) { + Sentry.getGlobalScope().setTag(key, value); + } + } + logger.info('Sentry initialised (TrueFoundry auth server)'); + return true; + } catch (errorFetchingAuthData) { + logger.error('Error in fetching sentry auth data; skipping Sentry initialization', { + error: errorFetchingAuthData instanceof Error ? errorFetchingAuthData.message : errorFetchingAuthData, + }); + return false; + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fcc27af86..b3b0a135a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -178,6 +178,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.30.0 version: 1.30.0(supports-color@8.1.1)(zod@4.4.3) + '@sentry/node': + specifier: ^10.74.0 + version: 10.74.0(@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.221.0(@opentelemetry/api@1.9.1))(supports-color@8.1.1) '@truefoundry/trueforge-core': specifier: workspace:* version: link:../trueforge-core @@ -2041,6 +2044,10 @@ packages: '@open-draft/until@2.1.0': resolution: {integrity: sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==} + '@opentelemetry/api-logs@0.220.0': + resolution: {integrity: sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api-logs@0.221.0': resolution: {integrity: sha512-OlanaW1vv7ufTqQ3/fPLI4arGt5ZoM+P8abOMki6uEYnpRazepSWDwDnnw+la7kE26SHVC18//SMccrDvLKOXQ==} engines: {node: '>=8.0.0'} @@ -2139,6 +2146,12 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/instrumentation@0.220.0': + resolution: {integrity: sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/instrumentation@0.221.0': resolution: {integrity: sha512-cCk80Z/iRDf/5gfsKMB4f74LqVA5yKETB/9ojPzVW/6/f70iu89nJvGxsFCxx4XfSohaOofkU19kiYm84AiAlw==} engines: {node: ^18.19.0 || >=20.6.0} @@ -3257,6 +3270,51 @@ packages: cpu: [x64] os: [win32] + '@sentry/conventions@0.16.0': + resolution: {integrity: sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ==} + engines: {node: '>=14'} + + '@sentry/core@10.74.0': + resolution: {integrity: sha512-u9rY8vcZfktccwm6LznfCZlqP5C9A+p76r4/pFS1grqpuTO0m21Cl8rosnlESrDGP/Xd9tfr91rWYk0jPH8jeQ==} + engines: {node: '>=18'} + + '@sentry/node-core@10.74.0': + resolution: {integrity: sha512-btgZXcGmkOGgojbxHo/gfGiyqYpzC9E8zVR78Q3MtM6Xnemk9gwJiQlhNmEX/FM+C36WBRPZrdcZcnMaJhbfJw==} + engines: {node: '>=18'} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + '@opentelemetry/core': ^1.30.1 || ^2.1.0 + '@opentelemetry/exporter-trace-otlp-http': '>=0.57.0 <1' + '@opentelemetry/instrumentation': '>=0.57.1 <1' + '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@opentelemetry/core': + optional: true + '@opentelemetry/exporter-trace-otlp-http': + optional: true + '@opentelemetry/instrumentation': + optional: true + '@opentelemetry/sdk-trace-base': + optional: true + + '@sentry/node@10.74.0': + resolution: {integrity: sha512-u1wsarTOWHn9CCev81Da5T4IQHZgdcosXRfX2+4DMII/lVJMBYesKixTQuMwXexjVG2+pkf16zTmgjZDa+75jA==} + engines: {node: '>=18'} + + '@sentry/opentelemetry@10.74.0': + resolution: {integrity: sha512-ua5mt0NDBfye+/ACKjAw9Ad2i+y42lOLvhxXZepZXjszblMz80MEsIZflMB7uLZUCTNH7MbZN8tlzCy8KsJKmQ==} + engines: {node: '>=18'} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + '@opentelemetry/core': ^1.30.1 || ^2.1.0 + '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0 + + '@sentry/server-utils@10.74.0': + resolution: {integrity: sha512-AHmPIGE8yVRyywpZRhAkm/H0cHTgsQSPhFZFbaeQLcYHFE/eOP9bAMh8Nj/Apr7WMagFXYHd04tR/63BxtLgKw==} + engines: {node: '>=18'} + '@sinclair/typebox@0.27.12': resolution: {integrity: sha512-hhyNJ+nbR6ZR7pToHvllEFun9TL0sbL+tk/ON75lo+Xas054uez98qRbsuNt7MBCyZKK4+8Yli/OAGZhmfBZ/g==} @@ -9329,6 +9387,10 @@ snapshots: '@open-draft/until@2.1.0': {} + '@opentelemetry/api-logs@0.220.0': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs@0.221.0': dependencies: '@opentelemetry/api': 1.9.1 @@ -9443,6 +9505,15 @@ snapshots: transitivePeerDependencies: - supports-color + '@opentelemetry/instrumentation@0.220.0(@opentelemetry/api@1.9.1)(supports-color@8.1.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.220.0 + import-in-the-middle: 3.3.3 + require-in-the-middle: 8.0.1(supports-color@8.1.1) + transitivePeerDependencies: + - supports-color + '@opentelemetry/instrumentation@0.221.0(@opentelemetry/api@1.9.1)(supports-color@8.1.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -10568,6 +10639,54 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.62.2': optional: true + '@sentry/conventions@0.16.0': {} + + '@sentry/core@10.74.0': + dependencies: + '@sentry/conventions': 0.16.0 + + '@sentry/node-core@10.74.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/instrumentation@0.220.0(@opentelemetry/api@1.9.1)(supports-color@8.1.1))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.1))': + dependencies: + '@sentry/conventions': 0.16.0 + '@sentry/core': 10.74.0 + '@sentry/opentelemetry': 10.74.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.1)) + import-in-the-middle: 3.3.3 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-http': 0.221.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.220.0(@opentelemetry/api@1.9.1)(supports-color@8.1.1) + '@opentelemetry/sdk-trace-base': 2.10.0(@opentelemetry/api@1.9.1) + + '@sentry/node@10.74.0(@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.221.0(@opentelemetry/api@1.9.1))(supports-color@8.1.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.220.0(@opentelemetry/api@1.9.1)(supports-color@8.1.1) + '@opentelemetry/sdk-trace-base': 2.10.0(@opentelemetry/api@1.9.1) + '@sentry/conventions': 0.16.0 + '@sentry/core': 10.74.0 + '@sentry/node-core': 10.74.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.221.0(@opentelemetry/api@1.9.1))(@opentelemetry/instrumentation@0.220.0(@opentelemetry/api@1.9.1)(supports-color@8.1.1))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.1)) + '@sentry/opentelemetry': 10.74.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.1)) + '@sentry/server-utils': 10.74.0 + import-in-the-middle: 3.3.3 + transitivePeerDependencies: + - '@opentelemetry/core' + - '@opentelemetry/exporter-trace-otlp-http' + - supports-color + + '@sentry/opentelemetry@10.74.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.1))': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.10.0(@opentelemetry/api@1.9.1) + '@sentry/conventions': 0.16.0 + '@sentry/core': 10.74.0 + + '@sentry/server-utils@10.74.0': + dependencies: + '@sentry/conventions': 0.16.0 + '@sentry/core': 10.74.0 + '@sinclair/typebox@0.27.12': {} '@sinclair/typebox@0.34.52': {} From 8f99d18849b597ba22782f8092a48e604c64b0f9 Mon Sep 17 00:00:00 2001 From: thesujai Date: Tue, 15 Sep 2026 14:54:43 +0530 Subject: [PATCH 2/4] refactor: simplify Sentry authentication data fetching in TrueFoundry --- .../src/truefoundry/initTrueFoundrySentry.ts | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts b/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts index 15c73452b..94fb7ff59 100644 --- a/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts +++ b/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts @@ -14,11 +14,7 @@ export interface TrueFoundrySentryInitConfig { TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS?: number | undefined; } -function isSentryNodeOptions(value: unknown): value is NodeOptions { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -async function fetchSentryAuth(config: TrueFoundrySentryInitConfig): Promise { +async function fetchSentryAuth(config: TrueFoundrySentryInitConfig): Promise { const url = `${config.TRUEFOUNDRY_AUTH_SERVER_URL}/api/v1/tenants/sentry-auth-data`; const urlParamsObj: Record = { serviceName: SENTRY_SERVICE_NAME, @@ -37,11 +33,7 @@ async function fetchSentryAuth(config: TrueFoundrySentryInitConfig): Promise { const { config, logger, version, tags } = input; try { - const sentryInitOptions = await fetchSentryAuth(config); - if (sentryInitOptions === undefined) { + const authData = await fetchSentryAuth(config); + if (authData === undefined || typeof authData !== 'object' || authData === null) { logger.error('Failed to fetch sentry config. Skipping initialization'); return false; } - sentryInitOptions.includeLocalVariables = false; - sentryInitOptions.integrations = []; + const sentryInitOptions: NodeOptions = { + includeLocalVariables: false, + integrations: [], + }; + Object.assign(sentryInitOptions, authData); Sentry.init(sentryInitOptions); Sentry.getGlobalScope().setTag('TRUEFORGE_VERSION', version); if (tags) { From d001b7a8aa94698949cd1e9b0c93f83a1204520f Mon Sep 17 00:00:00 2001 From: thesujai Date: Tue, 22 Sep 2026 22:49:28 +0530 Subject: [PATCH 3/4] feat: enhance Sentry integration for improved error reporting and customizable tags in TrueFoundry --- .changeset/trueforge-sentry-p1.md | 5 +- .../src/core/sandbox/Sandbox.ts | 4 -- packages/trueforge/src/config.ts | 26 +++++++ packages/trueforge/src/controller-main.ts | 5 +- packages/trueforge/src/controller.ts | 7 +- .../trueforge/src/controller/Controller.ts | 5 -- .../src/controller/scheduleDispatch.ts | 12 +++- packages/trueforge/src/main.ts | 14 ++-- .../trueforge/src/runtime/sessionResources.ts | 6 -- .../src/sentry/captureCriticalException.ts | 13 +--- packages/trueforge/src/sentry/index.ts | 13 ---- packages/trueforge/src/sentry/initSentry.ts | 46 +++++------- .../src/truefoundry/TrueFoundryAgentStore.ts | 14 +++- packages/trueforge/src/truefoundry/errors.ts | 9 ++- .../src/truefoundry/initTrueFoundrySentry.ts | 72 +++++++++++-------- 15 files changed, 130 insertions(+), 121 deletions(-) diff --git a/.changeset/trueforge-sentry-p1.md b/.changeset/trueforge-sentry-p1.md index ce53759d4..402cafcc9 100644 --- a/.changeset/trueforge-sentry-p1.md +++ b/.changeset/trueforge-sentry-p1.md @@ -1,6 +1,5 @@ --- -"@truefoundry/trueforge": minor -"@truefoundry/trueforge-core": patch +"@truefoundry/trueforge": patch --- -Add Sentry for P1 critical flows: TrueFoundry auth-server or SENTRY_DSN init, and agent-team captures for controller, dual-write, SFY hard failures, missing external_id, and sandbox init. +Add Sentry for critical-flow error reporting (TrueFoundry auth-server or SENTRY_DSN init) with configurable additional tags. diff --git a/packages/trueforge-core/src/core/sandbox/Sandbox.ts b/packages/trueforge-core/src/core/sandbox/Sandbox.ts index 82cbfba9b..47eac9169 100644 --- a/packages/trueforge-core/src/core/sandbox/Sandbox.ts +++ b/packages/trueforge-core/src/core/sandbox/Sandbox.ts @@ -101,7 +101,6 @@ export interface SandboxOptions { mcpConnectTimeoutMs: number; tracing: AgentTracing; logger: Logger; - onInitFailure?: ((error: unknown) => void) | undefined; } export const SANDBOX_EXEC_TOOL_NAME = 'exec'; @@ -211,7 +210,6 @@ export class Sandbox extends LocalToolMCP { private readonly logger: Logger; // Pre-resolved credential-store file content (null = clear / no git auth). private readonly resolvedGitCredentialsContent: string | null; - private readonly onInitFailure: ((error: unknown) => void) | undefined; private codeModeDispatcher: CodeModeDispatcher | undefined; private codeModeTransport: CodeModeTransport | undefined; /** Cached from transport.getClientInstall after sandbox init (when Code Mode is configured). */ @@ -236,7 +234,6 @@ export class Sandbox extends LocalToolMCP { this.requestTimeoutSeconds = Math.ceil(mcpBoundTimeoutMs / 1000) + NATS_REQUEST_TIMEOUT_BUFFER_SECONDS; this.logger = options.logger.child({ module: 'Sandbox' }); this.resolvedGitCredentialsContent = options.resolvedGitCredentialsContent ?? null; - this.onInitFailure = options.onInitFailure; if (this.existingSandboxId) { this.existingSandboxInfo = { sandbox_id: this.existingSandboxId }; @@ -531,7 +528,6 @@ export class Sandbox extends LocalToolMCP { ({ sandboxInfo, sandboxCreated } = await this.ensureReadySandbox()); } catch (e) { this.logger.error('Sandbox initialization failed', extractErrorLogFields(e)); - this.onInitFailure?.(e); const message = e instanceof Error ? e.message : 'Sandbox initialization failed'; const fallback = this.existingSandboxInfo; return toolResultResponse({ diff --git a/packages/trueforge/src/config.ts b/packages/trueforge/src/config.ts index 7004a7a8a..1aa4b3b1d 100644 --- a/packages/trueforge/src/config.ts +++ b/packages/trueforge/src/config.ts @@ -297,6 +297,30 @@ function parseTrueFoundrySandboxProvider(raw: string | undefined): 'daytona' | ' ); } +/** Parses `SENTRY_ADDITIONAL_TAGS` as a JSON object of string values. Unset/blank → `{}`. */ +function parseSentryAdditionalTags(raw: string | undefined): Record { + if (raw === undefined || raw.trim() === '') { + return {}; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw new Error('SENTRY_ADDITIONAL_TAGS must be a JSON object of string values', { cause: error }); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('SENTRY_ADDITIONAL_TAGS must be a JSON object of string values'); + } + const tags: Record = {}; + for (const [key, value] of Object.entries(parsed)) { + if (typeof value !== 'string') { + throw new Error(`SENTRY_ADDITIONAL_TAGS.${key} must be a string`); + } + tags[key] = value; + } + return tags; +} + /** Parses `POSTGRES_SSL_MODE`. Unset/blank → `''`. Unknown values throw. */ function validatePostgresSslMode(raw: string | undefined): PostgresSslMode | '' { const mode = raw?.trim() ?? ''; @@ -809,6 +833,7 @@ export interface SharedServerConfiguration { OUTBOUND_URL_BLOCKED_HOSTS: string[]; SENTRY_ENABLED: boolean; SENTRY_DSN: string | undefined; + SENTRY_ADDITIONAL_TAGS: Record; } export type StandaloneServerConfiguration = SharedServerConfiguration & { @@ -1113,6 +1138,7 @@ const shared: SharedServerConfiguration = { defaultValue: false, }), SENTRY_DSN: getEnv('SENTRY_DSN', { required: false }), + SENTRY_ADDITIONAL_TAGS: parseSentryAdditionalTags(getEnv('SENTRY_ADDITIONAL_TAGS', { required: false })), }; const configuration: ServerConfiguration = standalone diff --git a/packages/trueforge/src/controller-main.ts b/packages/trueforge/src/controller-main.ts index 833c4067a..9abe045a4 100644 --- a/packages/trueforge/src/controller-main.ts +++ b/packages/trueforge/src/controller-main.ts @@ -18,7 +18,7 @@ import { createDb } from './db/postgres/client'; import { PostgresScheduleStore } from './db/postgres/schedule-store/PostgresScheduleStore'; import { createControllerLogger } from './logger'; import { PACKAGE_VERSION } from './packageVersion'; -import { captureCriticalException, exitAfterFlushSentry, initSentry } from './sentry'; +import { initSentry } from './sentry'; try { const logger = createControllerLogger({ @@ -60,6 +60,5 @@ try { }); } catch (error) { console.error('Failed to start controller:', error instanceof Error ? error.message : error); - captureCriticalException(error, { tags: { module: 'controller', operation: 'boot' } }); - await exitAfterFlushSentry(1); + process.exit(1); } diff --git a/packages/trueforge/src/controller.ts b/packages/trueforge/src/controller.ts index 7c99aa467..61c13c8ee 100644 --- a/packages/trueforge/src/controller.ts +++ b/packages/trueforge/src/controller.ts @@ -3,7 +3,6 @@ import { Controller } from './controller/Controller'; import { scheduleDispatchLoop } from './controller/scheduleDispatch'; import type { IScheduleStore } from './db/scheduleStore'; import type { WithTransaction } from './db/transaction'; -import { captureCriticalException, exitAfterFlushSentry } from './sentry'; /** * Controller whose schedule loop hands runs to the server over HTTP @@ -53,11 +52,7 @@ export function runController(params: { // Passes only hold short transactions, so the deadline should never elapse. setTimeout(() => { logger.warn(`Controller drain timed out after ${String(gracefulTimeoutSeconds)}s, exiting`); - captureCriticalException(new Error('Controller drain timed out'), { - tags: { module: 'controller', operation: 'drain' }, - extra: { gracefulTimeoutSeconds }, - }); - void exitAfterFlushSentry(1); + process.exit(1); }, gracefulTimeoutSeconds * 1000).unref(); await controller.stop(); diff --git a/packages/trueforge/src/controller/Controller.ts b/packages/trueforge/src/controller/Controller.ts index 9b445552c..04b5de6bc 100644 --- a/packages/trueforge/src/controller/Controller.ts +++ b/packages/trueforge/src/controller/Controller.ts @@ -9,7 +9,6 @@ * process. Loops are written assuming this. */ import type { Logger } from 'winston'; -import { captureCriticalException } from '../sentry'; /** Reason passed to {@link AbortController.abort} when {@link Controller.stop} runs. */ export const CONTROLLER_STOPPED = 'controller-stopped'; @@ -121,10 +120,6 @@ export class Controller { return; } this.#logger.error('Control loop pass failed', { loop: loop.name, error }); - captureCriticalException(error, { - tags: { module: 'controller', operation: 'tick' }, - extra: { loop: loop.name }, - }); } })(); diff --git a/packages/trueforge/src/controller/scheduleDispatch.ts b/packages/trueforge/src/controller/scheduleDispatch.ts index 351fc9026..3a8c11d83 100644 --- a/packages/trueforge/src/controller/scheduleDispatch.ts +++ b/packages/trueforge/src/controller/scheduleDispatch.ts @@ -306,7 +306,11 @@ export async function dispatchScheduledRuns(params: { }); captureCriticalException(error, { tags: { module: 'scheduleDispatch', operation: 'handoff' }, - extra: { schedule_id: schedule.id, run_id: run.id }, + extra: { + tenant_id: run.tenant_id, + schedule_id: schedule.id, + run_id: run.id, + }, }); await finishScheduledRun({ store, @@ -336,7 +340,11 @@ export async function dispatchScheduledRuns(params: { }); captureCriticalException(error, { tags: { module: 'scheduleDispatch', operation: 'processRun' }, - extra: { schedule_id: run.schedule_id, run_id: run.id }, + extra: { + tenant_id: run.tenant_id, + schedule_id: run.schedule_id, + run_id: run.id, + }, }); } } diff --git a/packages/trueforge/src/main.ts b/packages/trueforge/src/main.ts index 5efb23560..8c8847fd4 100644 --- a/packages/trueforge/src/main.ts +++ b/packages/trueforge/src/main.ts @@ -93,7 +93,7 @@ import { PACKAGE_VERSION } from './packageVersion'; import { ActiveTurnRegistry } from './runtime/activeTurns'; import { EventSubscriptionRegistry } from './runtime/event-subscription'; import type { ConnectedRedis } from './runtime/redis'; -import { captureCriticalException, exitAfterFlushSentry, initSentry } from './sentry'; +import { initSentry } from './sentry'; import { printStandaloneStartupBanner } from './startupBanner'; import { InlineMcpServerStore } from './truefoundry/InlineMcpServerStore'; import { parseInlineMcpServers, parseInlineSkills, X_TFG_MCP, X_TFG_SKILLS } from './truefoundry/inlineResources'; @@ -780,8 +780,7 @@ try { server.on('error', (error: unknown) => { console.error('Failed to start server:', error instanceof Error ? error.message : error); - captureCriticalException(error, { tags: { module: 'main', operation: 'listen' } }); - void exitAfterFlushSentry(1); + process.exit(1); }); // Graceful drain is the safe default for built and direct execution. @@ -798,11 +797,7 @@ try { // Arm at the start of each shutdown; unref so this timer alone cannot keep the process alive. setTimeout(() => { logger.warn(`Drain timed out after ${String(configuration.GRACEFUL_TIMEOUT_SECONDS)}s, exiting`); - captureCriticalException(new Error('Server drain timed out'), { - tags: { module: 'main', operation: 'drain' }, - extra: { gracefulTimeoutSeconds: configuration.GRACEFUL_TIMEOUT_SECONDS }, - }); - void exitAfterFlushSentry(1); + process.exit(1); }, configuration.GRACEFUL_TIMEOUT_SECONDS * 1000).unref(); const closed = new Promise(resolve => { @@ -852,6 +847,5 @@ try { } } catch (error) { console.error('Failed to start server:', error instanceof Error ? error.message : error); - captureCriticalException(error, { tags: { module: 'main', operation: 'startup' } }); - await exitAfterFlushSentry(1); + process.exit(1); } diff --git a/packages/trueforge/src/runtime/sessionResources.ts b/packages/trueforge/src/runtime/sessionResources.ts index b6773d589..f2670ba92 100644 --- a/packages/trueforge/src/runtime/sessionResources.ts +++ b/packages/trueforge/src/runtime/sessionResources.ts @@ -24,7 +24,6 @@ import { LocalSandboxProvider } from '../sandbox/local/provider/LocalSandboxProv import { getCachedLocalSandboxSupport, isLocalSandboxFallbackEnabled } from '../sandbox/localRuntime'; import { toSandboxProviderFromRecord } from '../sandbox/providerUtils'; import type { ReasoningEffort } from '../schemas/modelProvider'; -import { captureCriticalException } from '../sentry'; import { hasConfiguredWebSearchProvider } from '../websearch/providers'; export interface McpConnection { @@ -296,11 +295,6 @@ export function buildTurnSandbox(input: { skillMounter: new SkillMounter({ skills: input.skills ?? [] }), tracing: input.tracing, logger: input.logger, - onInitFailure: error => { - captureCriticalException(error, { - tags: { module: 'sandbox', operation: 'init' }, - }); - }, }); } diff --git a/packages/trueforge/src/sentry/captureCriticalException.ts b/packages/trueforge/src/sentry/captureCriticalException.ts index 580e01a6a..062ae3397 100644 --- a/packages/trueforge/src/sentry/captureCriticalException.ts +++ b/packages/trueforge/src/sentry/captureCriticalException.ts @@ -1,20 +1,13 @@ import * as Sentry from '@sentry/node'; -import { isTrueFoundryModeEnabled } from '../config'; export function captureCriticalException( err: unknown, options?: { tags?: Record; extra?: Record }, ): void { Sentry.withScope(scope => { - scope.setTags({ - ...(isTrueFoundryModeEnabled() - ? { - priority: 'p1', - team: 'agent-team', - } - : {}), - ...options?.tags, - }); + if (options?.tags) { + scope.setTags(options.tags); + } if (options?.extra) { scope.setExtras(options.extra); } diff --git a/packages/trueforge/src/sentry/index.ts b/packages/trueforge/src/sentry/index.ts index f18754fef..c2a715c19 100644 --- a/packages/trueforge/src/sentry/index.ts +++ b/packages/trueforge/src/sentry/index.ts @@ -1,15 +1,2 @@ -import * as Sentry from '@sentry/node'; - export { captureCriticalException } from './captureCriticalException'; export { initSentry, type InitSentryOptions } from './initSentry'; - -export const SENTRY_FLUSH_TIMEOUT_MS = 2000; - -export async function flushSentry(timeoutMs: number = SENTRY_FLUSH_TIMEOUT_MS): Promise { - await Sentry.flush(timeoutMs); -} - -export async function exitAfterFlushSentry(exitCode: number): Promise { - await flushSentry(); - process.exit(exitCode); -} diff --git a/packages/trueforge/src/sentry/initSentry.ts b/packages/trueforge/src/sentry/initSentry.ts index d2e524bc8..0fd48a838 100644 --- a/packages/trueforge/src/sentry/initSentry.ts +++ b/packages/trueforge/src/sentry/initSentry.ts @@ -9,6 +9,13 @@ function isLocalLikeEnv(nodeEnv: string | undefined): boolean { return nodeEnv === 'development' || nodeEnv === 'test' || nodeEnv === 'local'; } +function applyGlobalTags(tags: Record): void { + const scope = Sentry.getGlobalScope(); + for (const [key, value] of Object.entries(tags)) { + scope.setTag(key, value); + } +} + export interface InitSentryOptions { tags?: Record; } @@ -23,46 +30,29 @@ export async function initSentry( return; } + const globalTags: Record = { + service: 'trueforge', + TRUEFORGE_VERSION: PACKAGE_VERSION, + ...config.SENTRY_ADDITIONAL_TAGS, + ...options?.tags, + }; + if (isTrueFoundryModeEnabled(config)) { - const authServerUrl = config.TRUEFOUNDRY_AUTH_SERVER_URL; - const apiKey = config.TRUEFOUNDRY_API_KEY; - if (authServerUrl === undefined || authServerUrl.trim() === '') { - logger.error('TRUEFOUNDRY_AUTH_SERVER_URL is required when SENTRY_ENABLED in TrueFoundry mode'); - return; - } - if (apiKey === undefined || apiKey.trim() === '') { - logger.error('TRUEFOUNDRY_API_KEY is required when SENTRY_ENABLED in TrueFoundry mode'); - return; - } - await initTrueFoundrySentry({ - config: { - TRUEFOUNDRY_AUTH_SERVER_URL: authServerUrl, - TRUEFOUNDRY_API_KEY: apiKey, - TRUEFOUNDRY_TENANT_NAME: config.TRUEFOUNDRY_TENANT_NAME, - TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS: config.TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS, - }, - logger, - version: PACKAGE_VERSION, - tags: options?.tags, - }); + await initTrueFoundrySentry({ logger, tags: globalTags }); return; } const dsn = config.SENTRY_DSN; if (dsn === undefined || dsn.trim() === '') { - logger.error('SENTRY_DSN is required when SENTRY_ENABLED outside TrueFoundry mode'); + logger.error('SENTRY_DSN is required when SENTRY_ENABLED=true'); return; } Sentry.init({ dsn, includeLocalVariables: false, + defaultIntegrations: false, integrations: [], }); - Sentry.getGlobalScope().setTag('TRUEFORGE_VERSION', PACKAGE_VERSION); - if (options?.tags) { - for (const [key, value] of Object.entries(options.tags)) { - Sentry.getGlobalScope().setTag(key, value); - } - } + applyGlobalTags(globalTags); logger.info('Sentry initialised'); } diff --git a/packages/trueforge/src/truefoundry/TrueFoundryAgentStore.ts b/packages/trueforge/src/truefoundry/TrueFoundryAgentStore.ts index ef9f48b58..f34b0d730 100644 --- a/packages/trueforge/src/truefoundry/TrueFoundryAgentStore.ts +++ b/packages/trueforge/src/truefoundry/TrueFoundryAgentStore.ts @@ -180,7 +180,12 @@ export class TrueFoundryAgentStore implements IAgentStore> }); captureCriticalException(aggregate, { tags: { module: 'TrueFoundryAgentStore', operation: 'dualWrite' }, - extra: { agent_id: created.id, tenant_id: input.tenant_id }, + extra: { + agent_id: created.id, + agent_name: created.name, + tenant_id: input.tenant_id, + external_id: externalId, + }, }); throw aggregate; } @@ -241,7 +246,12 @@ export class TrueFoundryAgentStore implements IAgentStore> ); captureCriticalException(aggregate, { tags: { module: 'TrueFoundryAgentStore', operation: 'dualWrite' }, - extra: { agent_id: input.id, tenant_id: input.tenant_id }, + extra: { + agent_id: input.id, + agent_name: previous.name, + tenant_id: input.tenant_id, + external_id: previous.external_id, + }, }); throw aggregate; } diff --git a/packages/trueforge/src/truefoundry/errors.ts b/packages/trueforge/src/truefoundry/errors.ts index a6eacb4bf..b0233b148 100644 --- a/packages/trueforge/src/truefoundry/errors.ts +++ b/packages/trueforge/src/truefoundry/errors.ts @@ -7,11 +7,18 @@ export const TRUEFOUNDRY_MANAGED_STATUS = 424 as const; export const TRUEFOUNDRY_MANAGED_MESSAGE = 'This resource is managed by TrueFoundry'; /** Require the remote identity needed for TrueFoundry agent operations. */ -export function requireTrueFoundryAgentExternalId(agent: Pick): string { +export function requireTrueFoundryAgentExternalId( + agent: Pick, +): string { if (agent.external_id === null) { const error = new HTTPException(500, { message: AGENT_EXTERNAL_ID_REQUIRED }); captureCriticalException(error, { tags: { module: 'truefoundry', operation: 'requireExternalId' }, + extra: { + agent_id: agent.id, + tenant_id: agent.tenant_id, + agent_name: agent.name, + }, }); throw error; } diff --git a/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts b/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts index 94fb7ff59..e160ac0c3 100644 --- a/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts +++ b/packages/trueforge/src/truefoundry/initTrueFoundrySentry.ts @@ -3,32 +3,30 @@ import * as Sentry from '@sentry/node'; import { fetch as undiciFetch } from 'undici'; import type { Logger } from 'winston'; -export const SENTRY_SERVICE_NAME = 'trueforge'; - -const AUTH_FETCH_TIMEOUT_MS = 10_000; +import configuration, { isTrueFoundryModeEnabled } from '../config'; +import { PACKAGE_VERSION } from '../packageVersion'; -export interface TrueFoundrySentryInitConfig { - TRUEFOUNDRY_AUTH_SERVER_URL: string; - TRUEFOUNDRY_API_KEY: string; - TRUEFOUNDRY_TENANT_NAME?: string | undefined; - TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS?: number | undefined; -} +export const SENTRY_SERVICE_NAME = 'trueforge'; -async function fetchSentryAuth(config: TrueFoundrySentryInitConfig): Promise { - const url = `${config.TRUEFOUNDRY_AUTH_SERVER_URL}/api/v1/tenants/sentry-auth-data`; +async function fetchSentryAuth(input: { + authServerUrl: string; + apiKey: string; + tenantName: string | undefined; + timeoutMs: number; +}): Promise { + const url = `${input.authServerUrl}/api/v1/tenants/sentry-auth-data`; const urlParamsObj: Record = { serviceName: SENTRY_SERVICE_NAME, }; - if (config.TRUEFOUNDRY_TENANT_NAME !== undefined && config.TRUEFOUNDRY_TENANT_NAME !== '') { - urlParamsObj['tenantName'] = config.TRUEFOUNDRY_TENANT_NAME; + if (input.tenantName !== undefined && input.tenantName !== '') { + urlParamsObj['tenantName'] = input.tenantName; } const params = new URLSearchParams(urlParamsObj); - const timeoutMs = config.TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS ?? AUTH_FETCH_TIMEOUT_MS; try { const res = await undiciFetch(`${url}?${params.toString()}`, { - headers: { Authorization: `Bearer ${config.TRUEFOUNDRY_API_KEY}` }, - signal: AbortSignal.timeout(timeoutMs), + headers: { Authorization: `Bearer ${input.apiKey}` }, + signal: AbortSignal.timeout(input.timeoutMs), }); if (res.status !== 200) { return undefined; @@ -36,32 +34,50 @@ async function fetchSentryAuth(config: TrueFoundrySentryInitConfig): Promise; - version: string; tags?: Record | undefined; }): Promise { - const { config, logger, version, tags } = input; + const { logger, tags } = input; + if (!isTrueFoundryModeEnabled(configuration)) { + logger.error('Sentry TrueFoundry init requires TrueFoundry mode; skipping'); + return false; + } + const authServerUrl = configuration.TRUEFOUNDRY_AUTH_SERVER_URL; + const apiKey = configuration.TRUEFOUNDRY_API_KEY; + if (authServerUrl === undefined || authServerUrl.trim() === '') { + logger.error('TRUEFOUNDRY_AUTH_SERVER_URL is required when SENTRY_ENABLED in TrueFoundry mode'); + return false; + } + if (apiKey === undefined || apiKey.trim() === '') { + logger.error('TRUEFOUNDRY_API_KEY is required when SENTRY_ENABLED in TrueFoundry mode'); + return false; + } + try { - const authData = await fetchSentryAuth(config); + const authData = await fetchSentryAuth({ + authServerUrl, + apiKey, + tenantName: configuration.TRUEFOUNDRY_TENANT_NAME, + timeoutMs: configuration.TRUEFOUNDRY_SERVICEFOUNDRY_HTTP_TIMEOUT_MS, + }); if (authData === undefined || typeof authData !== 'object' || authData === null) { logger.error('Failed to fetch sentry config. Skipping initialization'); return false; } - const sentryInitOptions: NodeOptions = { - includeLocalVariables: false, - integrations: [], - }; + const sentryInitOptions: NodeOptions = {}; Object.assign(sentryInitOptions, authData); + sentryInitOptions.includeLocalVariables = false; + sentryInitOptions.defaultIntegrations = false; + sentryInitOptions.integrations = []; Sentry.init(sentryInitOptions); - Sentry.getGlobalScope().setTag('TRUEFORGE_VERSION', version); + Sentry.getGlobalScope().setTag('TRUEFORGE_VERSION', PACKAGE_VERSION); if (tags) { for (const [key, value] of Object.entries(tags)) { Sentry.getGlobalScope().setTag(key, value); @@ -69,9 +85,9 @@ export async function initTrueFoundrySentry(input: { } logger.info('Sentry initialised (TrueFoundry auth server)'); return true; - } catch (errorFetchingAuthData) { + } catch (error) { logger.error('Error in fetching sentry auth data; skipping Sentry initialization', { - error: errorFetchingAuthData instanceof Error ? errorFetchingAuthData.message : errorFetchingAuthData, + error: error instanceof Error ? error.message : error, }); return false; } From 5a6f2ba2fee4cd1b9489bf68d00085485ef03b7e Mon Sep 17 00:00:00 2001 From: thesujai Date: Tue, 22 Sep 2026 22:58:42 +0530 Subject: [PATCH 4/4] refactor: remove Sentry error capturing from TrueFoundryServiceFoundryServerClient --- .../TrueFoundryServiceFoundryServerClient.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/trueforge/src/truefoundry/TrueFoundryServiceFoundryServerClient.ts b/packages/trueforge/src/truefoundry/TrueFoundryServiceFoundryServerClient.ts index 84a54c0a8..ab07a91b7 100644 --- a/packages/trueforge/src/truefoundry/TrueFoundryServiceFoundryServerClient.ts +++ b/packages/trueforge/src/truefoundry/TrueFoundryServiceFoundryServerClient.ts @@ -6,7 +6,6 @@ import type { Logger } from 'winston'; import { z } from 'zod'; import type { McpAuthStatus } from '../schemas/mcpServer'; -import { captureCriticalException } from '../sentry'; import { createInternalTlsDispatcher, normalizeInternalTlsUrl, type InternalTlsOptions } from './internalTls'; import { mapResolvedAgentSkillVersions, type ResolvedAgentSkillVersion } from './mapSfyAgentSkills'; import { parseSfyMcpAuthStatus, parseSfyMcpAuthorizeResult, type SfyMcpAuthSource } from './mapSfyMcpServers'; @@ -735,14 +734,9 @@ export class TrueFoundryServiceFoundryServerClient { } if (!response.ok) { const detail = await readServiceFoundryErrorMessage(response); - const error = new HTTPException(424, { + throw new HTTPException(424, { message: `TrueFoundry ServiceFoundry server request failed: ${detail ?? `HTTP ${String(response.status)}`}`, }); - captureCriticalException(error, { - tags: { module: 'TrueFoundryServiceFoundryServerClient', operation: 'requestJson' }, - extra: { url: input.url.href, method: input.method, status: response.status }, - }); - throw error; } if (response.status === 204) { return undefined; @@ -758,15 +752,10 @@ export class TrueFoundryServiceFoundryServerClient { url: input.url.href, ...extractErrorLogFields(error), }); - const httpError = new HTTPException(424, { + throw new HTTPException(424, { message: 'TrueFoundry ServiceFoundry server returned non-JSON', cause: error, }); - captureCriticalException(httpError, { - tags: { module: 'TrueFoundryServiceFoundryServerClient', operation: 'requestJson' }, - extra: { url: input.url.href, method: input.method }, - }); - throw httpError; } } }