From 7bbe3fe22fd394052232b15527a9d89920ea73ae Mon Sep 17 00:00:00 2001 From: Elioooon Date: Mon, 31 Aug 2026 17:54:20 +0800 Subject: [PATCH 1/8] fix: persist Daytona authentication failures --- .changeset/persist-daytona-auth-errors.md | 6 ++ .../core/sandbox/provider/DaytonaProvider.ts | 27 ++++++-- .../daytonaSnapshotRegistration.test.ts | 53 ++++++++++++---- .../trueforge/src/runtime/sessionResources.ts | 31 +++++++++- .../trueforge/src/sandbox/providerUtils.ts | 62 ++++++++++++++++--- .../tests/unit/sandbox/providerUtils.test.ts | 60 ++++++++++++++++++ 6 files changed, 212 insertions(+), 27 deletions(-) create mode 100644 .changeset/persist-daytona-auth-errors.md create mode 100644 packages/trueforge/tests/unit/sandbox/providerUtils.test.ts diff --git a/.changeset/persist-daytona-auth-errors.md b/.changeset/persist-daytona-auth-errors.md new file mode 100644 index 000000000..d03207257 --- /dev/null +++ b/.changeset/persist-daytona-auth-errors.md @@ -0,0 +1,6 @@ +--- +'@truefoundry/trueforge': patch +'@truefoundry/trueforge-core': patch +--- + +Persist failed Daytona credentials when sandbox operations return authorization errors. diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts index 2332325c6..73d3297e2 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts @@ -99,6 +99,7 @@ export interface DaytonaSandboxProviderOptions { /** Defaults to 1 hour (same as the gateway's max agent execution time). */ previewUrlExpirySeconds?: number; logger: Logger; + onError?: ((error: unknown) => Promise) | undefined; } export class DaytonaSandboxProvider implements SandboxProvider { @@ -118,6 +119,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { private readonly apiKey: string; private readonly apiUrl: string; private readonly logger: Logger; + private readonly onError: ((error: unknown) => Promise) | undefined; private readonly daytona: Daytona; private static readonly cachedSandboxes = new Map(); // De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip. @@ -138,6 +140,15 @@ export class DaytonaSandboxProvider implements SandboxProvider { this.natsBridgePort = options.natsBridgePort ?? DEFAULT_SANDBOX_NATS_WS_PORT; this.previewUrlExpirySeconds = options.previewUrlExpirySeconds ?? DEFAULT_PREVIEW_URL_EXPIRY_SECONDS; this.logger = options.logger.child({ module: 'DaytonaProvider' }); + this.onError = options.onError; + } + + private async reportError(error: unknown): Promise { + try { + await this.onError?.(error); + } catch (reportError) { + this.logger.error('Failed to report Daytona error', extractErrorLogFields(reportError)); + } } private async getOrCreateSandbox(sandboxId?: string): Promise<{ sandbox: Sandbox; defaultTimeoutMs: number }> { @@ -246,11 +257,16 @@ export class DaytonaSandboxProvider implements SandboxProvider { } async createSandbox(): Promise<{ sandboxId: string }> { - return context.with(suppressTracing(context.active()), async () => { - const { sandbox } = await this.getOrCreateSandbox(); - this.logger.debug(`Sandbox created: name=${sandbox.name}`); - return { sandboxId: sandbox.name }; - }); + try { + return await context.with(suppressTracing(context.active()), async () => { + const { sandbox } = await this.getOrCreateSandbox(); + this.logger.debug(`Sandbox created: name=${sandbox.name}`); + return { sandboxId: sandbox.name }; + }); + } catch (error) { + await this.reportError(error); + throw error; + } } /** Resolves undefined when no snapshot carries that name; auth/other failures throw. */ @@ -403,6 +419,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { if (e instanceof SandboxNotAvailableError) { throw e; } + await this.reportError(e); this.logger.error('Sandbox execution error', extractErrorLogFields(e)); const message = e instanceof Error ? e.message : 'Unknown error'; return { success: false, error: message }; diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts index 2c94b868d..bc135a8db 100644 --- a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts @@ -32,6 +32,23 @@ function makeProvider(): DaytonaSandboxProvider { }); } +function makeRuntimeProvider(client: Daytona, onError?: (error: unknown) => Promise): DaytonaSandboxProvider { + return new DaytonaSandboxProvider({ + client, + apiKey: 'dtn-test', + apiUrl: API_URL, + tenantName: 'test-tenant', + sandboxImage: 'registry.example.com/sandbox:029ea5ff', + timeoutMs: 1000, + autoStopIntervalInMinutes: 5, + autoArchiveIntervalInMinutes: 60, + autoDeleteIntervalInMinutes: 7200, + fileMaxBytesForDownload: 1024, + logger: makeSilentLogger(), + onError, + }); +} + function mockFetch({ status, body }: { status: number; body: unknown }): jest.SpiedFunction { return jest .spyOn(globalThis, 'fetch') @@ -81,22 +98,32 @@ describe('DaytonaSandboxProvider register-only snapshot create', () => { }); describe('DaytonaSandboxProvider exec', () => { + it('reports sandbox creation errors before rethrowing them', async () => { + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + jest.spyOn(client, 'create').mockRejectedValue(new DaytonaError('unauthorized', 401)); + const onError = jest.fn().mockResolvedValue(undefined); + const provider = makeRuntimeProvider(client, onError); + + await expect(provider.createSandbox()).rejects.toMatchObject({ statusCode: 401 }); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 })); + }); + + it('reports provider errors before converting them to failed exec results', async () => { + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('unauthorized', 401)); + const onError = jest.fn().mockResolvedValue(undefined); + const provider = makeRuntimeProvider(client, onError); + + await expect(provider.exec({ sandboxId: 'test-tenant.expired', command: 'true' })).resolves.toMatchObject({ + success: false, + }); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 })); + }); + it('rethrows SandboxNotAvailableError when the sandbox is gone', async () => { const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS)); - const provider = new DaytonaSandboxProvider({ - client, - apiKey: 'dtn-test', - apiUrl: API_URL, - tenantName: 'test-tenant', - sandboxImage: 'registry.example.com/sandbox:029ea5ff', - timeoutMs: 1000, - autoStopIntervalInMinutes: 5, - autoArchiveIntervalInMinutes: 60, - autoDeleteIntervalInMinutes: 7200, - fileMaxBytesForDownload: 1024, - logger: makeSilentLogger(), - }); + const provider = makeRuntimeProvider(client); await expect(provider.exec({ sandboxId: 'test-tenant.gone', command: 'true' })).rejects.toBeInstanceOf( SandboxNotAvailableError, diff --git a/packages/trueforge/src/runtime/sessionResources.ts b/packages/trueforge/src/runtime/sessionResources.ts index c1287be3e..a5e94feab 100644 --- a/packages/trueforge/src/runtime/sessionResources.ts +++ b/packages/trueforge/src/runtime/sessionResources.ts @@ -21,7 +21,15 @@ import type { ISandboxProviderStore } from '../db/sandboxProviderStore'; import type { ISkillStore } from '../db/skillStore'; import { LocalSandboxProvider } from '../sandbox/local/provider/LocalSandboxProvider'; import { getCachedLocalSandboxSupport, isLocalSandboxFallbackEnabled } from '../sandbox/localRuntime'; -import { toSandboxProviderFromRecord } from '../sandbox/providerUtils'; +<<<<<<< HEAD +import { + recordDaytonaAccessFailure, + toDaytonaSandboxProvider, + toSandboxProviderFromRecord, +} from '../sandbox/providerUtils'; +======= +import { recordDaytonaAccessFailure, toDaytonaSandboxProvider } from '../sandbox/providerUtils'; +>>>>>>> fix: persist Daytona authentication failures import type { ReasoningEffort } from '../schemas/modelProvider'; import { resolveWebSearchProvider } from '../websearch/providers'; @@ -239,7 +247,26 @@ export async function resolveSandboxProvider({ }): Promise { const record = await store.getSandboxProvider(tenant_id); if (record !== undefined) { - return toSandboxProviderFromRecord({ record, tenant_id, logger }); + if (record.manifest.type !== 'daytona') { + return toSandboxProviderFromRecord({ record, tenant_id, logger }); + } + // Clone from the snapshot that was actually built (persisted build_ref), not a name + // derived from the current image — otherwise an image bump breaks creation until rebuild. + return toDaytonaSandboxProvider({ + manifest: record.manifest, + tenant_id, + logger, + build_metadata: record.build_metadata, + onError: async error => { + await recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata: record.build_metadata, + expected_manifest: record.manifest, + }); + }, + }); } if (!configuration.STANDALONE) { return undefined; diff --git a/packages/trueforge/src/sandbox/providerUtils.ts b/packages/trueforge/src/sandbox/providerUtils.ts index f65f99350..f26ee1221 100644 --- a/packages/trueforge/src/sandbox/providerUtils.ts +++ b/packages/trueforge/src/sandbox/providerUtils.ts @@ -20,11 +20,15 @@ import { /** Daytona rejected the credentials (401 unauthorized); retrying the same key cannot succeed. */ export function isDaytonaAuthError(error: unknown): boolean { - return error instanceof DaytonaError && error.statusCode === 401; + return error instanceof DaytonaError + ? error.statusCode === 401 + : error instanceof Error && error.cause !== undefined && isDaytonaAuthError(error.cause); } export function isDaytonaPermissionError(error: unknown): boolean { - return error instanceof DaytonaError && error.statusCode === 403; + return error instanceof DaytonaError + ? error.statusCode === 403 + : error instanceof Error && error.cause !== undefined && isDaytonaPermissionError(error.cause); } /** @@ -40,11 +44,13 @@ export function toDaytonaSandboxProvider({ tenant_id, logger, build_metadata, + onError, }: { manifest: SandboxProviderManifest; tenant_id: string; logger: Logger; build_metadata?: SandboxBuildMetadata | null; + onError?: ((error: unknown) => Promise) | undefined; }): DaytonaSandboxProvider { const { apiKey, ...settings } = toDaytonaSandboxProviderInput(manifest); return new DaytonaSandboxProvider({ @@ -56,6 +62,7 @@ export function toDaytonaSandboxProvider({ buildRef: build_metadata?.['build_ref'], fileMaxBytesForDownload: configuration.SANDBOX_FILE_MAX_BYTES_FOR_DOWNLOAD, logger, + onError, }); } @@ -109,6 +116,30 @@ function sandboxStatusFromRecord(record: SandboxProviderRecord): SandboxStatus { }; } +export async function recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata, +}: { + store: ISandboxProviderStore; + tenant_id: string; + error: unknown; + build_metadata?: SandboxBuildMetadata | null; +}): Promise { + const status_reason = isDaytonaAuthError(error) + ? 'Daytona rejected the API key. Check the configured credentials.' + : isDaytonaPermissionError(error) + ? 'Daytona denied access. Check the API key permissions.' + : undefined; + if (status_reason === undefined) { + return undefined; + } + const next: SandboxStatus = { status: 'failed', status_reason, build_metadata: build_metadata ?? null }; + const updated = await store.updateSandboxStatus({ tenant_id, ...next }); + return updated ? sandboxStatusFromRecord(updated) : next; +} + // Daytona deactivates idle snapshots after 14 days; revalidate at 13 to stay a day ahead. const READY_REVALIDATE_INTERVAL_MS = 13 * 24 * 60 * 60 * 1000; @@ -149,11 +180,28 @@ export async function checkSnapshotStatus({ build_metadata: record.build_metadata, }); let build: SandboxBuild; - if (record.status === 'ready') { - // this is because image may have deactivated - build = await withTimeout(provider.buildImage(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox buildImage'); - } else { - build = await withTimeout(provider.getImageBuildStatus(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox getImageBuildStatus'); + try { + if (record.status === 'ready') { + // this is because image may have deactivated + build = await withTimeout(provider.buildImage(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox buildImage'); + } else { + build = await withTimeout( + provider.getImageBuildStatus(), + STATUS_REFRESH_TIMEOUT_MS, + 'sandbox getImageBuildStatus', + ); + } + } catch (error) { + const failed = await recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata: record.build_metadata, + }); + if (failed !== undefined) { + return failed; + } + throw error; } const next = toSandboxStatus(build); const updated = await store.updateSandboxStatus({ tenant_id, ...next }); diff --git a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts new file mode 100644 index 000000000..40e7a2268 --- /dev/null +++ b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts @@ -0,0 +1,60 @@ +jest.mock('@daytona/sdk', () => { + const actual = jest.requireActual('@daytona/sdk'); + return { ...actual, Daytona: jest.fn().mockImplementation(() => ({})) }; +}); + +import { DaytonaError } from '@daytona/sdk'; +import { DaytonaSandboxProvider } from '@truefoundry/trueforge-core/core'; +import { createLogger } from 'winston'; +import type { ISandboxProviderStore, SandboxProviderRecord } from '../../../src/db/sandboxProviderStore'; +import { checkSnapshotStatus } from '../../../src/sandbox/providerUtils'; + +const record: SandboxProviderRecord = { + tenant_id: 'tenant-1', + manifest: { + type: 'daytona', + auth: { api_key: 'dtn-revoked' }, + exec_timeout_ms: 60_000, + auto_stop_interval_in_minutes: 5, + auto_archive_interval_in_minutes: 60, + auto_delete_interval_in_minutes: 7200, + }, + status: 'pending', + status_reason: null, + build_metadata: null, + created_at: '2026-08-01T00:00:00.000Z', + updated_at: '2026-08-01T00:00:00.000Z', +}; + +function makeStore(): ISandboxProviderStore { + return { + getSandboxProvider: jest.fn().mockResolvedValue(record), + getSandboxProviderForUpdate: jest.fn(), + upsertSandboxProvider: jest.fn(), + updateSandboxStatus: jest.fn().mockImplementation(async input => ({ ...record, ...input })), + }; +} + +afterEach(() => { + jest.restoreAllMocks(); +}); + +it('persists Daytona authentication failures instead of surfacing a settings error', async () => { + jest + .spyOn(DaytonaSandboxProvider.prototype, 'getImageBuildStatus') + .mockRejectedValue(new DaytonaError('unauthorized', 401)); + const store = makeStore(); + + await expect( + checkSnapshotStatus({ store, tenant_id: record.tenant_id, logger: createLogger({ silent: true }) }), + ).resolves.toMatchObject({ + status: 'failed', + status_reason: 'Daytona rejected the API key. Check the configured credentials.', + }); + expect(store.updateSandboxStatus).toHaveBeenCalledWith({ + tenant_id: record.tenant_id, + status: 'failed', + status_reason: 'Daytona rejected the API key. Check the configured credentials.', + build_metadata: null, + }); +}); From 1ffdd4ecb260e8cda21dc29b6a8416d23695df53 Mon Sep 17 00:00:00 2001 From: Elioooon Date: Mon, 31 Aug 2026 18:13:11 +0800 Subject: [PATCH 2/8] fix: avoid stale sandbox status writes --- .../PostgresSandboxProviderStore.ts | 11 +++++++---- .../trueforge/src/db/sandboxProviderStore.ts | 1 + .../SqliteSandboxProviderStore.ts | 11 +++++++---- .../trueforge/src/runtime/sessionResources.ts | 6 +----- .../trueforge/src/sandbox/providerUtils.ts | 19 +++++++++++++++---- .../db/sandboxProviderStoreContractSuite.ts | 16 ++++++++++++++++ .../tests/unit/sandbox/providerUtils.test.ts | 1 + 7 files changed, 48 insertions(+), 17 deletions(-) diff --git a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts index c0ca35dfb..30cae739b 100644 --- a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts +++ b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts @@ -88,7 +88,8 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore, ): Promise { const db = transaction ?? this.#db; - const row = await db + const expectedUpdatedAt = input.expected_updated_at; + let query = db .updateTable('sandbox_provider') .set({ status: input.status, @@ -96,9 +97,11 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore>>>>>> fix: persist Daytona authentication failures import type { ReasoningEffort } from '../schemas/modelProvider'; import { resolveWebSearchProvider } from '../websearch/providers'; @@ -263,7 +259,7 @@ export async function resolveSandboxProvider({ tenant_id, error, build_metadata: record.build_metadata, - expected_manifest: record.manifest, + expected_updated_at: record.updated_at, }); }, }); diff --git a/packages/trueforge/src/sandbox/providerUtils.ts b/packages/trueforge/src/sandbox/providerUtils.ts index f26ee1221..6dc7f5180 100644 --- a/packages/trueforge/src/sandbox/providerUtils.ts +++ b/packages/trueforge/src/sandbox/providerUtils.ts @@ -121,11 +121,13 @@ export async function recordDaytonaAccessFailure({ tenant_id, error, build_metadata, + expected_updated_at, }: { store: ISandboxProviderStore; tenant_id: string; error: unknown; build_metadata?: SandboxBuildMetadata | null; + expected_updated_at?: string | undefined; }): Promise { const status_reason = isDaytonaAuthError(error) ? 'Daytona rejected the API key. Check the configured credentials.' @@ -136,8 +138,12 @@ export async function recordDaytonaAccessFailure({ return undefined; } const next: SandboxStatus = { status: 'failed', status_reason, build_metadata: build_metadata ?? null }; - const updated = await store.updateSandboxStatus({ tenant_id, ...next }); - return updated ? sandboxStatusFromRecord(updated) : next; + const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_updated_at }); + if (updated !== undefined) { + return sandboxStatusFromRecord(updated); + } + const current = await store.getSandboxProvider(tenant_id); + return current === undefined ? undefined : sandboxStatusFromRecord(current); } // Daytona deactivates idle snapshots after 14 days; revalidate at 13 to stay a day ahead. @@ -197,6 +203,7 @@ export async function checkSnapshotStatus({ tenant_id, error, build_metadata: record.build_metadata, + expected_updated_at: record.updated_at, }); if (failed !== undefined) { return failed; @@ -204,6 +211,10 @@ export async function checkSnapshotStatus({ throw error; } const next = toSandboxStatus(build); - const updated = await store.updateSandboxStatus({ tenant_id, ...next }); - return updated ? sandboxStatusFromRecord(updated) : next; + const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_updated_at: record.updated_at }); + if (updated !== undefined) { + return sandboxStatusFromRecord(updated); + } + const current = await store.getSandboxProvider(tenant_id); + return current === undefined ? undefined : sandboxStatusFromRecord(current); } diff --git a/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts b/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts index a337d1c0c..053e034e2 100644 --- a/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts +++ b/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts @@ -113,4 +113,20 @@ export function runSandboxProviderStoreContractSuite(getStore: () => ISandboxPro }), ).toBeUndefined(); }); + + it('updateSandboxStatus ignores a stale provider version', async () => { + const store = getStore(); + await store.upsertSandboxProvider(upsertInput()); + + expect( + await store.updateSandboxStatus({ + tenant_id: TENANT, + status: 'failed', + status_reason: 'stale failure', + build_metadata: BUILD_METADATA, + expected_updated_at: '1970-01-01T00:00:00.000Z', + }), + ).toBeUndefined(); + expect((await store.getSandboxProvider(TENANT))?.status).toBe('pending'); + }); } diff --git a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts index 40e7a2268..4eb9dc892 100644 --- a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts +++ b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts @@ -56,5 +56,6 @@ it('persists Daytona authentication failures instead of surfacing a settings err status: 'failed', status_reason: 'Daytona rejected the API key. Check the configured credentials.', build_metadata: null, + expected_updated_at: record.updated_at, }); }); From a49ba4054019b4dd46b473393381ce13ee3b5698 Mon Sep 17 00:00:00 2001 From: Elioooon Date: Mon, 31 Aug 2026 18:41:03 +0800 Subject: [PATCH 3/8] fix: compare sandbox provider manifests --- .../PostgresSandboxProviderStore.ts | 6 +++--- packages/trueforge/src/db/sandboxProviderStore.ts | 2 +- .../SqliteSandboxProviderStore.ts | 8 ++++---- packages/trueforge/src/runtime/sessionResources.ts | 7 ++++--- packages/trueforge/src/sandbox/providerUtils.ts | 10 +++++----- .../tests/db/sandboxProviderStoreContractSuite.ts | 4 ++-- .../trueforge/tests/unit/sandbox/providerUtils.test.ts | 2 +- 7 files changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts index 30cae739b..74bf9308d 100644 --- a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts +++ b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts @@ -88,7 +88,7 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore, ): Promise { const db = transaction ?? this.#db; - const expectedUpdatedAt = input.expected_updated_at; + const expectedManifest = input.expected_manifest; let query = db .updateTable('sandbox_provider') .set({ @@ -98,8 +98,8 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore, ): Promise { const db = transaction ?? this.#db; - const expectedUpdatedAt = input.expected_updated_at; + const expectedManifest = input.expected_manifest; let query = db .updateTable('sandbox_provider') .set({ @@ -101,8 +101,8 @@ export class SqliteSandboxProviderStore implements ISandboxProviderStore { const status_reason = isDaytonaAuthError(error) ? 'Daytona rejected the API key. Check the configured credentials.' @@ -138,7 +138,7 @@ export async function recordDaytonaAccessFailure({ return undefined; } const next: SandboxStatus = { status: 'failed', status_reason, build_metadata: build_metadata ?? null }; - const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_updated_at }); + const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_manifest }); if (updated !== undefined) { return sandboxStatusFromRecord(updated); } @@ -203,7 +203,7 @@ export async function checkSnapshotStatus({ tenant_id, error, build_metadata: record.build_metadata, - expected_updated_at: record.updated_at, + expected_manifest: record.manifest, }); if (failed !== undefined) { return failed; @@ -211,7 +211,7 @@ export async function checkSnapshotStatus({ throw error; } const next = toSandboxStatus(build); - const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_updated_at: record.updated_at }); + const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_manifest: record.manifest }); if (updated !== undefined) { return sandboxStatusFromRecord(updated); } diff --git a/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts b/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts index 053e034e2..c16d455b2 100644 --- a/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts +++ b/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts @@ -114,7 +114,7 @@ export function runSandboxProviderStoreContractSuite(getStore: () => ISandboxPro ).toBeUndefined(); }); - it('updateSandboxStatus ignores a stale provider version', async () => { + it('updateSandboxStatus ignores a stale provider manifest', async () => { const store = getStore(); await store.upsertSandboxProvider(upsertInput()); @@ -124,7 +124,7 @@ export function runSandboxProviderStoreContractSuite(getStore: () => ISandboxPro status: 'failed', status_reason: 'stale failure', build_metadata: BUILD_METADATA, - expected_updated_at: '1970-01-01T00:00:00.000Z', + expected_manifest: manifest({ exec_timeout_ms: 120000 }), }), ).toBeUndefined(); expect((await store.getSandboxProvider(TENANT))?.status).toBe('pending'); diff --git a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts index 4eb9dc892..e69756a6f 100644 --- a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts +++ b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts @@ -56,6 +56,6 @@ it('persists Daytona authentication failures instead of surfacing a settings err status: 'failed', status_reason: 'Daytona rejected the API key. Check the configured credentials.', build_metadata: null, - expected_updated_at: record.updated_at, + expected_manifest: record.manifest, }); }); From e8eed1c8e3e210334ad7222877147475d47e1296 Mon Sep 17 00:00:00 2001 From: Elioooon Date: Sat, 12 Sep 2026 10:02:26 +0800 Subject: [PATCH 4/8] fix(store): require explicit manifest guard --- packages/trueforge/src/db/sandboxProviderStore.ts | 2 +- .../trueforge/tests/db/sandboxProviderStoreContractSuite.ts | 2 ++ .../unit/truefoundry/TrueFoundrySandboxProviderStore.test.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/trueforge/src/db/sandboxProviderStore.ts b/packages/trueforge/src/db/sandboxProviderStore.ts index 6f1dc4d83..e696f46a0 100644 --- a/packages/trueforge/src/db/sandboxProviderStore.ts +++ b/packages/trueforge/src/db/sandboxProviderStore.ts @@ -38,7 +38,7 @@ export interface UpdateSandboxStatusInput { status: SandboxBuildStatus; status_reason: string | null; build_metadata: SandboxBuildMetadata | null; - expected_manifest?: StoredSandboxProviderManifest | undefined; + expected_manifest: StoredSandboxProviderManifest | undefined; } export interface ISandboxProviderStore { diff --git a/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts b/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts index c16d455b2..f17a2d52b 100644 --- a/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts +++ b/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts @@ -95,6 +95,7 @@ export function runSandboxProviderStoreContractSuite(getStore: () => ISandboxPro status: 'ready', status_reason: null, build_metadata: BUILD_METADATA, + expected_manifest: undefined, }); expect(updated?.status).toBe('ready'); @@ -110,6 +111,7 @@ export function runSandboxProviderStoreContractSuite(getStore: () => ISandboxPro status: 'ready', status_reason: null, build_metadata: BUILD_METADATA, + expected_manifest: undefined, }), ).toBeUndefined(); }); diff --git a/packages/trueforge/tests/unit/truefoundry/TrueFoundrySandboxProviderStore.test.ts b/packages/trueforge/tests/unit/truefoundry/TrueFoundrySandboxProviderStore.test.ts index 55a699b20..4d4625b93 100644 --- a/packages/trueforge/tests/unit/truefoundry/TrueFoundrySandboxProviderStore.test.ts +++ b/packages/trueforge/tests/unit/truefoundry/TrueFoundrySandboxProviderStore.test.ts @@ -118,6 +118,7 @@ describe('TrueFoundrySandboxProviderStore', () => { status: 'ready', status_reason: null, build_metadata: null, + expected_manifest: undefined, }), ); }); From 11dbf1d18cda7d8ff0f000fdeb338838df23000d Mon Sep 17 00:00:00 2001 From: Elioooon Date: Thu, 17 Sep 2026 07:03:10 +0800 Subject: [PATCH 5/8] fix(sandbox): scope cached clients to credentials --- .../core/sandbox/provider/DaytonaProvider.ts | 32 ++++++++++----- .../daytonaSnapshotRegistration.test.ts | 40 ++++++++++++++++++- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts index 73d3297e2..5fd8e666b 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts @@ -2,7 +2,7 @@ import type { Sandbox, Snapshot } from '@daytona/sdk'; import { Daytona, DaytonaError } from '@daytona/sdk'; import { context } from '@opentelemetry/api'; import { suppressTracing } from '@opentelemetry/core'; -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { join } from 'node:path/posix'; import type { Logger } from 'winston'; import { extractErrorLogFields } from '../../util/errorLogFields'; @@ -154,7 +154,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { private async getOrCreateSandbox(sandboxId?: string): Promise<{ sandbox: Sandbox; defaultTimeoutMs: number }> { if (sandboxId) { validateSandboxOwnedByTenant({ sandboxId, tenantName: this.tenantName }); - const cached = DaytonaSandboxProvider.cachedSandboxes.get(sandboxId); + const cached = DaytonaSandboxProvider.cachedSandboxes.get(this.sandboxCacheKey(sandboxId)); if (cached) { return cached; } @@ -171,18 +171,30 @@ export class DaytonaSandboxProvider implements SandboxProvider { }); const entry = { sandbox, defaultTimeoutMs: this.timeoutMs }; - DaytonaSandboxProvider.cachedSandboxes.set(sandbox.name, entry); + DaytonaSandboxProvider.cachedSandboxes.set(this.sandboxCacheKey(sandbox.name), entry); return entry; } + /** + * A Sandbox object carries the Daytona client that restored it. Include the client + * identity in the process-wide cache so a settings update with rotated credentials + * cannot reuse an object authenticated with the previous key. Hashing avoids keeping + * the raw API key as a Map key or exposing it through diagnostics. + */ + private sandboxCacheKey(sandboxId: string): string { + return createHash('sha256') + .update(`${this.tenantName}\u0000${this.apiUrl}\u0000${this.apiKey}\u0000${sandboxId}`) + .digest('hex'); + } + // Returns true iff the caller should retry: either we restarted a stopped sandbox, or the cache entry is missing and the retry will rebuild it via the cold path. - private static recoverSandboxIfStopped(sandboxId: string): Promise { - const existing = DaytonaSandboxProvider.inFlightRecoveries.get(sandboxId); + private static recoverSandboxIfStopped(cacheKey: string): Promise { + const existing = DaytonaSandboxProvider.inFlightRecoveries.get(cacheKey); if (existing) { return existing; } - const cached = DaytonaSandboxProvider.cachedSandboxes.get(sandboxId); + const cached = DaytonaSandboxProvider.cachedSandboxes.get(cacheKey); // Cache may have been evicted by a concurrent error path; signal retry so getOrCreateSandbox rebuilds via restoreExistingSandbox. if (!cached) { return Promise.resolve(true); @@ -197,10 +209,10 @@ export class DaytonaSandboxProvider implements SandboxProvider { await cached.sandbox.start(); return true; })().finally(() => { - DaytonaSandboxProvider.inFlightRecoveries.delete(sandboxId); + DaytonaSandboxProvider.inFlightRecoveries.delete(cacheKey); }); - DaytonaSandboxProvider.inFlightRecoveries.set(sandboxId, recovery); + DaytonaSandboxProvider.inFlightRecoveries.set(cacheKey, recovery); return recovery; } @@ -215,7 +227,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { let recovered: boolean; try { - recovered = await DaytonaSandboxProvider.recoverSandboxIfStopped(sandboxId); + recovered = await DaytonaSandboxProvider.recoverSandboxIfStopped(this.sandboxCacheKey(sandboxId)); } catch (recoveryError) { this.logger.error('Sandbox recovery failed', { ...extractErrorLogFields(recoveryError), @@ -415,7 +427,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { }; }); } catch (e: unknown) { - DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId); + DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId)); if (e instanceof SandboxNotAvailableError) { throw e; } diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts index bc135a8db..83fb1312f 100644 --- a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts @@ -32,10 +32,14 @@ function makeProvider(): DaytonaSandboxProvider { }); } -function makeRuntimeProvider(client: Daytona, onError?: (error: unknown) => Promise): DaytonaSandboxProvider { +function makeRuntimeProvider( + client: Daytona, + onError?: (error: unknown) => Promise, + apiKey = 'dtn-test', +): DaytonaSandboxProvider { return new DaytonaSandboxProvider({ client, - apiKey: 'dtn-test', + apiKey, apiUrl: API_URL, tenantName: 'test-tenant', sandboxImage: 'registry.example.com/sandbox:029ea5ff', @@ -129,4 +133,36 @@ describe('DaytonaSandboxProvider exec', () => { SandboxNotAvailableError, ); }); + + it('does not reuse a restored sandbox after the Daytona credentials rotate', async () => { + const sandboxId = 'test-tenant.rotated-credentials'; + const oldClient = new Daytona({ apiKey: 'dtn-old', useDeprecatedPolling: true }); + const newClient = new Daytona({ apiKey: 'dtn-new', useDeprecatedPolling: true }); + const oldSandbox = { + state: 'started', + process: { executeCommand: jest.fn().mockResolvedValue({ exitCode: 0, result: 'old' }) }, + }; + const newSandbox = { + state: 'started', + process: { executeCommand: jest.fn().mockResolvedValue({ exitCode: 0, result: 'new' }) }, + }; + jest.spyOn(oldClient, 'get').mockResolvedValue(oldSandbox as never); + jest.spyOn(newClient, 'get').mockResolvedValue(newSandbox as never); + + await expect( + makeRuntimeProvider(oldClient, undefined, 'dtn-old').exec({ sandboxId, command: 'true' }), + ).resolves.toMatchObject({ + success: true, + response: { result: 'old' }, + }); + await expect( + makeRuntimeProvider(newClient, undefined, 'dtn-new').exec({ sandboxId, command: 'true' }), + ).resolves.toMatchObject({ + success: true, + response: { result: 'new' }, + }); + + expect(oldClient.get).toHaveBeenCalledWith(sandboxId); + expect(newClient.get).toHaveBeenCalledWith(sandboxId); + }); }); From f553faf364f8890089a0d0a32df472c8802a5253 Mon Sep 17 00:00:00 2001 From: Elioooon Date: Thu, 17 Sep 2026 13:35:25 +0800 Subject: [PATCH 6/8] fix(sandbox): evict credential-scoped cache entries --- .../core/sandbox/provider/DaytonaProvider.ts | 6 ++-- .../daytonaSnapshotRegistration.test.ts | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts index 5fd8e666b..690481812 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts @@ -467,7 +467,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { if (e instanceof DaytonaError && e.statusCode === SANDBOX_NOT_FOUND_STATUS) { throw new SandboxFileNotFoundError(params.path); } - DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId); + DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId)); throw e; } }); @@ -481,7 +481,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { await sandbox.fs.uploadFile(params.content, params.remotePath); }); } catch (e: unknown) { - DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId); + DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId)); throw e; } }); @@ -497,7 +497,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { return signed.url; }); } catch (e: unknown) { - DaytonaSandboxProvider.cachedSandboxes.delete(params.sandboxId); + DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId)); this.logger.error('Failed to create signed preview URL', extractErrorLogFields(e)); throw e; } diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts index 83fb1312f..5d8007082 100644 --- a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts @@ -165,4 +165,33 @@ describe('DaytonaSandboxProvider exec', () => { expect(oldClient.get).toHaveBeenCalledWith(sandboxId); expect(newClient.get).toHaveBeenCalledWith(sandboxId); }); + + it('evicts the credential-scoped cache entry after a file operation fails', async () => { + const sandboxId = 'test-tenant.file-cache-eviction'; + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + const staleSandbox = { + state: 'started', + fs: { + getFileDetails: jest.fn().mockResolvedValue({ size: 1, isDir: false }), + downloadFile: jest.fn().mockRejectedValue(new Error('connection reset')), + }, + }; + const restoredSandbox = { + state: 'started', + fs: { + getFileDetails: jest.fn().mockResolvedValue({ size: 1, isDir: false }), + downloadFile: jest.fn().mockResolvedValue(Buffer.from('recovered')), + }, + }; + jest + .spyOn(client, 'get') + .mockResolvedValueOnce(staleSandbox as never) + .mockResolvedValueOnce(restoredSandbox as never); + const provider = makeRuntimeProvider(client); + + await expect(provider.downloadFile({ sandboxId, path: '/tmp/output' })).rejects.toThrow('connection reset'); + await expect(provider.downloadFile({ sandboxId, path: '/tmp/output' })).resolves.toEqual(Buffer.from('recovered')); + + expect(client.get).toHaveBeenCalledTimes(2); + }); }); From db91ef5f48d9d2b00bfe9201ad009fd393a1420e Mon Sep 17 00:00:00 2001 From: Elioooon Date: Thu, 17 Sep 2026 16:03:38 +0800 Subject: [PATCH 7/8] fix(sandbox): report Daytona file auth failures --- .../core/sandbox/provider/DaytonaProvider.ts | 3 ++ .../daytonaSnapshotRegistration.test.ts | 41 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts index 690481812..ff0c1e89b 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts @@ -468,6 +468,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { throw new SandboxFileNotFoundError(params.path); } DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId)); + await this.reportError(e); throw e; } }); @@ -482,6 +483,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { }); } catch (e: unknown) { DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId)); + await this.reportError(e); throw e; } }); @@ -498,6 +500,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { }); } catch (e: unknown) { DaytonaSandboxProvider.cachedSandboxes.delete(this.sandboxCacheKey(params.sandboxId)); + await this.reportError(e); this.logger.error('Failed to create signed preview URL', extractErrorLogFields(e)); throw e; } diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts index 5d8007082..2de142f36 100644 --- a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts @@ -194,4 +194,45 @@ describe('DaytonaSandboxProvider exec', () => { expect(client.get).toHaveBeenCalledTimes(2); }); + + it.each([ + [ + 'download', + async (provider: DaytonaSandboxProvider, sandboxId: string) => + provider.downloadFile({ sandboxId, path: '/tmp/output' }), + ], + [ + 'upload', + async (provider: DaytonaSandboxProvider, sandboxId: string) => + provider.uploadFile({ sandboxId, remotePath: '/tmp/output', content: Buffer.from('content') }), + ], + [ + 'preview', + async (provider: DaytonaSandboxProvider, sandboxId: string) => + ( + provider as unknown as { + getPreviewUrl(params: { sandboxId: string; port: number; expiresInSeconds: number }): Promise; + } + ).getPreviewUrl({ sandboxId, port: 4222, expiresInSeconds: 60 }), + ], + ])('reports Daytona authentication failures from %s operations', async (_operation, invoke) => { + const sandboxId = 'test-tenant.auth-failure'; + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + const unauthorized = new DaytonaError('unauthorized', 401); + const sandbox = { + state: 'started', + fs: { + getFileDetails: jest.fn().mockResolvedValue({ size: 1, isDir: false }), + downloadFile: jest.fn().mockRejectedValue(unauthorized), + uploadFile: jest.fn().mockRejectedValue(unauthorized), + }, + getSignedPreviewUrl: jest.fn().mockRejectedValue(unauthorized), + }; + jest.spyOn(client, 'get').mockResolvedValue(sandbox as never); + const onError = jest.fn().mockResolvedValue(undefined); + const provider = makeRuntimeProvider(client, onError); + + await expect(invoke(provider, sandboxId)).rejects.toMatchObject({ statusCode: 401 }); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 })); + }); }); From e76785c0dbe87e507e3dbdb5ff4398cb994f4d4f Mon Sep 17 00:00:00 2001 From: Elioooon Date: Thu, 17 Sep 2026 20:39:13 +0800 Subject: [PATCH 8/8] fix(sandbox): guard refresh status writes --- .../PostgresSandboxProviderStore.ts | 3 +++ .../trueforge/src/db/sandboxProviderStore.ts | 1 + .../SqliteSandboxProviderStore.ts | 3 +++ .../trueforge/src/runtime/sessionResources.ts | 1 + .../trueforge/src/sandbox/providerUtils.ts | 12 ++++++++-- .../tests/unit/sandbox/providerUtils.test.ts | 23 +++++++++++++++++++ 6 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts index 74bf9308d..975231c6b 100644 --- a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts +++ b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts @@ -101,6 +101,9 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore { + jest + .spyOn(DaytonaSandboxProvider.prototype, 'getImageBuildStatus') + .mockResolvedValue({ status: 'ready', metadata: { snapshot: 'snap-1' } } as never); + const failed = { + ...record, + status: 'failed' as const, + status_reason: 'Daytona rejected the API key. Check the configured credentials.', + updated_at: '2026-09-17T10:00:00.000Z', + }; + const store = makeStore(); + (store.updateSandboxStatus as jest.Mock).mockResolvedValue(undefined); + (store.getSandboxProvider as jest.Mock).mockResolvedValueOnce(record).mockResolvedValueOnce(failed); + + await expect( + checkSnapshotStatus({ store, tenant_id: record.tenant_id, logger: createLogger({ silent: true }) }), + ).resolves.toMatchObject({ status: 'failed', status_reason: failed.status_reason }); + expect(store.updateSandboxStatus).toHaveBeenCalledWith( + expect.objectContaining({ expected_updated_at: record.updated_at }), + ); +});