diff --git a/bun.lock b/bun.lock index 60b5189f7596..b4367a1889fd 100644 --- a/bun.lock +++ b/bun.lock @@ -119,7 +119,7 @@ "@effect/platform-node": "catalog:", "@opencode/client": "workspace:*", "@opencode/plugin": "workspace:*", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/schema": "workspace:*", "@opencode/server": "workspace:*", "@opencode/tui": "workspace:*", @@ -267,7 +267,7 @@ "@opencode/codemode": "workspace:*", "@opencode/plugin": "workspace:*", "@opencode/plugin-browser": "workspace:*", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/schema": "workspace:*", "@opencode/util": "workspace:*", "@parcel/watcher": "2.5.1", @@ -666,7 +666,7 @@ "devDependencies": { "@opencode/client": "workspace:*", "@opencode/plugin": "workspace:*", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@typescript/native": "catalog:", @@ -1957,23 +1957,23 @@ "@opencode/protocol": ["@opencode/protocol@workspace:packages/protocol"], - "@opencode/pty": ["@opencode/pty@0.4.0", "", { "optionalDependencies": { "@opencode/pty-darwin-arm64": "0.4.0", "@opencode/pty-darwin-x64": "0.4.0", "@opencode/pty-linux-arm64-gnu": "0.4.0", "@opencode/pty-linux-arm64-musl": "0.4.0", "@opencode/pty-linux-x64-gnu": "0.4.0", "@opencode/pty-linux-x64-musl": "0.4.0", "@opencode/pty-win32-arm64": "0.4.0", "@opencode/pty-win32-x64": "0.4.0" }, "bin": { "opencode-pty": "bin/opencode-pty.js" } }, "sha512-v7cLT71yfoNM5FlyPGwMjeqIJbr0DneGq4NpoOY5iAKpsevzOIcsI2E0fiDWajuk5xAFRxwyv4CpGqow7cu79Q=="], + "@opencode/pty": ["@opencode/pty@0.5.0", "", { "optionalDependencies": { "@opencode/pty-darwin-arm64": "0.5.0", "@opencode/pty-darwin-x64": "0.5.0", "@opencode/pty-linux-arm64-gnu": "0.5.0", "@opencode/pty-linux-arm64-musl": "0.5.0", "@opencode/pty-linux-x64-gnu": "0.5.0", "@opencode/pty-linux-x64-musl": "0.5.0", "@opencode/pty-win32-arm64": "0.5.0", "@opencode/pty-win32-x64": "0.5.0" }, "bin": { "opencode-pty": "bin/opencode-pty.js" } }, "sha512-2U0RfT5U1lp1BcegXLoSBYfBx9sBNw3uJ/pao4UWdNV6UDfmDNGzdc8m4K1Q4ByBkXhbh8hexa0JS7HZ0vSsgg=="], - "@opencode/pty-darwin-arm64": ["@opencode/pty-darwin-arm64@0.4.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-XzZZJcQeR42IRpxHbbPEo4AK7jdp9SpgPx234DZ6uLnl6yclXcixH70G2U88EhKtGAMyCPIXdBfvQf4OQTDvbQ=="], + "@opencode/pty-darwin-arm64": ["@opencode/pty-darwin-arm64@0.5.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-D+L/TnNUc35airWTBlLussnuDbZpKPQ2/uBHVkcycw6KrLYDb4hVoUJn8N5PXmLTnhmU64B4MUsdH9JoOZIi8w=="], - "@opencode/pty-darwin-x64": ["@opencode/pty-darwin-x64@0.4.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-NvQfygGcsGmDCIUJ/VHLcS6SduC93Y/qP2hrHjtBpiL2sSQyeCDcifNhLvqY/wxGFNNZ4GeZwwZqe7rMEuBJbQ=="], + "@opencode/pty-darwin-x64": ["@opencode/pty-darwin-x64@0.5.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-/JMtOIAY9WWszsauWnCwjl2Qr8PBihS94tLsxa8prZBVhbbpOzcFslQsqR8JnJ1gAVlYINxEVS79+QQPQ4NC9g=="], - "@opencode/pty-linux-arm64-gnu": ["@opencode/pty-linux-arm64-gnu@0.4.0", "", { "os": "linux", "cpu": "arm64", "libc": "glibc" }, "sha512-VZXkYD+CT+zdodmTnKn9/VlmswksLPBLJryeBsG4IXUFXTsATr7+PVygJk96FxCh5nq68H3a/UJgyiIUMl1XCQ=="], + "@opencode/pty-linux-arm64-gnu": ["@opencode/pty-linux-arm64-gnu@0.5.0", "", { "os": "linux", "cpu": "arm64", "libc": "glibc" }, "sha512-vNiHKopfKrX6vj8OmDXYxZ+5JEZwf5djDzt3LOqeYuwStQRX2rxcl2/zjrvd5i8rAJw30RpJ+PTdS1t08vqtLA=="], - "@opencode/pty-linux-arm64-musl": ["@opencode/pty-linux-arm64-musl@0.4.0", "", { "os": "linux", "cpu": "arm64", "libc": "musl" }, "sha512-hCx25Hn9sALdnkx2Y4aPGVw3glTbX61XRiJs+XETPPIO9QTaav4OS4k2JR2uHC6/u161xKdN82iJ+OYfwcGUSg=="], + "@opencode/pty-linux-arm64-musl": ["@opencode/pty-linux-arm64-musl@0.5.0", "", { "os": "linux", "cpu": "arm64", "libc": "musl" }, "sha512-aW3lNGjit3FNAfpEskJgvTrtE2Lfnukpn8KmUVPUBN/EbRhT56h/N3ghzHYc5jIExETy06Xpy9i2xZxZQy0McQ=="], - "@opencode/pty-linux-x64-gnu": ["@opencode/pty-linux-x64-gnu@0.4.0", "", { "os": "linux", "cpu": "x64", "libc": "glibc" }, "sha512-Xal/d0+xp6XE4iCGUAw4JTQ4uf5l8V0EIbeKq+e03cHJewOMKZk4fooQjxpZvQIToBHRtnCJAmd+EEhVxXiDDw=="], + "@opencode/pty-linux-x64-gnu": ["@opencode/pty-linux-x64-gnu@0.5.0", "", { "os": "linux", "cpu": "x64", "libc": "glibc" }, "sha512-XaMnBSh1QvJmcYKSOtx1fVvldZ+x212zAHaq8ovyafufqp6/Xl3AuqPCXwPFnycW2PfbUDwfZ7dBCXThQtOt3Q=="], - "@opencode/pty-linux-x64-musl": ["@opencode/pty-linux-x64-musl@0.4.0", "", { "os": "linux", "cpu": "x64", "libc": "musl" }, "sha512-dBVtKZoVKAN6/E+65GrJksgRKzPkPHUjWZBzf3mRANXt2adpdbV+/M1AMfPCZSju2mN1j9rNu8MCrxwr+ugEig=="], + "@opencode/pty-linux-x64-musl": ["@opencode/pty-linux-x64-musl@0.5.0", "", { "os": "linux", "cpu": "x64", "libc": "musl" }, "sha512-30LVc91GrJP8YD6chLic4+5nMsLz3hCYu2cxPQczSKyRWMyBjuwvCf040UzlRAQiGllp8KBb7nYa8SCtq2/49Q=="], - "@opencode/pty-win32-arm64": ["@opencode/pty-win32-arm64@0.4.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-IKJMUnS2aluWqRj13enMWomYtwbXq8rStcf/mUbxBoAGTDMtWR/ZrRmKb2ptR7KUqDjdSEa+8C8rczS7uGFl9g=="], + "@opencode/pty-win32-arm64": ["@opencode/pty-win32-arm64@0.5.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-BYVSytysZvXmIQcP9VbbXbOV417RLW/BATiAK8MMbHBF6fzqQlpAMTi4cQmWJKOlHgrREKeMBw786qZ8BES3Vg=="], - "@opencode/pty-win32-x64": ["@opencode/pty-win32-x64@0.4.0", "", { "os": "win32", "cpu": "x64" }, "sha512-P1QJLUX918/y8he8VEBSYoiN5bYhUnJtEaEpiE5fxoYsDP9nqAHXqIrGxTNNnda5jOBcCalxxy0WhL8qgeKOqQ=="], + "@opencode/pty-win32-x64": ["@opencode/pty-win32-x64@0.5.0", "", { "os": "win32", "cpu": "x64" }, "sha512-42OEc6vvlL/gXAairdWf4DLX6juPrpOycES9sp3iE3UfQ5KsRsuQ6F3HFFmSBkiZ/yUhJtKoJdJbquBTNEL+Ow=="], "@opencode/schema": ["@opencode/schema@workspace:packages/schema"], diff --git a/packages/cli/package.json b/packages/cli/package.json index ecc5abf437b5..00e249bb2a56 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -33,7 +33,7 @@ "@effect/platform-node": "catalog:", "@opencode/client": "workspace:*", "@opencode/plugin": "workspace:*", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/schema": "workspace:*", "@opencode/server": "workspace:*", "@opencode/tui": "workspace:*", diff --git a/packages/core/package.json b/packages/core/package.json index 07ab612afbab..055fc1979082 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -105,7 +105,7 @@ "@modelcontextprotocol/client": "2.3.1", "@modelcontextprotocol/core": "2.3.1", "@modelcontextprotocol/server": "2.3.1", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/ai": "workspace:*", "@opencode/codemode": "workspace:*", "@opencode/plugin": "workspace:*", diff --git a/packages/core/src/artifact.ts b/packages/core/src/artifact.ts index d59263696c39..5c184d291bc0 100644 --- a/packages/core/src/artifact.ts +++ b/packages/core/src/artifact.ts @@ -169,7 +169,9 @@ export interface Interface extends State.Transformable { readonly provider?: ProviderID }) => Effect.Effect readonly create: (input: CreateInput) => Effect.Effect - readonly get: (id: ID) => Effect.Effect + readonly get: ( + id: ID, + ) => Effect.Effect readonly info: (id: ID) => Effect.Effect readonly update: (id: ID, input: UpdateInput) => Effect.Effect readonly remove: (id: ID) => Effect.Effect diff --git a/packages/core/src/detaching.ts b/packages/core/src/detaching.ts new file mode 100644 index 000000000000..e15ad9967e34 --- /dev/null +++ b/packages/core/src/detaching.ts @@ -0,0 +1,51 @@ +export * as Detaching from "./detaching.js" + +import { Context, Effect, Layer } from "effect" +import { makeGlobalNode } from "@opencode/util/effect/app-node" + +/** + * Planned-shutdown state of this server process. + * + * A prepared daemon handoff means a new server will take over the daemon's processes. If this server + * then shuts down, it is detaching: work that can outlive it in the daemon is left running for the new + * server instead of being stopped. Without a prepared handoff (a normal quit), shutdown stops everything + * as before. + */ +export interface Interface { + /** Records that a daemon handoff was prepared successfully. */ + readonly handoffPrepared: Effect.Effect + /** Records that the prepared handoff no longer holds: the daemon was shut down, so nothing can outlive this server. */ + readonly handoffDropped: Effect.Effect + /** Called once shutdown starts, before any service is torn down. Returns whether this server is detaching. */ + readonly begin: Effect.Effect + readonly isDetaching: () => boolean + /** Whether shutdown has started, detaching or not. Interrupts from then on are not the user's. */ + readonly isStopping: () => boolean +} + +export class Service extends Context.Service()("@opencode/Detaching") {} + +export const make = Effect.sync(() => { + let prepared = false + let detaching = false + let stopping = false + return Service.of({ + handoffPrepared: Effect.sync(() => { + prepared = true + }), + handoffDropped: Effect.sync(() => { + prepared = false + }), + begin: Effect.sync(() => { + stopping = true + detaching = prepared + return detaching + }), + isDetaching: () => detaching, + isStopping: () => stopping, + }) +}) + +export const layer = Layer.effect(Service, make) + +export const node = makeGlobalNode({ service: Service, layer, deps: [] }) diff --git a/packages/core/src/file-retention.ts b/packages/core/src/file-retention.ts index a325f2165fc9..ee9f486e219c 100644 --- a/packages/core/src/file-retention.ts +++ b/packages/core/src/file-retention.ts @@ -7,6 +7,8 @@ export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( fs: FSUtil.Interface, files: ReadonlyArray, retention: Duration.Input, + // Runs before an expired file is removed. + beforeRemove?: (file: string) => Effect.Effect, ) { const cutoff = Date.now() - Duration.toMillis(retention) yield* Effect.forEach( @@ -16,6 +18,7 @@ export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( const info = yield* fs.stat(file).pipe(Effect.orElseSucceed(() => undefined)) const mtime = info && Option.getOrUndefined(info.mtime) if (!mtime || mtime.getTime() >= cutoff) return + if (beforeRemove) yield* beforeRemove(file) yield* fs.remove(file).pipe(Effect.ignore) }), { concurrency: 8, discard: true }, diff --git a/packages/core/src/generate.ts b/packages/core/src/generate.ts index a28ab6b32646..bb1c3e7dcbad 100644 --- a/packages/core/src/generate.ts +++ b/packages/core/src/generate.ts @@ -90,24 +90,26 @@ export const layer = Layer.effect( const decisionClient = yield* DecisionClient.Service const runText = Effect.fn("Generate.text")(function* (input: TextInput) { - const resolved = yield* resolver.resolve(input.model).pipe( - Effect.catchTag( - [ - "SessionRunnerModel.VariantUnavailableError", - "SessionRunnerModel.UnsupportedPackageError", - "SessionRunnerModel.ModelConfigurationError", - "SessionRunnerModel.ModelInitializationError", - "SessionRunnerModel.UnresolvedProviderVariablesError", - "SessionRunnerModel.UnsupportedCompactionError", - ], - (error) => - Effect.fail( - input.model - ? resolutionError(input.model, "chat", error) - : new UnavailableError({ message: error.message, service: error.providerID }), - ), - ), - ) + const resolved = yield* resolver + .resolve(input.model) + .pipe( + Effect.catchTag( + [ + "SessionRunnerModel.VariantUnavailableError", + "SessionRunnerModel.UnsupportedPackageError", + "SessionRunnerModel.ModelConfigurationError", + "SessionRunnerModel.ModelInitializationError", + "SessionRunnerModel.UnresolvedProviderVariablesError", + "SessionRunnerModel.UnsupportedCompactionError", + ], + (error) => + Effect.fail( + input.model + ? resolutionError(input.model, "chat", error) + : new UnavailableError({ message: error.message, service: error.providerID }), + ), + ), + ) if (!resolved && input.model) return yield* missing(input.model, "chat") if (!resolved) return yield* new UnavailableError({ message: "No model specified and no supported model is available" }) @@ -152,18 +154,20 @@ export const layer = Layer.effect( .pipe(Effect.mapError((error) => resolutionError(input.model, "image", error))) if (!model) return yield* missing(input.model, "image") // `ai.image.generate` turns an invalid request into a typed error; building it with `ai.image.request` would throw. - const response = yield* ai.image.generate({ - model, - prompt: input.prompt, - images: input.images?.map((file) => Media.base64(file.data, file.mime)), - n: input.n, - aspect: input.aspect, - resolution: input.resolution, - }).pipe( - Effect.provideService(ImageClient.Service, images), - ModelRetry.retryOneShot, - Effect.mapError((error) => providerError(input.model, error)), - ) + const response = yield* ai.image + .generate({ + model, + prompt: input.prompt, + images: input.images?.map((file) => Media.base64(file.data, file.mime)), + n: input.n, + aspect: input.aspect, + resolution: input.resolution, + }) + .pipe( + Effect.provideService(ImageClient.Service, images), + ModelRetry.retryOneShot, + Effect.mapError((error) => providerError(input.model, error)), + ) // URL results are downloaded so every image comes back inline, like the request's own files. // Materializing also detects the type of a downloaded image from its bytes when the URL didn't say. const outputs = yield* Effect.forEach(response.images, (asset) => @@ -190,19 +194,21 @@ export const layer = Layer.effect( .resolveDecision(input.model) .pipe(Effect.mapError((error) => resolutionError(input.model, "decision", error))) if (!model) return yield* missing(input.model, "decision") - const response = yield* ai.decision.generate({ - model, - input: input.input.map((part) => - part.type === "file" - ? { type: "media" as const, media: Media.base64(part.data, part.mime), filename: part.name } - : part, - ), - questions: input.questions, - }).pipe( - Effect.provideService(DecisionClient.Service, decisionClient), - ModelRetry.retryOneShot, - Effect.mapError((error) => providerError(input.model, error)), - ) + const response = yield* ai.decision + .generate({ + model, + input: input.input.map((part) => + part.type === "file" + ? { type: "media" as const, media: Media.base64(part.data, part.mime), filename: part.name } + : part, + ), + questions: input.questions, + }) + .pipe( + Effect.provideService(DecisionClient.Service, decisionClient), + ModelRetry.retryOneShot, + Effect.mapError((error) => providerError(input.model, error)), + ) const usage = response.usage return { answers: response.answers, diff --git a/packages/core/src/job.ts b/packages/core/src/job.ts index fd5d391b396b..c579e1c1c212 100644 --- a/packages/core/src/job.ts +++ b/packages/core/src/job.ts @@ -1,9 +1,23 @@ export * as Job from "./job.js" import { asc, eq, or } from "drizzle-orm" -import { Array, Cause, Clock, Context, Deferred, Effect, Exit, Layer, Schema, Scope, SynchronizedRef } from "effect" +import { + Array, + Cause, + Clock, + Context, + Deferred, + Effect, + Exit, + Layer, + Option, + Schema, + Scope, + SynchronizedRef, +} from "effect" import { makeGlobalNode } from "@opencode/util/effect/app-node" import { Database } from "./database/database.js" +import { Detaching } from "./detaching.js" import { JobList } from "./job/list.js" import { type Recovery, RecoverySchema } from "./job/recovery.js" import { JobTable } from "./job/sql.js" @@ -59,6 +73,8 @@ type State = { type FinishResult = { info?: Info done?: Deferred.Deferred + // Teardown interrupted the work: `done` is interrupted instead of settled. + interrupted?: boolean scope?: Scope.Closeable generation?: Scope.Closeable } @@ -187,6 +203,8 @@ function decrementSession(input: Map, sessionID: Sessi */ export const make = Effect.gen(function* () { const db = (yield* Database.Service).db + // Optional so registries built outside the application graph (tests, tools) need not provide it. + const detaching = Option.getOrUndefined(yield* Effect.serviceOption(Detaching.Service)) const state: State = { jobs: yield* SynchronizedRef.make(new Map()), scope: yield* Scope.Scope, @@ -248,6 +266,15 @@ export const make = Effect.gen(function* () { if (!job) return [{}, jobs] if (job.scope !== scope) return [{}, jobs] if (job.info.status !== "running") return [{ info: snapshot(job) }, jobs] + // Teardown (this registry closing, or any server shutdown) is not a terminal outcome: leave the job + // and its durable row running for the next server, and interrupt rather than settle its observers. + // A user cancel has already settled the job as cancelled before interrupting its work. + if ( + Exit.isFailure(exit) && + Cause.hasInterruptsOnly(exit.cause) && + (scope.state._tag === "Closed" || detaching?.isStopping()) + ) + return [{ done: job.done, scope: job.scope, interrupted: true }, jobs] const status: Exclude = Exit.isSuccess(exit) ? "completed" : Cause.hasInterruptsOnly(exit.cause) @@ -268,6 +295,7 @@ export const make = Effect.gen(function* () { return [{ info: snapshot(next), done: job.done, scope: job.scope }, new Map(jobs).set(id, next)] }), ) + if (result.interrupted && result.done) yield* Deferred.interrupt(result.done) if (result.info && result.done) yield* Deferred.succeed(result.done, result.info) if (result.scope) { yield* Scope.close(result.scope, Exit.void).pipe(Effect.forkIn(state.scope, { startImmediately: true })) @@ -295,6 +323,8 @@ export const make = Effect.gen(function* () { return [{ info: snapshot(existing) }, jobs] } const scope = yield* Scope.fork(state.scope, "parallel") + // Closing the registry interrupts the work without settling it; interrupt its observers too. + yield* Scope.addFinalizer(scope, Deferred.interrupt(done)) const job = { info: { id: input.id, @@ -551,4 +581,4 @@ export const make = Effect.gen(function* () { const layer = Layer.effect(Service, make) -export const node = makeGlobalNode({ service: Service, layer, deps: [Database.node] }) +export const node = makeGlobalNode({ service: Service, layer, deps: [Database.node, Detaching.node] }) diff --git a/packages/core/src/persistent-pty/daemon.ts b/packages/core/src/persistent-pty/daemon.ts index 526dc59c1a55..95382daf0a41 100644 --- a/packages/core/src/persistent-pty/daemon.ts +++ b/packages/core/src/persistent-pty/daemon.ts @@ -1,36 +1,59 @@ import { spawn } from "node:child_process" -import { mkdir, readFile, rename } from "node:fs/promises" +import { readFile } from "node:fs/promises" import net from "node:net" import path from "node:path" import { Cause, Data, Duration, Effect, Schema, Semaphore } from "effect" import type { Handoff } from "@opencode/schema/persistent-pty" -import { Session } from "@opencode/schema/session" -const ProtocolVersion = 7 +const ProtocolVersion = 8 const MaxFrameBytes = 8 * 1024 * 1024 const Lifecycle = Schema.Union([ Schema.Struct({ status: Schema.Literal("running") }), - Schema.Struct({ status: Schema.Literal("exited"), exit_code: Schema.NullOr(Schema.Number) }), + Schema.Struct({ + status: Schema.Literal("exited"), + exit_code: Schema.NullOr(Schema.Number), + signal: Schema.optional(Schema.NullOr(Schema.String)), + }), Schema.Struct({ status: Schema.Literal("failed"), message: Schema.String }), ]) -export const WireTerminal = Schema.Struct({ - id: Schema.Number, - pid: Schema.NullOr(Schema.Number), +const PtyInfo = Schema.Struct({ title: Schema.String, - user_title: Schema.optional(Schema.NullOr(Schema.String)), + user_title: Schema.NullOr(Schema.String), foreground_process: Schema.NullOr(Schema.String), - group_id: Session.ID, - command: Schema.Array(Schema.String), - cwd: Schema.String, cols: Schema.Number, rows: Schema.Number, +}) + +export const Metadata = Schema.Record(Schema.String, Schema.Unknown) +export type Metadata = typeof Metadata.Type + +const PipeInfo = Schema.Struct({ + output_path: Schema.String, + output_lines: Schema.Number, + output_error: Schema.optional(Schema.NullOr(Schema.String)), + metadata: Metadata, +}) + +export const ProcessInfo = Schema.Struct({ + id: Schema.Number, + mode: Schema.Literals(["pty", "pipe"]), + pid: Schema.NullOr(Schema.Number), + command: Schema.Array(Schema.String), + cwd: Schema.String, + group_id: Schema.String, lifecycle: Lifecycle, output_head: Schema.Number, output_tail: Schema.Number, + started_at: Schema.Number, + exited_at: Schema.optional(Schema.NullOr(Schema.Number)), + pty: Schema.optional(Schema.NullOr(PtyInfo)), + pipe: Schema.optional(Schema.NullOr(PipeInfo)), }) -export type WireTerminal = typeof WireTerminal.Type +export type ProcessInfo = typeof ProcessInfo.Type + +export type Signal = "SIGTERM" | "SIGINT" | "SIGKILL" const Registration = Schema.Struct({ instance_id: Schema.String, @@ -38,7 +61,7 @@ const Registration = Schema.Struct({ protocol: Schema.Number, socket: Schema.String, token: Schema.String, - features: Schema.optional(Schema.Array(Schema.String)), + features: Schema.Array(Schema.String), }) type Registration = typeof Registration.Type @@ -48,16 +71,17 @@ export const WireResponse = Schema.Union([ instance_id: Schema.String, pid: Schema.Number, protocol: Schema.Number, - features: Schema.optional(Schema.Array(Schema.String)), + features: Schema.Array(Schema.String), }), - Schema.Struct({ type: Schema.Literal("created"), terminal: WireTerminal }), - Schema.Struct({ type: Schema.Literal("terminals"), terminals: Schema.Array(WireTerminal) }), + Schema.Struct({ type: Schema.Literal("created"), process: ProcessInfo }), + Schema.Struct({ type: Schema.Literal("processes"), processes: Schema.Array(ProcessInfo) }), + Schema.Struct({ type: Schema.Literal("process"), process: ProcessInfo }), Schema.Struct({ type: Schema.Literal("ok") }), Schema.Struct({ type: Schema.Literal("owned") }), Schema.Struct({ type: Schema.Literal("handoff"), ticket: Schema.String, expires_at: Schema.Number }), Schema.Struct({ type: Schema.Literal("snapshot"), - terminal: WireTerminal, + process: ProcessInfo, text: Schema.String, checkpoint_base64: Schema.String, cursor_x: Schema.Number, @@ -65,14 +89,14 @@ export const WireResponse = Schema.Union([ }), Schema.Struct({ type: Schema.Literal("rows"), - terminal: WireTerminal, + process: ProcessInfo, lines: Schema.Array(Schema.String), cursor_x: Schema.Number, cursor_y: Schema.Number, }), Schema.Struct({ type: Schema.Literal("attached"), - terminal: WireTerminal, + process: ProcessInfo, role: Schema.Literals(["controller", "observer"]), generation: Schema.Number, requested_offset: Schema.Number, @@ -91,7 +115,12 @@ export const WireResponse = Schema.Union([ Schema.Struct({ type: Schema.Literal("exited"), exit_code: Schema.NullOr(Schema.Number), + signal: Schema.optional(Schema.NullOr(Schema.String)), final_offset: Schema.Number, + // Pipe processes only. + output_lines: Schema.optional(Schema.Number), + output_error: Schema.optional(Schema.NullOr(Schema.String)), + exited_at: Schema.Number, }), Schema.Struct({ type: Schema.Literal("controller_changed"), @@ -103,6 +132,7 @@ export const WireResponse = Schema.Union([ Schema.Struct({ type: Schema.Literal("error"), message: Schema.String }), ]) export type WireResponse = typeof WireResponse.Type +export type Exited = Extract export type Role = "controller" | "observer" @@ -130,7 +160,7 @@ export type SubscribeInput = { } export type DaemonAttachment = { - readonly terminal: WireTerminal + readonly terminal: ProcessInfo readonly role: Role readonly generation: number readonly replay: { @@ -144,8 +174,13 @@ export type DaemonAttachment = { readonly detach: () => void } +/** + * `connect`, `spawn`, `registration` and `protocol` mean the daemon could not be reached or started and the + * request never ran. `response` means the request was sent but its reply was lost. `request` is the daemon's + * own `error` reply. + */ export class DaemonError extends Data.TaggedError("PersistentPty.DaemonError")<{ - readonly kind: "connect" | "response" | "registration" | "protocol" | "spawn" + readonly kind: "connect" | "response" | "registration" | "protocol" | "spawn" | "request" readonly message: string readonly pid?: number }> {} @@ -153,13 +188,11 @@ export class DaemonError extends Data.TaggedError("PersistentPty.DaemonError")<{ export interface DaemonTransport { readonly request: (value: object, start?: boolean) => Effect.Effect readonly requestIfRunning: (value: object) => Effect.Effect - readonly supports: (feature: string) => Effect.Effect + /** Waits on its own connection, with no timeout, until the pipe process exits. Never starts the daemon. */ + readonly wait: (id: number) => Effect.Effect readonly shutdown: Effect.Effect readonly handoff: Effect.Effect - readonly subscribe: ( - id: number, - input: SubscribeInput, - ) => Effect.Effect + readonly subscribe: (id: number, input: SubscribeInput) => Effect.Effect } export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport")(function* ( @@ -174,6 +207,8 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" let registration: Registration | undefined let owner: Awaited> | undefined let closed = false + // Set once a daemon we spawned turned out to speak another protocol. + let incompatible: DaemonError | undefined yield* Effect.addFinalizer(() => Effect.sync(() => { closed = true @@ -229,8 +264,7 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" response.protocol !== ProtocolVersion ) return yield* Effect.fail(new DaemonError({ kind: "protocol", message: "opencode-pty registration mismatch" })) - // Daemons older than the features list omit it; trust the live daemon's answer over the file. - return { ...decoded, features: response.features ?? [] } + return decoded }) const spawnDaemon = Effect.fn("PersistentPty.daemon.spawn")(function* () { @@ -238,11 +272,11 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" const child = yield* Effect.tryPromise({ try: () => new Promise>((resolve, reject) => { - const child = spawn( - executable, - ["daemon", "--runtime-dir", root, "--name", id], - { detached: true, windowsHide: true, stdio: "ignore" }, - ) + const child = spawn(executable, ["daemon", "--runtime-dir", root, "--name", id], { + detached: true, + windowsHide: true, + stdio: "ignore", + }) child.once("spawn", () => { child.unref() resolve(child) @@ -258,6 +292,11 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" const found = yield* discover().pipe( Effect.map((value) => ({ value })), Effect.catch((error) => { + // The binary we just started speaks another protocol: waiting longer cannot help. + if (error.kind === "protocol" && error.pid !== undefined && error.pid === child.pid) { + incompatible = error + return Effect.fail(error) + } last = error return Effect.succeed(undefined) }), @@ -280,11 +319,13 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" ) }) - // An unreachable daemon can still hold this directory's lock, so retry once in a fresh directory. + // An unreachable daemon, or one left by another release, can still hold this directory's lock, so retry once in a + // fresh directory. A binary that cannot be spawned, or that speaks another protocol, fails the same way again. const start = () => spawnDaemon().pipe( Effect.catch((error) => Effect.gen(function* () { + if (error.kind === "spawn" || error === incompatible) return yield* Effect.fail(error) yield* Effect.logWarning("Starting opencode-pty in a fresh runtime directory", { cause: error.message }) abandon() return yield* spawnDaemon() @@ -307,7 +348,8 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" if (registration) return registration const found = yield* discover().pipe( Effect.catch((error) => { - if (shouldStart && error.kind === "connect") return start() + // Starting the same incompatible binary again would only fail again. + if (shouldStart && error.kind === "connect") return incompatible ? Effect.fail(incompatible) : start() return Effect.fail(error) }), ) @@ -351,9 +393,14 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" Effect.catch((error) => (error.kind === "connect" ? Effect.succeed(undefined) : Effect.fail(error))), ) - const supports = Effect.fn("PersistentPty.daemon.supports")(function* (feature: string) { - const current = yield* connect(false) - return current.features?.includes(feature) ?? false + const wait = Effect.fn("PersistentPty.daemon.wait")(function* (processID: number) { + const attempt = withRegistration(false, (current) => + Effect.tryPromise({ + try: (signal) => waitPromise(current, processID, signal), + catch: (cause) => (cause instanceof DaemonError ? cause : failure("connect", cause)), + }), + ) + return yield* attempt.pipe(Effect.catch((error) => (error.kind === "registration" ? attempt : Effect.fail(error)))) }) const shutdown = Effect.gen(function* () { @@ -398,10 +445,7 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" }), ) - const subscribe = Effect.fn("PersistentPty.daemon.subscribe")(function* ( - terminalID: number, - input: SubscribeInput, - ) { + const subscribe = Effect.fn("PersistentPty.daemon.subscribe")(function* (terminalID: number, input: SubscribeInput) { const attempt = withRegistration(false, (current) => Effect.tryPromise({ try: () => subscribePromise(current, terminalID, input), @@ -418,13 +462,10 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" yield* Effect.gen(function* () { if (inherited.expiresAt <= Date.now()) return yield* Effect.fail(new DaemonError({ kind: "registration", message: "PTY restart handoff expired" })) - // Older releases put the daemon directory outside the root; its directory name is still the ID. id = path.basename(inherited.directory) found = yield* discover(inherited.directory) if (found.instance_id !== inherited.instanceID) return yield* Effect.fail(new DaemonError({ kind: "registration", message: "PTY restart daemon changed" })) - if (path.resolve(inherited.directory) !== path.resolve(root, id)) - yield* migrate(inherited.directory, path.join(root, id)) yield* claim(found, inherited.ticket) }).pipe( Effect.catchCause((cause) => @@ -440,20 +481,7 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" ) } - return { request, requestIfRunning, supports, shutdown, handoff, subscribe } satisfies DaemonTransport -}) - -// Older releases kept registrations in purgeable temp directories. Renaming keeps the lock file's -// inode, so the still-running daemon keeps holding its lock in the new location. -const migrate = Effect.fn("PersistentPty.daemon.migrate")(function* (directory: string, target: string) { - yield* Effect.tryPromise({ - try: async () => { - await mkdir(target, { recursive: true, mode: 0o700 }) - await rename(path.join(directory, "service.lock"), path.join(target, "service.lock")) - await rename(path.join(directory, "service.json"), path.join(target, "service.json")) - }, - catch: (cause) => failure("registration", cause), - }) + return { request, requestIfRunning, wait, shutdown, handoff, subscribe } satisfies DaemonTransport }) async function openOwner(registration: Registration, ticket: string | undefined, signal: AbortSignal) { @@ -535,7 +563,7 @@ const oneShot = Effect.fn("PersistentPty.daemon.oneShot")(function* (registratio }) const response = yield* Effect.try({ try: () => decode(first), - catch: (cause) => failure("protocol", cause), + catch: (cause) => failure("response", cause), }) if (response.type === "error") return yield* Effect.fail(responseError(response.message)) return response @@ -632,7 +660,7 @@ async function subscribePromise( } let activated = false return { - terminal: response.terminal, + terminal: response.process, role: response.role, generation: response.generation, replay: { @@ -659,6 +687,29 @@ async function subscribePromise( } } +async function waitPromise(registration: Registration, id: number, signal: AbortSignal): Promise { + const socket = net.createConnection({ path: registration.socket }) + const abort = () => socket.destroy() + signal.addEventListener("abort", abort, { once: true }) + try { + await new Promise((resolve, reject) => { + socket.once("connect", resolve) + socket.once("error", reject) + }) + socket.write(encode({ token: registration.token, request: { op: "wait", id } })) + const frame = await decoder(socket).next() + // The connection was open, so a missing reply means the daemon went away while the process ran. + if (frame.done) throw failure("connect", "opencode-pty closed before the process exited") + const response = decode(frame.value) + if (response.type === "error") throw responseError(response.message) + if (response.type !== "exited") throw failure("response", `unexpected opencode-pty response: ${response.type}`) + return response + } finally { + signal.removeEventListener("abort", abort) + socket.destroy() + } +} + function encode(value: unknown) { const payload = Buffer.from(JSON.stringify(value)) if (payload.length > MaxFrameBytes) throw new Error("opencode-pty frame too large") @@ -691,7 +742,7 @@ function decode(payload: Uint8Array) { } function responseError(message: string) { - return new DaemonError({ kind: message === "authentication failed" ? "registration" : "protocol", message }) + return new DaemonError({ kind: message === "authentication failed" ? "registration" : "request", message }) } function failure(kind: DaemonError["kind"], cause: unknown) { diff --git a/packages/core/src/persistent-pty/index.ts b/packages/core/src/persistent-pty/index.ts index 1728996636db..32536b1451cc 100644 --- a/packages/core/src/persistent-pty/index.ts +++ b/packages/core/src/persistent-pty/index.ts @@ -16,21 +16,26 @@ import { import { Session } from "@opencode/schema/session" import { SessionEvent } from "@opencode/schema/session-event" import { Bus } from "../bus.js" +import { Detaching } from "../detaching.js" import { Pty } from "@opencode/schema/pty" import { Global } from "@opencode/util/global" import { ShellSelect } from "../shell/select.js" import { makeDaemonTransport, type DaemonAttachment, + type DaemonError, type DaemonTransport, + type Exited, + type Metadata, + type ProcessInfo, type Role, + type Signal, type SubscribeInput, type WireResponse, - type WireTerminal, } from "./daemon.js" import { resolveBinary } from "#persistent-pty-binary" -export type { Role, StreamEvent } from "./daemon.js" +export type { Metadata, Role, Signal, StreamEvent } from "./daemon.js" export { Handoff } from "@opencode/schema/persistent-pty" export { available } from "#persistent-pty-binary" @@ -54,22 +59,73 @@ export type Attachment = { readonly detach: () => void } +/** The daemon could not be reached or started; the request never ran. */ export class UnavailableError extends Schema.TaggedError()("PersistentPty.UnavailableError", { message: Schema.String, }) {} +/** The daemon rejected the request, or its reply was lost after it was sent, so it may have run. */ +export class RequestError extends Schema.TaggedError()("PersistentPty.RequestError", { + message: Schema.String, +}) {} + export class NotFoundError extends Schema.TaggedError()("PersistentPty.NotFoundError", { ptyID: Pty.ID, }) {} +export class ProcessNotFoundError extends Schema.TaggedError()( + "PersistentPty.ProcessNotFoundError", + { processID: Schema.Number }, +) {} + export class InvalidCwdError extends Schema.TaggedError()("PersistentPty.InvalidCwdError", { cwd: Schema.String, message: Schema.String, }) {} -export class UnsupportedError extends Schema.TaggedError()("PersistentPty.UnsupportedError", { - feature: Schema.String, -}) {} +/** A non-PTY process run by the daemon, its output written to `outputPath`. */ +export type PipeInfo = { + readonly id: number + readonly pid: number | undefined + readonly command: readonly string[] + readonly cwd: string + readonly groupID: string + readonly lifecycle: + | { readonly status: "running" } + | { readonly status: "exited"; readonly exitCode: number | undefined; readonly signal: string | undefined } + | { readonly status: "failed"; readonly message: string } + /** Bytes counted so far; the file can hold a little more. Pair only bytes up to here with `outputLines`. */ + readonly outputTail: number + readonly outputLines: number + readonly outputPath: string + readonly outputError: string | undefined + readonly metadata: Metadata + readonly startedAt: number + readonly exitedAt: number | undefined +} + +export type PipeExit = { + readonly exitCode: number | undefined + readonly signal: string | undefined + /** Runs past the file's size when `outputError` is set; use the file's real size instead. */ + readonly finalOffset: number + readonly outputLines: number + readonly outputError: string | undefined + readonly exitedAt: number +} + +export type CreatePipeInput = { + readonly program: string + readonly args: readonly string[] + /** Absolute. */ + readonly cwd: string + /** The child's whole environment; the daemon's own is not inherited. */ + readonly env: Readonly> + readonly groupID: string + /** Absolute; truncated on open, symlinks refused. */ + readonly outputPath: string + readonly metadata: Metadata +} export interface Interface { readonly list: (sessionID?: Session.ID) => Effect.Effect @@ -110,19 +166,27 @@ export interface Interface { rows: number, data: Uint8Array, ) => Effect.Effect - readonly rename: ( - id: Pty.ID, - title: string | null, - ) => Effect.Effect + readonly rename: (id: Pty.ID, title: string | null) => Effect.Effect readonly snapshot: (id: Pty.ID) => Effect.Effect readonly read: (sessionID: Session.ID, lines?: number) => Effect.Effect readonly remove: (id: Pty.ID) => Effect.Effect readonly shutdown: () => Effect.Effect readonly handoff: () => Effect.Effect - readonly attach: ( - id: Pty.ID, - input: SubscribeInput, - ) => Effect.Effect + readonly attach: (id: Pty.ID, input: SubscribeInput) => Effect.Effect + /** Starts the daemon if needed. Fall back to another spawner only on `UnavailableError`. */ + readonly createPipe: (input: CreatePipeInput) => Effect.Effect + /** Resolves when the process exits, on its own connection with no timeout; interrupting closes it. */ + readonly waitPipe: (id: number) => Effect.Effect + /** `ProcessNotFoundError` for an unknown id, a PTY, or when no daemon is running. */ + readonly getPipe: (id: number) => Effect.Effect + /** Signals the process group without forgetting the process; `waitPipe` then reports the signal. */ + readonly signalPipe: (id: number, signal: Signal) => Effect.Effect + /** Replaces the stored metadata. */ + readonly setPipeMetadata: (id: number, metadata: Metadata) => Effect.Effect + /** Kills the process if it still runs and forgets it; the output file is left alone. */ + readonly terminatePipe: (id: number) => Effect.Effect + /** Never starts the daemon; empty when none is running. */ + readonly listPipes: () => Effect.Effect } export class Service extends Context.Service()("@opencode/PersistentPty") {} @@ -133,6 +197,7 @@ const makeLayer = (options: Options = {}) => Effect.gen(function* () { const bus = yield* Bus.Service const global = yield* Global.Service + const detaching = yield* Detaching.Service const context = yield* Effect.context() const runFork = Effect.runForkWith(context) let binary: Promise | undefined @@ -151,18 +216,18 @@ const makeLayer = (options: Options = {}) => const current = new Map() const list = Effect.fn("PersistentPty.list")(function* (sessionID?: Session.ID) { - const response = yield* optionalRequest(daemon, { op: "list" }) + const response = yield* optionalRequest(daemon, { op: "list", mode: "pty" }) if (!response) return [] - if (response.type !== "terminals") return yield* unexpected(response) - return response.terminals + if (response.type !== "processes") return yield* unexpected(response) + return response.processes .map(toInfo) .filter((terminal) => sessionID === undefined || terminal.sessionID === sessionID) }) const get = Effect.fn("PersistentPty.get")(function* (id: Pty.ID) { - const found = (yield* list()).find((terminal) => terminal.id === id) - if (!found) return yield* new NotFoundError({ ptyID: id }) - return found + const found = yield* lookup(daemon, fromID(id)).pipe(Effect.mapError(unavailable)) + if (found?.mode !== "pty") return yield* new NotFoundError({ ptyID: id }) + return toInfo(found) }) const create = Effect.fn("PersistentPty.create")(function* ( @@ -181,7 +246,7 @@ const makeLayer = (options: Options = {}) => const response = yield* request( daemon, { - op: "create", + op: "create_pty", program: input.command ?? ShellSelect.environment(global.bin), args: input.args, cwd, @@ -194,7 +259,7 @@ const makeLayer = (options: Options = {}) => true, ) if (response.type !== "created") return yield* unexpected(response) - const terminal = toInfo(response.terminal) + const terminal = toInfo(response.process) yield* bus.publish(Added, { sessionID, terminal }) return terminal }) @@ -217,8 +282,9 @@ const makeLayer = (options: Options = {}) => ), ) - const resize = Effect.fn("PersistentPty.resize")((id: Pty.ID, cols: number, rows: number, attachmentID?: string) => - command(id, "resize", { attachment_id: attachmentID ?? null, cols, rows }, true), + const resize = Effect.fn("PersistentPty.resize")( + (id: Pty.ID, cols: number, rows: number, attachmentID?: string) => + command(id, "resize", { attachment_id: attachmentID ?? null, cols, rows }, true), ) const control = Effect.fn("PersistentPty.control")( @@ -238,9 +304,6 @@ const makeLayer = (options: Options = {}) => const rename = Effect.fn("PersistentPty.rename")(function* (id: Pty.ID, title: string | null) { const terminal = yield* get(id) - // Daemons without the feature drop the connection on unknown requests. - if (!(yield* daemon.supports("rename").pipe(Effect.mapError(unavailable)))) - return yield* new UnsupportedError({ feature: "rename" }) const response = yield* request(daemon, { op: "rename", id: fromID(id), title }) if (response.type !== "ok") return yield* unexpected(response) const renamed = yield* get(id) @@ -253,7 +316,7 @@ const makeLayer = (options: Options = {}) => const response = yield* request(daemon, { op: "snapshot", id: fromID(id) }) if (response.type !== "snapshot") return yield* unexpected(response) return { - info: toInfo(response.terminal), + info: toInfo(response.process), text: response.text, checkpoint: Buffer.from(response.checkpoint_base64, "base64"), cursor: { x: response.cursor_x, y: response.cursor_y }, @@ -273,7 +336,7 @@ const makeLayer = (options: Options = {}) => // Let the daemon choose the live height in the same snapshot when lines is omitted. const response = yield* request(daemon, { op: "read_rows", id: fromID(id), rows: lines }) if (response.type !== "rows") return yield* unexpected(response) - const info = toInfo(response.terminal) + const info = toInfo(response.process) return { ptyID: info.id, title: info.title, @@ -300,11 +363,17 @@ const makeLayer = (options: Options = {}) => const shutdown = Effect.fn("PersistentPty.shutdown")(function* () { const response = yield* daemon.shutdown.pipe(Effect.mapError(unavailable)) current.clear() + yield* detaching.handoffDropped if (!response) return if (response.type !== "ok") return yield* unexpected(response) }) - const handoff = Effect.fn("PersistentPty.handoff")(() => daemon.handoff.pipe(Effect.mapError(unavailable))) + const handoff = Effect.fn("PersistentPty.handoff")(() => + daemon.handoff.pipe( + Effect.tap((prepared) => (prepared ? detaching.handoffPrepared : Effect.void)), + Effect.mapError(unavailable), + ), + ) const removeVisibleExit = (id: Pty.ID) => { if (removing.has(id)) return @@ -367,9 +436,10 @@ const makeLayer = (options: Options = {}) => }, }) .pipe(Effect.mapError(unavailable)) - if (attachment.role === "controller") current.set(attachment.terminal.group_id, id) + const info = toInfo(attachment.terminal) + if (attachment.role === "controller") current.set(info.sessionID, id) return { - info: toInfo(attachment.terminal), + info, role: attachment.role, generation: attachment.generation, replay: attachment.replay, @@ -378,6 +448,61 @@ const makeLayer = (options: Options = {}) => } }) + const createPipe = Effect.fn("PersistentPty.createPipe")(function* (input: CreatePipeInput) { + const response = yield* daemon + .request( + { + op: "create_pipe", + program: input.program, + args: input.args, + cwd: input.cwd, + env: input.env, + group_id: input.groupID, + output_path: input.outputPath, + metadata: input.metadata, + }, + true, + ) + .pipe(Effect.mapError(pipeError)) + if (response.type !== "created") return yield* unexpectedPipe(response) + return toPipeInfo(response.process) + }) + + const waitPipe = Effect.fn("PersistentPty.waitPipe")(function* (id: number) { + return toPipeExit(yield* daemon.wait(id).pipe(Effect.mapError(pipeError))) + }) + + const getPipe = Effect.fn("PersistentPty.getPipe")(function* (id: number) { + const found = yield* lookup(daemon, id).pipe(Effect.mapError(unavailable)) + if (found?.mode !== "pipe") return yield* new ProcessNotFoundError({ processID: id }) + return toPipeInfo(found) + }) + + const pipeCommand = (value: object) => + Effect.gen(function* () { + const response = yield* daemon.request(value).pipe(Effect.mapError(pipeError)) + if (response.type !== "ok") return yield* unexpectedPipe(response) + }) + + const signalPipe = Effect.fn("PersistentPty.signalPipe")((id: number, signal: Signal) => + pipeCommand({ op: "signal", id, signal }), + ) + + const setPipeMetadata = Effect.fn("PersistentPty.setPipeMetadata")((id: number, metadata: Metadata) => + pipeCommand({ op: "set_metadata", id, metadata }), + ) + + const terminatePipe = Effect.fn("PersistentPty.terminatePipe")((id: number) => + pipeCommand({ op: "terminate", id }), + ) + + const listPipes = Effect.fn("PersistentPty.listPipes")(function* () { + const response = yield* daemon.requestIfRunning({ op: "list", mode: "pipe" }).pipe(Effect.mapError(pipeError)) + if (!response) return [] + if (response.type !== "processes") return yield* unexpectedPipe(response) + return response.processes.map(toPipeInfo) + }) + return Service.of({ list, get, @@ -393,16 +518,24 @@ const makeLayer = (options: Options = {}) => shutdown, handoff, attach, + createPipe, + waitPipe, + getPipe, + signalPipe, + setPipeMetadata, + terminatePipe, + listPipes, }) }), ) export const layer = makeLayer() +export const layerWith = (options: Options) => makeLayer(options) export const configured = (options?: Options) => makeGlobalNode({ service: Service, layer: options === undefined ? layer : makeLayer(options), - deps: [Bus.node, Global.node], + deps: [Bus.node, Global.node, Detaching.node], }) export const node = configured() @@ -412,33 +545,96 @@ const request = (daemon: DaemonTransport, value: object, start = false) => const optionalRequest = (daemon: DaemonTransport, value: object) => daemon.requestIfRunning(value).pipe(Effect.mapError(unavailable)) +// `get` fails only for an unknown id, and no running daemon has no processes. +const lookup = (daemon: DaemonTransport, id: number) => + daemon.requestIfRunning({ op: "get", id }).pipe( + Effect.catchIf( + (error) => error.kind === "request", + () => Effect.succeed(undefined), + ), + Effect.flatMap((response) => { + if (!response) return Effect.succeed(undefined) + if (response.type !== "process") + return Effect.fail(new Error(`unexpected opencode-pty response: ${response.type}`)) + return Effect.succeed(response.process) + }), + ) + const unexpected = (response: WireResponse) => Effect.fail(new UnavailableError({ message: `unexpected opencode-pty response: ${response.type}` })) const unavailable = (error: unknown) => new UnavailableError({ message: error instanceof Error ? error.message : String(error) }) -function toInfo(value: WireTerminal): Info { +// Only a failure before the request was sent is safe to retry somewhere else. +const pipeError = (error: DaemonError) => + error.kind === "request" || error.kind === "response" + ? new RequestError({ message: error.message }) + : new UnavailableError({ message: error.message }) + +const unexpectedPipe = (response: WireResponse) => + Effect.fail(new RequestError({ message: `unexpected opencode-pty response: ${response.type}` })) + +function toInfo(value: ProcessInfo): Info { + if (!value.pty) throw new Error(`opencode-pty process ${value.id} is not a terminal`) const status = value.lifecycle.status return { ...Pty.Info.make({ id: toID(value.id), - title: value.title, + title: value.pty.title, command: value.command[0] || "", args: value.command.slice(1), cwd: value.cwd, status: status === "running" ? "running" : "exited", pid: value.pid ?? 0, - ...(status === "exited" ? { exitCode: value.lifecycle.exit_code ?? undefined } : {}), + ...(value.lifecycle.status === "exited" ? { exitCode: value.lifecycle.exit_code ?? undefined } : {}), }), - sessionID: value.group_id, - userTitle: value.user_title ?? null, - foregroundProcess: value.foreground_process, - size: { cols: value.cols, rows: value.rows }, + // Terminals are always created with a session ID as their group. + sessionID: Session.ID.make(value.group_id), + userTitle: value.pty.user_title, + foregroundProcess: value.pty.foreground_process, + size: { cols: value.pty.cols, rows: value.pty.rows }, output: { head: value.output_head, tail: value.output_tail }, } } +function toPipeInfo(value: ProcessInfo): PipeInfo { + if (!value.pipe) throw new Error(`opencode-pty process ${value.id} is not a pipe process`) + return { + id: value.id, + pid: value.pid ?? undefined, + command: value.command, + cwd: value.cwd, + groupID: value.group_id, + lifecycle: + value.lifecycle.status === "exited" + ? { + status: "exited", + exitCode: value.lifecycle.exit_code ?? undefined, + signal: value.lifecycle.signal ?? undefined, + } + : value.lifecycle, + outputTail: value.output_tail, + outputLines: value.pipe.output_lines, + outputPath: value.pipe.output_path, + outputError: value.pipe.output_error ?? undefined, + metadata: value.pipe.metadata, + startedAt: value.started_at, + exitedAt: value.exited_at ?? undefined, + } +} + +function toPipeExit(value: Exited): PipeExit { + return { + exitCode: value.exit_code ?? undefined, + signal: value.signal ?? undefined, + finalOffset: value.final_offset, + outputLines: value.output_lines ?? 0, + outputError: value.output_error ?? undefined, + exitedAt: value.exited_at, + } +} + function toID(value: number) { return Pty.ID.make(`pty_persistent_${value}`) } diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index a5fed55d5344..a77e460c529a 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -53,6 +53,8 @@ import { Global } from "@opencode/util/global" import { Image } from "../image.js" import { InstructionDiscovery } from "../instruction-discovery.js" import { Integration } from "../integration.js" +import { Detaching } from "../detaching.js" +import { LocationLifecycle } from "../location-lifecycle.js" import { Job } from "../job.js" import { KV } from "../kv.js" import { Location } from "../location.js" @@ -146,6 +148,7 @@ const services = [ Image.Service, InstructionDiscovery.Service, Integration.Service, + Detaching.Service, Job.Service, KV.Service, LLMClient.Service, @@ -166,6 +169,7 @@ const services = [ SubagentJob.Service, SharedCache.Service, Shell.Service, + Shell.Daemon, ShellSelect.Service, Snapshot.Service, Skill.Service, @@ -207,7 +211,10 @@ export const requirements = LayerNode.group([ Image.node, InstructionDiscovery.node, Integration.node, + Detaching.node, Job.node, + LocationLifecycle.node, + Shell.daemonNode, KV.node, llmClient, Location.node, diff --git a/packages/core/src/schedule.ts b/packages/core/src/schedule.ts index 16dad0ac5ba0..8182768a02d1 100644 --- a/packages/core/src/schedule.ts +++ b/packages/core/src/schedule.ts @@ -2,7 +2,20 @@ export * as Schedule from "./schedule.js" import { randomUUID } from "node:crypto" import { and, asc, eq, gt, isNull, lte, min, or, sql } from "drizzle-orm" -import { Cause, Clock, Context, Cron, DateTime, Effect, Latch, Layer, Option, Result, Schedule as EffectSchedule, Schema } from "effect" +import { + Cause, + Clock, + Context, + Cron, + DateTime, + Effect, + Latch, + Layer, + Option, + Result, + Schedule as EffectSchedule, + Schema, +} from "effect" import { Node, makeGlobalNode } from "@opencode/util/effect/app-node" import { LayerNode } from "@opencode/util/effect/layer-node" import { @@ -317,15 +330,10 @@ const layer = Layer.effect( const note = outcome.invalid ? `[Schedule paused after invalid check: ${outcome.error}]` : `[Schedule paused after ${strikes} failed checks: ${outcome.error}]` - const id = yield* admitWake( - row, - session.id, - run, - due, - late, - `${row.action.prompt}\n\n${note}`, - { error: outcome.error, failures: strikes }, - ) + const id = yield* admitWake(row, session.id, run, due, late, `${row.action.prompt}\n\n${note}`, { + error: outcome.error, + failures: strikes, + }) yield* db .update(ScheduleTable) .set({ @@ -624,12 +632,7 @@ const layer = Layer.effect( }) } - const firstDue = Effect.fnUntraced(function* ( - when: When, - hasCheck: boolean, - zone: DateTime.TimeZone, - now: number, - ) { + const firstDue = Effect.fnUntraced(function* (when: When, hasCheck: boolean, zone: DateTime.TimeZone, now: number) { switch (when.type) { case "in": return now + (yield* duration(when.in)) @@ -688,8 +691,7 @@ const layer = Layer.effect( if (!config.enabled) return yield* new DisabledError() const check = yield* resolveStoredCheck(input) const when = input.when ?? (check?.mode === "until" ? DEFAULT_UNTIL_EVERY : undefined) - if (when === undefined) - return yield* new InvalidError({ reason: "when is required unless until is set" }) + if (when === undefined) return yield* new InvalidError({ reason: "when is required unless until is set" }) const zone = yield* resolveZone(when.tz ?? config.timezone) const now = yield* Clock.currentTimeMillis const due = yield* firstDue(when, check !== undefined, zone, now) @@ -724,7 +726,8 @@ const layer = Layer.effect( ) if (!init.ok) { // A code check that throws or fails at creation is broken (e.g. wrong MCP response field); fail right away. - if (init.invalid || check.condition.type === "code") return yield* new InvalidError({ reason: init.error }) + if (init.invalid || check.condition.type === "code") + return yield* new InvalidError({ reason: init.error }) return UNSEEDED } if (check.mode === "until" && init.wake) diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index 3bf4bae98888..a7557fde90f9 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -1,12 +1,21 @@ export * as SessionRestart from "./restart.js" -import { Context, Effect, Layer } from "effect" +import { Context, Effect, Layer, Option, Schema } from "effect" +import { isReadonlyArrayNonEmpty } from "effect/Array" import { makeGlobalNode } from "@opencode/util/effect/app-node" +import { Shell as ShellSchema } from "@opencode/schema/shell" import { Bus } from "../../bus.js" +import { Database } from "../../database/database.js" +import { Detaching } from "../../detaching.js" +import { Instance } from "../../instance/service.js" import { Job } from "../../job.js" import { Session } from "../../session.js" +import { Shell } from "../../shell.js" +import { ShellTool } from "../../tool/plugin/shell.js" import { SessionEvent } from "../event.js" import { SessionExecution } from "../execution.js" +import { SessionHistory } from "../history.js" +import { SessionMessage } from "../message.js" import { SessionSchema } from "../schema.js" import { SessionStore } from "../store.js" import { ImageNotice } from "../../image/notice.js" @@ -72,6 +81,10 @@ export const layer = (options?: Options) => const bus = yield* Bus.Service const jobs = yield* Job.Service const sessions = yield* Session.Service + const instances = yield* Instance.Service + const db = (yield* Database.Service).db + // Optional so restart built outside the application graph (tests) need not provide it. + const detaching = Option.getOrUndefined(yield* Effect.serviceOption(Detaching.Service)) const scope = yield* Effect.scope const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS @@ -99,10 +112,7 @@ export const layer = (options?: Options) => return true }) - const restartMessage = ( - background: Job.Background, - recovery: Exclude, - ) => { + const restartMessage = (background: Job.Background, recovery: Exclude) => { const state = background.status === "running" ? "cancelled" : background.status if (recovery.kind === "shell") { const text = @@ -164,6 +174,102 @@ export const layer = (options?: Options) => yield* jobs.completeBackground(background.notificationID) }) + // The Shell of a session's Location, booted so it takes back the commands the daemon still runs for it. + const shellOf = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) { + const session = yield* store.get(sessionID) + if (!session) return undefined + return yield* Effect.serviceOption(Shell.Service).pipe( + instances.provide(session), + Effect.map(Option.getOrUndefined), + // The session's directory is gone: its commands are reported like any lost background work. + Effect.catchTag("FileSystem.DirectoryNotFoundError", () => Effect.succeed(undefined)), + ) + }) + + const reattach = ( + shell: Shell.Interface, + sessionID: SessionSchema.ID, + info: ShellSchema.Info, + suspended: ReadonlySet, + notificationID?: SessionMessage.ID, + ) => + ShellTool.reattach({ + shell, + jobs, + sessions, + scope, + sessionID, + info, + notificationID, + resume: () => (suspended.has(sessionID) ? false : undefined), + leaveRunning: () => detaching?.isDetaching() === true && shell.detachable(info.id), + }) + + // A shell job the previous server left running is watched again if its command is still known (running or + // finished during the restart); otherwise it is reported as lost. + const recoverShellJob = Effect.fnUntraced(function* ( + background: Job.Background, + recovery: Extract, + suspended: ReadonlySet, + reattached: Set, + ) { + const shell = background.status === "running" ? yield* shellOf(recovery.sessionID) : undefined + const info = + shell && Schema.is(ShellSchema.ID)(recovery.shellID) + ? yield* shell.get(recovery.shellID).pipe(Effect.option) + : Option.none() + if (!shell || Option.isNone(info)) return yield* recoverToolJob(background, recovery) + reattached.add(info.value.id) + yield* reattach(shell, recovery.sessionID, info.value, suspended, background.notificationID) + }) + + // A server killed while shutting down may leave a foreground shell call unsettled, its command still running + // in the daemon. Settle the call as moved to the background and watch the command, as the old server would have. + const settleBackgroundedCalls = Effect.fnUntraced(function* ( + sessionID: SessionSchema.ID, + suspended: ReadonlySet, + reattached: ReadonlySet, + ) { + const calls = (yield* SessionHistory.incompleteAssistants(db, sessionID)).flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => + part.type === "tool" && part.name === ShellTool.name && part.state.status === "running" + ? [{ assistantMessageID: message.id, part }] + : [], + ) + : [], + ) + if (calls.length === 0) return + const shell = yield* shellOf(sessionID) + if (!shell) return + yield* Effect.forEach( + calls, + Effect.fnUntraced(function* (call) { + const info = yield* shell.byToolCall({ + sessionID, + assistantMessageID: call.assistantMessageID, + callID: call.part.id, + }) + if (!info) return + const result = ShellTool.backgroundToolResult(info.id, info.file) + if (!isReadonlyArrayNonEmpty(result.content)) return + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID: call.assistantMessageID, + id: call.part.id, + content: result.content, + metadata: { shellID: info.id, status: "running", truncated: false }, + executed: call.part.executed === true, + }) + yield* shell.timeout(info.id, 0).pipe(Effect.ignore) + // Killed after backgrounding the job but before settling the call: the job was already reattached. + if (reattached.has(info.id) || (yield* jobs.get(info.id))) return + yield* reattach(shell, sessionID, info, suspended) + }), + { discard: true }, + ) + }) + const recoverSubagent = Effect.fnUntraced(function* ( background: Job.Background, recovery: Extract, @@ -232,13 +338,37 @@ export const layer = (options?: Options) => [...(yield* store.listSuspended()), ...children].filter((sessionID) => !active.has(sessionID)), ) yield* store.releaseChildClaims(children) + const recoverable = Effect.fnUntraced(function* (background: Job.Background) { + return (yield* jobs.get(background.id))?.status !== "running" + }) + // Shell and tool outcomes first, then calls a killed server left unsettled, and only then recovered children, + // whose first resume would otherwise fail their own unsettled shell calls. + const reattached = new Set() + yield* Effect.forEach( + pending, + Effect.fnUntraced(function* (background) { + const recovery = background.recovery + if (recovery.kind === "subagent" || !(yield* recoverable(background))) return + if (recovery.kind === "shell") return yield* recoverShellJob(background, recovery, suspended, reattached) + yield* recoverToolJob(background, recovery) + }), + { discard: true }, + ) + yield* Effect.forEach( + suspended, + (sessionID) => + settleBackgroundedCalls(sessionID, suspended, reattached).pipe( + Effect.catch((error) => + Effect.logWarning("failed to settle backgrounded shell calls", { sessionID, error }), + ), + ), + { discard: true }, + ) yield* Effect.forEach( - // Admit shell outcomes before a recovered child can start its first model request. - pending.toSorted((a, b) => Number(a.recovery.kind === "subagent") - Number(b.recovery.kind === "subagent")), + pending, Effect.fnUntraced(function* (background) { - if ((yield* jobs.get(background.id))?.status === "running") return const recovery = background.recovery - if (recovery.kind !== "subagent") return yield* recoverToolJob(background, recovery) + if (recovery.kind !== "subagent" || !(yield* recoverable(background))) return yield* recoverSubagent(background, recovery, suspended) }), { discard: true }, @@ -264,5 +394,14 @@ export const layer = (options?: Options) => export const node = makeGlobalNode({ service: Service, layer: layer(), - deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node], + deps: [ + SessionStore.node, + SessionExecution.node, + Bus.node, + Job.node, + Session.node, + Instance.node, + Database.node, + Detaching.node, + ], }) diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index 9243c6702b58..bfc101132983 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -15,6 +15,7 @@ import { Permission } from "../../permission.js" import { Snapshot } from "../../snapshot.js" import { Tool } from "../../tool.js" import { ToolOutput } from "../../tool-output.js" +import { ToolInterrupted } from "../../tool/interrupted.js" import { QuestionTool } from "../../tool/plugin/question.js" import { StepFailedError } from "../error.js" import { SessionEvent } from "../event.js" @@ -117,12 +118,23 @@ export const make = Effect.gen(function* () { if (firstOutputAt === undefined && publisher.record().outputStarted) firstOutputAt = yield* Clock.currentTimeMillis if (event.type !== "tool-call" || event.providerExecuted) return + const interrupted: ToolInterrupted.Slot = {} toolRuns.push({ call: event, fiber: yield* Effect.uninterruptibleMask((restore) => - restore(executeTool(event)).pipe( + restore(executeTool(event).pipe(Effect.provideService(ToolInterrupted.Slot, interrupted))).pipe( Effect.flatMap(toolOutput.truncate), Effect.flatMap((outcome) => publisher.toolExecution(event.id, event.name, outcome)), + // A tool can settle its call even when interrupted; the fiber still exits interrupted, so the + // step ends as interrupted instead of starting another request. + Effect.catchCause((cause) => { + const result = interrupted.result + if (!result || !Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause) + return toolOutput.truncate(result).pipe( + Effect.flatMap((outcome) => publisher.toolExecution(event.id, event.name, outcome)), + Effect.andThen(Effect.failCause(cause)), + ) + }), Effect.catchTag("Tool.Error", (error) => publisher.failTool(event.id, toSessionError(error), error.metadata).pipe(Effect.asVoid), ), diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index e7cfea7b5176..2844a7f05a67 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -1,7 +1,20 @@ export * as Shell from "./shell.js" import path from "path" -import { Context, Deferred, Duration, Effect, Fiber, Latch, Layer, Schema, Schedule, Stream } from "effect" +import { + Context, + Deferred, + Duration, + Effect, + Exit, + Fiber, + Latch, + Layer, + Option, + Schema, + Schedule, + Stream, +} from "effect" import { ChildProcess } from "effect/process" import { produce } from "immer" import { Shell } from "@opencode/schema/shell" @@ -14,6 +27,7 @@ import { Environment } from "./environment/index.js" import { FileRetention } from "./file-retention.js" import { Location } from "./location.js" import { Global } from "@opencode/util/global" +import { PersistentPty } from "./persistent-pty.js" import { ShellSelect } from "./shell/select.js" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" import { PluginHooks } from "./plugin/hooks.js" @@ -32,16 +46,48 @@ export class NotFoundError extends Schema.TaggedError()("Shell.No const EXITED_LIMIT = 25 export const RETENTION = Duration.days(7) export const DIRECTORY = "shell" +const STOP_GRACE = Duration.seconds(3) +const LOST = "Command was killed: the process daemon stopped" type Info = Shell.Info +export type StopReason = "timeout" | "killed" + +/** The tool call that started a command, so a server that took over can settle it. */ +export const ToolCall = Schema.Struct({ + sessionID: Schema.String, + assistantMessageID: Schema.String, + callID: Schema.String, +}) +export type ToolCall = typeof ToolCall.Type + type CreateInput = Shell.CreateInput & { shell?: string + toolCall?: ToolCall + // A call is waiting on it. A server that takes it over after a restart moves it to the background instead. + foreground?: boolean } -type Active = { +// What the server needs to take a daemon-run command back after a restart, kept in the daemon's metadata. +const Stored = Schema.Struct({ + id: Shell.ID, + command: Schema.String, + shell: Schema.String, + location: Schema.Struct({ directory: Schema.String, workspaceID: Schema.optionalKey(Schema.String) }), + metadata: Shell.Metadata, + toolCall: Schema.optional(ToolCall), + timeoutDeadline: Schema.optional(Schema.Number), + stopReason: Schema.optional(Schema.Literals(["timeout", "killed"])), + foreground: Schema.optional(Schema.Boolean), +}) +type Stored = typeof Stored.Type +const decodeStored = Schema.decodeUnknownOption(Stored) + +// A command the server spawned itself. It dies with its Location. +type Spawned = { + kind: "spawned" // Immutable snapshot; lifecycle updates replace it via immer `produce`. info: Info - file: string + // Bytes and newlines written so far. size: number newlines: number // Resolves with the terminal Info once the command exits, times out, or is killed. A wait @@ -49,10 +95,35 @@ type Active = { done: Deferred.Deferred timeoutFiber?: Fiber.Fiber timeout?: (duration: number) => Effect.Effect + stop?: (reason: StopReason) => Effect.Effect } +// A command the process daemon runs, which writes and counts its output file. It belongs to the server, not to +// the Location that started it, so it outlives a Location that closes or reloads. +type Run = { + kind: "daemon" + info: Info + process: number + toolCall?: ToolCall + timeoutDeadline?: number + stopReason?: StopReason + // A call waits on it, until it moves to the background (its timeout cleared). + foreground: boolean + // From the exit: newlines in the file, and why the daemon stopped writing it early. + outputLines?: number + outputError?: string + // Why it was killed without a signal of its own. + killed?: string + done: Deferred.Deferred + timeoutFiber?: Fiber.Fiber + stopping?: Fiber.Fiber + wait?: Fiber.Fiber +} + +type Command = Spawned | Run + type Ended = { - status: "exited" | "timeout" + status: "exited" | StopReason exit?: number signal?: string completed: number @@ -61,10 +132,13 @@ type Ended = { /** * Location-owned non-interactive shell command process service. * - * Each `create` spawns one shell command, captures combined stdout/stderr to a + * Each `create` starts one shell command, captures combined stdout/stderr to a * file, and returns an ID. Clients poll `get` for status and `output` for * file-backed output by cursor. No session, message, or permission state lives * here; callers (e.g. `ShellTool`) own that association and store the shell ID. + * + * Local commands run in the process daemon when it is available, so they survive a planned server + * restart; this service takes them back when it starts. Elsewhere they are spawned by the server. */ export interface Interface { readonly create: ( @@ -78,10 +152,22 @@ export interface Interface { // NotFoundError if the command is unknown or is removed before it terminates. readonly wait: (id: Shell.ID) => Effect.Effect // A known shell's terminal state and bounded tail. Missing capture remains distinct from its exit status. - readonly result: (started: Shell.Info) => Effect.Effect + // With `wait`, gives up waiting for the exit after that long and returns the output so far, still running. + readonly result: ( + started: Shell.Info, + options?: { readonly wait?: Duration.Input }, + ) => Effect.Effect // Replaces the running command's timeout from now; zero clears it. Ignored once the command has // ended or timed out, even while its process is still being killed or its output captured. readonly timeout: (id: Shell.ID, duration: number) => Effect.Effect + // Stops a running command gracefully (SIGTERM, then SIGKILL after 3 s) without forgetting it: it ends + // with the given status and stays observable, file included, until retention evicts it. Returns at + // once; `wait` observes the exit. Ignored once the command has ended or is already stopping. + readonly stop: (id: Shell.ID, reason: StopReason) => Effect.Effect + // Whether the command's process can outlive this server, so a planned shutdown leaves it running. + readonly detachable: (id: Shell.ID) => boolean + // The command a tool call started, while it is still known here (including one taken back after a restart). + readonly byToolCall: (call: ToolCall) => Effect.Effect readonly output: (id: Shell.ID, input?: Shell.OutputInput) => Effect.Effect readonly remove: (id: Shell.ID) => Effect.Effect } @@ -91,6 +177,7 @@ export class Service extends Context.Service()("@opencode/Sh export const cleanup = Effect.fn("Shell.cleanup")(function* () { const fs = yield* FSUtil.Service const global = yield* Global.Service + const pty = yield* PersistentPty.Service const directory = path.join(global.data, DIRECTORY) const projects = yield* fs.readDirectoryEntries(directory).pipe( Effect.map((entries) => entries.filter((entry) => entry.type === "Directory")), @@ -111,7 +198,32 @@ export const cleanup = Effect.fn("Shell.cleanup")(function* () { ), { concurrency: 8 }, ) - yield* FileRetention.cleanup(fs, files.flat(), RETENTION) + // Never starts the daemon. A command it still runs keeps its file; a finished one is forgotten before deletion. + const processes = new Map( + (yield* pty.listPipes().pipe(Effect.orElseSucceed(() => []))).map((process) => [process.outputPath, process]), + ) + yield* FileRetention.cleanup( + fs, + files.flat().filter((file) => processes.get(file)?.lifecycle.status !== "running"), + RETENTION, + (file) => { + const process = processes.get(file) + return process ? pty.terminatePipe(process.id).pipe(Effect.ignore) : Effect.void + }, + ) + // Finished processes whose file is gone, or that ended long ago, are of no use to any Location: forget them. + const cutoff = Date.now() - Duration.toMillis(RETENTION) + yield* Effect.forEach( + processes.values(), + (process) => + Effect.gen(function* () { + if (process.lifecycle.status === "running") return + const stale = (process.exitedAt ?? process.startedAt) < cutoff + if (!stale && (yield* fs.exists(process.outputPath).pipe(Effect.orElseSucceed(() => true)))) return + yield* pty.terminatePipe(process.id).pipe(Effect.ignore) + }), + { concurrency: 8, discard: true }, + ) }) const cleanupLayer = Layer.effectDiscard( @@ -121,7 +233,372 @@ const cleanupLayer = Layer.effectDiscard( const cleanupNode = makeGlobalNode({ name: "shell-output-cleanup", layer: cleanupLayer, - deps: [FSUtil.node, Global.node], + deps: [FSUtil.node, Global.node, PersistentPty.node], +}) + +/** One Location's daemon commands, as `Daemon.at` hands them to its `Shell`. */ +interface Runs { + readonly get: (id: Shell.ID) => Run | undefined + readonly list: () => Run[] + /** Fails with `UnavailableError` only when the daemon could not be reached or started: spawning instead is safe. */ + readonly create: (input: { + readonly info: Info + readonly args: readonly string[] + readonly env: Readonly> + readonly timeout: number + readonly toolCall: ToolCall | undefined + readonly foreground: boolean + }) => Effect.Effect + readonly timeout: (run: Run, duration: number) => Effect.Effect + readonly stop: (run: Run, reason: StopReason) => Effect.Effect + readonly remove: (id: Shell.ID) => Effect.Effect +} + +export interface DaemonInterface { + /** A Location's daemon commands. The first call for a Location in this server takes its commands back from the daemon. */ + readonly at: (location: Location.Interface) => Effect.Effect + /** Runs `effect` until it ends or the server stops, outliving the caller's Location (e.g. a job's completion notice). */ + readonly watch: (effect: Effect.Effect) => Effect.Effect +} + +/** The process daemon's shell commands, owned by the server and grouped by Location. */ +export class Daemon extends Context.Service()("@opencode/Shell/Daemon") {} + +const daemonLayer = Layer.effect( + Daemon, + Effect.gen(function* () { + const bus = yield* Bus.Service + const global = yield* Global.Service + const pty = yield* PersistentPty.Service + const scope = yield* Effect.scope + const context = yield* Effect.context() + const { unlink } = yield* Effect.promise(() => import("fs/promises")) + // Keyed by canonical Location. `recovered` resolves once the Location's commands were taken back. + const places = new Map }>() + // Fibers run with the server's context in its scope, so a Location closing never interrupts them. + const fork = (effect: Effect.Effect) => + effect.pipe(Effect.forkIn(scope, { startImmediately: true }), Effect.setContext(context)) + + // On shutdown the processes keep running for the next server: only stop waiting on them. + yield* Effect.addFinalizer(() => + Effect.forEach( + places.values(), + (place) => Effect.forEach(place.runs.list(), (run) => Deferred.interrupt(run.done), { discard: true }), + { discard: true }, + ), + ) + + const place = (location: Location.Interface) => { + const runs = new Map() + const exitOrder: Shell.ID[] = [] + const ref = Location.Ref.make({ directory: location.directory, workspaceID: location.workspaceID }) + const outputDir = path.join(global.data, DIRECTORY, location.project.id) + + const persisted = (run: Run): Stored => ({ + id: run.info.id, + command: run.info.command, + shell: run.info.shell, + location: { + directory: location.directory, + ...(location.workspaceID !== undefined ? { workspaceID: location.workspaceID } : {}), + }, + metadata: run.info.metadata, + toolCall: run.toolCall, + timeoutDeadline: run.timeoutDeadline, + stopReason: run.stopReason, + foreground: run.foreground, + }) + + // Records the command's latest timeout and stop reason in the daemon, so a restart sees them. + const store = (run: Run) => + pty + .setPipeMetadata(run.process, persisted(run)) + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to update shell command metadata", { id: run.info.id, error: error.message }), + ), + ) + + const removeRun = Effect.fnUntraced(function* (id: Shell.ID) { + const run = runs.get(id) + const index = exitOrder.indexOf(id) + if (index !== -1) exitOrder.splice(index, 1) + if (!run) return + runs.delete(id) + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + if (run.stopping) yield* Fiber.interrupt(run.stopping) + // Stop waiting first, so forgetting the process is not mistaken for losing it. + if (run.wait) yield* Fiber.interrupt(run.wait) + // Forget the process before deleting its file. This also kills what is left of its group, even after a + // successful exit, which a spawned command would leave running: the daemon cannot forget a process otherwise. + yield* pty.terminatePipe(run.process).pipe(Effect.ignore) + yield* Effect.promise(() => unlink(run.info.file).catch(() => {})) + yield* Deferred.fail(run.done, new NotFoundError({ id })) + // Every shell reports exactly one exit; removing a running one ends it as killed. + if (run.info.status === "running") + yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }, { location: ref }) + yield* bus.publish(Shell.Event.Deleted, { id }, { location: ref }) + }) + + const evict = Effect.fnUntraced(function* () { + while (exitOrder.length > EXITED_LIMIT) { + const oldest = exitOrder[0] + if (!oldest) break + yield* removeRun(oldest) + } + }) + + const finish = Effect.fnUntraced(function* (run: Run, outcome: Ended) { + const id = run.info.id + // A removed command already failed its waiters and published its exit and deletion. + if (runs.get(id) !== run) return + run.info = produce(run.info, (draft) => { + draft.status = outcome.status + if (outcome.exit !== undefined) draft.exit = outcome.exit + if (outcome.signal !== undefined) draft.signal = outcome.signal + draft.time.completed = outcome.completed + }) + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + // Resolve waiters before retention eviction, so an evicted command still reports its + // terminal Info rather than the removal NotFoundError. + yield* Deferred.succeed(run.done, run.info) + yield* bus.publish( + Shell.Event.Exited, + { id, ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), status: outcome.status }, + { location: ref }, + ) + exitOrder.push(id) + yield* evict() + }) + + // Graceful, like spawned commands: SIGTERM, then SIGKILL if it has not exited 3 s later. It runs on its own, + // so interrupting the caller (a timer being retimed, removed or torn down) never cancels a recorded stop. + const stop = (run: Run, reason: StopReason) => + Effect.gen(function* () { + if (run.stopReason !== undefined || run.info.status !== "running") return + run.stopReason = reason + run.timeoutDeadline = undefined + run.stopping = yield* fork( + Effect.gen(function* () { + yield* store(run) + yield* pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) + const ended = yield* Deferred.await(run.done).pipe(Effect.timeoutOption(STOP_GRACE), Effect.exit) + if (Exit.isSuccess(ended) && Option.isNone(ended.value)) + yield* pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore) + }), + ) + }) + + const arm = Effect.fnUntraced(function* (run: Run, duration: number) { + run.timeoutFiber = yield* fork( + Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop(run, "timeout"))), + ) + }) + + const timeout = Effect.fnUntraced(function* (run: Run, duration: number) { + if (run.stopReason !== undefined || run.info.status !== "running") return + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + run.timeoutDeadline = duration === 0 ? undefined : Date.now() + duration + // Clearing the timeout is how a call moves its command to the background. + if (duration === 0) run.foreground = false + yield* store(run) + if (duration !== 0) yield* arm(run, duration) + }) + + // Watches the exit, and the timeout (in ms from now, none if undefined). + const attach = Effect.fnUntraced(function* (run: Run, duration: number | undefined) { + run.wait = yield* fork( + pty.waitPipe(run.process).pipe( + Effect.map((exit): Ended => { + run.outputLines = exit.outputLines + run.outputError = exit.outputError + // A stopped command reports its reason, not the signal it was stopped with, as spawned ones do. + if (run.stopReason) return { status: run.stopReason, completed: exit.exitedAt } + return { + status: "exited", + ...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}), + ...(exit.signal !== undefined ? { signal: exit.signal } : {}), + completed: exit.exitedAt, + } + }), + // The daemon went away (and its processes with it), or forgot the process: it was killed. + Effect.catch((error) => + Effect.logWarning("lost a shell command's process", { id: run.info.id, error: error.message }).pipe( + Effect.andThen( + Effect.sync((): Ended => { + run.killed = LOST + return { status: "killed", completed: Date.now() } + }), + ), + ), + ), + // Like spawned commands: what a failed command left running in its group is stopped with it, while a + // successful one may leave background processes (`cmd &`) behind. + Effect.tap((outcome) => + outcome.status === "exited" && outcome.exit !== undefined && outcome.exit !== 0 + ? pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) + : Effect.void, + ), + Effect.flatMap((outcome) => finish(run, outcome)), + ), + ) + // A stop interrupted by a restart lost its grace period: kill now. + if (run.stopReason !== undefined) { + run.stopping = yield* fork(pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore)) + return + } + if (duration === undefined) return + if (duration > 0) return yield* arm(run, duration) + yield* stop(run, "timeout") + }) + + const create: Runs["create"] = Effect.fnUntraced(function* (input) { + const run: Run = { + kind: "daemon", + info: input.info, + process: 0, + toolCall: input.toolCall, + foreground: input.foreground, + timeoutDeadline: input.timeout > 0 ? input.info.time.started + input.timeout : undefined, + done: Deferred.makeUnsafe(), + } + const created = yield* pty.createPipe({ + program: input.info.shell, + args: input.args, + cwd: input.info.cwd, + env: input.env, + groupID: location.project.id, + outputPath: input.info.file, + metadata: persisted(run), + }) + run.process = created.id + run.info = produce(input.info, (draft) => { + if (created.pid !== undefined) draft.pid = created.pid + // The daemon's clock, so the command reports the same start after a restart. + draft.time.started = created.startedAt + }) + runs.set(run.info.id, run) + // Timed from now: starting the daemon may have taken a while. + yield* attach(run, input.timeout > 0 ? input.timeout : undefined) + yield* bus.publish(Shell.Event.Created, { info: run.info }, { location: ref }) + return run.info + }) + + // Takes back this Location's commands, left by a previous server. Two Locations can share a project's output + // directory, so match the Location itself. + const recover = Effect.fnUntraced(function* () { + const processes = yield* pty + .listPipes() + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to list shell commands to recover", { error: error.message }).pipe( + Effect.as([]), + ), + ), + ) + const finished: Run[] = [] + for (const process of processes) { + const stored = Option.getOrUndefined(decodeStored(process.metadata)) + if ( + !stored || + stored.location.directory !== location.directory || + stored.location.workspaceID !== location.workspaceID || + path.dirname(process.outputPath) !== outputDir || + runs.has(stored.id) + ) + continue + const running = process.lifecycle.status === "running" + const exited = process.lifecycle.status === "exited" ? process.lifecycle : undefined + const status = running ? "running" : (stored.stopReason ?? (exited ? "exited" : "killed")) + const run: Run = { + kind: "daemon", + info: { + id: stored.id, + status, + command: stored.command, + cwd: process.cwd, + shell: stored.shell, + file: process.outputPath, + ...(process.pid !== undefined ? { pid: process.pid } : {}), + ...(status === "exited" && exited?.exitCode !== undefined ? { exit: exited.exitCode } : {}), + ...(status === "exited" && exited?.signal !== undefined ? { signal: exited.signal } : {}), + metadata: stored.metadata, + time: { + started: process.startedAt, + ...(running ? {} : { completed: process.exitedAt ?? process.startedAt }), + }, + }, + process: process.id, + toolCall: stored.toolCall, + timeoutDeadline: stored.timeoutDeadline, + stopReason: stored.stopReason, + foreground: stored.foreground === true, + ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), + ...(process.lifecycle.status === "failed" && !stored.stopReason + ? { killed: `Command was killed: ${process.lifecycle.message}` } + : {}), + done: Deferred.makeUnsafe(), + } + runs.set(stored.id, run) + if (running) { + // Its call died with the previous server, so taking it over moves it to the background, which clears + // its timeout. Otherwise an expired deadline fires at once. + const deadline = stored.foreground ? undefined : stored.timeoutDeadline + yield* attach(run, deadline === undefined ? undefined : Math.max(0, deadline - Date.now())) + continue + } + yield* Deferred.succeed(run.done, run.info) + finished.push(run) + } + exitOrder.push( + ...finished + .toSorted((a, b) => (a.info.time.completed ?? 0) - (b.info.time.completed ?? 0)) + .map((run) => run.info.id), + ) + yield* evict() + }) + + const handle: Runs = { + get: (id) => runs.get(id), + list: () => Array.from(runs.values()), + create, + timeout, + stop, + remove: removeRun, + } + return { runs: handle, recover } + } + + const at = Effect.fnUntraced(function* (location: Location.Interface) { + const key = `${location.directory}\0${location.workspaceID ?? ""}` + const existing = places.get(key) + if (existing) { + yield* Deferred.await(existing.recovered) + return existing.runs + } + const placed = place(location) + const entry = { runs: placed.runs, recovered: Deferred.makeUnsafe() } + places.set(key, entry) + yield* placed.recover().pipe(Effect.ensuring(Deferred.succeed(entry.recovered, undefined))) + return placed.runs + }) + + return Daemon.of({ + at, + watch: (effect) => + fork(effect.pipe(Effect.catch((error) => Effect.logError("shell watcher failed", { error })))).pipe( + Effect.asVoid, + ), + }) + }), +) + +export const daemonNode = makeGlobalNode({ + service: Daemon, + layer: daemonLayer, + deps: [Bus.node, Global.node, PersistentPty.node], }) const layer = () => @@ -136,56 +613,100 @@ const layer = () => const hooks = yield* PluginHooks.Service const environments = yield* SessionEnvironment.Service const config = yield* Config.Service + const daemon = yield* Daemon const context = yield* Effect.context() const runFork = Effect.runForkWith(context) - const commands = new Map() + // Commands this server spawned for the Location. Daemon commands live in `runs`, owned by the server. + const spawned = new Map() const exitOrder: Shell.ID[] = [] + // Remote workspaces and Windows keep spawning commands from the server. + const daemonEnabled = + location.workspaceID === undefined && process.platform !== "win32" && PersistentPty.available + // Taken back before the service is returned, so session restart recovery sees them. + const runs = daemonEnabled ? yield* daemon.at(location) : undefined const outputDir = path.join(global.data, DIRECTORY, location.project.id) - const { mkdir, unlink } = yield* Effect.promise(() => import("fs/promises")) + const { mkdir, unlink, open } = yield* Effect.promise(() => import("fs/promises")) const { createWriteStream, createReadStream } = yield* Effect.promise(() => import("fs")) yield* Effect.promise(() => mkdir(outputDir, { recursive: true })) + // Spawned commands end with their Location; daemon commands are left to the server. yield* Effect.addFinalizer(() => Effect.gen(function* () { - for (const command of commands.values()) { + for (const command of spawned.values()) { if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - // Teardown interrupts pending commands; it is not a terminal command failure. + // Teardown interrupts pending commands; it is not a terminal command failure. It never stops or + // removes a command itself: a spawned process dies with its handle's scope once `done` settles. yield* Deferred.interrupt(command.done) } - commands.clear() + spawned.clear() exitOrder.length = 0 }), ) + const find = (id: Shell.ID): Command | undefined => spawned.get(id) ?? runs?.get(id) + const require = Effect.fnUntraced(function* (id: Shell.ID) { - const command = commands.get(id) + const command = find(id) if (!command) return yield* new NotFoundError({ id }) return command }) - const removeCommand = Effect.fnUntraced(function* (id: Shell.ID) { - const command = commands.get(id) + const removeSpawned = Effect.fnUntraced(function* (id: Shell.ID) { + const command = spawned.get(id) const index = exitOrder.indexOf(id) if (index !== -1) exitOrder.splice(index, 1) if (!command) return - commands.delete(id) + spawned.delete(id) if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) // Unblock any wait still pending when the command is removed before it terminated. yield* Deferred.fail(command.done, new NotFoundError({ id })) - yield* Effect.promise(() => unlink(command.file).catch(() => {})) + yield* Effect.promise(() => unlink(command.info.file).catch(() => {})) // Every shell reports exactly one exit; removing a running one ends it as killed. if (command.info.status === "running") yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }) yield* bus.publish(Shell.Event.Deleted, { id }) }) + // Records how a spawned command ended, once, unless it was removed meanwhile. + const finish = Effect.fnUntraced(function* (command: Spawned, outcome: Ended) { + const id = command.info.id + if (spawned.get(id) !== command) return + command.info = produce(command.info, (draft) => { + draft.status = outcome.status + if (outcome.exit !== undefined) draft.exit = outcome.exit + if (outcome.signal !== undefined) draft.signal = outcome.signal + draft.time.completed = outcome.completed + }) + // Resolve waiters before retention eviction, so an evicted command still reports its + // terminal Info rather than the removal NotFoundError. + yield* Deferred.succeed(command.done, command.info) + yield* bus.publish(Shell.Event.Exited, { + id, + ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), + status: outcome.status, + }) + exitOrder.push(id) + while (exitOrder.length > EXITED_LIMIT) { + const oldest = exitOrder[0] + if (!oldest) break + yield* removeSpawned(oldest) + } + // Keep exited history data-only. + const timeoutFiber = command.timeoutFiber + command.timeout = undefined + command.stop = undefined + command.timeoutFiber = undefined + if (timeoutFiber) yield* Fiber.interrupt(timeoutFiber) + }) + const remove = Effect.fn("Shell.remove")(function* (id: Shell.ID) { - yield* require(id) - yield* removeCommand(id) + const command = yield* require(id) + if (command.kind === "daemon") return yield* runs!.remove(id) + yield* removeSpawned(id) }) const list = Effect.fn("Shell.list")(function* () { - return Array.from(commands.values()) + return [...spawned.values(), ...(runs?.list() ?? [])] .filter((command) => command.info.status === "running") .map((command) => command.info) }) @@ -200,43 +721,149 @@ const layer = () => const timeout = Effect.fn("Shell.timeout")(function* (id: Shell.ID, duration: number) { const command = yield* require(id) - if (command.info.status !== "running" || !command.timeout) return command.info - yield* command.timeout(duration) + if (command.kind === "daemon") yield* runs!.timeout(command, duration) + if (command.kind === "spawned" && command.info.status === "running" && command.timeout) + yield* command.timeout(duration) return command.info }) + const stop = Effect.fn("Shell.stop")(function* (id: Shell.ID, reason: StopReason) { + const command = yield* require(id) + if (command.kind === "daemon") yield* runs!.stop(command, reason) + if (command.kind === "spawned" && command.info.status === "running" && command.stop) yield* command.stop(reason) + return command.info + }) + + const detachable = (id: Shell.ID) => find(id)?.kind === "daemon" + + const byToolCall = Effect.fn("Shell.byToolCall")(function* (call: ToolCall) { + return runs + ?.list() + .find( + (run) => + run.toolCall?.sessionID === call.sessionID && + run.toolCall.assistantMessageID === call.assistantMessageID && + run.toolCall.callID === call.callID, + )?.info + }) + + const readSpawned = (file: string, start: number, length: number) => + length <= 0 + ? Effect.succeed(Buffer.alloc(0)) + : Effect.promise( + () => + new Promise((resolve) => { + const buffer = Buffer.alloc(length) + const stream = createReadStream(file, { start, end: start + length - 1 }) + let offset = 0 + stream.on("data", (chunk: string | Buffer) => { + const bytes = Buffer.from(chunk) + bytes.copy(buffer, offset) + offset += bytes.length + }) + stream.on("end", () => resolve(buffer.subarray(0, offset))) + stream.on("error", () => resolve(Buffer.alloc(0))) + }), + ) + + // Reads a daemon command's file, which is the only record of its output. `size` comes from the same open + // handle; a missing file reads as empty. + const readFile = (file: string, start: (size: number) => number, limit: number) => + Effect.promise(async () => { + const handle = await open(file, "r").catch(() => undefined) + if (!handle) return { size: 0, start: 0, bytes: Buffer.alloc(0) } + try { + const size = (await handle.stat()).size + const from = Math.max(0, start(size)) + if (from >= size) return { size, start: from, bytes: Buffer.alloc(0) } + const buffer = Buffer.alloc(Math.min(limit, size - from)) + const read = await handle.read(buffer, 0, buffer.length, from) + return { size, start: from, bytes: buffer.subarray(0, read.bytesRead) } + } catch { + return { size: 0, start: 0, bytes: Buffer.alloc(0) } + } finally { + await handle.close() + } + }) + const output = Effect.fnUntraced(function* (id: Shell.ID, input?: Shell.OutputInput) { const command = yield* require(id) const cursor = input?.cursor ?? 0 const limit = input?.limit ?? 65536 + if (command.kind === "daemon") { + const page = yield* readFile(command.info.file, () => cursor, limit) + if (cursor >= page.size) return { output: "", cursor: page.size, size: page.size, truncated: false } + return { + output: page.bytes.toString("utf8"), + cursor: page.start + page.bytes.length, + size: page.size, + truncated: false, + } + } if (cursor >= command.size) return { output: "", cursor: command.size, size: command.size, truncated: false } const start = Math.max(0, cursor) - const length = Math.min(limit, command.size - start) - const buffer = Buffer.alloc(length) - const bytesRead = yield* Effect.promise( + const bytes = yield* readSpawned(command.info.file, start, Math.min(limit, command.size - start)) + return { + output: bytes.toString("utf8"), + cursor: start + bytes.length, + size: command.size, + truncated: false, + } + }) + + // Only for a command whose exit report was lost; the daemon counts newlines otherwise. + const countNewlines = (file: string) => + Effect.promise( () => new Promise((resolve) => { - const stream = createReadStream(command.file, { start, end: start + length - 1 }) - let offset = 0 + let count = 0 + const stream = createReadStream(file) stream.on("data", (chunk: string | Buffer) => { const bytes = Buffer.from(chunk) - bytes.copy(buffer, offset) - offset += bytes.length + for (let index = bytes.indexOf(10); index !== -1; index = bytes.indexOf(10, index + 1)) count++ }) - stream.on("end", () => resolve(offset)) - stream.on("error", () => resolve(0)) + stream.on("end", () => resolve(count)) + stream.on("error", () => resolve(count)) }), ) + + // The last `maxBytes` of a finished command's output, the file's size and its newline count. + const tail = Effect.fnUntraced(function* (command: Command, maxBytes: number) { + if (command.kind === "spawned") { + const start = Math.max(0, command.size - maxBytes) + const bytes = yield* readSpawned(command.info.file, start, command.size - start) + return { + text: bytes.toString("utf8"), + size: command.size, + newlines: command.newlines, + error: undefined, + killed: undefined, + } + } + const page = yield* readFile(command.info.file, (size) => size - maxBytes, maxBytes) return { - output: buffer.subarray(0, bytesRead).toString("utf8"), - cursor: start + bytesRead, - size: command.size, - truncated: false, + text: page.bytes.toString("utf8"), + size: page.size, + // The daemon's count, unless the command ended without its exit report (the daemon went away). + newlines: command.outputLines ?? (yield* countNewlines(command.info.file)), + error: command.outputError, + killed: command.killed, } }) - const result = Effect.fn("Shell.result")(function* (started: Shell.Info) { - const info = yield* wait(started.id).pipe( + const result = Effect.fn("Shell.result")(function* ( + started: Shell.Info, + options?: { readonly wait?: Duration.Input }, + ) { + const ended = wait(started.id).pipe(Effect.map(Option.some)) + const info = yield* ( + options?.wait === undefined + ? ended + : ended.pipe( + Effect.timeoutOrElse({ duration: options.wait, orElse: () => Effect.succeed(Option.none()) }), + ) + ).pipe( + Effect.flatMap((info) => (Option.isSome(info) ? Effect.succeed(info.value) : get(started.id))), Effect.catchTag("Shell.NotFoundError", () => Effect.succeed({ ...started, status: "killed" as const, time: { ...started.time, completed: Date.now() } }), ), @@ -245,18 +872,22 @@ const layer = () => const limits = Config.latest(yield* config.entries(), "tool_output") const maxLines = limits?.max_lines ?? ToolOutput.MAX_LINES const maxBytes = limits?.max_bytes ?? ToolOutput.MAX_BYTES - const latest = yield* output(info.id, { cursor: Number.MAX_SAFE_INTEGER }) - const page = yield* output(info.id, { cursor: Math.max(0, latest.size - maxBytes), limit: maxBytes }) - const lines = page.output.split("\n") - if (page.output.endsWith("\n")) lines.pop() - const truncated = latest.size > maxBytes || lines.length > maxLines - const text = lines.length > maxLines ? lines.slice(-maxLines).join("\n") : page.output - const total = (yield* require(info.id)).newlines + (page.output.endsWith("\n") ? 0 : 1) - const shown = Math.min(lines.length, maxLines) - const notice = truncated - ? `\n\n[showing lines ${total - shown + 1}-${total} of ${total}; full output saved to ${info.file}]` - : "" - return { output: `${text || "(no output)"}${notice}`, truncated } + const page = yield* tail(yield* require(info.id), maxBytes) + const lines = page.text.split("\n") + if (page.text.endsWith("\n")) lines.pop() + const truncated = page.size > maxBytes || lines.length > maxLines + const text = lines.length > maxLines ? lines.slice(-maxLines).join("\n") : page.text + const empty = page.size === 0 + const shown = empty ? 0 : Math.min(lines.length, maxLines) + const total = empty ? 0 : Math.max(page.newlines + (page.text.endsWith("\n") ? 0 : 1), shown) + const notice = truncated ? `\n\n${ShellResult.savedNotice({ shown, total }, info.file)}` : "" + const error = page.error ? `\n\n[output file ${info.file} stopped early: ${page.error}]` : "" + return { + output: `${text || "(no output)"}${notice}${error}`, + truncated, + lines: { shown, total }, + ...(page.killed ? { killed: page.killed } : {}), + } }).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined))) return { info, capture } }) @@ -283,6 +914,8 @@ const layer = () => } yield* hooks.trigger("shell", "create.before", invocation) if (before) yield* before(invocation) + // Absolute for local commands, so the command reports the same cwd after a restart, wherever it runs. + if (location.workspaceID === undefined) invocation.cwd = path.resolve(location.directory, invocation.cwd) const id = Shell.ID.ascending() const args = ShellSelect.args(invocation.shell, invocation.command) @@ -299,10 +932,38 @@ const layer = () => time: { started: Date.now() }, } + if (runs) { + const created = yield* runs + .create({ + info, + args, + // The daemon replaces the environment, so variables removed by hooks stay removed, like the spawner. + env: Object.fromEntries( + Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined), + ), + timeout: invocation.timeout, + toolCall: input.toolCall, + foreground: input.foreground === true, + }) + .pipe( + Effect.map(Option.some), + Effect.catchTags({ + "PersistentPty.UnavailableError": (error) => + Effect.logWarning("process daemon unavailable; spawning shell command instead", { + error: error.message, + }).pipe(Effect.as(Option.none())), + // The daemon may have run it: never run it again here. + "PersistentPty.RequestError": (cause) => + Effect.fail(new AppProcess.AppProcessError({ command: invocation.command, cause })), + }), + ) + if (Option.isSome(created)) return created.value + } + // Spawn through the Environment and stream combined output to the file. The handle is scope-bound, so // the managing fiber keeps its scope open until the command terminates (it awaits `done` at the // end). `create` returns once `ready` resolves with the registered command. - const ready = Deferred.makeUnsafe() + const ready = Deferred.makeUnsafe() runFork( Effect.scoped( Effect.gen(function* () { @@ -313,22 +974,22 @@ const layer = () => env: invocation.env, stdin: "ignore", detached: process.platform !== "win32", - forceKillAfter: Duration.seconds(3), + forceKillAfter: STOP_GRACE, }), ) .pipe( Effect.mapError((cause) => new AppProcess.AppProcessError({ command: invocation.command, cause })), ) - const command: Active = { + const command: Spawned = { + kind: "spawned", info: produce(info, (draft) => { draft.pid = handle.pid }), - file, size: 0, newlines: 0, done: Deferred.makeUnsafe(), } - commands.set(id, command) + spawned.set(id, command) const stream = createWriteStream(file) const outputDone = Latch.makeUnsafe() @@ -368,6 +1029,15 @@ const layer = () => const end = (outcome: Omit) => Effect.suspend(() => Deferred.succeed(ended, { ...outcome, completed: Date.now() })) + // A recorded stop must always kill. The kill runs detached, so interrupting the caller (a timer + // being retimed, removed or torn down) neither cancels it nor waits for the process to die. + const stop = (reason: StopReason) => + Effect.gen(function* () { + if (!(yield* end({ status: reason }))) return + runFork(handle.kill({ forceKillAfter: STOP_GRACE }).pipe(Effect.catch(() => Effect.void))) + }).pipe(Effect.uninterruptible) + command.stop = stop + command.timeout = (duration) => Effect.gen(function* () { if (Deferred.isDoneUnsafe(ended)) return @@ -375,18 +1045,7 @@ const layer = () => command.timeoutFiber = undefined if (duration === 0) return command.timeoutFiber = runFork( - Effect.sleep(Duration.millis(duration)).pipe( - // A recorded timeout must always kill. The kill runs detached, so interrupting this - // timer (retime, remove, teardown) neither cancels it nor waits for the process to die. - Effect.andThen( - Effect.gen(function* () { - if (!(yield* end({ status: "timeout" }))) return - runFork( - handle.kill({ forceKillAfter: Duration.seconds(3) }).pipe(Effect.catch(() => Effect.void)), - ) - }).pipe(Effect.uninterruptible), - ), - ), + Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop("timeout"))), ) }) @@ -409,33 +1068,7 @@ const layer = () => // Report the terminal status only once captured output is complete, so a reader that // sees it never reads a partial file. yield* outputDone.await - // A removed command already failed its waiters and published its exit and deletion. - if (commands.get(id) !== command) return - command.info = produce(command.info, (draft) => { - draft.status = outcome.status - if (outcome.exit !== undefined) draft.exit = outcome.exit - if (outcome.signal !== undefined) draft.signal = outcome.signal - draft.time.completed = outcome.completed - }) - // Resolve waiters before retention eviction, so an evicted command still reports its - // terminal Info rather than the removal NotFoundError. - yield* Deferred.succeed(command.done, command.info) - yield* bus.publish(Shell.Event.Exited, { - id, - ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), - status: outcome.status, - }) - exitOrder.push(id) - while (exitOrder.length > EXITED_LIMIT) { - const oldest = exitOrder[0] - if (!oldest) break - yield* removeCommand(oldest) - } - // Keep exited history data-only. - const timeoutFiber = command.timeoutFiber - command.timeout = undefined - command.timeoutFiber = undefined - if (timeoutFiber) yield* Fiber.interrupt(timeoutFiber) + yield* finish(command, outcome) }), ) @@ -452,7 +1085,19 @@ const layer = () => return command.info }) - return Service.of({ create, list, get, wait, result, timeout, output, remove }) + return Service.of({ + create, + list, + get, + wait, + result, + timeout, + stop, + detachable, + byToolCall, + output, + remove, + }) }), ) @@ -468,6 +1113,7 @@ export const node = makeLocationNode({ PluginHooks.node, SessionEnvironment.node, Config.node, + daemonNode, cleanupNode, ], }) diff --git a/packages/core/src/shell/result.ts b/packages/core/src/shell/result.ts index b9dc352e86fa..484f81297c19 100644 --- a/packages/core/src/shell/result.ts +++ b/packages/core/src/shell/result.ts @@ -2,12 +2,24 @@ export * as ShellResult from "./result.js" import type { Shell } from "@opencode/schema/shell" +export type Lines = { shown: number; total: number } + export type Result = { info: Shell.Info - capture: { output: string; truncated: boolean } | undefined + // `lines` counts the tail's lines (`shown`) and the file's (`total`), zero for an empty file. + // `killed` says why a command was killed when no signal of its own tells, e.g. the process daemon stopped. + capture: { output: string; truncated: boolean; lines?: Lines; killed?: string } | undefined } -type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean } +type Output = { + output: string + truncated: boolean + exit?: number + signal?: string + timeout?: boolean + // Killed without an exit code or signal of its own: stopped, or lost with the process daemon. + killed?: string +} const missing = "Shell command output is no longer available." export const unavailable: Shell.Output = { @@ -17,18 +29,47 @@ export const unavailable: Shell.Output = { truncated: false, } -export function output(result: Result): Output { +export function output(result: Result, reportKilled = true): Output { return { output: result.capture?.output ?? unavailable.output, truncated: result.capture?.truncated ?? false, ...(result.info.exit !== undefined ? { exit: result.info.exit } : {}), ...(result.info.signal !== undefined ? { signal: result.info.signal } : {}), ...(result.info.status === "timeout" ? { timeout: true } : {}), + // A removed command has no capture and keeps today's plain "unavailable" result. + ...(reportKilled && result.info.status === "killed" && result.capture && result.info.signal === undefined + ? { killed: result.capture.killed ?? "Command was killed" } + : {}), + } +} + +export function savedNotice(lines: Lines, file: string) { + if (lines.total === 0) return `[no output; output file: ${file}]` + return `[showing lines ${lines.total - lines.shown + 1}-${lines.total} of ${lines.total}; full output saved to ${file}]` +} + +// The result of a command stopped because the user interrupted its call. It always names the shell +// and its output file, which `output` leaves out when the output fits. +export function interrupted(result: Result): Output { + // Saying it was interrupted already says it was killed. + const captured = output(result, false) + const lines = result.capture && !result.capture.truncated ? result.capture.lines : undefined + // An empty capture is all notice: no "(no output)" before "[no output; …]". + const text = lines?.total === 0 ? "" : `${captured.output.replace(/\n$/, "")}\n\n` + return { + ...captured, + output: [ + lines ? `${text}${savedNotice(lines, result.info.file)}` : captured.output, + result.info.status === "running" + ? `Command interrupted by the user (shell ID: ${result.info.id}); it is still being stopped.` + : `Command interrupted by the user (shell ID: ${result.info.id}).`, + ].join("\n\n"), } } -export function notice(output: Pick) { +export function notice(output: Pick) { if (output.timeout) return "Timed out before completion" + if (output.killed !== undefined) return output.killed if (output.signal !== undefined) return `Killed by ${output.signal}` if (output.exit !== undefined && output.exit !== 0) return `Exited with code ${output.exit}` } @@ -39,6 +80,7 @@ export function metadata(output: Output) { ...(output.exit !== undefined ? { exit: output.exit } : {}), ...(output.signal !== undefined ? { signal: output.signal } : {}), ...(output.timeout !== undefined ? { timeout: output.timeout } : {}), + ...(output.killed !== undefined ? { killed: true } : {}), } } diff --git a/packages/core/src/tool.ts b/packages/core/src/tool.ts index 463723972d6f..2219d1a915ec 100644 --- a/packages/core/src/tool.ts +++ b/packages/core/src/tool.ts @@ -18,6 +18,7 @@ import { SessionSchema } from "./session/schema.js" import { State } from "./state.js" import { definition, effectiveName, execute, normalizedName, normalizeContent } from "./tool/runtime.js" import { Wildcard } from "./util/wildcard.js" +import { ToolInterrupted } from "./tool/interrupted.js" export class RegistrationError extends Schema.TaggedError()("Tool.RegistrationError", { name: Schema.String, @@ -272,7 +273,9 @@ const layer = Layer.effect( const codeModeTool = codeModeEnabled ? CodeModeTool.create( codeModeInventory, - (name, tool, input, context) => run(tool, name, input, context), + // A child call cannot settle the outer CodeMode call with its own interrupted result. + (name, tool, input, context) => + run(tool, name, input, context).pipe(Effect.provideService(ToolInterrupted.Slot, undefined)), permission, ) : undefined diff --git a/packages/core/src/tool/interrupted.ts b/packages/core/src/tool/interrupted.ts new file mode 100644 index 000000000000..83c9811a8f2a --- /dev/null +++ b/packages/core/src/tool/interrupted.ts @@ -0,0 +1,22 @@ +export * as ToolInterrupted from "./interrupted.js" + +import { Context, Effect } from "effect" +import type { NormalizedResult } from "../tool.js" + +/** + * Lets a tool whose call is interrupted still settle it with a result. + * + * An interrupted fiber fails as soon as it re-enters an interruptible region, so a tool that finishes its + * work uninterruptibly cannot return a value through the caller. It reports the result here instead, + * before returning; the session step settles the call with it and still ends as interrupted. + */ +export type Slot = { result?: NormalizedResult } + +export const Slot = Context.Reference("@opencode/ToolInterrupted/Slot", { + defaultValue: () => undefined, +}) + +export const report = Effect.fnUntraced(function* (result: NormalizedResult) { + const slot = yield* Slot + if (slot) slot.result = result +}) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index d433087033b5..2385bc668044 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -4,22 +4,29 @@ import { ToolFailure } from "@opencode/ai/effect" import type { Context } from "@opencode/plugin/effect/plugin" import type { SessionHooks } from "@opencode/plugin/effect/session" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" +import type { Shell as ShellSchema } from "@opencode/schema/shell" import type { Tool } from "@opencode/schema/tool" -import { Deferred, Effect, Schema, Scope } from "effect" +import { Cause, Deferred, Effect, Exit, Option, Schema, Scope } from "effect" +import { Detaching } from "../../detaching.js" import { Environment } from "../../environment/index.js" import { Job } from "../../job.js" +import { LocationLifecycle } from "../../location-lifecycle.js" import { FileAccess } from "../../file-access.js" import { Permission } from "../../permission.js" import { NonNegativeInt } from "../../schema.js" import { Session } from "../../session.js" +import { SessionMessage } from "../../session/message.js" import { SessionSchema } from "../../session/schema.js" import { Shell } from "../../shell.js" import { ShellParse } from "../../shell/parse.js" import { ShellSelect } from "../../shell/select.js" import { ShellResult } from "../../shell/result.js" +import { ToolInterrupted } from "../interrupted.js" export const name = "shell" export const DEFAULT_TIMEOUT_MS = 2 * 60 * 1_000 +// How long an interrupted call waits for its stopped command to exit before settling with the output so far. +const INTERRUPT_WAIT = "500 millis" const BACKGROUND_INSTRUCTION = "You will be notified automatically when the command finishes. The notification will include the command's output. Unless the user explicitly asks otherwise, DO NOT poll for completion, even if you need the final result to continue. Repeatedly sleeping and reading or searching the output file is polling, not useful work. You may read the current output if it lets you do useful work now, but do not repeatedly check it while waiting for the command to finish. Keep working on anything that does not depend on the result. If you have nothing else to do, end your response; you will be resumed automatically when the command finishes." @@ -64,6 +71,8 @@ const StructuredOutput = Schema.Struct({ shellID: Schema.optionalKey(Schema.String), truncated: Schema.Boolean, timeout: Schema.optionalKey(Schema.Boolean), + interrupted: Schema.optionalKey(Schema.Boolean), + killed: Schema.optionalKey(Schema.String), }) const Output = Schema.Struct({ @@ -87,6 +96,7 @@ const toolResult = (output: Output) => { status: output.status, ...ShellResult.metadata(output), ...(output.shellID !== undefined ? { shellID: output.shellID } : {}), + ...(output.interrupted !== undefined ? { interrupted: output.interrupted } : {}), }, } } @@ -98,18 +108,162 @@ const backgroundResult = (shellID: string, file: string) => ({ status: "running" as const, }) +// The model's result for a finished command. +const completed = (shell: Shell.Interface, info: ShellSchema.Info, timeout?: number) => + Effect.gen(function* () { + const result = yield* shell.result(info) + if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) + const output = ShellResult.output(result) + return { + ...output, + output: + output.timeout && timeout !== undefined + ? `${output.output}\n\nCommand exceeded timeout of ${timeout} ms. Retry with a larger timeout if the command is expected to take longer.` + : output.output, + status: "completed" as const, + } + }) + +// Tells the session how a background command ended, once its job settles. +const notify = Effect.fnUntraced(function* ( + sessions: Session.Interface, + jobs: Job.Interface, + input: { + readonly sessionID: SessionSchema.ID + readonly id: string + readonly shellID: string + readonly command: string + readonly settled: Deferred.Deferred + // Evaluated at delivery; `false` admits the notice without waking the session. + readonly resume?: () => false | undefined + }, +) { + const waited = yield* jobs.wait({ id: input.id }).pipe(Effect.exit) + // Teardown interrupts the job without settling it; its durable row is reported by the next server. + if (Exit.isFailure(waited)) + return Cause.hasInterrupts(waited.cause) ? undefined : yield* Effect.failCause(waited.cause) + const info = waited.value.info + if (!info || info.status === "running") return + const output = info.status === "completed" ? yield* Deferred.await(input.settled) : undefined + const text = output + ? resultMessages(output).join("\n\n") + : info.status === "error" + ? (info.error ?? "Command failed") + : "Cancelled" + const resume = input.resume?.() + yield* sessions.synthetic({ + ...(info.notificationID ? { id: info.notificationID } : {}), + sessionID: input.sessionID, + description: input.command, + ...ShellResult.notification({ + jobID: input.id, + shellID: input.shellID, + command: input.command, + state: info.status, + text, + output, + }), + ...(resume === false ? { resume } : {}), + }) + if (info.notificationID) yield* jobs.completeBackground(info.notificationID) +}) + +const startJob = ( + jobs: Job.Interface, + input: { + readonly sessionID: SessionSchema.ID + readonly info: ShellSchema.Info + readonly notificationID?: SessionMessage.ID + readonly run: Effect.Effect + }, +) => + jobs.start({ + // CodeMode children share a tool-call ID, but each shell must own its job. + id: input.info.id, + type: name, + title: input.info.command, + metadata: { sessionID: input.sessionID, shellID: input.info.id }, + recovery: { + kind: "shell", + sessionID: input.sessionID, + shellID: input.info.id, + command: input.info.command, + }, + ...(input.notificationID ? { notificationID: input.notificationID } : {}), + run: input.run, + }) + +/** The tool result of a call whose command moved to the background. */ +export const backgroundToolResult = (shellID: string, file: string) => toolResult(backgroundResult(shellID, file)) + +/** + * Watches a command taken over from a previous server as a background job of its session, and notifies the + * session when it finishes, as for a command the model ran in the background. A finished command notifies at once. + */ +export const reattach = Effect.fnUntraced(function* (input: { + readonly shell: Shell.Interface + readonly jobs: Job.Interface + readonly sessions: Session.Interface + readonly scope: Scope.Scope + readonly sessionID: SessionSchema.ID + readonly info: ShellSchema.Info + // The job's existing durable row, if the previous server had backgrounded it. + readonly notificationID?: SessionMessage.ID + readonly resume?: () => false | undefined + // Whether a planned shutdown leaves the command running; otherwise cancelling the job stops it. + readonly leaveRunning: () => boolean +}) { + const settled = yield* Deferred.make() + const id = input.info.id + yield* startJob(input.jobs, { + sessionID: input.sessionID, + info: input.info, + notificationID: input.notificationID, + run: completed(input.shell, input.info).pipe( + Effect.tap((output) => Deferred.succeed(settled, output)), + Effect.map((output) => resultMessages(output).join("\n\n")), + Effect.onInterrupt(() => + input.leaveRunning() ? Effect.void : input.shell.stop(id, "killed").pipe(Effect.ignore), + ), + ), + }) + yield* input.jobs.background(id) + const notifying = notify(input.sessions, input.jobs, { + sessionID: input.sessionID, + id, + shellID: id, + command: input.info.command, + settled, + resume: input.resume, + }) + // A finished command's result is admitted before recovery moves on, like a recovered subagent's. + const current = yield* input.shell.get(id).pipe(Effect.option) + if (Option.isSome(current) && current.value.status !== "running") + return yield* notifying.pipe( + Effect.catch((error) => Effect.logWarning("failed to report a recovered shell command", { id, error })), + ) + yield* notifying.pipe(Effect.forkIn(input.scope, { startImmediately: true })) +}) + export const Plugin = { id: "opencode.tool.shell", effect: Effect.fn("ShellTool.Plugin")(function* (ctx: Context) { const sessions = yield* Session.Service const jobs = yield* Job.Service - const scope = yield* Scope.Scope const environment = yield* Environment.Service const access = yield* FileAccess.Service const shell = yield* Shell.Service + const daemon = yield* Shell.Daemon const shellSelect = yield* ShellSelect.Service const compatibleShell = shellSelect.resolve({ priority: "compat" }) const permission = yield* Permission.Service + const detaching = yield* Detaching.Service + const lifecycle = Option.getOrUndefined(yield* Effect.serviceOption(LocationLifecycle.Service)) + + // A planned shutdown leaves commands that can outlive this server running for the next one. + const leaveRunning = (detachable: boolean) => detaching.isDetaching() && detachable + // Shutting down or closing the Location interrupts calls too, but that is not the user cancelling them. + const closing = () => detaching.isStopping() || lifecycle?.isClosed() === true const prepare = Effect.fn("ShellTool.prepare")(function* (invocation: ShellCreateBefore, context: Tool.Context) { const target = yield* access.resolve({ path: invocation.cwd, kind: "directory" }) @@ -142,39 +296,44 @@ export const Plugin = { }) const notifyWhenDone = Effect.fn("ShellTool.notifyWhenDone")( - function* ( - sessionID: SessionSchema.ID, - id: string, - shellID: string, - command: string, - settled: Deferred.Deferred, - ) { - const info = (yield* jobs.wait({ id })).info - if (!info || info.status === "running") return - const output = info.status === "completed" ? yield* Deferred.await(settled) : undefined - const text = output - ? resultMessages(output).join("\n\n") - : info.status === "error" - ? (info.error ?? "Command failed") - : "Cancelled" - yield* sessions.synthetic({ - ...(info.notificationID ? { id: info.notificationID } : {}), - sessionID, - description: command, - ...ShellResult.notification({ - jobID: id, - shellID, - command, - state: info.status, - text, - output, - }), - }) - if (info.notificationID) yield* jobs.completeBackground(info.notificationID) - }, - Effect.forkIn(scope, { startImmediately: true }), + // Not in this Location's scope: a daemon command outlives a Location that closes while the server runs, and + // its session must still hear how it ended. + (sessionID: SessionSchema.ID, id: string, shellID: string, command: string, settled: Deferred.Deferred) => + daemon.watch(notify(sessions, jobs, { sessionID, id, shellID, command, settled })), ) + // Settles an interrupted foreground call. The interrupt only reaches the step through ToolInterrupted. + const interrupted = Effect.fnUntraced(function* ( + info: ShellSchema.Info, + jobID: string, + detachable: boolean, + cause: Cause.Cause, + ) { + if (leaveRunning(detachable)) { + yield* jobs.background(jobID) + yield* shell.timeout(info.id, 0).pipe(Effect.ignore) + const output = backgroundResult(info.id, info.file) + yield* ToolInterrupted.report(toolResult(output)) + return output + } + // Settled as interrupted by the step, as before; the command ends with the job's scope. + if (closing()) return yield* Effect.failCause(cause) + yield* shell.stop(info.id, "killed").pipe(Effect.ignore) + // The stop escalates to SIGKILL on its own; the session does not wait for that to settle the call. + const result = yield* shell.result(info, { wait: INTERRUPT_WAIT }) + // The stopped job settles from the same exit; this only releases it. + yield* jobs.cancel(jobID) + // No `shellID`: clients read it as a background call and show the shell's live output instead of this result. + // The text names the shell and its output file. + const output = { + ...ShellResult.interrupted(result), + interrupted: true, + status: "completed" as const, + } + yield* ToolInterrupted.report(toolResult(output)) + return output + }) + yield* ctx.tool .transform((editor) => editor.add({ @@ -184,80 +343,76 @@ export const Plugin = { input: Input, output: Output, execute: (input, context) => - Effect.gen(function* () { - const timeout = input.background === true ? (input.timeout ?? 0) : (input.timeout ?? DEFAULT_TIMEOUT_MS) - let finalTimeout = timeout - const info = yield* shell.create( - { - command: input.command, - cwd: input.workdir, - timeout, - shell: yield* compatibleShell, - metadata: { sessionID: context.sessionID }, - }, - (invocation) => - Effect.gen(function* () { - invocation.env.AGENT = "1" - invocation.env.OPENCODE = "1" - invocation.env.AI_AGENT ||= "opencode" - invocation.env.OPENCODE_SESSION_ID = context.sessionID - finalTimeout = yield* prepare(invocation, context) - }), - ) - yield* context.progress({ shellID: info.id }) + // Uninterruptible from spawning to blocking, so an interrupt cannot orphan a started command: + // it stops (or, while detaching, backgrounds) the command and settles the call with its output. + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const timeout = input.background === true ? (input.timeout ?? 0) : (input.timeout ?? DEFAULT_TIMEOUT_MS) + let finalTimeout = timeout + const info = yield* shell.create( + { + command: input.command, + cwd: input.workdir, + timeout, + shell: yield* compatibleShell, + metadata: { sessionID: context.sessionID }, + foreground: input.background !== true, + toolCall: { + sessionID: context.sessionID, + assistantMessageID: context.messageID, + callID: context.id, + }, + }, + (invocation) => + Effect.gen(function* () { + invocation.env.AGENT = "1" + invocation.env.OPENCODE = "1" + invocation.env.AI_AGENT ||= "opencode" + invocation.env.OPENCODE_SESSION_ID = context.sessionID + finalTimeout = yield* restore(prepare(invocation, context)) + }), + ) + // Read once: teardown may forget the command before an interrupt is handled. + const detachable = shell.detachable(info.id) + yield* context.progress({ shellID: info.id }) - const settled = yield* Deferred.make() - const run = Effect.gen(function* () { - const result = yield* shell.result(info) - if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) - const output = ShellResult.output(result) - return { - ...output, - output: output.timeout - ? `${output.output}\n\nCommand exceeded timeout of ${finalTimeout} ms. Retry with a larger timeout if the command is expected to take longer.` - : output.output, - status: "completed" as const, - } - }).pipe( - Effect.tap((output) => Deferred.succeed(settled, output)), - Effect.map((output) => resultMessages(output).join("\n\n")), - Effect.onInterrupt(() => shell.remove(info.id).pipe(Effect.ignore)), - ) - const job = yield* jobs.start({ - // CodeMode children share a tool-call ID, but each shell must own its job. - id: info.id, - type: name, - title: info.command, - metadata: { sessionID: context.sessionID, shellID: info.id }, - recovery: { - kind: "shell", - sessionID: context.sessionID, - shellID: info.id, - command: info.command, - }, - run, - }) + const settled = yield* Deferred.make() + const run = completed(shell, info, finalTimeout).pipe( + Effect.tap((output) => Deferred.succeed(settled, output)), + Effect.map((output) => resultMessages(output).join("\n\n")), + Effect.onInterrupt(() => + leaveRunning(detachable) ? Effect.void : shell.stop(info.id, "killed").pipe(Effect.ignore), + ), + ) + const job = yield* startJob(jobs, { sessionID: context.sessionID, info, run }) - if (input.background === true) { - yield* jobs.background(job.id) - yield* notifyWhenDone(context.sessionID, job.id, info.id, info.command, settled) - return backgroundResult(info.id, info.file) - } + if (input.background === true) { + yield* jobs.background(job.id) + yield* notifyWhenDone(context.sessionID, job.id, info.id, info.command, settled) + return backgroundResult(info.id, info.file) + } - const result = yield* jobs - .block({ id: job.id, sessionID: context.sessionID }) - .pipe(Effect.onInterrupt(() => jobs.cancel(job.id).pipe(Effect.ignore))) - if (result?.type === "backgrounded") { - yield* shell.timeout(info.id, 0) - yield* notifyWhenDone(context.sessionID, job.id, info.id, info.command, settled) - return backgroundResult(info.id, info.file) - } - if (result?.info.status === "error") - return yield* Effect.fail(new Error(result.info.error ?? "Command failed")) - if (result?.info.status === "cancelled") return yield* Effect.fail(new Error("Command cancelled")) + const blocked = yield* restore(jobs.block({ id: job.id, sessionID: context.sessionID })).pipe( + Effect.exit, + ) + if (Exit.isFailure(blocked)) { + if (Cause.hasInterrupts(blocked.cause)) + return yield* interrupted(info, job.id, detachable, blocked.cause) + return yield* Effect.failCause(blocked.cause) + } + const result = blocked.value + if (result?.type === "backgrounded") { + yield* shell.timeout(info.id, 0) + yield* notifyWhenDone(context.sessionID, job.id, info.id, info.command, settled) + return backgroundResult(info.id, info.file) + } + if (result?.info.status === "error") + return yield* Effect.fail(new Error(result.info.error ?? "Command failed")) + if (result?.info.status === "cancelled") return yield* Effect.fail(new Error("Command cancelled")) - return yield* Deferred.await(settled) - }).pipe( + return yield* Deferred.await(settled) + }), + ).pipe( Effect.map(toolResult), Effect.mapError( (error) => new ToolFailure({ message: `Unable to execute command: ${input.command}`, error }), diff --git a/packages/core/test/detaching.test.ts b/packages/core/test/detaching.test.ts new file mode 100644 index 000000000000..1143668d7f7c --- /dev/null +++ b/packages/core/test/detaching.test.ts @@ -0,0 +1,38 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { Detaching } from "@opencode/core/detaching" +import { it } from "./lib/effect" + +describe("Detaching", () => { + it.effect("detaches only on a shutdown after a prepared handoff", () => + Effect.gen(function* () { + const quit = yield* Detaching.make + expect(yield* quit.begin).toBe(false) + const detaching = yield* Detaching.make + yield* detaching.handoffPrepared + expect(detaching.isDetaching()).toBe(false) + expect(yield* detaching.begin).toBe(true) + expect(detaching.isDetaching()).toBe(true) + }), + ) + + it.effect("does not detach once the prepared handoff was dropped", () => + Effect.gen(function* () { + const detaching = yield* Detaching.make + yield* detaching.handoffPrepared + yield* detaching.handoffDropped + expect(yield* detaching.begin).toBe(false) + expect(detaching.isStopping()).toBe(true) + }), + ) + + it.effect("starts stopping on any shutdown, detaching or not", () => + Effect.gen(function* () { + const detaching = yield* Detaching.make + expect(detaching.isStopping()).toBe(false) + expect(yield* detaching.begin).toBe(false) + expect(detaching.isStopping()).toBe(true) + expect(detaching.isDetaching()).toBe(false) + }), + ) +}) diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index c4a5e57794dc..605f5fb28c59 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -1,10 +1,11 @@ +import { appendFileSync, writeFileSync } from "node:fs" import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import net from "node:net" import os from "node:os" import path from "node:path" import { Effect } from "effect" -export const pong = { type: "pong", instance_id: "test", pid: process.pid, protocol: 7 } +export const pong = { type: "pong", instance_id: "test", pid: process.pid, protocol: 8, features: [] } export function temporaryDirectory() { return Effect.acquireRelease( @@ -22,14 +23,18 @@ export function listen( const sockets = new Set() const server = net.createServer((socket) => { sockets.add(socket) + // A client may close its end (an interrupted wait) just before a reply is written. + socket.on("error", () => {}) socket.once("close", () => sockets.delete(socket)) - void readRequest(socket) - .then((envelope) => { - const response = - envelope.request.op === "own" ? { type: "owned" } : handle(socket, envelope.request, envelope.token) + const serve = (): Promise => + readRequest(socket).then((envelope) => { + const owning = envelope.request.op === "own" + const response = owning ? { type: "owned" } : handle(socket, envelope.request, envelope.token) if (response) socket.write(frame(response)) + // The ownership connection stays open for later requests, such as handoff preparation. + if (owning) return serve() }) - .catch(() => socket.destroy()) + void serve().catch(() => socket.destroy()) }) await new Promise((resolve, reject) => { server.once("error", reject) @@ -81,7 +86,7 @@ function readRequest(socket: net.Socket) { }) } -function frame(value: object) { +export function frame(value: object) { const payload = Buffer.from(JSON.stringify(value)) const output = Buffer.allocUnsafe(payload.length + 4) output.writeUInt32BE(payload.length) @@ -89,70 +94,164 @@ function frame(value: object) { return output } -export function writeRegistration( - directory: string, - socket: string, - instance = "test", - token = "test", - features?: string[], -) { +export function writeRegistration(directory: string, socket: string, instance = "test", token = "test") { return writeFile( path.join(directory, "service.json"), - JSON.stringify({ instance_id: instance, pid: process.pid, protocol: 7, socket, token, features }), + JSON.stringify({ instance_id: instance, pid: process.pid, protocol: 8, socket, token, features: [] }), ) } -// An in-process opencode-pty that a server adopts through a handoff, so no daemon is spawned. Without -// features it answers like a daemon that predates the features list. -export const fakeDaemon = Effect.fn(function* (features?: string[]) { +type Process = { + id: number + mode: "pty" | "pipe" + pid: number | null + command: string[] + cwd: string + group_id: string + lifecycle: { status: "running" | "exited"; exit_code?: number | null; signal?: string } + output_head: number + output_tail: number + started_at: number + exited_at: number | null + pty?: { title: string; user_title: string | null; foreground_process: string | null; cols: number; rows: number } + pipe?: { output_path: string; output_lines: number; output_error?: string; metadata: Record } +} + +// An in-process opencode-pty that a server adopts through a handoff, so no daemon is spawned. Pipe processes +// run nothing: they stay running until `exit` or a signal ends them (SIGTERM spares those in `trap`), and +// write only what `output` gives them. +export const fakeDaemon = Effect.fn(function* () { const base = yield* temporaryDirectory() const root = path.join(base, "root") const directory = path.join(root, "inherited") const socket = path.join(base, "daemon.sock") - const terminals: Array & { user_title?: string | null }> = [] + const processes: Process[] = [] + const waiters = new Map() const failTerminate = new Set() + const trap = new Set() + const failCreate = { message: undefined as string | undefined } const ops: string[] = [] + const requests: Record[] = [] let nextID = 1 + + const find = (id: unknown) => processes.find((item) => item.id === id) + const exit = (id: number, exitCode: number | null, signal?: string) => { + const found = find(id) + if (!found || found.lifecycle.status !== "running") return + found.lifecycle = { status: "exited", exit_code: exitCode, signal } + found.exited_at = Date.now() + const reply = exited(found) + waiters.get(id)?.forEach((connection) => connection.end(frame(reply))) + waiters.delete(id) + } + + // Appends to the output file and counts it, as the daemon's reader does; after `fail`, the file stops growing. + const output = (id: number, text: string) => { + const found = find(id) + if (!found?.pipe) return + if (found.pipe.output_error === undefined) { + appendFileSync(found.pipe.output_path, text) + found.pipe.output_lines += text.split("\n").length - 1 + } + found.output_tail += Buffer.byteLength(text) + } + // The daemon going away under a waiting server: the wait connection closes without a reply. + const drop = (id: number) => { + waiters.get(id)?.forEach((connection) => connection.destroy()) + waiters.delete(id) + } + const fail = (id: number, message: string) => { + const found = find(id) + if (found?.pipe) found.pipe.output_error = message + } + yield* listen(socket, (connection, request) => { ops.push(String(request.op)) - if (request.op === "ping") return features ? { ...pong, features } : pong - if (request.op === "list") return { type: "terminals", terminals } - if (request.op === "create") { - const base = terminal(nextID++, String(request.group_id)) - // A daemon that advertises features always serializes user_title. - const created = features ? { ...base, user_title: null } : base - terminals.push(created) - return { type: "created", terminal: created } + requests.push(request) + if (request.op === "ping") return pong + if (request.op === "list") + return { type: "processes", processes: processes.filter((item) => item.mode === request.mode) } + if (request.op === "get") { + const found = find(request.id) + if (!found) return { type: "error", message: `terminal ${String(request.id)} not found` } + return { type: "process", process: found } + } + if (request.op === "create_pty") { + const created = terminal(nextID++, String(request.group_id)) + processes.push(created) + return { type: "created", process: created } + } + if (request.op === "create_pipe") { + if (typeof request.cwd !== "string" || !path.isAbsolute(request.cwd)) + return { type: "error", message: "cwd must be absolute" } + if (failCreate.message !== undefined) return { type: "error", message: failCreate.message } + const created = pipe(nextID++, request) + writeFileSync(String(request.output_path), "") + processes.push(created) + return { type: "created", process: created } } if (request.op === "control") return { type: "ok" } - if (request.op === "terminate") { - if (failTerminate.has(`pty_persistent_${request.id}`)) return { type: "error", message: "terminate failed" } - const index = terminals.findIndex((item) => item.id === request.id) - if (index === -1) return { type: "error", message: `terminal not found: ${String(request.id)}` } - terminals.splice(index, 1) + if (request.op === "prepare_handoff") return { type: "handoff", ticket: "next", expires_at: Date.now() + 60_000 } + if (request.op === "rename") { + const found = find(request.id) + if (!found?.pty) return { type: "error", message: "terminal not found" } + found.pty.user_title = typeof request.title === "string" && request.title !== "" ? request.title : null + found.pty.title = found.pty.user_title ?? "test" return { type: "ok" } } - if (request.op === "rename" && features?.includes("rename")) { - const found = terminals.find((item) => item.id === request.id) - if (!found) return { type: "error", message: "terminal not found" } - found.user_title = typeof request.title === "string" && request.title !== "" ? request.title : null - found.title = found.user_title ?? "test" + if (request.op === "wait") { + const found = find(request.id) + if (!found?.pipe) return { type: "error", message: `terminal ${String(request.id)} not found` } + if (found.lifecycle.status !== "running") return exited(found) + waiters.set(found.id, [...(waiters.get(found.id) ?? []), connection]) + connection.once("close", () => + waiters.set( + found.id, + (waiters.get(found.id) ?? []).filter((waiter) => waiter !== connection), + ), + ) + return undefined + } + if (request.op === "signal") { + const found = find(request.id) + if (!found?.pipe) return { type: "error", message: `terminal ${String(request.id)} not found` } + if (request.signal !== "SIGTERM" || !trap.has(found.id)) exit(found.id, null, String(request.signal)) return { type: "ok" } } - // Old daemons cannot parse unknown requests and drop the connection. - if (request.op === "rename") { - connection.destroy() - return undefined + if (request.op === "set_metadata") { + const found = find(request.id) + if (!found?.pipe || !isRecord(request.metadata)) + return { type: "error", message: `terminal ${String(request.id)} not found` } + found.pipe.metadata = request.metadata + return { type: "ok" } + } + if (request.op === "terminate") { + if (failTerminate.has(`pty_persistent_${request.id}`)) return { type: "error", message: "terminate failed" } + const index = processes.findIndex((item) => item.id === request.id) + if (index === -1) return { type: "error", message: `terminal ${String(request.id)} not found` } + const [removed] = processes.splice(index, 1) + waiters.get(removed.id)?.forEach((waiter) => waiter.end(frame({ type: "error", message: "terminated" }))) + waiters.delete(removed.id) + return { type: "ok" } } - return { type: "error", message: `unexpected request: ${String(request.op)}` } + return { type: "error", message: `unsupported request: ${String(request.op)}` } }) yield* Effect.promise(async () => { await mkdir(directory, { recursive: true }) - await writeRegistration(directory, socket, "test", "test", features) + await writeRegistration(directory, socket) }) return { failTerminate, + failCreate, + trap, ops, + requests, + exit, + output, + fail, + drop, + processes, + waiters: (id: number) => waiters.get(id)?.length ?? 0, options: { root, handoff: { directory, instanceID: "test", ticket: "ticket", expiresAt: Date.now() + 60_000 }, @@ -160,20 +259,53 @@ export const fakeDaemon = Effect.fn(function* (features?: string[]) { } }) -export function terminal(id: number, groupID = "ses_test") { +export function terminal(id: number, groupID = "ses_test"): Process { return { id, + mode: "pty", pid: null, - title: "test", - foreground_process: null, - group_id: groupID, command: ["test"], cwd: "/tmp", - cols: 80, - rows: 24, + group_id: groupID, + lifecycle: { status: "running" }, + output_head: 0, + output_tail: 0, + started_at: Date.now(), + exited_at: null, + pty: { title: "test", user_title: null, foreground_process: null, cols: 80, rows: 24 }, + } +} + +function pipe(id: number, request: Record): Process { + return { + id, + mode: "pipe", + pid: 1_000 + id, + command: [String(request.program), ...(Array.isArray(request.args) ? request.args.map(String) : [])], + cwd: String(request.cwd), + group_id: String(request.group_id), lifecycle: { status: "running" }, output_head: 0, output_tail: 0, + started_at: Date.now(), + exited_at: null, + pipe: { + output_path: String(request.output_path), + output_lines: 0, + metadata: isRecord(request.metadata) ? request.metadata : {}, + }, + } +} + +function exited(process: Process) { + return { + type: "exited", + exit_code: process.lifecycle.exit_code ?? null, + signal: process.lifecycle.signal, + final_offset: process.output_tail, + output_lines: process.pipe?.output_lines, + output_error: process.pipe?.output_error, + exited_at: process.exited_at ?? Date.now(), } } diff --git a/packages/core/test/fixture/persistent-pty.ts b/packages/core/test/fixture/persistent-pty.ts new file mode 100644 index 000000000000..c81232f00e08 --- /dev/null +++ b/packages/core/test/fixture/persistent-pty.ts @@ -0,0 +1,8 @@ +import { Effect, Layer } from "effect" +import { PersistentPty } from "@opencode/core/persistent-pty" + +// No process daemon: shell commands fall back to the Environment's spawner, and nothing is ever running in it. +export const disabledPersistentPtyLayer = Layer.mock(PersistentPty.Service, { + createPipe: () => Effect.fail(new PersistentPty.UnavailableError({ message: "process daemon disabled" })), + listPipes: () => Effect.succeed([]), +}) diff --git a/packages/core/test/fixture/shell-daemon.ts b/packages/core/test/fixture/shell-daemon.ts new file mode 100644 index 000000000000..fabb399779bb --- /dev/null +++ b/packages/core/test/fixture/shell-daemon.ts @@ -0,0 +1,122 @@ +import path from "node:path" +import { Context, Effect, Exit, Layer, Scope } from "effect" +import { ChildProcessSpawner } from "effect/process" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { AbsolutePath } from "@opencode/core/schema" +import { Shell } from "@opencode/core/shell" +import { Bus } from "@opencode/core/bus" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { LocationLifecycle } from "@opencode/core/location-lifecycle" +import { Global } from "@opencode/util/global" +import { hostEnvironmentLayer } from "./environment" +import { location } from "./location" +import { fakeDaemon } from "./persistent-pty-daemon" +import { tmpdirScoped } from "./tmpdir" + +// One fake daemon and data directory shared by every Location (and server) a test builds. +export const daemonShells = Effect.fn(function* () { + const daemon = yield* fakeDaemon() + const data = (yield* tmpdirScoped()).path + const global = Global.layerWith({ data, state: path.join(data, "state") }) + const context = yield* Layer.build( + AppNodeBuilder.build(PersistentPty.configured(daemon.options), [Global.node.replace(global)]), + ) + const pty = Context.get(context, PersistentPty.Service) + let spawned = 0 + const environment = Layer.effect( + Environment.Service, + Effect.gen(function* () { + const host = yield* Environment.Service + return Environment.Service.of({ + ...host, + spawner: ChildProcessSpawner.make(() => + Effect.sync(() => spawned++).pipe(Effect.andThen(Effect.die("the spawner must not be used"))), + ), + }) + }), + ).pipe(Layer.provide(hostEnvironmentLayer)) + // `server` shares one server's daemon commands between Locations; without it the Location gets a server of its own. + const shellLayer = (directory: string, server?: Shell.DaemonInterface) => + AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node]), [ + ...(server ? [Shell.daemonNode.replace(Layer.succeed(Shell.Daemon, server))] : []), + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(environment), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]) + // The server that `openLocation` Locations share, as long as the test runs. + const serverContext = yield* Layer.build( + AppNodeBuilder.build(LayerNode.group([Shell.daemonNode, Bus.node]), [ + Global.node.replace(global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]), + ) + const server = Context.get(serverContext, Shell.Daemon) + // A Location's Shell, on a server of its own, that stays open for the rest of the test. + const openShell = (directory: string) => + Effect.gen(function* () { + const context = yield* Layer.build(shellLayer(directory)) + return Context.get(context, Shell.Service) + }) + // A Location's Shell and its services, closed by `close` as a reload or an eviction does while the server runs. + const openLocation = (directory: string) => + Effect.gen(function* () { + const scope = yield* Scope.make() + const context = yield* Layer.buildWithScope(shellLayer(directory, server), scope) + return { + shell: Context.get(context, Shell.Service), + bus: Context.get(context, Bus.Service), + close: Context.get(context, LocationLifecycle.Service).shutdown.pipe( + Effect.andThen(Scope.close(scope, Exit.void)), + ), + } + }) + // Runs `body` against the Location's Shell on a server of its own, then stops that server, as a restart does. + const withShell = ( + directory: string, + body: (shell: Shell.Interface) => Effect.Effect, + ) => + Effect.scoped( + Effect.gen(function* () { + const shell = yield* Shell.Service + return yield* body(shell) + }).pipe(Effect.provide(shellLayer(directory))), + ) + const directory = (yield* tmpdirScoped()).path + return { + daemon, + data, + directory, + global, + pty, + withShell, + openShell, + openLocation, + // Where the server's daemon commands report. + serverBus: Context.get(serverContext, Bus.Service), + spawned: () => spawned, + created: () => daemon.requests.filter((request) => request.op === "create_pipe"), + signals: (id: number) => + daemon.requests.flatMap((request) => (request.op === "signal" && request.id === id ? [request.signal] : [])), + } +}) + +// Polls in real time until `check` holds: the fake daemon answers over a socket, which the test clock does not drive. +export const until = (check: (() => boolean) | Effect.Effect) => + Effect.gen(function* () { + for (let attempt = 0; attempt < 2_000; attempt++) { + if (typeof check === "function" ? check() : yield* check) return + yield* Effect.promise(() => Bun.sleep(2)) + } + return yield* Effect.die(new Error("condition never held")) + }) diff --git a/packages/core/test/job.test.ts b/packages/core/test/job.test.ts index 044159bf471a..17e350902525 100644 --- a/packages/core/test/job.test.ts +++ b/packages/core/test/job.test.ts @@ -1,5 +1,6 @@ import { describe, expect } from "bun:test" import { Database } from "@opencode/core/database/database" +import { Detaching } from "@opencode/core/detaching" import jobMigration from "@opencode/core/database/migration/20261030054400_job" import { Job } from "@opencode/core/job" import { JobTable } from "@opencode/core/job/sql" @@ -457,6 +458,79 @@ describe("Job", () => { }), ) + it.live("interrupts observers without settling a background job when its registry is torn down", () => + Effect.gen(function* () { + const scope = yield* Scope.make() + const jobs = yield* Job.make.pipe(Scope.provide(scope)) + const job = yield* jobs.start({ + id: "shell_teardown", + type: "shell", + recovery: { + kind: "shell", + sessionID: SessionSchema.ID.make("ses_teardown"), + shellID: "shell_teardown", + command: "sleep 60", + }, + run: Effect.never, + }) + const background = yield* jobs.background(job.id) + const observer = yield* jobs.wait({ id: job.id }).pipe(Effect.forkChild({ startImmediately: true })) + + yield* Scope.close(scope, Exit.void) + + const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second")) + expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true) + const current = yield* Job.make + expect(yield* current.pendingBackground).toMatchObject([ + { id: job.id, notificationID: background?.notificationID, status: "running" }, + ]) + if (background?.notificationID) yield* current.completeBackground(background.notificationID) + }), + ) + + for (const handoff of [true, false]) + it.live( + `leaves interrupted work unsettled on ${handoff ? "a planned restart" : "a normal quit"}, but still cancels on request`, + () => + Effect.gen(function* () { + const detaching = yield* Detaching.make + const jobs = yield* Job.make.pipe(Effect.provideService(Detaching.Service, detaching)) + const recovery = (id: string) => ({ + kind: "shell" as const, + sessionID: SessionSchema.ID.make("ses_detaching"), + shellID: id, + command: "sleep 60", + }) + const interrupt = yield* Deferred.make() + const detached = yield* jobs.start({ + id: "shell_detached", + type: "shell", + recovery: recovery("shell_detached"), + run: Deferred.await(interrupt).pipe(Effect.andThen(Effect.interrupt)), + }) + const cancelled = yield* jobs.start({ + id: "shell_cancelled", + type: "shell", + recovery: recovery("shell_cancelled"), + run: Effect.never, + }) + const background = yield* jobs.background(detached.id) + const observer = yield* jobs.wait({ id: detached.id }).pipe(Effect.forkChild({ startImmediately: true })) + + if (handoff) yield* detaching.handoffPrepared + expect(yield* detaching.begin).toBe(handoff) + expect(detaching.isStopping()).toBe(true) + yield* Deferred.succeed(interrupt, undefined) + + const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second")) + expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true) + expect(yield* jobs.pendingBackground).toMatchObject([{ id: detached.id, status: "running" }]) + expect(yield* jobs.cancel(cancelled.id)).toMatchObject({ status: "cancelled" }) + expect((yield* jobs.wait({ id: cancelled.id })).info?.status).toBe("cancelled") + if (background?.notificationID) yield* jobs.completeBackground(background.notificationID) + }), + ) + it.live("cancels running session jobs and removes unacknowledged SQLite job rows on cancelSession", () => Effect.gen(function* () { const jobs = yield* Job.Service diff --git a/packages/core/test/persistent-pty-daemon.test.ts b/packages/core/test/persistent-pty-daemon.test.ts index 70eb640737c6..f1f03a45cb03 100644 --- a/packages/core/test/persistent-pty-daemon.test.ts +++ b/packages/core/test/persistent-pty-daemon.test.ts @@ -1,11 +1,11 @@ import { expect } from "bun:test" import { spawn } from "node:child_process" -import { existsSync } from "node:fs" -import { mkdir, stat, writeFile } from "node:fs/promises" +import { mkdir, writeFile } from "node:fs/promises" +import net from "node:net" import path from "node:path" -import { Effect } from "effect" +import { Effect, Fiber } from "effect" import { makeDaemonTransport } from "../src/persistent-pty/daemon" -import { listen, pong, temporaryDirectory, terminal, writeRegistration } from "./fixture/persistent-pty-daemon" +import { frame, listen, pong, temporaryDirectory, terminal, writeRegistration } from "./fixture/persistent-pty-daemon" import { it } from "./lib/effect" it.live("rediscovers a same-protocol daemon after its registration rotates", () => @@ -19,7 +19,7 @@ it.live("rediscovers a same-protocol daemon after its registration rotates", () if (receivedToken !== token) return { type: "error", message: "authentication failed" } if (request.op === "ping") return { ...pong, instance_id: instance } if (request.op === "create") creates++ - return request.op === "list" ? { type: "terminals", terminals: [] } : { type: "ok" } + return request.op === "list" ? { type: "processes", processes: [] } : { type: "ok" } }) yield* Effect.promise(() => writeRegistration(directory, socketPath, instance, token)) const daemon = yield* makeDaemonTransport(root, "daemon") @@ -30,7 +30,7 @@ it.live("rediscovers a same-protocol daemon after its registration rotates", () yield* Effect.promise(() => writeRegistration(directory, socketPath, instance, token)) const running = yield* daemon.requestIfRunning({ op: "list" }) - expect(running).toEqual({ type: "terminals", terminals: [] }) + expect(running).toEqual({ type: "processes", processes: [] }) token = "newest-token" instance = "newest-instance" @@ -50,11 +50,11 @@ it.live("rediscovers a rotated registration when acquiring a subscription", () = yield* listen(socketPath, (_socket, request, receivedToken) => { if (receivedToken !== token) return { type: "error", message: "authentication failed" } if (request.op === "ping") return { ...pong, instance_id: instance } - if (request.op !== "subscribe") return { type: "terminals", terminals: [] } + if (request.op !== "subscribe") return { type: "processes", processes: [] } subscriptions++ return { type: "attached", - terminal: terminal(1), + process: terminal(1), role: "observer", generation: 1, requested_offset: 0, @@ -85,30 +85,75 @@ it.live("rediscovers a rotated registration when acquiring a subscription", () = }), ) -it.live("refreshes the daemon's features when its registration rotates", () => +it.live("waits on its own connection until the process exits", () => Effect.gen(function* () { const { root, directory } = yield* temporaryRuntime() const socketPath = path.join(directory, "daemon.sock") - let token = "old-token" - let instance = "old-instance" - let features: string[] | undefined - yield* listen(socketPath, (_socket, request, receivedToken) => { - if (receivedToken !== token) return { type: "error", message: "authentication failed" } - if (request.op === "ping") return { ...pong, instance_id: instance, ...(features ? { features } : {}) } - return { type: "terminals", terminals: [] } + const waiting: net.Socket[] = [] + yield* listen(socketPath, (socket, request) => { + if (request.op === "ping") return pong + if (request.op === "wait" && request.id === 1) { + waiting.push(socket) + return undefined + } + return { type: "error", message: "terminal 2 not found" } }) - yield* Effect.promise(() => writeRegistration(directory, socketPath, instance, token)) + yield* Effect.promise(() => writeRegistration(directory, socketPath)) const daemon = yield* makeDaemonTransport(root, "daemon") - expect(yield* daemon.supports("rename")).toBeFalse() - token = "new-token" - instance = "new-instance" - features = ["rename"] - yield* Effect.promise(() => writeRegistration(directory, socketPath, instance, token, features)) - // The cached registration is replaced once a request is rejected, as rename's lookup does first. - yield* daemon.request({ op: "list" }) + const missing = yield* Effect.flip(daemon.wait(2)) + expect(missing).toMatchObject({ kind: "request", message: "terminal 2 not found" }) + + const fiber = yield* daemon.wait(1).pipe(Effect.forkScoped({ startImmediately: true })) + yield* poll(() => waiting.length === 1) + waiting[0].end( + frame({ type: "exited", exit_code: null, signal: "SIGTERM", final_offset: 3, output_lines: 1, exited_at: 7 }), + ) + expect(yield* Fiber.join(fiber)).toEqual({ + type: "exited", + exit_code: null, + signal: "SIGTERM", + final_offset: 3, + output_lines: 1, + exited_at: 7, + }) + }), +) + +it.live("closes the wait connection when the wait is interrupted", () => + Effect.gen(function* () { + const { root, directory } = yield* temporaryRuntime() + const socketPath = path.join(directory, "daemon.sock") + const waiting: net.Socket[] = [] + yield* listen(socketPath, (socket, request) => { + if (request.op === "ping") return pong + waiting.push(socket) + return undefined + }) + yield* Effect.promise(() => writeRegistration(directory, socketPath)) + const daemon = yield* makeDaemonTransport(root, "daemon") + + const fiber = yield* daemon.wait(1).pipe(Effect.forkScoped({ startImmediately: true })) + yield* poll(() => waiting.length === 1) + const closed = new Promise((resolve) => waiting[0].once("close", () => resolve())) + yield* Fiber.interrupt(fiber) + yield* Effect.promise(() => closed) + }), +) - expect(yield* daemon.supports("rename")).toBeTrue() +it.live("reports a daemon that goes away during a wait as unreachable", () => + Effect.gen(function* () { + const { root, directory } = yield* temporaryRuntime() + const socketPath = path.join(directory, "daemon.sock") + yield* listen(socketPath, (socket, request) => { + if (request.op === "ping") return pong + socket.destroy() + return undefined + }) + yield* Effect.promise(() => writeRegistration(directory, socketPath)) + const daemon = yield* makeDaemonTransport(root, "daemon") + + expect((yield* Effect.flip(daemon.wait(1))).kind).toBe("connect") }), ) @@ -119,7 +164,7 @@ it.live("retries a start-required request when connection fails before dispatch" const secondSocket = path.join(directory, "second.sock") const first = yield* listen(firstSocket, (_socket, request) => { if (request.op === "ping") return pong - return { type: "terminals", terminals: [] } + return { type: "processes", processes: [] } }) yield* Effect.promise(() => writeRegistration(directory, firstSocket)) const daemon = yield* makeDaemonTransport(root, "daemon") @@ -175,7 +220,14 @@ it.live("rejects incompatible daemons without replacing or killing them", () => yield* Effect.promise(() => writeFile( path.join(directory, "service.json"), - JSON.stringify({ instance_id: "old", pid: existing.pid, protocol: 6, socket: "/unused", token: "old" }), + JSON.stringify({ + instance_id: "old", + pid: existing.pid, + protocol: 7, + socket: "/unused", + token: "old", + features: [], + }), ), ) const daemon = yield* makeDaemonTransport(root, "daemon", () => Promise.resolve("/missing/opencode-pty")) @@ -183,7 +235,7 @@ it.live("rejects incompatible daemons without replacing or killing them", () => const optional = yield* Effect.flip(daemon.requestIfRunning({ op: "list" })) expect(optional).toMatchObject({ kind: "protocol", - message: "opencode-pty protocol mismatch: daemon=6, client=7", + message: "opencode-pty protocol mismatch: daemon=7, client=8", pid: existing.pid, }) @@ -195,36 +247,6 @@ it.live("rejects incompatible daemons without replacing or killing them", () => }), ) -it.live("moves an inherited legacy registration into the runtime root and keeps its lock file", () => - Effect.gen(function* () { - const base = yield* temporaryDirectory() - const legacy = path.join(base, "legacy", "inherited") - const socketPath = path.join(base, "daemon.sock") - yield* listen(socketPath, (_socket, request) => { - if (request.op === "ping") return pong - return { type: "terminals", terminals: [] } - }) - yield* Effect.promise(async () => { - await mkdir(legacy, { recursive: true }) - await writeFile(path.join(legacy, "service.lock"), "") - await writeRegistration(legacy, socketPath) - }) - const lock = yield* Effect.promise(() => stat(path.join(legacy, "service.lock"))) - const daemon = yield* makeDaemonTransport(path.join(base, "root"), "fresh", undefined, { - directory: legacy, - instanceID: "test", - ticket: "ticket", - expiresAt: Date.now() + 60_000, - }) - - const migrated = path.join(base, "root", "inherited") - expect(existsSync(path.join(legacy, "service.json"))).toBeFalse() - expect((yield* Effect.promise(() => stat(path.join(migrated, "service.lock")))).ino).toBe(lock.ino) - expect(existsSync(path.join(migrated, "service.json"))).toBeTrue() - expect(yield* daemon.request({ op: "list" })).toEqual({ type: "terminals", terminals: [] }) - }), -) - it.live("starts fresh instead of failing when an inherited registration is gone or expired", () => Effect.gen(function* () { const base = yield* temporaryDirectory() @@ -264,6 +286,13 @@ it.live("stops a discovered inherited daemon when adoption fails", () => }), ) +function poll(ready: () => boolean) { + return Effect.gen(function* () { + for (let attempt = 0; attempt < 200 && !ready(); attempt++) yield* Effect.sleep(10) + expect(ready()).toBeTrue() + }) +} + function temporaryRuntime() { return Effect.gen(function* () { const root = yield* temporaryDirectory() @@ -272,3 +301,36 @@ function temporaryRuntime() { return { root, directory } }) } + +if (process.platform !== "win32") + it.live("gives up at once on a daemon binary that speaks another protocol, and never starts it again", () => + Effect.gen(function* () { + const root = yield* temporaryDirectory() + const runtime = path.join(root, "runtime") + const spawns = path.join(root, "spawns") + const binary = path.join(root, "old-opencode-pty") + // Registers like an older release would, then idles. + yield* Effect.promise(() => + writeFile( + binary, + [ + "#!/bin/sh", + `echo spawn >> ${JSON.stringify(spawns)}`, + 'mkdir -p "$3/$5"', + `printf '{"instance_id":"old","pid":%s,"protocol":7,"socket":"none","token":"t","features":[]}' $$ > "$3/$5/service.json"`, + "exec sleep 30", + ].join("\n"), + { mode: 0o755 }, + ), + ) + const daemon = yield* makeDaemonTransport(runtime, "daemon", () => Promise.resolve(binary)) + const started = Date.now() + const first = yield* daemon.request({ op: "list", mode: "pipe" }, true).pipe(Effect.flip) + expect(first.kind).toBe("protocol") + const second = yield* daemon.request({ op: "list", mode: "pipe" }, true).pipe(Effect.flip) + expect(second.kind).toBe("protocol") + expect(Date.now() - started).toBeLessThan(2_000) + const count = yield* Effect.promise(() => Bun.file(spawns).text()) + expect(count.trim().split("\n")).toHaveLength(1) + }), + ) diff --git a/packages/core/test/persistent-pty-session.test.ts b/packages/core/test/persistent-pty-session.test.ts index e154ebe85b5d..54e4d0ca2816 100644 --- a/packages/core/test/persistent-pty-session.test.ts +++ b/packages/core/test/persistent-pty-session.test.ts @@ -17,6 +17,7 @@ import { SessionStore } from "@opencode/core/session/store" import { SessionEnvironment } from "@opencode/core/session/environment" import { LocationServiceMap } from "@opencode/core/location-services" import { Removed, Updated } from "@opencode/schema/persistent-pty" +import { Pty } from "@opencode/schema/pty" import { fakeDaemon } from "./fixture/persistent-pty-daemon" import { it } from "./lib/effect" import { globalProjectNode } from "./lib/project" @@ -89,7 +90,7 @@ describe("PersistentPty session cleanup", () => { describe("PersistentPty rename", () => { it.live("renames a terminal, publishes the update and clears the user title", () => Effect.gen(function* () { - const daemon = yield* fakeDaemon(["rename"]) + const daemon = yield* fakeDaemon() yield* Effect.gen(function* () { const pty = yield* PersistentPty.Service const bus = yield* Bus.Service @@ -107,31 +108,115 @@ describe("PersistentPty rename", () => { }).pipe(Effect.provide(layer(daemon))) }), ) +}) + +describe("PersistentPty pipe processes", () => { + const input = { + program: "/bin/sh", + args: ["-c", "echo hi"], + cwd: "/tmp", + env: { PATH: "/usr/bin" }, + groupID: "proj_test", + outputPath: "/tmp/sh_test.out", + metadata: { id: "sh_test", command: "echo hi" }, + } + + it.live("creates, inspects, signals and waits for a pipe process", () => + Effect.gen(function* () { + const daemon = yield* fakeDaemon() + yield* Effect.gen(function* () { + const pty = yield* PersistentPty.Service + const created = yield* pty.createPipe(input) + expect(daemon.requests.at(-1)).toEqual({ + op: "create_pipe", + program: "/bin/sh", + args: ["-c", "echo hi"], + cwd: "/tmp", + env: { PATH: "/usr/bin" }, + group_id: "proj_test", + output_path: "/tmp/sh_test.out", + metadata: { id: "sh_test", command: "echo hi" }, + }) + expect(created).toMatchObject({ + groupID: "proj_test", + outputPath: "/tmp/sh_test.out", + lifecycle: { status: "running" }, + metadata: input.metadata, + exitedAt: undefined, + }) + + yield* pty.setPipeMetadata(created.id, { id: "sh_test", status: "killed" }) + expect((yield* pty.getPipe(created.id)).metadata).toEqual({ id: "sh_test", status: "killed" }) + expect((yield* pty.listPipes()).map((item) => item.id)).toEqual([created.id]) + + const waiting = yield* pty.waitPipe(created.id).pipe(Effect.forkScoped({ startImmediately: true })) + yield* pty.signalPipe(created.id, "SIGTERM") + expect(yield* Fiber.join(waiting)).toMatchObject({ exitCode: undefined, signal: "SIGTERM", outputLines: 0 }) + // A finished process answers a later wait at once and keeps its record until it is terminated. + expect((yield* pty.waitPipe(created.id)).signal).toBe("SIGTERM") + expect((yield* pty.getPipe(created.id)).lifecycle).toEqual({ + status: "exited", + exitCode: undefined, + signal: "SIGTERM", + }) + + yield* pty.terminatePipe(created.id) + expect(yield* Effect.flip(pty.getPipe(created.id))).toEqual( + new PersistentPty.ProcessNotFoundError({ processID: created.id }), + ) + expect(yield* pty.listPipes()).toEqual([]) + }).pipe(Effect.provide(layer(daemon))) + }), + ) - it.live("refuses to rename on a daemon without the rename feature and never sends it", () => + it.live("keeps pipe processes and terminals apart", () => Effect.gen(function* () { const daemon = yield* fakeDaemon() yield* Effect.gen(function* () { const pty = yield* PersistentPty.Service const directory = yield* tmpdirScoped() - const terminal = yield* pty.create(Session.ID.make("ses_rename"), { + const terminal = yield* pty.create(Session.ID.make("ses_pipe"), { args: [], cwd: directory.path, title: "test", env: {}, }) - expect(terminal.userTitle).toBeNull() + const pipe = yield* pty.createPipe(input) - const error = yield* Effect.flip(pty.rename(terminal.id, "build")) + expect((yield* pty.list()).map((item) => item.id)).toEqual([terminal.id]) + expect((yield* pty.listPipes()).map((item) => item.id)).toEqual([pipe.id]) + const ptyID = Pty.ID.make(`pty_persistent_${pipe.id}`) + expect(yield* Effect.flip(pty.get(ptyID))).toEqual(new PersistentPty.NotFoundError({ ptyID })) + }).pipe(Effect.provide(layer(daemon))) + }), + ) - expect(error).toEqual(new PersistentPty.UnsupportedError({ feature: "rename" })) - expect(daemon.ops).not.toContain("rename") - expect((yield* pty.get(terminal.id)).title).toBe("test") + it.live("reports daemon rejections as request errors and a terminated wait as one", () => + Effect.gen(function* () { + const daemon = yield* fakeDaemon() + yield* Effect.gen(function* () { + const pty = yield* PersistentPty.Service + expect(yield* Effect.flip(pty.createPipe({ ...input, cwd: "relative" }))).toEqual( + new PersistentPty.RequestError({ message: "cwd must be absolute" }), + ) + const created = yield* pty.createPipe(input) + const waiting = yield* pty.waitPipe(created.id).pipe(Effect.forkScoped({ startImmediately: true })) + yield* poll(() => daemon.waiters(created.id) === 1) + yield* pty.terminatePipe(created.id) + expect(yield* Fiber.join(waiting).pipe(Effect.flip)).toEqual( + new PersistentPty.RequestError({ message: "terminated" }), + ) }).pipe(Effect.provide(layer(daemon))) }), ) }) +const poll = (ready: () => boolean) => + Effect.gen(function* () { + for (let attempt = 0; attempt < 200 && !ready(); attempt++) yield* Effect.sleep(10) + expect(ready()).toBeTrue() + }) + const nextUpdated = (bus: Bus.Interface) => bus .subscribe(Updated) diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index edc90d4495cd..823f95fe7a94 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -22,6 +22,11 @@ import { SessionMessage } from "@opencode/core/session/message" import { SessionRunner } from "@opencode/core/session/runner/index" import { SessionInboxTable, SessionTable } from "@opencode/core/session/sql" import { SessionStore } from "@opencode/core/session/store" +import { Shell } from "@opencode/core/shell" +import { Agent } from "@opencode/core/agent" +import { ID } from "@opencode/core/model" +import { Provider } from "@opencode/core/provider" +import { daemonShells } from "./fixture/shell-daemon" import { Cause, Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect" import { eq } from "drizzle-orm" import { testEffect } from "./lib/effect" @@ -464,6 +469,279 @@ describe("SessionRestart background recovery", () => { }), ) + it.live("watches shell jobs whose commands survived the restart", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const parent = Session.ID.make("ses_background_shell_survived") + yield* seedSessions(database, [parent]) + const daemon = yield* daemonShells() + const before = yield* daemon.withShell("/project", (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running", cwd: "/tmp" }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished", cwd: "/tmp" }) + return { running, finished } + }), + ) + yield* seedBackground(jobs, parent, [ + { id: before.running.id, shellID: before.running.id, command: "running" }, + { id: before.finished.id, shellID: before.finished.id, command: "finished" }, + ]) + // The finished command ended while no server watched it. + daemon.daemon.output(2, "done\n") + daemon.daemon.exit(2, 0) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const notices = Effect.gen(function* () { + return (yield* store.context(parent)).filter( + (message) => message.type === "synthetic" && message.metadata?.source === "shell", + ) + }) + while ((yield* notices).length < 1) yield* Effect.sleep(5) + expect(yield* notices).toMatchObject([ + { + text: expect.stringContaining("done"), + metadata: { source: "shell", shellID: before.finished.id, state: "completed", exit: 0 }, + }, + ]) + expect((yield* restarted.get(before.running.id))?.status).toBe("running") + expect((yield* restarted.pendingBackground).map((item) => item.id)).toEqual([before.running.id]) + + daemon.daemon.exit(1, 2) + while ((yield* notices).length < 2) yield* Effect.sleep(5) + expect((yield* notices)[1]).toMatchObject({ + text: expect.stringContaining("Exited with code 2"), + metadata: { source: "shell", shellID: before.running.id, state: "completed", exit: 2 }, + }) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + + it.live("settles a shell call a killed server left running as moved to the background", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const parent = Session.ID.make("ses_background_shell_killed") + yield* seedSessions(database, [parent], { time_suspended: Date.now() }) + const assistantMessageID = SessionMessage.ID.create() + yield* bus.publish(SessionEvent.Step.Started, { + sessionID: parent, + assistantMessageID, + agent: Agent.ID.make("build"), + model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") }, + started: 0, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { + sessionID: parent, + assistantMessageID, + id: "call-shell", + name: "shell", + }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID: parent, + assistantMessageID, + id: "call-shell", + input: { command: "sleep 60" }, + executed: false, + }) + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "sleep 60", + cwd: "/tmp", + timeout: 120_000, + foreground: true, + toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, + }), + ) + // Its deadline passed while no server ran: taking it over must not time it out before settling the call. + const pipe = daemon.daemon.processes[0]?.pipe + if (pipe) pipe.metadata = { ...pipe.metadata, timeoutDeadline: Date.now() - 1 } + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const [assistant] = (yield* store.context(parent)).filter((message) => message.type === "assistant") + expect(assistant?.type === "assistant" ? assistant.content : []).toMatchObject([ + { + type: "tool", + id: "call-shell", + state: { + status: "completed", + content: [ + { + type: "text", + text: `Command moved to the background (shell ID: ${info.id}).\nOutput is streaming to: ${info.file}`, + }, + { type: "text", text: expect.stringContaining("You will be notified automatically") }, + ], + }, + }, + ]) + expect((yield* restarted.get(info.id))?.status).toBe("running") + expect(daemon.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(daemon.daemon.processes[0]?.pipe?.metadata).toMatchObject({ foreground: false }) + expect(daemon.signals(1)).toEqual([]) + + daemon.daemon.exit(1, 0) + const notices = Effect.gen(function* () { + return (yield* store.context(parent)).filter( + (message) => message.type === "synthetic" && message.metadata?.source === "shell", + ) + }) + while ((yield* notices).length < 1) yield* Effect.sleep(5) + expect((yield* notices)[0]).toMatchObject({ metadata: { source: "shell", shellID: info.id, state: "completed" } }) + }), + ) + + for (const exited of [false, true]) + it.live( + `reattaches once a ${exited ? "finished" : "running"} shell a killed server backgrounded without settling its call`, + () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const jobs = yield* Job.Service + const parent = Session.ID.make(`ses_background_shell_unsettled_${exited}`) + yield* seedSessions(database, [parent], { time_suspended: Date.now() }) + const assistantMessageID = yield* seedShellCall(bus, parent, "call-shell") + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "work", + cwd: "/tmp", + foreground: true, + toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, + }), + ) + // The old server persisted the background job, then died before the step recorded the call's result. + yield* seedBackground(jobs, parent, [{ id: info.id, shellID: info.id, command: "work" }]) + const [row] = yield* jobs.pendingBackground + if (exited) { + daemon.daemon.output(1, "finished output\n") + daemon.daemon.exit(1, 0) + } + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + expect(yield* shellCallState(store, parent)).toMatchObject({ status: "completed" }) + const notices = shellNotices(store, parent) + // A finished command's result is admitted during recovery, before any session resumes. + if (exited) + expect((yield* SessionInbox.list(database.db, parent)).length + (yield* notices).length).toBeGreaterThan(0) + if (!exited) { + expect((yield* restarted.pendingBackground).map((item) => item.notificationID)).toEqual([ + row?.notificationID, + ]) + daemon.daemon.exit(1, 0) + } + while ((yield* notices).length < 1) yield* Effect.sleep(5) + yield* Effect.sleep(50) + expect(yield* notices).toMatchObject([{ id: row?.notificationID, metadata: { shellID: info.id } }]) + if (exited) expect(yield* notices).toMatchObject([{ text: expect.stringContaining("finished output") }]) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + + it.live("settles a recovered child's unsettled shell call before the child resumes", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const jobs = yield* Job.Service + const parent = Session.ID.make("ses_background_shell_child_parent") + const child = Session.ID.make("ses_background_shell_child") + yield* seedSessions(database, [parent]) + yield* seedSessions(database, [child], { parent_id: parent, time_suspended: Date.now() }) + yield* jobs.start({ + id: child, + type: "subagent", + recovery: { + kind: "subagent", + parentSessionID: parent, + childSessionID: child, + agent: "explore", + description: "Inspect recovery", + }, + run: Effect.never, + }) + yield* jobs.background(child) + const assistantMessageID = yield* seedShellCall(bus, child, "call-child-shell") + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "child work", + cwd: "/tmp", + foreground: true, + toolCall: { sessionID: child, assistantMessageID, callID: "call-child-shell" }, + }), + ) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const seen = yield* Deferred.make() + const context = yield* buildExecution( + scope, + ({ sessionID }) => + sessionID === child + ? shellCallState(store, child).pipe(Effect.flatMap((state) => Deferred.succeed(seen, state))) + : Effect.void, + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + expect(yield* Deferred.await(seen).pipe(Effect.timeout("2 seconds"))).toMatchObject({ status: "completed" }) + expect((yield* restarted.get(info.id))?.status).toBe("running") + // Cancelling a reattached job stops its command. + yield* restarted.cancel(info.id) + while (daemon.signals(1).length === 0) yield* Effect.sleep(5) + expect(daemon.signals(1)).toEqual(["SIGTERM"]) + }), + ) + it.effect("preserves locally running background work", () => Effect.gen(function* () { const database = yield* Database.Service @@ -1378,6 +1656,7 @@ function buildExecution( drain: (input: Parameters[0]) => Effect.Effect, options?: SessionRestart.Options, overrideJobs?: Job.Interface, + shell?: Shell.Interface, ) { return Effect.gen(function* () { const database = yield* Database.Service @@ -1408,9 +1687,12 @@ function buildExecution( LocationServiceMap.Service, LayerMap.make( () => - // The local execution test only needs the Session runner from the Location graph. + // The local execution test only needs the Session runner (and the Shell, if given) from the Location graph. // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion - runner as unknown as Layer.Layer, + (shell ? Layer.merge(runner, Layer.succeed(Shell.Service, shell)) : runner) as unknown as Layer.Layer< + LocationServices, + FileSystem.DirectoryNotFoundError + >, ), ) return yield* Layer.buildWithScope( @@ -1430,3 +1712,49 @@ function buildExecution( ) }) } + +/** Records a shell call the model made that is still running, as a killed server leaves it. */ +function seedShellCall(bus: Bus.Interface, sessionID: Session.ID, callID: string) { + return Effect.gen(function* () { + const assistantMessageID = SessionMessage.ID.create() + yield* bus.publish(SessionEvent.Step.Started, { + sessionID, + assistantMessageID, + agent: Agent.ID.make("build"), + model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") }, + started: 0, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { sessionID, assistantMessageID, id: callID, name: "shell" }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID, + id: callID, + input: { command: "work" }, + executed: false, + }) + return assistantMessageID + }) +} + +function shellCallState(store: SessionStore.Interface, sessionID: Session.ID) { + return store.context(sessionID).pipe( + Effect.map((messages) => + messages.flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => (part.type === "tool" ? [part.state] : [])) + : [], + ), + ), + Effect.map((states) => states[0]), + ) +} + +function shellNotices(store: SessionStore.Interface, sessionID: Session.ID) { + return store + .context(sessionID) + .pipe( + Effect.map((messages) => + messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"), + ), + ) +} diff --git a/packages/core/test/session-owned.test.ts b/packages/core/test/session-owned.test.ts index 8d68b5b49cfb..19393e5c3891 100644 --- a/packages/core/test/session-owned.test.ts +++ b/packages/core/test/session-owned.test.ts @@ -125,7 +125,7 @@ const setup = Effect.fnUntraced(function* (options?: { const services = Layer.mergeAll( Layer.succeed(PluginHooks.Service, hooks), Layer.mock(Image.Service, {}), - options?.shell ?? Layer.mock(Shell.Service, {}), + options?.shell ?? Layer.mock(Shell.Service, { detachable: () => false }), ) const servicesFor = (ref: Location.Ref) => { locations.push(ref) @@ -508,6 +508,7 @@ describe("Session-owned handles", () => { }) const fixture = yield* setup({ shell: Layer.mock(Shell.Service, { + detachable: () => false, create: (input) => Effect.sync(() => { expect(input).toEqual({ diff --git a/packages/core/test/session-step.test.ts b/packages/core/test/session-step.test.ts index 513263e59007..532c136cd66f 100644 --- a/packages/core/test/session-step.test.ts +++ b/packages/core/test/session-step.test.ts @@ -18,10 +18,11 @@ import { SessionStep } from "@opencode/core/session/runner/step" import { SessionMessageTable, SessionTable } from "@opencode/core/session/sql" import { Snapshot } from "@opencode/core/snapshot" import { ToolOutput } from "@opencode/core/tool-output" +import { ToolInterrupted } from "@opencode/core/tool/interrupted" import { Money } from "@opencode/schema/money" import { LayerNode } from "@opencode/util/effect/layer-node" import { asc, eq } from "drizzle-orm" -import { Effect, Exit, Layer } from "effect" +import { Deferred, Effect, Exit, Fiber, Layer } from "effect" import { testEffect } from "./lib/effect" const it = testEffect( @@ -227,3 +228,85 @@ it.effect("records the cost the endpoint reports instead of the catalog price", expect(yield* db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get()).toMatchObject({ cost: 0 }) }), ) + +it.live("settles an interrupted tool with the result it reports and still ends interrupted", () => + Effect.gen(function* () { + const db = (yield* Database.Service).db + const llm = yield* TestLLM.Test + const sessionID = Session.ID.create() + const assistantMessageID = SessionMessage.ID.create() + const steps = yield* SessionStep.make.pipe( + Effect.provide( + Layer.mock(Snapshot.Service)({ + capture: () => Effect.succeed(undefined), + files: () => Effect.succeed([]), + }), + ), + ) + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + yield* db + .insert(SessionTable) + .values({ id: sessionID, project_id: Project.ID.global, slug: "step", directory: "/project", version: "test" }) + .run() + const model = SessionRunnerModel.resolved( + LanguageModel.make({ id: "test-model", provider: "test", route: OpenAIChat.route }), + { + capabilities: { tools: true, input: ["text"], output: ["text"] }, + limit: { context: 100_000, output: 1_000 }, + cost: [], + }, + ) + yield* llm.push( + TestLLM.complete( + { reason: { normalized: "tool-calls" }, usage: { inputTokens: 1, outputTokens: 1 } }, + LLMEvent.toolCall({ id: "call-interrupted", name: "test", input: {} }), + ), + ) + const started = yield* Deferred.make() + const attempt = yield* steps + .attempt({ + isLocationClosed: () => false, + sessionID, + assistantMessageID, + agent: Agent.defaultID, + model, + prepared: { + retry: () => Effect.void, + request: ai.request({ model: model.model, prompt: "Run one tool" }), + options: {}, + // Like the shell tool: finish the work uninterruptibly, then report the result. + executeTool: () => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* Deferred.succeed(started, undefined) + yield* restore(Effect.never).pipe(Effect.exit) + const result = { content: [{ type: "text" as const, text: "partial output" }] } + yield* ToolInterrupted.report(result) + return result + }), + ), + }, + retry: () => Effect.succeed({ retry: false }), + recoverContinuation: true, + recoverOverflow: Effect.succeed(false), + }) + .pipe(Effect.forkChild({ startImmediately: true })) + yield* Deferred.await(started) + yield* Fiber.interrupt(attempt) + const exit = yield* Fiber.await(attempt) + expect(Exit.hasInterrupts(exit)).toBe(true) + expect(yield* llm.requests()).toHaveLength(1) + const message = yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.id, assistantMessageID)) + .get() + expect(message?.data).toMatchObject({ + error: { type: "aborted", message: "Step interrupted" }, + content: [{ type: "tool", state: { status: "completed", content: [{ type: "text", text: "partial output" }] } }], + }) + }), +) diff --git a/packages/core/test/shell-cleanup.test.ts b/packages/core/test/shell-cleanup.test.ts index 9af9ca64af26..a807b69d8307 100644 --- a/packages/core/test/shell-cleanup.test.ts +++ b/packages/core/test/shell-cleanup.test.ts @@ -6,13 +6,16 @@ import { Shell } from "@opencode/core/shell" import { LayerNode } from "@opencode/util/effect/layer-node" import { FSUtil } from "@opencode/util/fs-util" import { Global } from "@opencode/util/global" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { withTempDir } from "./fixture/tmpdir" import { it } from "./lib/effect" const withStore = (body: (fs: FSUtil.Interface, root: string) => Effect.Effect) => withTempDir((tmp) => { - const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node]), [ + const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [ Global.node.replace(Global.layerWith({ data: tmp.path })), + PersistentPty.node.replace(disabledPersistentPtyLayer), ]) return Effect.gen(function* () { const fs = yield* FSUtil.Service diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts new file mode 100644 index 000000000000..35b6d32a9428 --- /dev/null +++ b/packages/core/test/shell-daemon.test.ts @@ -0,0 +1,513 @@ +import { describe, expect } from "bun:test" +import { existsSync, readdirSync, realpathSync } from "node:fs" +import { mkdir, rm, utimes, writeFile } from "node:fs/promises" +import path from "node:path" +import { Effect, Layer } from "effect" +import { TestClock } from "effect/testing" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { AbsolutePath } from "@opencode/core/schema" +import { Shell } from "@opencode/core/shell" +import { Global } from "@opencode/util/global" +import { FSUtil } from "@opencode/util/fs-util" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { AppProcess } from "@opencode/util/process" +import { hostEnvironmentLayer } from "./fixture/environment" +import { location } from "./fixture/location" +import { daemonShells, until } from "./fixture/shell-daemon" +import { tmpdirScoped } from "./fixture/tmpdir" +import { binaryPath } from "@opencode/pty" +import { it } from "./lib/effect" + +const unix = process.platform !== "win32" +const test = unix ? it.live : it.live.skip +// On the test clock, for timers. +const timed = unix ? it.effect : it.effect.skip + +const lines = (count: number) => Array.from({ length: count }, (_, index) => `line ${index + 1}\n`).join("") + +describe("Shell on the process daemon", () => { + test("runs a command with the server's environment, an absolute cwd and its recovery metadata", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* Effect.promise(() => mkdir(path.join(env.directory, "sub"))) + process.env.OPENCODE_TEST_REMOVED = "1" + yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create( + { + shell: "/bin/sh", + command: "echo hi", + cwd: "sub", + timeout: 60_000, + metadata: { sessionID: "ses_test" }, + toolCall: { sessionID: "ses_test", assistantMessageID: "msg_test", callID: "call_test" }, + }, + (invocation) => + Effect.sync(() => { + delete invocation.env.OPENCODE_TEST_REMOVED + invocation.env.OPENCODE_TEST_UNSET = undefined + }), + ) + const [request] = env.created() + expect(request).toMatchObject({ + program: "/bin/sh", + args: ["-c", "echo hi"], + cwd: path.join(env.directory, "sub"), + output_path: info.file, + metadata: { + id: info.id, + command: "echo hi", + shell: "/bin/sh", + location: { directory: env.directory }, + metadata: { sessionID: "ses_test" }, + toolCall: { sessionID: "ses_test", assistantMessageID: "msg_test", callID: "call_test" }, + timeoutDeadline: expect.any(Number), + }, + }) + expect(request?.env).toMatchObject({ TERM: "xterm-256color", OPENCODE_TERMINAL: "1" }) + expect(request?.env).not.toHaveProperty("OPENCODE_TEST_REMOVED") + expect(request?.env).not.toHaveProperty("OPENCODE_TEST_UNSET") + expect(info).toMatchObject({ status: "running", pid: 1001, cwd: path.join(env.directory, "sub") }) + expect(shell.detachable(info.id)).toBe(true) + expect(env.spawned()).toBe(0) + expect( + yield* shell.byToolCall({ sessionID: "ses_test", assistantMessageID: "msg_test", callID: "call_test" }), + ).toMatchObject({ id: info.id }) + + env.daemon.output(1, "first\nsecond") + expect(yield* shell.output(info.id)).toEqual({ + output: "first\nsecond", + cursor: 12, + size: 12, + truncated: false, + }) + expect(yield* shell.output(info.id, { cursor: 6, limit: 3 })).toEqual({ + output: "sec", + cursor: 9, + size: 12, + truncated: false, + }) + expect(yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).toEqual({ + output: "", + cursor: 12, + size: 12, + truncated: false, + }) + expect((yield* shell.list()).map((item) => item.id)).toEqual([info.id]) + + env.daemon.exit(1, 0) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "exited", exit: 0, time: { completed: expect.any(Number) } }, + capture: { output: "first\nsecond", truncated: false, lines: { shown: 2, total: 2 } }, + }) + expect(yield* shell.list()).toEqual([]) + }), + ) + })) + + test("counts lines like the spawner and reads only what reached the file after a write error", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const long = yield* shell.create({ shell: "/bin/sh", command: "long" }) + env.daemon.output(1, lines(3000)) + env.daemon.exit(1, 0) + const result = yield* shell.result(long) + expect(result.capture?.lines).toEqual({ shown: 2000, total: 3000 }) + expect(result.capture?.output).toEndWith( + `line 3000\n\n[showing lines 1001-3000 of 3000; full output saved to ${long.file}]`, + ) + + const failed = yield* shell.create({ shell: "/bin/sh", command: "failed" }) + env.daemon.output(2, "kept\n") + env.daemon.fail(2, "No space left on device") + env.daemon.output(2, "lost\n") + env.daemon.exit(2, 1) + expect(yield* shell.result(failed)).toMatchObject({ + info: { status: "exited", exit: 1 }, + capture: { + output: `kept\n\n\n[output file ${failed.file} stopped early: No space left on device]`, + lines: { shown: 1, total: 1 }, + }, + }) + }), + ) + })) + + test("reports a command lost with the daemon as killed, with what it wrote", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "work" }) + env.daemon.output(1, "partial\n") + yield* until(() => env.daemon.waiters(1) > 0) + env.daemon.drop(1) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "killed" }, + capture: { output: "partial\n", killed: "Command was killed: the process daemon stopped" }, + }) + }), + ) + })) + + test("stops what a failed command left in its group, and leaves a successful one's", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const succeeded = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 0" }) + const failed = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 3" }) + yield* until(() => env.daemon.waiters(1) > 0 && env.daemon.waiters(2) > 0) + env.daemon.exit(1, 0) + env.daemon.exit(2, 3) + expect(yield* shell.wait(failed.id)).toMatchObject({ status: "exited", exit: 3 }) + expect(yield* shell.wait(succeeded.id)).toMatchObject({ status: "exited", exit: 0 }) + expect(env.signals(2)).toEqual(["SIGTERM"]) + expect(env.signals(1)).toEqual([]) + }), + ) + })) + + test("fails a command the daemon rejected instead of spawning it", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.failCreate.message = "output path is a symlink" + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const error = yield* shell.create({ shell: "/bin/sh", command: "echo hi" }).pipe(Effect.flip) + expect(error).toBeInstanceOf(AppProcess.AppProcessError) + expect(env.spawned()).toBe(0) + expect(yield* shell.list()).toEqual([]) + }), + ) + })) + + // On the test clock: the timeout and the grace period pass without waiting for them. + timed("stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.trap.add(1) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "trap '' TERM; sleep 60", timeout: 10 }) + yield* TestClock.adjust("10 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect((yield* shell.get(info.id)).status).toBe("running") + // The grace period starts once the daemon acknowledged SIGTERM, so keep the clock moving until it ends. + yield* until(TestClock.adjust("1 second").pipe(Effect.map(() => env.signals(1).length > 1))) + expect((yield* shell.wait(info.id)).status).toBe("timeout") + expect(env.signals(1)).toEqual(["SIGTERM", "SIGKILL"]) + expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ stopReason: "timeout" }) + expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "timeout" }, + capture: { output: "(no output)" }, + }) + expect((yield* shell.result(info)).info).not.toHaveProperty("signal") + expect(env.daemon.ops).not.toContain("terminate") + expect(existsSync(info.file)).toBe(true) + }), + ) + }), + ) + + test("forgets a command in the daemon before deleting its file, on remove and on eviction", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "sleep 60" }) + yield* shell.remove(running.id) + expect(env.daemon.requests.filter((request) => request.op === "terminate")).toEqual([ + expect.objectContaining({ id: 1 }), + ]) + expect(existsSync(running.file)).toBe(false) + expect(env.daemon.processes).toEqual([]) + expect(yield* shell.result(running)).toMatchObject({ info: { status: "killed" }, capture: undefined }) + + const finished = yield* Effect.forEach(Array.from({ length: 26 }), (_, index) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: `echo ${index}` }) + env.daemon.exit(index + 2, 0) + yield* shell.wait(info.id) + return info + }), + ) + // Waiters resolve before retention runs, which forgets the process and then deletes its file. + yield* until(() => !existsSync(finished[0]!.file)) + expect(yield* shell.get(finished[0]!.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + expect(existsSync(finished[0]!.file)).toBe(false) + expect(env.daemon.processes.map((process) => process.id)).not.toContain(2) + expect(existsSync(finished[1]!.file)).toBe(true) + }), + ) + })) + + test("takes its commands back after a restart and leaves other Locations' alone", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.trap.add(4) + const before = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running", metadata: { tag: "a" } }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) + env.daemon.output(2, "done\n") + env.daemon.exit(2, 3) + const exited = yield* shell.wait(finished.id) + const expiring = yield* shell.create({ shell: "/bin/sh", command: "expiring", timeout: 100 }) + const stopping = yield* shell.create({ shell: "/bin/sh", command: "stopping" }) + yield* shell.stop(stopping.id, "killed") + yield* until(() => env.signals(4).length > 0) + return { running, exited, expiring, stopping } + }), + ) + // Closing the Location only detached from its commands. + expect(env.daemon.ops).not.toContain("terminate") + expect(env.signals(1)).toEqual([]) + expect(env.signals(4)).toEqual(["SIGTERM"]) + // The expiring command's deadline passes while no server runs. + const expiring = env.daemon.processes.find((process) => process.id === 3)?.pipe + if (expiring) expiring.metadata = { ...expiring.metadata, timeoutDeadline: Date.now() - 1 } + + const other = (yield* tmpdirScoped()).path + yield* env.withShell(other, (shell) => + Effect.gen(function* () { + expect(yield* shell.get(before.running.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + expect(yield* shell.list()).toEqual([]) + }), + ) + + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + expect(yield* shell.get(before.running.id)).toMatchObject({ + status: "running", + command: "running", + shell: "/bin/sh", + file: before.running.file, + metadata: { tag: "a" }, + pid: before.running.pid, + }) + expect(yield* shell.get(before.exited.id)).toEqual({ ...before.exited, cwd: env.directory }) + expect((yield* shell.result(before.exited)).capture).toMatchObject({ output: "done\n", lines: { total: 1 } }) + // The deadline passed during the restart, and the interrupted stop goes straight to SIGKILL. + expect((yield* shell.wait(before.expiring.id)).status).toBe("timeout") + expect(env.signals(3)).toEqual(["SIGTERM"]) + expect((yield* shell.wait(before.stopping.id)).status).toBe("killed") + expect(env.signals(4)).toEqual(["SIGTERM", "SIGKILL"]) + + env.daemon.exit(1, 0) + expect(yield* shell.wait(before.running.id)).toMatchObject({ status: "exited", exit: 0 }) + }), + ) + })) + + // A reload or an eviction closes a Location while the server keeps running. + timed("a Location closed while the server runs keeps enforcing its commands' timeouts", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const closing = yield* env.openLocation(env.directory) + const info = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 50 }) + yield* closing.close + expect(env.signals(1)).toEqual([]) + yield* TestClock.adjust("50 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect(yield* closing.shell.wait(info.id)).toMatchObject({ status: "timeout" }) + }), + ) + + timed("a closed Location and its replacement see the same commands, with one timer and one report each", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const closing = yield* env.openLocation(env.directory) + const backgrounded = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 }) + const stopped = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) + const removed = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) + yield* closing.close + let exits = 0 + yield* env.serverBus.listen((event) => + Effect.sync(() => { + if (event.type === "shell.exited") exits++ + }), + ) + const replacement = yield* env.openLocation(env.directory) + expect((yield* replacement.shell.list()).map((item) => item.id)).toEqual([ + backgrounded.id, + stopped.id, + removed.id, + ]) + + // The call that started it, still in the closed Location, moves it to the background and clears its timeout. + yield* closing.shell.timeout(backgrounded.id, 0) + yield* TestClock.adjust("100 millis") + expect(env.signals(1)).toEqual([]) + + // Cancelling its job in the closed Location stops it; both see why it ended, and it is reported once. + yield* closing.shell.stop(stopped.id, "killed") + expect(yield* replacement.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) + expect(yield* closing.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) + yield* until(() => exits === 1) + + // Removed through one, it is gone from both. + yield* replacement.shell.remove(removed.id) + expect(yield* closing.shell.get(removed.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + // The server watched them all along: the replacement took nothing back and waited on nothing more. + expect(env.daemon.ops.filter((op) => op === "wait")).toHaveLength(3) + yield* replacement.close + }), + ) + + timed("a server that takes over a foreground command times out only what moved to the background", () => + Effect.gen(function* () { + const env = yield* daemonShells() + // The old server dies without a shutdown: one call still waits on its command, another backgrounded its own. + const before = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const waited = yield* shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100, foreground: true }) + const backgrounded = yield* shell.create({ + shell: "/bin/sh", + command: "sleep 60", + timeout: 100, + foreground: true, + }) + yield* shell.timeout(backgrounded.id, 30_000) + yield* shell.timeout(backgrounded.id, 0) + yield* shell.timeout(backgrounded.id, 100) + return { waited, backgrounded } + }), + ) + expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ + foreground: true, + timeoutDeadline: expect.any(Number), + }) + expect(env.daemon.processes[1]?.pipe?.metadata).toMatchObject({ foreground: false }) + // Both deadlines pass while no server runs. + for (const process of env.daemon.processes) + if (process.pipe) process.pipe.metadata = { ...process.pipe.metadata, timeoutDeadline: Date.now() - 1 } + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + yield* until(() => env.signals(2).length > 0) + expect((yield* shell.wait(before.backgrounded.id)).status).toBe("timeout") + // The restart moves the waited-on command to the background, so its old timeout no longer applies. + yield* TestClock.adjust("1 hour") + expect((yield* shell.get(before.waited.id)).status).toBe("running") + expect(env.signals(1)).toEqual([]) + }), + ) + }), + ) + + test("the sweep keeps a running command's file and forgets a finished one before deleting its file", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const ids = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running" }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) + env.daemon.exit(2, 0) + yield* shell.wait(finished.id) + const lostFile = yield* shell.create({ shell: "/bin/sh", command: "lost file" }) + env.daemon.exit(3, 0) + yield* shell.wait(lostFile.id) + const old = yield* shell.create({ shell: "/bin/sh", command: "old" }) + env.daemon.exit(4, 0) + yield* shell.wait(old.id) + return { running, finished, lostFile, old } + }), + ) + // A finished process whose file was deleted elsewhere, and one that ended more than 7 days ago. + yield* Effect.promise(() => rm(ids.lostFile.file)) + const ended = env.daemon.processes.find((process) => process.id === 4) + if (ended) ended.exited_at = Date.now() - 8 * 24 * 60 * 60 * 1_000 + const expired = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000) + yield* Effect.promise(() => utimes(ids.running.file, expired, expired)) + yield* Effect.promise(() => utimes(ids.finished.file, expired, expired)) + const orphan = path.join(path.dirname(ids.running.file), "sh_0123456789abORPHANORPHANOR.out") + yield* Effect.promise(() => writeFile(orphan, "orphan")) + yield* Effect.promise(() => utimes(orphan, expired, expired)) + + yield* Shell.cleanup().pipe( + Effect.provide( + AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [ + Global.node.replace(env.global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, env.pty)), + ]), + ), + ) + + expect(existsSync(ids.running.file)).toBe(true) + expect(existsSync(ids.finished.file)).toBe(false) + expect(existsSync(orphan)).toBe(false) + expect(env.daemon.processes.map((process) => process.id)).toEqual([1]) + // The old process's file is recent, so it stays. + expect(existsSync(ids.old.file)).toBe(true) + })) +}) + +// Against the real opencode-pty, like server/test/persistent-pty.test.ts. +const binary = unix ? (process.env.OPENCODE_PTY_BIN ?? binaryPath) : undefined +const real = binary !== undefined && existsSync(binary) ? it.live : it.live.skip + +describe("Shell on a real process daemon", () => { + real( + "starts the daemon on the first command, never on boot or sweep, and runs it with the server's environment", + () => + Effect.gen(function* () { + const data = (yield* tmpdirScoped()).path + const directory = (yield* tmpdirScoped()).path + const state = path.join(data, "state") + const global = Global.layerWith({ data, state }) + const started = () => existsSync(path.join(state, "pty")) && readdirSync(path.join(state, "pty")).length > 0 + process.env.OPENCODE_TEST_REMOVED = "1" + yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) + yield* Effect.gen(function* () { + const shell = yield* Shell.Service + yield* Shell.cleanup() + expect(started()).toBe(false) + const info = yield* shell.create( + { + shell: "/bin/sh", + command: 'printf "%s|%s|%s\\n" "${OPENCODE_TEST_REMOVED:-unset}" "$TERM" "$(pwd)"; printf tail', + }, + (invocation) => + Effect.sync(() => { + delete invocation.env.OPENCODE_TEST_REMOVED + }), + ) + expect(started()).toBe(true) + expect(shell.detachable(info.id)).toBe(true) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "exited", exit: 0 }, + capture: { + output: `unset|xterm-256color|${realpathSync(directory)}\ntail`, + lines: { shown: 2, total: 2 }, + }, + }) + }).pipe( + Effect.scoped, + Effect.provide( + AppNodeBuilder.build(LayerNode.group([Shell.node, FSUtil.node, Global.node, PersistentPty.node]), [ + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(hostEnvironmentLayer), + ]), + ), + ) + }), + { timeout: 20_000 }, + ) +}) diff --git a/packages/core/test/shell-result.test.ts b/packages/core/test/shell-result.test.ts new file mode 100644 index 000000000000..a9ba2d6706e9 --- /dev/null +++ b/packages/core/test/shell-result.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from "bun:test" +import { ShellResult } from "@opencode/core/shell/result" +import { Shell } from "@opencode/schema/shell" + +const info = (status: Shell.Status, extra: Partial = {}): Shell.Info => ({ + id: Shell.ID.make("sh_result"), + status, + command: "work", + cwd: "/tmp", + shell: "/bin/sh", + file: "/tmp/sh_result.out", + metadata: {}, + time: { started: 0, completed: 1 }, + ...extra, +}) +const capture = { output: "partial\n", truncated: false, lines: { shown: 1, total: 1 } } + +describe("ShellResult", () => { + test("reports a command killed without a signal of its own", () => { + const killed = ShellResult.output({ info: info("killed"), capture }) + expect(killed).toMatchObject({ killed: "Command was killed" }) + expect(ShellResult.notice(killed)).toBe("Command was killed") + expect(ShellResult.metadata(killed)).toMatchObject({ killed: true }) + + const lost = ShellResult.output({ + info: info("killed"), + capture: { ...capture, killed: "Command was killed: the process daemon stopped" }, + }) + expect(ShellResult.notice(lost)).toBe("Command was killed: the process daemon stopped") + }) + + test("keeps a removed command's result unchanged", () => { + const removed = ShellResult.output({ info: info("killed"), capture: undefined }) + expect(removed).toEqual({ output: ShellResult.unavailable.output, truncated: false }) + expect(ShellResult.notice(removed)).toBeUndefined() + }) + + test("an interrupted result says it was interrupted, not also that it was killed", () => { + const result = ShellResult.interrupted({ info: info("killed"), capture }) + expect(result).not.toHaveProperty("killed") + expect(ShellResult.notice(result)).toBeUndefined() + expect(result.output).toBe( + "partial\n\n[showing lines 1-1 of 1; full output saved to /tmp/sh_result.out]\n\nCommand interrupted by the user (shell ID: sh_result).", + ) + }) + + test("an interrupted command without output says so once", () => { + const result = ShellResult.interrupted({ + info: info("killed"), + capture: { output: "(no output)", truncated: false, lines: { shown: 0, total: 0 } }, + }) + expect(result.output).toBe( + "[no output; output file: /tmp/sh_result.out]\n\nCommand interrupted by the user (shell ID: sh_result).", + ) + }) +}) diff --git a/packages/core/test/shell-retention.test.ts b/packages/core/test/shell-retention.test.ts index 3f776e4bb275..e650568496d1 100644 --- a/packages/core/test/shell-retention.test.ts +++ b/packages/core/test/shell-retention.test.ts @@ -10,6 +10,8 @@ import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" import { Workspace } from "@opencode/core/workspace" import { Global } from "@opencode/util/global" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { hostEnvironmentLayer } from "./fixture/environment" import { tempGlobalLayer } from "./fixture/global" import { location, tempLocationLayer } from "./fixture/location" @@ -95,6 +97,7 @@ it.live("eviction makes progress past an already-removed shell", () => Global.node.replace(tempGlobalLayer), Config.node.replace(Config.testLayer()), Environment.node.replace(environment), + PersistentPty.node.replace(disabledPersistentPtyLayer), ]), ), ) diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index e4879804c580..ad1e4d353d3f 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -18,10 +18,12 @@ import { Global } from "@opencode/util/global" import { Location } from "@opencode/core/location" import { FileAccess } from "@opencode/core/file-access" import { LocationServiceMap } from "@opencode/core/location-service-map" +import { LocationLifecycle } from "@opencode/core/location-lifecycle" import { Model } from "@opencode/core/model" import { Provider } from "@opencode/core/provider" import { AbsolutePath } from "@opencode/core/schema" import { Agent } from "@opencode/core/agent" +import { Detaching } from "@opencode/core/detaching" import { Job } from "@opencode/core/job" import { Session } from "@opencode/core/session" import { SessionEvent } from "@opencode/core/session/event" @@ -36,8 +38,12 @@ import { Shell } from "@opencode/core/shell" import { ShellSelect } from "@opencode/core/shell/select" import { ID } from "@opencode/schema/shell" import { ShellTool } from "@opencode/core/tool/plugin/shell" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" +import { fakeDaemon } from "./fixture/persistent-pty-daemon" import { ToolOutput } from "@opencode/core/tool-output" import { Tool } from "@opencode/core/tool" +import { ToolInterrupted } from "@opencode/core/tool/interrupted" import { tmpdir, tmpdirScoped, withTempDir } from "./fixture/tmpdir" import { tempGlobalLayer } from "./fixture/global" import { offlineModels } from "./fixture/models" @@ -135,16 +141,21 @@ const shellPluginSupervisor = makeLocationNode({ FileAccess.node, Permission.node, Session.node, + Detaching.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], }) const nodes = LayerNode.group([ + PersistentPty.node, + Shell.daemonNode, Database.node, Bus.node, + Detaching.node, Job.node, Session.node, SessionExecution.node, @@ -153,25 +164,92 @@ const nodes = LayerNode.group([ FSUtil.node, Global.node, ]) -const replacements = [ +const daemonReplacements = [ SessionExecution.node.replace(executionNode), Permission.node.replace(permission), Global.node.replace(tempGlobalLayer), offlineModels, ] satisfies LayerNode.Replacements +// Most suites spawn commands from the server, so CI starts no daemon for them; `parity` runs a subset on both. +const replacements = [ + ...daemonReplacements, + PersistentPty.node.replace(disabledPersistentPtyLayer), +] satisfies LayerNode.Replacements const productionIt = testEffect(AppNodeBuilder.build(nodes, replacements)) const it = testEffect( AppNodeBuilder.build(nodes, [...replacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), ) +// On the real process daemon (when its binary is compatible; otherwise this falls back to spawning). +const daemonIt = testEffect( + AppNodeBuilder.build(nodes, [...daemonReplacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), +) +const parity = { + live: (name: string, body: Parameters[1], options?: Parameters[2]) => { + it.live(`${name} (spawned)`, body, options) + daemonIt.live(`${name} (daemon)`, body, options) + }, + skip: (name: string, body: Parameters[1], options?: Parameters[2]) => + it.live.skip(name, body, options), +} +// On an in-process fake daemon, so daemon-only behavior is covered without the real binary. +let fake: Effect.Success> | undefined +const fakeDaemonIt = testEffect( + AppNodeBuilder.build(nodes, [ + ...daemonReplacements, + PluginSupervisor.node.replace(shellPluginSupervisor), + PersistentPty.node.replace( + makeGlobalNode({ + service: PersistentPty.Service, + layer: Layer.unwrap( + fakeDaemon().pipe( + Effect.map((daemon) => { + fake = daemon + return PersistentPty.layerWith(daemon.options) + }), + ), + ), + deps: [Bus.node, Global.node, Detaching.node], + }), + ), + ]), +) + const permissionIt = testEffect( AppNodeBuilder.build(LayerNode.group([nodes, PermissionSaved.node]), [ SessionExecution.node.replace(executionNode), Global.node.replace(tempGlobalLayer), PluginSupervisor.node.replace(shellPluginSupervisor), + PersistentPty.node.replace(disabledPersistentPtyLayer), offlineModels, ]), ) +// Stands in for a daemon-backed shell: every command reports that it can outlive the server. +const detachableShellSupervisor = makeLocationNode({ + name: "test/shell-plugins-detachable", + layer: Layer.effectDiscard( + registerToolPlugin(ShellTool.Plugin).pipe( + Effect.updateService(Shell.Service, (shell) => Shell.Service.of({ ...shell, detachable: () => true })), + ), + ), + deps: [ + Config.node, + Environment.node, + FileAccess.node, + Permission.node, + Session.node, + Detaching.node, + Job.node, + Shell.node, + Shell.daemonNode, + ShellSelect.node, + Tool.node, + ], +}) +const detachableIt = testEffect( + AppNodeBuilder.build(nodes, [...replacements, PluginSupervisor.node.replace(detachableShellSupervisor)]), +) + const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({ sessionID, ...toolIdentity, @@ -179,6 +257,19 @@ const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({ }) const isWindows = process.platform === "win32" + +// The process daemon keeps a finished command's zombie until the command is forgotten. +const running = (pid: number) => { + try { + process.kill(pid, 0) + } catch { + return false + } + return !Bun.spawnSync(["ps", "-o", "stat=", "-p", String(pid)]) + .stdout.toString() + .trim() + .startsWith("Z") +} const cwdCommand = isWindows ? "(Get-Location).Path; Start-Sleep -Milliseconds 100" : "pwd" const helloCommand = isWindows ? "[Console]::Out.Write('hello'); Start-Sleep -Milliseconds 100" : "printf hello" const stderrCommand = isWindows @@ -1170,8 +1261,10 @@ describe("ShellTool", () => { FileAccess.node, Permission.node, Session.node, + Detaching.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], @@ -1241,7 +1334,7 @@ describe("ShellTool", () => { ) } - it.live("keeps non-zero exits useful", () => + parity.live("keeps non-zero exits useful", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1283,7 +1376,7 @@ describe("ShellTool", () => { { timeout: 15_000 }, ) - it.live("uses configured line limits", () => + parity.live("uses configured line limits", () => withTempDir((tmp) => { reset() return Effect.gen(function* () { @@ -1354,7 +1447,7 @@ describe("ShellTool", () => { { timeout: 15_000 }, ) - it.live("returns the shell id for a background command", () => + parity.live("returns the shell id for a background command", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1402,6 +1495,114 @@ describe("ShellTool", () => { }), ) + fakeDaemonIt.live("keeps a background daemon command's job across a Location reload and reports how it ended", () => + withTempDir((tmp) => + Effect.gen(function* () { + reset() + const sessions = yield* Session.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const locations = yield* LocationServiceMap.Service + const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) }) + yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel }) + const admitted = yield* bus.subscribe(SessionEvent.InboxEnqueued).pipe( + Stream.filter((event) => event.data.sessionID === sessionID && event.data.item.type === "synthetic"), + Stream.take(2), + Stream.runCollect, + Effect.forkScoped({ startImmediately: true }), + ) + const started = yield* Effect.gen(function* () { + const plugins = yield* Plugin.Service + yield* plugins.awaitActivation + const registry = yield* Tool.Service + const shell = yield* Shell.Service + const run = (id: string) => + Effect.gen(function* () { + const settled = yield* executeTool(registry, call({ command: "work", background: true }, id)) + return yield* shell.get(ID.make(String(settled.metadata?.shellID))) + }) + return { finishing: yield* run("call-finishing"), cancelled: yield* run("call-cancelled") } + }).pipe(Effect.provide(locations.get(location))) + const process = (info: { file: string }) => { + const found = fake?.processes.find((item) => item.pipe?.output_path === info.file) + if (!found) throw new Error("no daemon process for the command") + return found.id + } + + // A reload closes the Location and boots its replacement, which takes the commands back. + yield* locations.invalidate(location) + const replacement = yield* Effect.gen(function* () { + return yield* Shell.Service + }).pipe(Effect.provide(locations.get(location))) + expect((yield* replacement.list()).map((info) => info.id).toSorted()).toEqual( + [started.finishing.id, started.cancelled.id].toSorted(), + ) + expect((yield* jobs.get(started.finishing.id))?.status).toBe("running") + + // Cancelling the session's job still stops its command. + yield* jobs.cancel(started.cancelled.id) + while (!fake?.ops.includes("signal")) yield* Effect.promise(() => Bun.sleep(2)) + expect(fake?.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: process(started.cancelled), signal: "SIGTERM" }), + ) + + fake?.output(process(started.finishing), "done\n") + fake?.exit(process(started.finishing), 0) + const notices = Array.from(yield* Fiber.join(admitted)).map((event) => event.data.item.payload) + expect(notices).toContainEqual( + expect.objectContaining({ + text: expect.stringContaining("done"), + metadata: expect.objectContaining({ shellID: started.finishing.id, state: "completed", exit: 0 }), + }), + ) + expect(notices).toContainEqual( + expect.objectContaining({ + metadata: expect.objectContaining({ shellID: started.cancelled.id, state: "cancelled" }), + }), + ) + expect((yield* jobs.get(started.finishing.id))?.status).toBe("completed") + expect((yield* replacement.get(started.finishing.id)).status).toBe("exited") + yield* locations.invalidate(location) + }), + ), + ) + + it.live("leaves a background job running for the next server, without a notice, on a normal quit", () => + withTempDir((tmp) => + Effect.gen(function* () { + reset() + const sessions = yield* Session.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const detaching = yield* Detaching.Service + const locations = yield* LocationServiceMap.Service + const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) }) + yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel }) + const notices: unknown[] = [] + yield* bus.listen((event) => + Effect.sync(() => { + if (event.type === SessionEvent.InboxEnqueued.type) notices.push(event) + }), + ) + const shellID = yield* Effect.gen(function* () { + const plugins = yield* Plugin.Service + yield* plugins.awaitActivation + const registry = yield* Tool.Service + const settled = yield* executeTool(registry, call({ command: "sleep 60", background: true }, "call-quit")) + return String(settled.metadata?.shellID) + }).pipe(Effect.provide(locations.get(location))) + // The server quits without a handoff: shutdown starts, then the Location closes. + expect(yield* detaching.begin).toBe(false) + yield* locations.invalidate(location) + // The job's observers are interrupted, not told it was cancelled. + expect(Exit.hasInterrupts(yield* jobs.wait({ id: shellID }).pipe(Effect.exit))).toBe(true) + yield* Effect.yieldNow + expect((yield* jobs.pendingBackground).map((item) => [item.id, item.status])).toEqual([[shellID, "running"]]) + expect(notices).toEqual([]) + }), + ), + ) + it.live("preserves a background command's non-zero exit", () => withTempDir((tmp) => { reset() @@ -1512,7 +1713,8 @@ describe("ShellTool", () => { ) const signalTest = isWindows ? it.live.skip : it.live - signalTest( + const signalParity = isWindows ? parity.skip : parity.live + signalParity( "escalates a shell timeout when the ready process ignores SIGTERM", () => Effect.gen(function* () { @@ -1544,7 +1746,7 @@ describe("ShellTool", () => { expect((yield* shell.wait(info.id).pipe(Effect.timeout("10 seconds"))).status).toBe("timeout") const pid = info.pid if (pid === undefined) throw new Error("Expected shell PID") - expect(() => process.kill(pid, 0)).toThrow() + expect(running(pid)).toBe(false) }), (info) => Effect.try(() => { @@ -1560,6 +1762,51 @@ describe("ShellTool", () => { { timeout: 30_000 }, ) + signalParity( + "stops a timed-out command with SIGTERM first and SIGKILL after the grace period", + () => + withTempDir((tmp) => + withSession(tmp.path, () => + Effect.gen(function* () { + const shell = yield* Shell.Service + const info = yield* shell.create({ + shell: "/bin/sh", + command: "trap 'echo got TERM' TERM; echo ready; while :; do sleep 0.1; done", + timeout: 0, + }) + while ((yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).size === 0) + yield* Effect.sleep(Duration.millis(10)) + const start = Date.now() + yield* shell.timeout(info.id, 1) + const ended = yield* shell.wait(info.id).pipe(Effect.timeout("10 seconds")) + expect(ended.status).toBe("timeout") + expect(Date.now() - start).toBeGreaterThanOrEqual(2_900) + expect((yield* shell.output(info.id)).output).toContain("got TERM") + }), + ), + ), + { timeout: 30_000 }, + ) + + signalTest("stops a command without forgetting it", () => + withTempDir((tmp) => + withSession(tmp.path, () => + Effect.gen(function* () { + const shell = yield* Shell.Service + const info = yield* shell.create({ shell: "/bin/sh", command: "echo kept; sleep 60", timeout: 0 }) + while ((yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).size === 0) + yield* Effect.sleep(Duration.millis(10)) + yield* shell.stop(info.id, "killed") + expect((yield* shell.wait(info.id).pipe(Effect.timeout("5 seconds"))).status).toBe("killed") + expect(yield* shell.get(info.id)).toMatchObject({ status: "killed" }) + expect(yield* Effect.promise(() => fs.readFile(info.file, "utf8"))).toBe("kept\n") + // Stopping an ended command changes nothing. + expect((yield* shell.stop(info.id, "timeout")).status).toBe("killed") + }), + ), + ), + ) + if (!isWindows) { it.live("settles a shell terminated by an external signal", () => withTempDir((tmp) => { @@ -1590,7 +1837,190 @@ describe("ShellTool", () => { ) } - it.live("backgrounds a foreground command when the session is signaled", () => + // Runs a foreground call the way a session step does, and interrupts it once its output is on disk. + const interruptCall = ( + registry: Tool.Interface, + command: string, + id: string, + onStarted: Effect.Effect = Effect.void, + waitForOutput = true, + ) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const slot: ToolInterrupted.Slot = {} + const started = yield* Deferred.make() + const fiber = yield* executeTool(registry, { + ...call({ command }, id), + progress: (update) => + typeof update.shellID === "string" + ? Deferred.succeed(started, update.shellID).pipe(Effect.asVoid) + : Effect.void, + }).pipe(Effect.provideService(ToolInterrupted.Slot, slot), Effect.forkChild({ startImmediately: true })) + const shellID = ID.make(yield* Deferred.await(started)) + while (waitForOutput && (yield* shell.output(shellID, { cursor: Number.MAX_SAFE_INTEGER })).size === 0) + yield* Effect.sleep(Duration.millis(10)) + yield* onStarted + yield* Fiber.interrupt(fiber) + const exit = yield* Fiber.await(fiber) + return { shellID, exit, result: slot.result } + }) + + if (!isWindows) { + parity.live("keeps the output of an interrupted call and settles it with the tail, file and shell ID", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const jobs = yield* Job.Service + const { shellID, exit, result } = yield* interruptCall( + registry, + "printf 'first\\nsecond\\n'; sleep 60", + "call-interrupted", + ) + // The interrupt still reaches the caller; the result travels through ToolInterrupted. + expect(Exit.hasInterrupts(exit)).toBe(true) + const info = yield* shell.get(shellID) + expect(info.status).toBe("killed") + expect(yield* Effect.promise(() => fs.readFile(info.file, "utf8"))).toBe("first\nsecond\n") + expect(result?.metadata).toMatchObject({ interrupted: true, truncated: false, status: "completed" }) + // A shell ID marks a background call, whose live output clients show instead of the result. + expect(result?.metadata).not.toHaveProperty("shellID") + expect(result?.content).toEqual([ + { + type: "text", + text: `first\nsecond\n\n[showing lines 1-2 of 2; full output saved to ${info.file}]\n\nCommand interrupted by the user (shell ID: ${shellID}).`, + }, + ]) + expect((yield* jobs.get(shellID))?.status).not.toBe("running") + expect(yield* jobs.pendingBackground).toEqual([]) + }), + ) + }), + ) + + parity.live( + "settles an interrupted call without waiting for a command that ignores SIGTERM", + () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const started = Date.now() + const { shellID, result } = yield* interruptCall( + registry, + "trap '' TERM; printf 'kept\\n'; while :; do sleep 1; done", + "call-interrupted-stubborn", + ) + // Bounded by the interrupt's short wait, not by the 3 s grace before SIGKILL. + expect(Date.now() - started).toBeLessThan(2_500) + const info = yield* shell.get(shellID) + expect(result?.content).toEqual([ + { + type: "text", + text: `kept\n\n[showing lines 1-1 of 1; full output saved to ${info.file}]\n\nCommand interrupted by the user (shell ID: ${shellID}); it is still being stopped.`, + }, + ]) + // The stop still ends it. + expect((yield* shell.wait(shellID)).status).toBe("killed") + }), + ) + }), + { timeout: 15_000 }, + ) + + fakeDaemonIt.live("a prepared handoff moves a daemon-run foreground call to the background at shutdown", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const jobs = yield* Job.Service + const pty = yield* PersistentPty.Service + const detaching = yield* Detaching.Service + const { shellID, exit, result } = yield* interruptCall( + registry, + "sleep 60", + "call-detaching-daemon", + Effect.gen(function* () { + expect(yield* pty.handoff()).toMatchObject({ ticket: "next" }) + expect(yield* detaching.begin).toBe(true) + }), + false, + ) + expect(shell.detachable(shellID)).toBe(true) + expect(Exit.hasInterrupts(exit)).toBe(true) + const info = yield* shell.get(shellID) + expect(info.status).toBe("running") + expect(result?.metadata).toMatchObject({ shellID, status: "running" }) + const process = fake?.processes.find((item) => item.pipe?.output_path === info.file) + expect(process?.lifecycle.status).toBe("running") + // Moved to the background: a next server neither waits on it nor times it out. + expect(process?.pipe?.metadata).toMatchObject({ id: shellID, foreground: false }) + expect(process?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(process?.pipe?.metadata).not.toHaveProperty("stopReason") + expect(fake?.ops).not.toContain("signal") + expect((yield* jobs.pendingBackground).map((item) => item.id)).toEqual([shellID]) + }), + ) + }), + ) + + parity.live("leaves a call interrupted, not cancelled by the user, when its Location closes", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const lifecycle = yield* LocationLifecycle.Service + const { shellID, exit, result } = yield* interruptCall( + registry, + "printf ready; sleep 60", + "call-location-closed", + lifecycle.shutdown, + ) + expect(Exit.hasInterrupts(exit)).toBe(true) + expect(result).toBeUndefined() + expect((yield* shell.get(shellID)).status).toBe("running") + yield* shell.remove(shellID) + }), + ) + }), + ) + + for (const handoff of [true, false]) + it.live( + `leaves a spawned command's call interrupted, not cancelled by the user, on ${handoff ? "a planned restart" : "a normal quit"}`, + () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const detaching = yield* Detaching.Service + const { shellID, exit, result } = yield* interruptCall( + registry, + "printf ready; sleep 60", + `call-stopping-spawned-${handoff}`, + Effect.gen(function* () { + if (handoff) yield* detaching.handoffPrepared + yield* detaching.begin + }), + ) + expect(shell.detachable(shellID)).toBe(false) + // The step settles it as interrupted; no "interrupted by the user" result, no stop to wait for. + expect(Exit.hasInterrupts(exit)).toBe(true) + expect(result).toBeUndefined() + expect((yield* shell.get(shellID)).status).toBe("running") + yield* shell.remove(shellID) + }), + ) + }), + ) + } + + parity.live("backgrounds a foreground command when the session is signaled", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1638,3 +2068,42 @@ describe("ShellTool", () => { }), ) }) + +describe("ShellTool detaching", () => { + detachableIt.live("moves a detachable command to the background when interrupted while detaching", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const jobs = yield* Job.Service + const detaching = yield* Detaching.Service + const slot: ToolInterrupted.Slot = {} + const started = yield* Deferred.make() + const fiber = yield* executeTool(registry, { + ...call({ command: idleCommand }, "call-detaching"), + progress: (update) => + typeof update.shellID === "string" + ? Deferred.succeed(started, update.shellID).pipe(Effect.asVoid) + : Effect.void, + }).pipe(Effect.provideService(ToolInterrupted.Slot, slot), Effect.forkChild({ startImmediately: true })) + const shellID = ID.make(yield* Deferred.await(started)) + yield* detaching.handoffPrepared + expect(yield* detaching.begin).toBe(true) + yield* Fiber.interrupt(fiber) + + const info = yield* shell.get(shellID) + expect(info.status).toBe("running") + expect(slot.result?.content[0]).toEqual({ + type: "text", + text: `Command moved to the background (shell ID: ${shellID}).\nOutput is streaming to: ${info.file}`, + }) + expect(slot.result?.metadata).toMatchObject({ status: "running", shellID }) + // Backgrounded durably for the next server, without a completion notification of its own. + expect(yield* jobs.pendingBackground).toMatchObject([{ id: shellID, status: "running" }]) + yield* shell.remove(shellID) + }), + ) + }), + ) +}) diff --git a/packages/server/package.json b/packages/server/package.json index efdc624ac1ad..da5fcf020ea9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -37,7 +37,7 @@ "effect": "catalog:" }, "devDependencies": { - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/client": "workspace:*", "@opencode/plugin": "workspace:*", "@tsconfig/bun": "catalog:", diff --git a/packages/server/src/handlers/persistent-pty.ts b/packages/server/src/handlers/persistent-pty.ts index e861cfdecc2d..d57340829e9e 100644 --- a/packages/server/src/handlers/persistent-pty.ts +++ b/packages/server/src/handlers/persistent-pty.ts @@ -3,7 +3,6 @@ import { PtyTicket } from "@opencode/core/pty/ticket" import { ForbiddenError, InvalidRequestError, - PtyFeatureUnsupportedError, PtyNotFoundError, ServiceUnavailableError, } from "@opencode/protocol/errors" @@ -113,19 +112,7 @@ export const PersistentPtyHandler = HttpApiBuilder.group(Api, "server.experiment .resize(ctx.params.ptyID, ctx.payload.size.cols, ctx.payload.size.rows, ctx.payload.attachmentID) .pipe(mapTerminalError) if (ctx.payload.title === undefined) return { data: yield* pty.get(ctx.params.ptyID).pipe(mapTerminalError) } - return { - data: yield* pty.rename(ctx.params.ptyID, ctx.payload.title).pipe( - Effect.catchTags({ - "PersistentPty.NotFoundError": notFound, - "PersistentPty.UnsupportedError": (error) => - new PtyFeatureUnsupportedError({ - feature: error.feature, - message: "Terminal renaming will be available after OpenCode's terminal service restarts", - }), - "PersistentPty.UnavailableError": unavailable, - }), - ), - } + return { data: yield* pty.rename(ctx.params.ptyID, ctx.payload.title).pipe(mapTerminalError) } }), ) .handle( diff --git a/packages/server/src/process.ts b/packages/server/src/process.ts index 71aff3ecb6ac..8f15371f3857 100644 --- a/packages/server/src/process.ts +++ b/packages/server/src/process.ts @@ -2,6 +2,7 @@ export * as ServerProcess from "./process" import { NodeHttpServer } from "@effect/platform-node" import { Bus } from "@opencode/core/bus" +import { Detaching } from "@opencode/core/detaching" import { SessionRestart } from "@opencode/core/session/execution/restart" import { Schedule } from "@opencode/core/schedule" import { InstallationEvent } from "@opencode/schema/installation-event" @@ -88,6 +89,8 @@ export const start = Effect.fn("ServerProcess.start")(function* ( const parentScope = yield* Scope.Scope const applicationScope = yield* Scope.fork(parentScope) + let detaching: Detaching.Interface | undefined + yield* enterDetachingOnShutdown(() => detaching) yield* Effect.addFinalizer(() => status.beginStopping.pipe( Effect.andThen(Ref.set(application, Option.none())), @@ -115,6 +118,7 @@ export const start = Effect.fn("ServerProcess.start")(function* ( } // Schedules fire from any server process (`service`, `opencode serve`, or `--standalone`); the table holds all their state. yield* Effect.forkIn(Context.get(context, Schedule.Service).run, applicationScope) + detaching = Context.get(context, Detaching.Service) yield* Ref.set(application, Option.some(Context.get(context, HttpRouter.HttpRouter).asHttpEffect())) yield* status.ready const bus = Context.get(context, Bus.Service) @@ -146,6 +150,14 @@ export const start = Effect.fn("ServerProcess.start")(function* ( return yield* Effect.raceFirst(boot, shutdown.await.pipe(Effect.andThen(Effect.interrupt))) }) +/** + * Decides on shutdown whether this server is detaching (a daemon handoff was prepared), before anything + * is torn down. Must be added to the parent scope after the application scope was forked from it: + * finalizers run in reverse order, so it then runs before the application scope closes. + */ +export const enterDetachingOnShutdown = (detaching: () => Detaching.Interface | undefined) => + Effect.addFinalizer(() => Effect.suspend(() => detaching()?.begin ?? Effect.void)) + function listen(options: { readonly hostname: string; readonly port: Option.Option }) { if (Option.isSome(options.port)) return bind(options.hostname, options.port.value) const next = (port: number): ReturnType => diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index f0430053f0b1..dec20ac21b60 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -20,6 +20,8 @@ import { Instance } from "@opencode/core/instance/service" import { SessionTransfer } from "@opencode/core/session/transfer" import { InstructionEntry } from "@opencode/core/session/instruction-entry" import { ShellSelect } from "@opencode/core/shell/select" +import { Shell } from "@opencode/core/shell" +import { Detaching } from "@opencode/core/detaching" import { Job } from "@opencode/core/job" import { KV } from "@opencode/core/kv" import { Mcp } from "@opencode/core/mcp/index" @@ -62,6 +64,7 @@ const applicationServiceNodes = [ Bus.node, EventLogger.node, httpClient, + Detaching.node, Job.node, Project.node, Project.sweepNode, @@ -75,6 +78,8 @@ const applicationServiceNodes = [ PermissionSaved.node, PtyTicket.node, PersistentPty.node, + // Daemon-run shell commands belong to the server, so a Location closing never takes them down. + Shell.daemonNode, Credential.node, WellKnown.node, ServerPairing.node, diff --git a/packages/server/test/persistent-pty.test.ts b/packages/server/test/persistent-pty.test.ts index 2be137bb61bf..d4b728543342 100644 --- a/packages/server/test/persistent-pty.test.ts +++ b/packages/server/test/persistent-pty.test.ts @@ -208,7 +208,7 @@ smoke( 20_000, ) -it.live("reports renaming as unsupported on a daemon without the rename feature", () => +it.live("renames without a feature check and hides unknown terminals", () => Effect.gen(function* () { const fixture = yield* testDirectory() const daemon = yield* fakeDaemon() @@ -222,7 +222,7 @@ it.live("reports renaming as unsupported on a daemon without the rename feature" fs: { filewatcher: false }, }) const base = HttpServer.formatAddress(server.address) - const created = yield* request(base, "POST", "/api/experimental/session/ses_old_daemon/terminal", { + const created = yield* request(base, "POST", "/api/experimental/session/ses_v8_daemon/terminal", { args: [], cwd: fixture.root, title: "created", @@ -231,28 +231,19 @@ it.live("reports renaming as unsupported on a daemon without the rename feature" const terminal = Schema.decodeUnknownSync(PersistentPty.Info)(created.data) expect(terminal.userTitle).toBeNull() - const response = yield* Effect.promise(async () => { - const response = await fetch(new URL(`/api/experimental/persistent-pty/${terminal.id}`, base), { - method: "PUT", - headers: { authorization: `Basic ${btoa("opencode:secret")}`, "content-type": "application/json" }, - body: JSON.stringify({ title: "build" }), - }) - return { status: response.status, body: await response.json() } - }) + const renamed = yield* request(base, "PUT", `/api/experimental/persistent-pty/${terminal.id}`, { title: "build" }) + expect(renamed.data).toMatchObject({ title: "build", userTitle: "build" }) + expect(daemon.ops).toContain("rename") + expect(daemon.ops).not.toContain("list") - expect(response).toEqual({ - status: 501, - body: { - _tag: "PtyFeatureUnsupportedError", - feature: "rename", - message: "Terminal renaming will be available after OpenCode's terminal service restarts", - }, - }) - expect(daemon.ops).not.toContain("rename") - expect((yield* request(base, "GET", `/api/experimental/persistent-pty/${terminal.id}`)).data).toMatchObject({ - title: "test", - userTitle: null, + const missing = yield* Effect.promise(async () => { + const response = await fetch(new URL("/api/experimental/persistent-pty/pty_persistent_99", base), { + headers: { authorization: `Basic ${btoa("opencode:secret")}` }, + }) + await response.arrayBuffer() + return response.status }) + expect(missing).toBe(404) }), ) @@ -573,12 +564,11 @@ smoke( ) smoke( - "migrates a legacy runtime directory on restart and boots fresh when the registration is gone", + "boots fresh on restart when the handed-off registration is gone", () => Effect.gen(function* () { const fixture = yield* testDirectory() const scope = yield* Scope.Scope - const legacy = path.join(fixture.root, "legacy") const options = { hostname: "127.0.0.1", port: 0, @@ -586,50 +576,32 @@ smoke( app: { version: "test-version" }, database: { path: fixture.database }, fs: { filewatcher: false }, + pty: { root: fixture.directory }, } - const sessionID = Session.ID.make("ses_persistent_pty_migration") + const sessionID = Session.ID.make("ses_persistent_pty_gone") const originalScope = yield* Scope.fork(scope) - const original = yield* ServerProcess.start({ ...options, pty: { root: legacy } }).pipe( + const original = yield* ServerProcess.start(options).pipe( Effect.provideService(Scope.Scope, originalScope), ) const base = HttpServer.formatAddress(original.address) const first = Schema.decodeUnknownSync(PersistentPty.Info)( (yield* request(base, "POST", `/api/experimental/session/${sessionID}/terminal`, { command: "/bin/sh", - args: ["-c", "stty -echo; printf before-migration; exec cat"], + args: ["-c", "stty -echo; printf before-restart; exec cat"], cwd: process.cwd(), - title: "legacy", + title: "first", env: {}, })).data, ) - expect(yield* waitForText(base, first.id, "before-migration")).toContain("before-migration") - const legacyHandoff = Schema.decodeUnknownSync(PersistentPty.Handoff)( + expect(yield* waitForText(base, first.id, "before-restart")).toContain("before-restart") + const handoff = Schema.decodeUnknownSync(PersistentPty.Handoff)( (yield* request(base, "POST", "/api/experimental/persistent-pty/handoff")).handoff, ) yield* Scope.close(originalScope, Exit.void) - - const migratedScope = yield* Scope.fork(scope) - const migrated = yield* ServerProcess.start({ - ...options, - pty: { root: fixture.directory, handoff: legacyHandoff }, - }).pipe(Effect.provideService(Scope.Scope, migratedScope)) - const migratedBase = HttpServer.formatAddress(migrated.address) - const directory = path.join(fixture.directory, path.basename(legacyHandoff.directory)) - expect(existsSync(path.join(directory, "service.json"))).toBeTrue() - expect(existsSync(path.join(legacyHandoff.directory, "service.json"))).toBeFalse() - expect( - (yield* request(migratedBase, "GET", `/api/experimental/session/${sessionID}/terminal`)).data, - ).toMatchObject([{ id: first.id, pid: first.pid }]) - - const handoff = Schema.decodeUnknownSync(PersistentPty.Handoff)( - (yield* request(migratedBase, "POST", "/api/experimental/persistent-pty/handoff")).handoff, - ) - expect(handoff.directory).toBe(directory) - yield* Scope.close(migratedScope, Exit.void) // Simulate macOS purging the registration while the daemon waits for its successor. - yield* Effect.promise(() => fs.rm(path.join(directory, "service.json"))) + yield* Effect.promise(() => fs.rm(path.join(handoff.directory, "service.json"))) - const fresh = yield* ServerProcess.start({ ...options, pty: { root: fixture.directory, handoff } }) + const fresh = yield* ServerProcess.start({ ...options, pty: { ...options.pty, handoff } }) const freshBase = HttpServer.formatAddress(fresh.address) expect((yield* request(freshBase, "GET", `/api/experimental/session/${sessionID}/terminal`)).data).toEqual([]) const second = Schema.decodeUnknownSync(PersistentPty.Info)( diff --git a/packages/server/test/process.test.ts b/packages/server/test/process.test.ts index 20ada8bae073..b797e01aee49 100644 --- a/packages/server/test/process.test.ts +++ b/packages/server/test/process.test.ts @@ -1,5 +1,6 @@ import { expect } from "bun:test" -import { Effect } from "effect" +import { Detaching } from "@opencode/core/detaching" +import { Effect, Exit, Scope } from "effect" import { HttpServer, HttpServerRequest, HttpServerResponse } from "effect/http" import { it } from "../../core/test/lib/effect" import { ServerProcess } from "../src/process" @@ -259,3 +260,24 @@ it.live("server info lists remote URLs alongside local addresses while they are expect((yield* info()).urls).toEqual([base]) }), ) + +it.live("enters detaching on shutdown before the application scope closes", () => + Effect.gen(function* () { + for (const prepared of [true, false]) { + const detaching = yield* Detaching.make + if (prepared) yield* detaching.handoffPrepared + const parent = yield* Scope.make() + // Same order as ServerProcess.start: fork the application scope, then register the hook. + const application = yield* Scope.fork(parent) + const seen: boolean[] = [] + yield* Scope.addFinalizer( + application, + Effect.sync(() => seen.push(detaching.isDetaching())), + ) + yield* ServerProcess.enterDetachingOnShutdown(() => detaching).pipe(Scope.provide(parent)) + expect(detaching.isDetaching()).toBe(false) + yield* Scope.close(parent, Exit.void) + expect(seen).toEqual([prepared]) + } + }), +)