diff --git a/.yarn/versions/f57c4e47.yml b/.yarn/versions/f57c4e47.yml new file mode 100644 index 000000000000..8fca1d8ee7ed --- /dev/null +++ b/.yarn/versions/f57c4e47.yml @@ -0,0 +1,6 @@ +releases: + "@yarnpkg/shell": patch + +declined: + - "@yarnpkg/cli" + - "@yarnpkg/core" diff --git a/packages/yarnpkg-shell/sources/index.ts b/packages/yarnpkg-shell/sources/index.ts index d715ac5aa56d..5d4a3745034b 100644 --- a/packages/yarnpkg-shell/sources/index.ts +++ b/packages/yarnpkg-shell/sources/index.ts @@ -9,7 +9,7 @@ import {setTimeout} import EntryCommand from './commands/entry'; import {ShellError} from './errors'; import * as globUtils from './globUtils'; -import {createOutputStreamsWithPrefix, makeBuiltin, makeProcess} from './pipe'; +import {createOutputStreamsWithPrefix, isPipeStdin, makeBuiltin, makeProcess} from './pipe'; import {Handle, ProcessImplementation, ProtectedStream, Stdio, start, Pipe} from './pipe'; export {EntryCommand}; @@ -54,6 +54,7 @@ export type ShellState = { stdin: Readable; stdout: Writable; stderr: Writable; + closeStdinOnRedirectionOnly: boolean; variables: {[key: string]: string}; nextBackgroundJobIndex: number; backgroundJobs: Array>; @@ -64,39 +65,68 @@ enum StreamType { Writable = 0b10, } -function getFileDescriptorStream(fd: number, type: StreamType, state: ShellState) { +function isWritableClosed(stream: Writable) { + return stream.destroyed || (stream as Writable & {closed?: boolean}).closed === true; +} + +function throwBadFileDescriptor(fd: number): never { + throw new ShellError(`Bad file descriptor: "${fd}"`); +} + +function getFileDescriptorStream(fd: number, type: StreamType, state: ShellState, {endReadable = false}: {endReadable?: boolean} = {}) { const stream = new PassThrough({autoDestroy: true}); + const pipeToWritable = (target: Writable) => { + const onTargetClose = () => { + stream.destroy(); + }; + const onTargetError = (error: Error) => { + stream.destroy(error); + }; + + target.on(`close`, onTargetClose); + target.on(`error`, onTargetError); + stream.on(`close`, () => { + target.off(`close`, onTargetClose); + target.off(`error`, onTargetError); + }); + + if (isWritableClosed(target)) { + stream.destroy(); + } else { + stream.pipe(target, {end: false}); + } + }; switch (fd) { case Pipe.STDIN: { if ((type & StreamType.Readable) === StreamType.Readable) - state.stdin.pipe(stream, {end: false}); + state.stdin.pipe(stream, {end: endReadable}); - if ((type & StreamType.Writable) === StreamType.Writable && state.stdin instanceof Writable) { - stream.pipe(state.stdin, {end: false}); + if ((type & StreamType.Writable) === StreamType.Writable) { + throwBadFileDescriptor(fd); } } break; case Pipe.STDOUT: { if ((type & StreamType.Readable) === StreamType.Readable) - state.stdout.pipe(stream, {end: false}); + throwBadFileDescriptor(fd); if ((type & StreamType.Writable) === StreamType.Writable) { - stream.pipe(state.stdout, {end: false}); + pipeToWritable(state.stdout); } } break; case Pipe.STDERR: { if ((type & StreamType.Readable) === StreamType.Readable) - state.stderr.pipe(stream, {end: false}); + throwBadFileDescriptor(fd); if ((type & StreamType.Writable) === StreamType.Writable) { - stream.pipe(state.stderr, {end: false}); + pipeToWritable(state.stderr); } } break; default: { - throw new ShellError(`Bad file descriptor: "${fd}"`); + throwBadFileDescriptor(fd); } } @@ -141,6 +171,10 @@ const BUILTINS = new Map([ return 0; }], + [`__ysh_validate_redirects`, async (args: Array, opts: ShellOptions, state: ShellState) => { + return 0; + }], + [`false`, async (args: Array, opts: ShellOptions, state: ShellState) => { return 1; }], @@ -193,8 +227,172 @@ const BUILTINS = new Map([ let stderr = state.stderr; const inputs: Array<() => Readable> = []; + const validateInputs: Array<() => Promise> = []; const outputs: Array = []; const errors: Array = []; + let endReadableFdStreams = false; + let readsFromStdinFd = false; + + const makeCloseTargets = (stream: Writable, targets: Array) => { + if (targets.length === 0) + return async () => {}; + + const absorbStreamError = () => {}; + stream.on(`error`, absorbStreamError); + + const closePromises = targets.map(target => { + return new Promise((resolve, reject) => { + let settled = false; + + const cleanup = () => { + target.off(`error`, onError); + target.off(`close`, onClose); + }; + + const settle = (next: () => void) => { + if (settled) + return; + + settled = true; + cleanup(); + next(); + }; + + const onError = (error: Error) => { + if (!stream.destroyed) + stream.destroy(error); + + settle(() => { + reject(error); + }); + }; + + const onClose = () => { + settle(() => { + resolve(); + }); + }; + + if (isWritableClosed(target)) { + settle(() => { + resolve(); + }); + } else { + target.on(`error`, onError); + target.on(`close`, onClose); + } + }); + }); + closePromises.forEach(closePromise => { + closePromise.catch(() => {}); + }); + + return async () => { + if (!stream.destroyed) + stream.end(); + + try { + await Promise.all(closePromises); + } finally { + stream.off(`error`, absorbStreamError); + } + }; + }; + + const copyStream = async (source: Readable, target: Writable) => { + const isSinkClosedError = (error: Error) => { + return [`EPIPE`, `ERR_STREAM_DESTROYED`, `ERR_STREAM_WRITE_AFTER_END`].includes( + (error as NodeJS.ErrnoException).code ?? ``, + ); + }; + + const writeChunk = async (chunk: Buffer | string) => { + return await new Promise((resolve, reject) => { + let settled = false; + + const silenceFutureSinkClosedErrors = () => { + target.on(`error`, error => { + if (!isSinkClosedError(error)) { + throw error; + } + }); + }; + + const cleanup = () => { + target.off(`error`, onError); + target.off(`close`, onClose); + }; + + const settle = (next: () => void) => { + if (settled) + return; + + settled = true; + cleanup(); + next(); + }; + + const onError = (error: Error) => { + if (isSinkClosedError(error)) { + silenceFutureSinkClosedErrors(); + settle(() => { + resolve(false); + }); + } else { + settle(() => { + reject(error); + }); + } + }; + + const onClose = () => { + settle(() => { + resolve(false); + }); + }; + + target.on(`error`, onError); + target.on(`close`, onClose); + + try { + target.write(chunk, error => { + if (typeof error === `undefined` || error === null) { + settle(() => { + resolve(true); + }); + } else if (isSinkClosedError(error)) { + silenceFutureSinkClosedErrors(); + settle(() => { + resolve(false); + }); + } else { + settle(() => { + reject(error); + }); + } + }); + } catch (error) { + if (error instanceof Error && isSinkClosedError(error)) { + silenceFutureSinkClosedErrors(); + settle(() => { + resolve(false); + }); + } else { + settle(() => { + reject(error); + }); + } + } + }); + }; + + for await (const chunk of source) { + if (!await writeChunk(chunk)) { + source.destroy(); + return; + } + } + }; let t = 0; @@ -236,8 +434,13 @@ const BUILTINS = new Map([ for (let u = t; u < last; ++t, ++u) { switch (type) { case `<`: { + const inputPath = ppath.resolve(state.cwd, npath.toPortablePath(args[u])); pushInput(() => { - return opts.baseFs.createReadStream(ppath.resolve(state.cwd, npath.toPortablePath(args[u]))); + return opts.baseFs.createReadStream(inputPath); + }); + validateInputs.push(async () => { + const fd = await opts.baseFs.openPromise(inputPath, `r`); + await opts.baseFs.closePromise(fd); }); } break; case `<<<`: { @@ -251,7 +454,14 @@ const BUILTINS = new Map([ }); } break; case `<&`: { - pushInput(() => getFileDescriptorStream(Number(args[u]), StreamType.Readable, state)); + const fd = Number(args[u]); + readsFromStdinFd ||= fd === Pipe.STDIN; + pushInput(() => getFileDescriptorStream(fd, StreamType.Readable, state, {endReadable: endReadableFdStreams})); + validateInputs.push(async () => { + if (fd !== Pipe.STDIN) { + throwBadFileDescriptor(fd); + } + }); } break; case `>`: @@ -282,7 +492,11 @@ const BUILTINS = new Map([ } } - if (inputs.length > 0) { + const commandArgs = args.slice(t + 1); + const validatesOnly = commandArgs.length === 1 && commandArgs[0] === `__ysh_validate_redirects`; + endReadableFdStreams = commandArgs.length === 0; + + if (inputs.length > 0 && !validatesOnly) { const pipe = new PassThrough(); stdin = pipe; @@ -292,6 +506,9 @@ const BUILTINS = new Map([ } else { const input = inputs[n](); input.pipe(pipe, {end: false}); + input.on(`error`, error => { + pipe.destroy(error); + }); input.on(`end`, () => { bindInput(n + 1); }); @@ -319,39 +536,62 @@ const BUILTINS = new Map([ } } - const exitCode = await start(makeCommandAction(args.slice(t + 1), opts, state), { - stdin: new ProtectedStream(stdin), - stdout: new ProtectedStream(stdout), - stderr: new ProtectedStream(stderr), - }).run(); + const closeStdoutTargets = makeCloseTargets(stdout, outputs); + const closeStderrTargets = makeCloseTargets(stderr, errors); - // Close all the outputs (since the shell never closes the output stream) - await Promise.all(outputs.map(output => { - // Wait until the output got flushed to the disk - return new Promise((resolve, reject) => { - output.on(`error`, error => { - reject(error); - }); - output.on(`close`, () => { - resolve(); - }); - output.end(); - }); - })); - - // Close all the errors (since the shell never closes the error stream) - await Promise.all(errors.map(err => { - // Wait until the error got flushed to the disk - return new Promise((resolve, reject) => { - err.on(`error`, error => { - reject(error); - }); - err.on(`close`, () => { - resolve(); - }); - err.end(); - }); - })); + const closeTargets = async () => { + let firstError: unknown = null; + + for (const closeTarget of [closeStdoutTargets, closeStderrTargets]) { + try { + await closeTarget(); + } catch (error) { + firstError ??= error; + } + } + + if (firstError !== null) { + throw firstError; + } + }; + + const closeUnusedStdin = () => { + if (state.closeStdinOnRedirectionOnly && !readsFromStdinFd && isPipeStdin(state.stdin)) { + state.stdin.destroy(); + } + }; + + const validateInputRedirections = async () => { + for (const validateInput of validateInputs) { + await validateInput(); + } + }; + + let exitCode = 0; + try { + if (commandArgs.length === 0) { + closeUnusedStdin(); + + if (inputs.length > 0) { + await copyStream(stdin, stdout); + } + } else { + if (validatesOnly) + await validateInputRedirections(); + + exitCode = await start(makeCommandAction( + commandArgs, + opts, + state, + ), { + stdin: new ProtectedStream(stdin), + stdout: new ProtectedStream(stdout), + stderr: new ProtectedStream(stderr), + }).run(); + } + } finally { + await closeTargets(); + } return exitCode; }], @@ -685,21 +925,99 @@ function makeCommandAction(args: Array, opts: ShellOptions, state: Shell }); } +function isRedirectionOnlyCommand(args: Array) { + if (args[0] !== `__ysh_set_redirects`) + return false; + + let separatorIndex = 1; + while (separatorIndex < args.length && args[separatorIndex] !== `--`) { + const count = Number(args[separatorIndex + 1]); + if (!Number.isInteger(count) || count < 0) + return false; + + separatorIndex += 2 + count; + } + + return args[separatorIndex] === `--` && separatorIndex === args.length - 1; +} + +function persistEnvironmentAfterSuccess(action: ProcessImplementation, activeState: ShellState, environment: {[key: string]: string}) { + return (stdio: Stdio) => { + const handle = action(stdio); + + return { + stdin: handle.stdin, + promise: handle.promise.then(exitCode => { + activeState.environment = {...activeState.environment, ...environment}; + return exitCode; + }), + }; + }; +} + function makeSubshellAction(ast: ShellLine, opts: ShellOptions, state: ShellState) { return (stdio: Stdio) => { - const stdin = new PassThrough(); - const promise = executeShellLine(ast, opts, cloneState(state, {stdin})); + const pipedStdin = stdio[0] === `pipe` + ? new PassThrough() + : null; + const stdin = pipedStdin ?? (stdio[0] instanceof Readable ? stdio[0] : state.stdin); + const stdout = stdio[1] instanceof Writable ? stdio[1] : state.stdout; + const stderr = stdio[2] instanceof Writable ? stdio[2] : state.stderr; + const returnedStdin = pipedStdin ?? new PassThrough(); + if (pipedStdin === null) + returnedStdin.end(); + + const promise = executeShellLine(ast, opts, cloneState(state, { + stdin, + stdout, + stderr, + closeStdinOnRedirectionOnly: false, + })).finally(() => { + if (pipedStdin !== null) { + pipedStdin.destroy(); + } else if (state.closeStdinOnRedirectionOnly && isPipeStdin(stdin)) { + stdin.destroy(); + } + }); - return {stdin, promise}; + return {stdin: returnedStdin, promise}; }; } function makeGroupAction(ast: ShellLine, opts: ShellOptions, state: ShellState) { return (stdio: Stdio) => { - const stdin = new PassThrough(); - const promise = executeShellLine(ast, opts, state); + const pipedStdin = stdio[0] === `pipe` + ? new PassThrough() + : null; + const stdin = pipedStdin ?? (stdio[0] instanceof Readable ? stdio[0] : state.stdin); + const stdout = stdio[1] instanceof Writable ? stdio[1] : state.stdout; + const stderr = stdio[2] instanceof Writable ? stdio[2] : state.stderr; + const returnedStdin = pipedStdin ?? new PassThrough(); + if (pipedStdin === null) + returnedStdin.end(); + + const previousStdin = state.stdin; + const previousStdout = state.stdout; + const previousStderr = state.stderr; + const previousCloseStdinOnRedirectionOnly = state.closeStdinOnRedirectionOnly; + state.stdin = stdin; + state.stdout = stdout; + state.stderr = stderr; + state.closeStdinOnRedirectionOnly = false; + const promise = executeShellLine(ast, opts, state).finally(() => { + state.stdin = previousStdin; + state.stdout = previousStdout; + state.stderr = previousStderr; + state.closeStdinOnRedirectionOnly = previousCloseStdinOnRedirectionOnly; + + if (pipedStdin !== null) { + pipedStdin.destroy(); + } else if (previousCloseStdinOnRedirectionOnly && isPipeStdin(stdin)) { + stdin.destroy(); + } + }); - return {stdin, promise}; + return {stdin: returnedStdin, promise}; }; } @@ -736,11 +1054,18 @@ async function executeCommandChainImpl(node: CommandChain, opts: ShellOptions, s switch (current.type) { case `command`: { const args = await interpolateArguments(current.args, opts, state); - const environment = await applyEnvVariables(current.envs, opts, state); - action = current.envs.length - ? makeCommandAction(args, opts, cloneState(activeState, {environment})) - : makeCommandAction(args, opts, activeState); + if (current.envs.length) { + const environment = await applyEnvVariables(current.envs, opts, state); + + if (isRedirectionOnlyCommand(args)) { + action = persistEnvironmentAfterSuccess(makeCommandAction([...args, `__ysh_validate_redirects`], opts, activeState), activeState, environment); + } else { + action = makeCommandAction(args, opts, cloneState(activeState, {environment})); + } + } else { + action = makeCommandAction(args, opts, activeState); + } } break; case `subshell`: { @@ -1080,6 +1405,7 @@ export async function execute(command: string, args: Array = [], { stdin, stdout, stderr, + closeStdinOnRedirectionOnly: true, variables: Object.assign({}, variables, { [`?`]: 0, }), diff --git a/packages/yarnpkg-shell/sources/pipe.ts b/packages/yarnpkg-shell/sources/pipe.ts index b92addf530a6..ca30c7d3a065 100644 --- a/packages/yarnpkg-shell/sources/pipe.ts +++ b/packages/yarnpkg-shell/sources/pipe.ts @@ -27,6 +27,16 @@ export type ProcessImplementation = ( const activeChildren = new Set(); +const PIPE_STDIN = Symbol(`PIPE_STDIN`); + +type PipeStdin = Readable & { + [PIPE_STDIN]?: boolean; +}; + +export function isPipeStdin(stream: Readable) { + return (stream as PipeStdin)[PIPE_STDIN] === true; +} + function sigintHandler() { // We don't want SIGINT to kill our process; we want it to kill the // innermost process, whose end will cause our own to exit. @@ -38,6 +48,22 @@ function sigtermHandler() { } } +function bindChildOutput(child: ChildProcess, source: Readable, target: Transform) { + const closeSource = () => { + source.destroy(); + }; + + const cleanup = () => { + target.off(`close`, closeSource); + target.off(`error`, closeSource); + }; + + target.on(`close`, closeSource); + target.on(`error`, closeSource); + child.on(`close`, cleanup); + child.on(`error`, cleanup); +} + export function makeProcess(name: string, args: Array, opts: ShellOptions, spawnOpts: any): ProcessImplementation { return (stdio: Stdio) => { const stdin = stdio[0] instanceof Transform @@ -67,10 +93,14 @@ export function makeProcess(name: string, args: Array, opts: ShellOption if (stdio[0] instanceof Transform) stdio[0].pipe(child.stdin!); - if (stdio[1] instanceof Transform) + if (stdio[1] instanceof Transform) { child.stdout!.pipe(stdio[1], {end: false}); - if (stdio[2] instanceof Transform) + bindChildOutput(child, child.stdout!, stdio[1]); + } + if (stdio[2] instanceof Transform) { child.stderr!.pipe(stdio[2], {end: false}); + bindChildOutput(child, child.stderr!, stdio[2]); + } return { stdin: child.stdin!, @@ -121,17 +151,25 @@ export function makeProcess(name: string, args: Array, opts: ShellOption export function makeBuiltin(builtin: (opts: any) => Promise): ProcessImplementation { return (stdio: Stdio) => { - const stdin = stdio[0] === `pipe` + const hasPipeStdin = stdio[0] === `pipe`; + const stdin = hasPipeStdin ? new PassThrough() : stdio[0]; + if (hasPipeStdin) + (stdin as PipeStdin)[PIPE_STDIN] = true; + return { stdin, promise: Promise.resolve().then(() => builtin({ stdin, stdout: stdio[1], stderr: stdio[2], - })), + })).finally(() => { + if (hasPipeStdin) { + stdin.destroy(); + } + }), }; }; } diff --git a/packages/yarnpkg-shell/tests/shell.test.ts b/packages/yarnpkg-shell/tests/shell.test.ts index be7b68bd6079..f044224b06f9 100644 --- a/packages/yarnpkg-shell/tests/shell.test.ts +++ b/packages/yarnpkg-shell/tests/shell.test.ts @@ -20,6 +20,29 @@ const expectResult = async (promise: Promise, {exitCode = 0, stdout = ``, s return await expect(promise).resolves.toMatchObject({exitCode, stdout, stderr}); }; +const withTimeout = async (promise: Promise, duration: number, message: string) => { + let timeout: ReturnType | undefined; + + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = globalThis.setTimeout(() => { + reject(new Error(message)); + }, duration); + + if (typeof timeout !== `number`) { + timeout.unref?.(); + } + }), + ]); + } finally { + if (typeof timeout !== `undefined`) { + globalThis.clearTimeout(timeout); + } + } +}; + const bufferResult = async (command: string, args: Array = [], options: Partial & {tty?: boolean} = {}): Promise => { const stdout = new PassThrough(); const stderr = new PassThrough(); @@ -163,6 +186,19 @@ describe(`Shell`, () => { }); }); + it(`should not retain process output listeners`, async () => { + const stdout = new PassThrough(); + const stderr = new PassThrough(); + const command = Array.from({length: 12}, () => `node -e ""`).join(`; `); + + await expect(execute(command, [], {stdout, stderr})).resolves.toEqual(0); + + expect(stdout.listenerCount(`close`)).toEqual(0); + expect(stdout.listenerCount(`error`)).toEqual(0); + expect(stderr.listenerCount(`close`)).toEqual(0); + expect(stderr.listenerCount(`error`)).toEqual(0); + }); + it(`should support empty string as argument`, async () => { await expectResult(bufferResult( `node -pe "process.argv[2]" "" 1`, @@ -386,6 +422,30 @@ describe(`Shell`, () => { }); }); + it(`should allow subshells to read inherited stdin`, async () => { + const stdin = new PassThrough(); + stdin.end(`hello world`); + + await expectResult(bufferResult( + `( cat )`, + [], + {stdin}, + ), { + stdout: `hello world`, + }); + + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + await xfs.writeFilePromise(file, `hello world\n`); + + await expectResult(bufferResult( + `( cat ) < "${file}"`, + ), { + stdout: `hello world\n`, + }); + }); + }); + it(`should support redirections on groups (one command)`, async () => { await xfs.mktempPromise(async tmpDir => { const file = ppath.join(tmpDir, `file`); @@ -414,6 +474,30 @@ describe(`Shell`, () => { }); }); + it(`should allow groups to read inherited stdin`, async () => { + const stdin = new PassThrough(); + stdin.end(`hello world`); + + await expectResult(bufferResult( + `{ cat; }`, + [], + {stdin}, + ), { + stdout: `hello world`, + }); + + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + await xfs.writeFilePromise(file, `hello world\n`); + + await expectResult(bufferResult( + `{ cat; } < "${file}"`, + ), { + stdout: `hello world\n`, + }); + }); + }); + it(`shouldn't allow subshells to mutate the state of the parent shell`, async () => { await expectResult(bufferResult( `(FOO=hello); echo $FOO`, @@ -659,6 +743,111 @@ describe(`Shell`, () => { }); }); + it(`should support env assignment without command and with redirection`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + + await expectResult(bufferResult([ + `FOO=1`, + `FOO=2 > "${file}"`, + `echo $FOO`, + ].join(` ; `)), { + stdout: `2\n`, + }); + + await expect(xfs.readFilePromise(file, `utf8`)).resolves.toEqual(``); + }); + }); + + it(`should not copy input when env assignment without command has redirection`, async () => { + await xfs.mktempPromise(async tmpDir => { + const source = ppath.join(tmpDir, `source`); + const target = ppath.join(tmpDir, `target`); + await xfs.writeFilePromise(source, `hello world\n`); + + await expectResult(bufferResult([ + `FOO=1`, + `FOO=2 < "${source}"`, + `FOO=3 < "${source}" > "${target}"`, + `echo $FOO`, + ].join(` ; `)), { + stdout: `3\n`, + }); + + await expect(xfs.readFilePromise(target, `utf8`)).resolves.toEqual(``); + }); + }); + + it(`should support env assignment without command and with stdin fd redirection`, async () => { + const stdin = new PassThrough(); + stdin.end(`hello world\n`); + + await expectResult(withTimeout( + bufferResult([ + `FOO=1`, + `FOO=2 <&0`, + `echo $FOO`, + ].join(` ; `), [], {stdin}), + 1000, + `Timed out waiting for commandless assignment stdin fd redirection`, + ), { + stdout: `2\n`, + }); + }); + + ifNotWin32It(`should not drain env assignment without command input redirections`, async () => { + await expectResult(withTimeout( + bufferResult([ + `FOO=1`, + `FOO=2 < /dev/zero`, + `echo $FOO`, + ].join(` ; `)), + 1000, + `Timed out waiting for commandless assignment input redirection`, + ), { + stdout: `2\n`, + }); + }); + + it(`should fail env assignment without command when input redirection fails`, async () => { + await xfs.mktempPromise(async tmpDir => { + const missing = ppath.join(tmpDir, `missing`); + + await expect(bufferResult([ + `FOO=1`, + `FOO=2 < "${missing}"`, + `echo $FOO`, + ].join(` ; `))).rejects.toThrowError(`ENOENT: no such file or directory, open`); + }); + }); + + it(`should support env assignment without command and with a -- redirection target`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `--`); + + await expectResult(bufferResult([ + `FOO=1`, + `FOO=2 > "${file}"`, + `echo $FOO`, + ].join(` ; `)), { + stdout: `2\n`, + }); + + await expect(xfs.readFilePromise(file, `utf8`)).resolves.toEqual(``); + }); + }); + + it(`should not persist env assignment when commandless redirection fails`, async () => { + await expectResult(bufferResult([ + `FOO=1`, + `FOO=2 >&42`, + `echo $FOO`, + ].join(` ; `)), { + stdout: `1\n`, + stderr: `Bad file descriptor: "42"\n`, + }); + }); + it(`should evaluate variables once before starting execution`, async () => { await expectResult(bufferResult([ `FOO=1`, @@ -785,6 +974,113 @@ describe(`Shell`, () => { }); }); + it(`should support redirection-only commands (file)`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + await xfs.writeFilePromise(file, `hello world\n`); + + await expectResult(bufferResult( + `< "${file}"`, + ), { + stdout: `hello world\n`, + }); + }); + }); + + it(`should support redirection-only commands from stdin fd`, async () => { + for (const redirection of [`<&0`, `0<&0`]) { + const stdin = new PassThrough(); + stdin.end(`hello world\n`); + + await expectResult(withTimeout( + bufferResult(redirection, [], {stdin}), + 1000, + `Timed out waiting for ${redirection}`, + ), { + stdout: `hello world\n`, + }); + } + }); + + it(`should throw recoverable errors for input redirections from output fds`, async () => { + for (const [command, fd] of [[`cat <&1`, 1], [`cat <&2`, 2], [`<&1`, 1], [`<&2`, 2]] as const) { + await expectResult(withTimeout( + bufferResult(command), + 1000, + `Timed out waiting for ${command}`, + ), { + exitCode: 1, + stderr: `Bad file descriptor: "${fd}"\n`, + }); + } + }); + + it(`should support large redirection-only copies`, async () => { + await xfs.mktempPromise(async tmpDir => { + const source = ppath.join(tmpDir, `source`); + const target = ppath.join(tmpDir, `target`); + const contents = Buffer.alloc(4 * 1024 * 1024, `x`); + + await xfs.writeFilePromise(source, contents); + + await expectResult(bufferResult( + `< "${source}" > "${target}"`, + ), { + stdout: ``, + }); + + const output = await xfs.readFilePromise(target); + expect(output.length).toEqual(contents.length); + expect(output.equals(contents)).toEqual(true); + }); + }); + + it(`should throw on redirection-only copies to inexistent folders`, async () => { + await xfs.mktempPromise(async tmpDir => { + const source = ppath.join(tmpDir, `source`); + const target = ppath.join(tmpDir, `missing`, `target`); + + await xfs.writeFilePromise(source, `hello world\n`); + + await expect(bufferResult( + `< "${source}" > "${target}"`, + )).rejects.toThrowError(`ENOENT: no such file or directory, open`); + }); + }); + + it(`should throw on redirection-only commands with missing inputs`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `missing`); + + await expect(bufferResult( + `< "${file}"`, + )).rejects.toThrowError(`ENOENT: no such file or directory, open`); + }); + }); + + it(`should support redirection-only commands in subshells (file)`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + await xfs.writeFilePromise(file, `hello world\n`); + + await expectResult(bufferResult( + `echo "$(< "${file}")"`, + ), { + stdout: `hello world\n`, + }); + }); + }); + + it(`should throw on redirection-only commands with missing inputs in subshells`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `missing`); + + await expect(bufferResult( + `echo "$(< "${file}")"`, + )).rejects.toThrowError(`ENOENT: no such file or directory, open`); + }); + }); + it(`should support input redirections to fd (file)`, async () => { await xfs.mktempPromise(async tmpDir => { const file = ppath.join(tmpDir, `file`); @@ -883,6 +1179,21 @@ describe(`Shell`, () => { }); }); + it(`should support redirection-only commands (overwrite)`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + await xfs.writeFilePromise(file, `hello world\n`); + + await expectResult(bufferResult( + `> "${file}"`, + ), { + stdout: ``, + }); + + await expect(xfs.readFilePromise(file, `utf8`)).resolves.toEqual(``); + }); + }); + it(`shouldn't affect unrelated commands`, async () => { await xfs.mktempPromise(async tmpDir => { const file = ppath.join(tmpDir, `file`); @@ -1020,6 +1331,21 @@ describe(`Shell`, () => { }); }); + it(`should support redirection-only commands (append)`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + await xfs.writeFilePromise(file, `foo bar baz\n`); + + await expectResult(bufferResult( + `>> "${file}"`, + ), { + stdout: ``, + }); + + await expect(xfs.readFilePromise(file, `utf8`)).resolves.toEqual(`foo bar baz\n`); + }); + }); + it(`should support output redirections from fd (stdout)`, async () => { await xfs.mktempPromise(async tmpDir => { const file = ppath.join(tmpDir, `file`); @@ -1143,6 +1469,18 @@ describe(`Shell`, () => { stderr: `Bad file descriptor: "42"\n`, }); }); + + await xfs.mktempPromise(async tmpDir => { + await expectResult(bufferResult( + `echo "hello world" >& 0 && echo OK || echo KO`, + [], + {cwd: tmpDir}, + ), { + exitCode: 0, + stdout: `KO\n`, + stderr: `Bad file descriptor: "0"\n`, + }); + }); }); }); }); @@ -1197,6 +1535,197 @@ describe(`Shell`, () => { }); }); + it(`should pipe the result of a command into a group`, async () => { + await expectResult(bufferResult([ + `node -e 'process.stdout.write("hello world")'`, + `{ cat; }`, + ].join(` | `)), { + stdout: `hello world`, + }); + }); + + ifNotWin32It(`should allow redirection-only producers when consumers close early`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + await xfs.writeFilePromise(file, Buffer.alloc(4 * 1024 * 1024, `x`)); + + await expectResult(bufferResult( + `< "${file}" | head -c 1`, + ), { + stdout: `x`, + }); + }); + }); + + it(`should stop redirection-only producers when builtin consumers exit`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + await xfs.writeFilePromise(file, Buffer.alloc(4 * 1024 * 1024, `x`)); + + await expectResult(withTimeout( + bufferResult(`< "${file}" | true`), + 1000, + `Timed out waiting for redirection-only producer to stop`, + ), { + stdout: ``, + }); + + await expectResult(withTimeout( + bufferResult(`< "${file}" | echo ok`), + 1000, + `Timed out waiting for redirection-only producer to stop`, + ), { + stdout: `ok\n`, + }); + }); + }); + + it(`should stop fd-redirected producers when builtin consumers exit`, async () => { + await expectResult(withTimeout( + bufferResult(`node -e 'process.stdout.write("hello world")' >&1 | echo ok`), + 10000, + `Timed out waiting for stdout fd-redirected producer to stop`, + ), { + stdout: `ok\n`, + }); + + await expectResult(withTimeout( + bufferResult(`node -e 'process.stderr.write("hello world")' 2>&1 | echo ok`), + 10000, + `Timed out waiting for stderr fd-redirected producer to stop`, + ), { + stdout: `ok\n`, + }); + }); + + ifNotWin32It(`should let native producers continue after builtin consumers close`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + const producer = `node -e 'process.stdout.end(); setTimeout(() => require("fs").writeFileSync(${JSON.stringify(file)}, "done"), 200);'`; + + await expectResult(bufferResult(`${producer} | true`), { + stdout: ``, + }); + + await expect(xfs.readFilePromise(file, `utf8`)).resolves.toEqual(`done`); + }); + }); + + it(`should support piped redirection-only stdin fd copies`, async () => { + await expectResult(withTimeout( + bufferResult(`node -e 'process.stdout.write("hello world")' | <&0`), + 10000, + `Timed out waiting for piped stdin fd redirection`, + ), { + stdout: `hello world`, + }); + }); + + ifNotWin32It(`should close unused pipeline input in redirection-only commands`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + const producer = `node -e 'process.stdout.on("error", error => { if (error.code === "EPIPE") process.exit(0); throw error; }); const chunk = "x".repeat(1024); function write() { while (process.stdout.write(chunk)); process.stdout.once("drain", write); } write(); setTimeout(() => process.exit(2), 5000);'`; + + await expectResult(withTimeout( + bufferResult(`${producer} | > "${file}"`), + 10000, + `Timed out waiting for redirection-only command to close unused pipeline input`, + ), { + stdout: ``, + stderr: ``, + exitCode: 0, + }); + }); + }); + + ifNotWin32It(`should close unused piped subshell input after redirection-only commands`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + const producer = `node -e 'process.stdout.on("error", error => { if (error.code === "EPIPE") process.exit(0); throw error; }); const chunk = "x".repeat(1024); function write() { while (process.stdout.write(chunk)); process.stdout.once("drain", write); } write(); setTimeout(() => process.exit(2), 5000);'`; + + await expectResult(withTimeout( + bufferResult(`${producer} | ( > "${file}" )`), + 10000, + `Timed out waiting for subshell to close unused pipeline input`, + ), { + stdout: ``, + stderr: ``, + exitCode: 0, + }); + }); + }); + + ifNotWin32It(`should close unused piped group input after redirection-only commands`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + const producer = `node -e 'process.stdout.on("error", error => { if (error.code === "EPIPE") process.exit(0); throw error; }); const chunk = "x".repeat(1024); function write() { while (process.stdout.write(chunk)); process.stdout.once("drain", write); } write(); setTimeout(() => process.exit(2), 5000);'`; + + await expectResult(withTimeout( + bufferResult(`${producer} | { > "${file}"; }`), + 10000, + `Timed out waiting for group to close unused pipeline input`, + ), { + stdout: ``, + stderr: ``, + exitCode: 0, + }); + }); + }); + + it(`should keep piped subshell input after redirection-only commands`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + + await expectResult(bufferResult( + `node -e 'process.stdout.write("hello world")' | ( > "${file}"; cat )`, + ), { + stdout: `hello world`, + }); + + await expect(xfs.readFilePromise(file, `utf8`)).resolves.toEqual(``); + }); + }); + + it(`should keep redirected piped subshell input after redirection-only commands`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + const output = ppath.join(tmpDir, `output`); + + await expectResult(withTimeout( + bufferResult( + `node -e 'process.stdout.write("hello world")' | ( > "${file}"; cat ) > "${output}"`, + ), + 10000, + `Timed out waiting for redirected subshell to preserve pipeline input`, + ), { + stdout: ``, + }); + + await expect(xfs.readFilePromise(file, `utf8`)).resolves.toEqual(``); + await expect(xfs.readFilePromise(output, `utf8`)).resolves.toEqual(`hello world`); + }); + }); + + it(`should keep redirected piped group input after redirection-only commands`, async () => { + await xfs.mktempPromise(async tmpDir => { + const file = ppath.join(tmpDir, `file`); + const output = ppath.join(tmpDir, `output`); + + await expectResult(withTimeout( + bufferResult( + `node -e 'process.stdout.write("hello world")' | { > "${file}"; cat; } > "${output}"`, + ), + 10000, + `Timed out waiting for redirected group to preserve pipeline input`, + ), { + stdout: ``, + }); + + await expect(xfs.readFilePromise(file, `utf8`)).resolves.toEqual(``); + await expect(xfs.readFilePromise(output, `utf8`)).resolves.toEqual(`hello world`); + }); + }); + it(`should pipe the stdout of a command into another`, async () => { await expectResult(bufferResult([ `node -e 'process.stdout.write("Hello World");'`,