fix(fspy): avoid recursive syscall initialization

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+- **Fixed** On Linux, concurrent system calls in tracked programs, such as `pnpm dlx`, no longer cause a stack overflow ([#854](https://github.com/voidzero-dev/vite-task/pull/854)).
 - **Fixed** Cached tasks can now run scripts whose shebang names an interpreter path longer than 126 bytes. Before, the path was cut short and the script failed to start or ran the wrong program. On macOS, text after a `#` in the shebang line is now ignored, as it is when the script runs on its own ([#788](https://github.com/voidzero-dev/vite-task/pull/788)).
 - **Fixed** On macOS, tasks with automatic file-access tracking can now run Node.js single executable applications, such as pnpm 11's standalone `pnpm` binary, which previously aborted with `Assertion failed: (magic) == (kMagic)`, and `deno compile` outputs, which previously failed with `Did not find magic bytes` ([#840](https://github.com/voidzero-dev/vite-task/pull/840)).
 - **Fixed** Reinstalling dependencies with pnpm, or installing them at another path or with another pnpm version, no longer causes cache misses, and tasks that run `pnpm run` or `pnpm exec` no longer miss the cache after pnpm rechecks dependencies ([#823](https://github.com/voidzero-dev/vite-task/pull/823), [#824](https://github.com/voidzero-dev/vite-task/pull/824), [#825](https://github.com/voidzero-dev/vite-task/pull/825), [#850](https://github.com/voidzero-dev/vite-task/pull/850)).

crates/fspy/tests/rust_std.rs

@@ -51,6 +51,40 @@ async fn metadata() -> anyhow::Result<()> {
     Ok(())
 }
 
+#[cfg(all(target_os = "linux", not(target_env = "musl")))]
+#[test(tokio::test)]
+async fn syscall_during_dlopen() {
+    let tmp_dir = tempfile::tempdir().unwrap();
+    let fifo_path = tmp_dir.path().join("fifo").to_str().unwrap().to_owned();
+    nix::unistd::mkfifo(fifo_path.as_str(), nix::sys::stat::Mode::S_IRWXU).unwrap();
+
+    track_fn!(fifo_path, |fifo_path: String| {
+        // The interposer used to resolve libc's syscall with dlsym, which waits for dlopen,
+        // so a second thread calling syscall waited for the first through futex, re-entering
+        // the interposer until its stack overflowed.
+        fn call_syscall_during_dlopen(fifo_path: &str) {
+            // This waits for dlopen to open the FIFO, which it reads until writers close.
+            let _writer = OpenOptions::new().write(true).open(fifo_path).unwrap();
+            // SAFETY: getpid takes no arguments.
+            unsafe { libc::syscall(libc::SYS_getpid) };
+        }
+        // Opening files and spawning threads can also wait for dlopen, so do both first.
+        File::open("/").unwrap();
+        let path = fifo_path.clone();
+        std::thread::spawn(move || {
+            let c = std::ffi::CString::new(path.as_str()).unwrap();
+            // SAFETY: dlopen fails on the FIFO's empty contents.
+            std::thread::spawn(move || unsafe {
+                libc::dlopen(c.as_ptr(), libc::RTLD_NOW);
+            });
+            call_syscall_during_dlopen(&path);
+        });
+        call_syscall_during_dlopen(&fifo_path);
+    })
+    .await
+    .unwrap();
+}
+
 #[test(tokio::test)]
 async fn readdir() -> anyhow::Result<()> {
     let tmpdir = tempfile::tempdir()?;

crates/fspy_preload_unix/Cargo.toml

@@ -19,5 +19,8 @@ allocator-api2 = { workspace = true }
 fspy_nostd_alloc = { workspace = true }
 static_cell = { workspace = true }
 
+[target.'cfg(target_os = "linux")'.dependencies]
+syscalls = { workspace = true }
+
 [lints]
 workspace = true

crates/fspy_preload_unix/src/interceptions/linux_syscall.rs

@@ -48,6 +48,48 @@ unsafe extern "C" fn syscall(syscall_no: c_long, mut args: ...) -> c_long {
             };
         }
     }
-    // SAFETY: forwarding the syscall to the original libc syscall function with the extracted arguments
-    unsafe { syscall::original()(syscall_no, a0, a1, a2, a3, a4, a5) }
+    // Do not resolve libc's syscall through LazyLock: concurrent first calls
+    // wait through futex, which calls this interposer again before the lock
+    // is initialized. Forward directly to the kernel instead.
+    #[expect(
+        clippy::cast_sign_loss,
+        clippy::cast_possible_truncation,
+        reason = "c_long and usize have the same width on supported Linux targets; preserve their bit patterns"
+    )]
+    // SAFETY: forward the syscall number and arguments supplied by the caller.
+    let result = unsafe {
+        syscalls::raw::syscall6(
+            syscall_no as usize,
+            a0 as usize,
+            a1 as usize,
+            a2 as usize,
+            a3 as usize,
+            a4 as usize,
+            a5 as usize,
+        )
+    };
+    match syscalls::Errno::from_ret(result) {
+        #[expect(clippy::cast_possible_wrap, reason = "preserve the syscall result's bit pattern")]
+        Ok(value) => value as c_long,
+        Err(errno) => {
+            // SAFETY: __errno_location returns this thread's errno storage.
+            unsafe { *libc::__errno_location() = errno.into_raw() };
+            -1
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::syscall;
+
+    #[test]
+    fn error_sets_errno() {
+        // SAFETY: close rejects an invalid descriptor without accessing memory.
+        unsafe {
+            *libc::__errno_location() = 0;
+            assert_eq!(syscall(libc::SYS_close, -1_i64, 0_i64, 0_i64, 0_i64, 0_i64, 0_i64), -1);
+            assert_eq!(*libc::__errno_location(), libc::EBADF);
+        }
+    }
 }