diff --git a/AGENTS.md b/AGENTS.md index a567ce8..a8e56ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,6 +119,7 @@ and `cargo build --release` directly; CI runs the gate on both systems. | what macOS lists and measures in bulk | `crates/disktree-core/src/macos.rs` — `unsafe` for `getattrlistbulk` | | what Windows lists, measures and compares differently | `crates/disktree-core/src/windows.rs` — `unsafe` for Win32 | | reading a whole NTFS drive from its file table | `crates/disktree-core/src/mft.rs` | +| GNU du arguments, walk, output and index | `crates/disktree-core/src/du/` | | a key, a screen transition, a mark | `crates/disktree-app/src/state.rs` | | spacing, type and size | `crates/disktree-app/src/ui.rs` — tokens only, no `px` in layout | | the mosaic's painting or labels | `crates/disktree-app/src/treemap_view.rs` | diff --git a/Cargo.lock b/Cargo.lock index 71482b5..2e8d78e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1428,6 +1428,7 @@ dependencies = [ "gpui-kit", "gpui-omarchy", "rustc-hash 2.1.3", + "rustix", "tempfile", "windows-sys 0.61.2", ] @@ -1437,6 +1438,7 @@ name = "disktree-core" version = "0.11.0" dependencies = [ "chrono", + "libc", "rayon", "rustc-hash 2.1.3", "rustix", diff --git a/Cargo.toml b/Cargo.toml index 30dfed0..611f299 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,10 @@ [workspace] resolver = "3" -members = ["crates/disktree-app", "crates/disktree-core", "xtask"] +members = [ + "crates/disktree-app", + "crates/disktree-core", + "xtask", +] [workspace.package] version = "0.11.0" diff --git a/README.md b/README.md index 8044269..8c85657 100644 --- a/README.md +++ b/README.md @@ -372,6 +372,85 @@ tested: fsmonitor, hooks and pager off, and a checkout that defines its own filter drivers is not asked for its status at all ("changes unknown"). +## As `du` + +`disktree --du` is GNU du: the same options, output, messages and exit +status. It runs headless from the existing `disktree` binary, so it needs no +display and adds no second binary to install. Everything after `--du` is a +GNU du argument. + +```sh +disktree --du -sh ~/src +ln -s "$(command -v disktree)" ~/.local/bin/du # early on PATH +du -sh ~/src +``` + +When invoked through a link whose basename is `du`, the same binary enters +this mode automatically. Existing scripts and coding agents can keep calling +`du` without learning a new command. + +It follows coreutils 9.11. `crates/disktree-app/tests/gnu.rs` runs 78 argument +sets against the GNU du it finds (`gdu` from Homebrew's coreutils, or a `du` +that is GNU's) and compares output, messages and exit status. GNU du 9.4 and +later match, apart from two corner cases GNU changed since then: +`--max-depth` below zero, and the wording of a bad `--time-style`. + +It is faster three ways: + +- The walk runs on every core. Each directory is still visited in the order + GNU's `fts` visits it, so hard links, `-x`, `--exclude`, depths and + thresholds come out the same. +- On macOS, `getattrlistbulk` reads a directory's entries and their sizes in + one system call instead of one per file. +- Every walk is kept as a snapshot, in `~/Library/Caches/disktree/du` on + macOS and `~/.cache/disktree/du` elsewhere (`DISKTREE_DU_INDEX_DIR` moves + it). With `--max-age=AGE`, a snapshot younger than `AGE` is brought up to + date instead of walked again. On macOS the FSEvents journal names every + directory that changed since, and only those are read. Elsewhere every + directory is stat'ed and read again if it changed. + +On a 670 GB tree of 6.65 million entries (Apple M5 Max, APFS, warm cache): + +| command | time | +| --- | --- | +| GNU du 9.11, `du -s` | 148 s | +| `disktree --du -s` | 25 s | +| `disktree --du -s --max-age=1h`, through the journal | 2 to 3 s | +| `disktree --du -s --max-age=1h`, by directory | 3.4 s | + +Without `--max-age` every answer is a full walk, so it is what GNU du would +say. With it, two things can be missed. A file still open for writing is not +in the journal until it is closed, so files modified within an hour of the +snapshot are stat'ed again every time, which covers the usual case, a log or +a build still being written. And by directory, a file that grew in place +without its directory changing is only caught if it was one of those recent +files. Either way `AGE` counts from the last full walk, and a catch-up does +not reset it, so nothing can go unseen for longer than `AGE`. `--fresh` walks +everything again. + +Known differences from GNU du, all rare: + +- `du >&-` exits 0: Rust reopens a closed standard output on `/dev/null` + before `du` starts, so the write error GNU reports never happens. +- Hard links to one file in the same directory of more than 10,000 entries + can be listed in another order, since GNU sorts those with the C library's + unstable `qsort`. +- Numbers use the C locale's `.` and no digit grouping, and `--time` formats + go through chrono rather than the C library's `strftime`: `full-iso`, + `long-iso` and `iso` match, but `%Z` prints an offset and `%x` a two-digit + year. +- In a UTF-8 locale, `--exclude` wildcards match bytes rather than + characters, and diagnostics do not escape control characters inside + `‘...’` quotes. + +`--json` prints the same entries as one document, each with its size, its +kind, why it can be had back, the command that gives the space back (`cargo +clean`, `pnpm store prune`, …) and when it was last written, plus `as_of` +and `source` (`walk`, `journal` or `directories`) for the whole answer. +Behind a `du` symlink the same switches come from the environment: +`DISKTREE_DU_JSON=1`, `DISKTREE_DU_MAX_AGE=1h`, `DISKTREE_DU_FRESH=1`, and +`DISKTREE_DU_INDEX=0` to keep no snapshots. + ## On Hyprland Hyprland tiles new windows, so disktree opens into whatever tile it is given. @@ -406,6 +485,8 @@ gone while their neighbours are not. | `crates/disktree-app/src/treemap_view.rs` | painting the mosaic and its labels | | `crates/disktree-app/src/ui.rs` | the spacing, type and size scale, in `rem` | | `crates/disktree-app/src/tests.rs` | end-to-end tests through a real window | +| `crates/disktree-core/src/du` | GNU du on a parallel walk and its snapshots | +| `crates/disktree-app/tests/gnu.rs` | comparison against GNU du | | `packaging/`, `assets/`, `Makefile` | the desktop entry, the icon, and install | The interface follows the diff --git a/crates/disktree-app/Cargo.toml b/crates/disktree-app/Cargo.toml index 4bc758a..649fa83 100644 --- a/crates/disktree-app/Cargo.toml +++ b/crates/disktree-app/Cargo.toml @@ -31,3 +31,6 @@ gpui-kit = { version = "=0.6.6", default-features = false, features = [ "assets", "test-support", ] } + +[target.'cfg(unix)'.dev-dependencies] +rustix = { version = "1.1", features = ["fs"] } diff --git a/crates/disktree-app/src/main.rs b/crates/disktree-app/src/main.rs index 92fa2f7..5686d6a 100644 --- a/crates/disktree-app/src/main.rs +++ b/crates/disktree-app/src/main.rs @@ -25,7 +25,9 @@ mod ui; mod views; mod widgets; -use std::path::PathBuf; +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; #[cfg(target_os = "macos")] use std::{ io::IsTerminal as _, @@ -81,15 +83,42 @@ options: -h, --help show this help "; -fn main() -> Result<()> { +fn main() -> Result { #[cfg(windows)] console::attach(); - let outcome = run(); + let mut argv = std::env::args_os(); + let argv0 = argv.next().unwrap_or_else(|| OsString::from("disktree")); + let outcome = match du_invocation(&argv0, argv) { + Some((program, arguments)) => { + Ok(disktree_core::du::run(&program, &arguments)) + } + None => run().map(|()| ExitCode::SUCCESS), + }; #[cfg(windows)] console::detach(); outcome } +/// Select the `du` personality before loading settings or starting GPUI. +/// +/// Keeping headless personalities at this boundary also leaves one obvious +/// dispatch point for a future text-tree mode such as issue #74 proposes. +fn du_invocation( + argv0: &OsStr, + mut arguments: impl Iterator, +) -> Option<(Vec, Vec)> { + if Path::new(argv0).file_name() == Some(OsStr::new("du")) { + return Some(( + argv0.to_owned().into_encoded_bytes(), + arguments.collect(), + )); + } + if arguments.next().as_deref() == Some(OsStr::new("--du")) { + return Some((b"du".to_vec(), arguments.collect())); + } + None +} + fn run() -> Result<()> { let saved = power::settings_path() .map_or_else( diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index 3d7f151..33e3971 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -5,6 +5,7 @@ //! that panics while painting, a binding that never fires, a removal that //! reports success without removing anything. +use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; use disktree_core::removal::RemovalMode; @@ -48,6 +49,29 @@ fn options() -> ScanOptions { } } +#[test] +fn du_mode_is_selected_only_at_the_headless_dispatch_point() { + let args = [OsString::from("--du"), OsString::from("-sh")]; + let (program, rest) = + super::du_invocation(OsStr::new("/usr/bin/disktree"), args.into_iter()) + .expect("--du mode"); + assert_eq!(program, b"du"); + assert_eq!(rest, [OsString::from("-sh")]); + + let args = [OsString::from("-sh")]; + let (program, rest) = + super::du_invocation(OsStr::new("/tmp/du"), args.into_iter()) + .expect("du multicall"); + assert_eq!(program, b"/tmp/du"); + assert_eq!(rest, [OsString::from("-sh")]); + + let args = [OsString::from("--help")]; + assert!( + super::du_invocation(OsStr::new("/usr/bin/disktree"), args.into_iter()) + .is_none() + ); +} + type Window = VisualTestContext; /// Open a window over a real scan of `root`, without the background walk. diff --git a/crates/disktree-app/tests/gnu.rs b/crates/disktree-app/tests/gnu.rs new file mode 100644 index 0000000..d60e944 --- /dev/null +++ b/crates/disktree-app/tests/gnu.rs @@ -0,0 +1,524 @@ +//! `disktree --du` against GNU du itself: the same fixture and arguments, +//! and the output, the diagnostics and the exit status compared byte for +//! byte. Each case runs three times: without an index, with the index the +//! first run left, and with `--max-age` answering from it. +//! +//! GNU du is found as `gdu` (Homebrew's coreutils) or as a `du` that says it +//! is GNU. Without one the comparison is skipped, loudly. +#![cfg(unix)] + +use std::os::unix::fs::{PermissionsExt as _, symlink}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use rustix::fs::{Mode, OFlags, mkdirat, openat}; + +/// The coreutils release `disktree --du` follows. Older ones differ in a few +/// places, listed in [`CHANGED_SINCE`]. +const FOLLOWS: (u32, u32) = (9, 11); + +/// The oldest GNU du compared against. Before 9.2, `--apparent-size` counted +/// a directory's own size, and messages began with the full `argv[0]`. +const OLDEST: (u32, u32) = (9, 4); + +/// Cases whose GNU output changed after [`OLDEST`], compared only against a +/// GNU du as new as [`FOLLOWS`]. +const CHANGED_SINCE: &[&[&str]] = &[ + // The maximum depth became signed: -1 no longer wraps to "everything". + &["-d", "-1"], + // The list of valid time styles is worded differently. + &["--time-style=bogus", "--time"], +]; + +fn version(gnu: &Path) -> (u32, u32) { + let output = Command::new(gnu).arg("--version").output().expect("run"); + let text = String::from_utf8_lossy(&output.stdout); + let number = text + .lines() + .next() + .and_then(|line| line.rsplit(' ').next()) + .unwrap_or_default(); + let mut parts = number.split('.').map(|part| part.parse().unwrap_or(0)); + (parts.next().unwrap_or(0), parts.next().unwrap_or(0)) +} + +fn gnu_du() -> Option { + for name in ["gdu", "du"] { + let Ok(output) = Command::new(name).arg("--version").output() else { + continue; + }; + if String::from_utf8_lossy(&output.stdout).contains("GNU coreutils") { + let path = Command::new("sh") + .args(["-c", &format!("command -v {name}")]) + .output() + .ok()?; + let path = String::from_utf8_lossy(&path.stdout).trim().to_owned(); + return Some(PathBuf::from(path)); + } + } + None +} + +fn write(path: &Path, bytes: usize) { + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(path, vec![b'x'; bytes]).expect("write"); +} + +/// A tree with each thing du treats specially. +fn fixture(root: &Path) { + write(&root.join("a/small"), 10); + write(&root.join("a/four-k"), 4096); + write(&root.join("a/b/c/deep"), 70_000); + write(&root.join("a/b/mid"), 5000); + write(&root.join("big"), 3 * 1024 * 1024 + 7); + write(&root.join("with space/x y"), 100); + write(&root.join("quote's/f"), 1); + write(&root.join("keep/logs/app.log"), 12_345); + write(&root.join("keep/logs/old.log.gz"), 999); + std::fs::create_dir_all(root.join("empty")).expect("mkdir"); + + // A sparse file: allocation and apparent size disagree. + let sparse = std::fs::File::create(root.join("sparse")).expect("sparse"); + sparse.set_len(10 * 1024 * 1024).expect("set_len"); + + // Hard links: within one directory, across directories, and so across + // operands. + write(&root.join("links/original"), 50_000); + std::fs::hard_link(root.join("links/original"), root.join("links/second")) + .expect("link"); + std::fs::create_dir_all(root.join("elsewhere")).expect("mkdir"); + std::fs::hard_link( + root.join("links/original"), + root.join("elsewhere/third"), + ) + .expect("link"); + + // Symbolic links: to a file, to a directory, to nothing, and a loop. + symlink("../big", root.join("a/to-big")).expect("symlink"); + symlink("../a/b", root.join("keep/to-b")).expect("symlink"); + symlink("nowhere", root.join("a/dangling")).expect("symlink"); + symlink("..", root.join("a/b/c/up")).expect("symlink"); + + // More than 10,000 entries: fts sorts such a directory by inode. + let many = root.join("many"); + std::fs::create_dir_all(&many).expect("mkdir"); + for n in 0..10_050 { + std::fs::write(many.join(format!("f{n:05}")), b"").expect("write"); + } + // A subdirectory among them, so the order of directories shows. + write(&many.join("zz-sub/inner"), 3000); + write(&many.join("aa-sub/inner"), 3000); + + // Deeper than PATH_MAX (1024 bytes on macOS) from the root, built a + // level at a time since no single path can reach the bottom. + let dir_flags = OFlags::RDONLY | OFlags::DIRECTORY; + std::fs::create_dir(root.join("deep")).expect("mkdir"); + let mut at = + openat(rustix::fs::CWD, root.join("deep"), dir_flags, Mode::empty()) + .expect("open"); + for level in 0..6 { + let name = format!("{level}{}", "d".repeat(200)); + mkdirat(&at, name.as_str(), Mode::from_raw_mode(0o755)) + .expect("mkdirat"); + at = openat(&at, name.as_str(), dir_flags, Mode::empty()) + .expect("openat"); + } + let file = openat( + &at, + "bottom", + OFlags::WRONLY | OFlags::CREATE, + Mode::from_raw_mode(0o644), + ) + .expect("create"); + rustix::io::write(&file, &[b'x'; 5000]).expect("write"); +} + +struct Harness { + ours: PathBuf, + gnu: PathBuf, + cwd: PathBuf, + index: PathBuf, +} + +fn run( + program: &Path, + cwd: &Path, + args: &[&str], + env: &[(&str, &Path)], +) -> Output { + let mut command = Command::new(program); + if program == Path::new(env!("CARGO_BIN_EXE_disktree")) { + command.arg("--du"); + } + command + .args(args) + .current_dir(cwd) + .env("TZ", "UTC") + .env("LC_ALL", "C") + .env_remove("DU_BLOCK_SIZE") + .env_remove("BLOCK_SIZE") + .env_remove("BLOCKSIZE") + .env_remove("POSIXLY_CORRECT") + .env_remove("TIME_STYLE") + .env_remove("DISKTREE_DU_MAX_AGE") + .env_remove("DISKTREE_DU_FRESH") + .env_remove("DISKTREE_DU_JSON"); + for (name, value) in env { + command.env(name, value); + } + command.output().expect("run") +} + +fn show(output: &Output) -> String { + format!( + "status {:?}\n--- stdout\n{}--- stderr\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// Standard error with the program's name replaced by a placeholder: the +/// full path `getopt` and the "Try ... --help" line repeat, and the name +/// messages start with, which is the basename since coreutils 9.2 and the +/// full path before. +fn stderr(output: &Output, program: &Path) -> Vec { + let base = program.file_name().expect("name").to_string_lossy(); + String::from_utf8_lossy(&output.stderr) + .replace(&program.display().to_string(), "DU") + .lines() + .map(|line| match line.strip_prefix(&format!("{base}:")) { + Some(rest) => format!("DU:{rest}\n"), + None => format!("{line}\n"), + }) + .collect::() + .into_bytes() +} + +impl Harness { + /// Compare one argument set; a mismatch is returned, not raised, so + /// that one run reports every case that differs. + fn check(&self, args: &[&str]) -> Option { + let gnu = run(&self.gnu, &self.cwd, args, &[]); + let index = self.index.as_path(); + let one = Path::new("1"); + let hour = Path::new("1h"); + // Through the environment, as behind a `du` symlink: appended + // flags would land after a `--`. + let passes: [(&str, Vec<(&str, &Path)>); 3] = [ + ( + "cold", + vec![ + ("DISKTREE_DU_INDEX_DIR", index), + ("DISKTREE_DU_FRESH", one), + ], + ), + ("indexed", vec![("DISKTREE_DU_INDEX_DIR", index)]), + ( + "--max-age", + vec![ + ("DISKTREE_DU_INDEX_DIR", index), + ("DISKTREE_DU_MAX_AGE", hour), + ], + ), + ]; + for (pass, env) in passes { + let ours = run(&self.ours, &self.cwd, args, &env); + if ours.stdout != gnu.stdout + || stderr(&ours, &self.ours) != stderr(&gnu, &self.gnu) + || ours.status.code() != gnu.status.code() + { + return Some(format!( + "du {} ({pass})\n=== GNU\n{}\n=== disktree\n{}", + args.join(" "), + show(&gnu), + show(&ours) + )); + } + } + None + } +} + +#[test] +fn matches_gnu_du() { + let Some(gnu) = gnu_du() else { + eprintln!("SKIPPED: no GNU du (install coreutils for gdu)"); + return; + }; + let version = version(&gnu); + if version < OLDEST { + eprintln!("SKIPPED: GNU du {version:?} predates {OLDEST:?}"); + return; + } + let temp = tempfile::TempDir::new().expect("tempdir"); + // Canonical, so both tools print the same path when given it. + let base = temp.path().canonicalize().expect("canonical"); + let tree = base.join("tree"); + fixture(&tree); + let locked = tree.join("locked"); + write(&locked.join("secret"), 100); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)) + .expect("chmod"); + // Readable but not searchable: its names can be listed, not stat'ed. + let listed = tree.join("listed"); + write(&listed.join("one"), 100); + write(&listed.join("two"), 100); + std::fs::set_permissions(&listed, std::fs::Permissions::from_mode(0o644)) + .expect("chmod"); + + // A link named `du` selects the multicall personality. The comparison + // normalizes program names because Homebrew installs GNU du as `gdu`. + let bin = base.join("bin"); + std::fs::create_dir_all(&bin).expect("mkdir"); + let ours = bin.join("du"); + symlink(env!("CARGO_BIN_EXE_disktree"), &ours).expect("symlink"); + let harness = Harness { + ours, + gnu, + cwd: tree, + index: base.join("index"), + }; + + let exclude_file = base.join("excludes"); + std::fs::write(&exclude_file, "*.gz \r\n\n\nmid\n").expect("write"); + let exclude_from = format!("--exclude-from={}", exclude_file.display()); + let names0 = base.join("names0"); + std::fs::write(&names0, b"a\0big\0\0links\0").expect("write"); + let files0 = format!("--files0-from={}", names0.display()); + + let cases: &[&[&str]] = &[ + &[], + &["."], + &["-a"], + &["-a", "."], + &["-s"], + &["-sh", "a", "big", "sparse", "links", "many"], + &["-ah"], + &["--si", "-a"], + &["-ab"], + &["-a", "--apparent-size"], + &["-ak"], + &["-am"], + &["-a", "-B1K"], + &["-a", "-BK"], + &["-a", "-BKB"], + &["-a", "-B", "MiB"], + &["-a", "--block-size=human-readable"], + &["-c", "a", "links", "elsewhere"], + &["-sc", "links", "elsewhere", "links"], + &["-al", "links", "elsewhere"], + &["-d", "1"], + &["--max-depth=0", "-a"], + &["-d", "2", "-a", "a"], + &["-S"], + &["-Sa"], + &["-S", "--apparent-size", "-a"], + &["--inodes"], + &["--inodes", "-a", "-h"], + &["-a", "--exclude=*.log"], + &["-a", "--exclude=b"], + &["-a", "--exclude=a/b/c"], + &["-a", &exclude_from], + &["-a", "-t", "4K"], + &["-a", "-t", "-4K"], + &["-a", "-0"], + &["-aL", "a"], + &["-aL", "keep"], + &["-aH", "keep/to-b"], + &["-a", "keep/to-b"], + &["-aD", "a/to-big"], + &["-s", "a/dangling"], + &["-sL", "a/dangling"], + &["-aLl", "a"], + &["-ax"], + &["-a", "--time"], + &["-a", "--time=ctime", "--time-style=full-iso"], + &["-s", "--time", "--time-style=+%Y %j %N"], + &["-a", "--time-style=iso", "--time"], + &["-s", "missing", "a", "", "big"], + &["-s", "a//"], + &["-s", &files0], + &["-s", "locked"], + &["-a", "locked"], + &["-as"], + &["-s", "-d", "1"], + &["-s", "-d", "0"], + &["-d", "x"], + &["-d", "-1"], + &["-B", "0"], + &["-t", "1x"], + &["-t", "-0"], + &["--s"], + &["--foo", "-z"], + &["--all=1"], + &["--time=x"], + &["--time-style=bogus", "--time"], + &["--files0-from=/nonexistent"], + &["-X", "/nonexistent"], + &["-s", "--files0-from=/dev/null", "a"], + &["--max-d=1", "--sum"], + &["--", "-a"], + &["a", "-s"], + &["--files0-from=a", "-c"], + &["--s=1"], + &["-a", "listed"], + &["-s", "deep"], + &["-a", "--exclude=./a/b", "./a"], + &["-a", "--exclude=./a/b", "a"], + ]; + let failures: Vec = cases + .iter() + .filter(|case| version >= FOLLOWS || !CHANGED_SINCE.contains(case)) + .filter_map(|case| harness.check(case)) + .collect(); + + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)) + .expect("chmod"); + std::fs::set_permissions(&listed, std::fs::Permissions::from_mode(0o755)) + .expect("chmod"); + assert!( + failures.is_empty(), + "{} of {} cases differ:\n\n{}", + failures.len(), + cases.len(), + failures.join("\n") + ); +} + +#[test] +fn json_describes_what_du_prints() { + let temp = tempfile::TempDir::new().expect("tempdir"); + let root = temp.path(); + write(&root.join("proj/Cargo.toml"), 10); + write(&root.join("proj/target/debug/app"), 50_000); + write(&root.join("proj/src/main.rs"), 100); + let output = run( + Path::new(env!("CARGO_BIN_EXE_disktree")), + root, + &["--json", "-d", "1", "proj"], + &[("DISKTREE_DU_INDEX_DIR", &root.join("index"))], + ); + let text = String::from_utf8_lossy(&output.stdout); + assert!(output.status.success(), "{}", show(&output)); + assert!(text.contains("\"source\": \"walk\""), "{text}"); + assert!( + text.contains("\"reclaim\": \"build output\"") + && text.contains("cargo clean --manifest-path proj/Cargo.toml"), + "{text}" + ); +} + +/// Set a file's modification time `seconds` into the past. +fn age(path: &Path, seconds: u64) { + let when = + std::time::SystemTime::now() - std::time::Duration::from_secs(seconds); + std::fs::File::options() + .append(true) + .open(path) + .and_then(|file| file.set_modified(when)) + .expect("set mtime"); +} + +fn source_of(output: &Output) -> String { + let text = String::from_utf8_lossy(&output.stdout); + let at = text.find("\"source\": \"").expect("a source") + 11; + text[at..].split('"').next().unwrap_or_default().to_owned() +} + +/// `--max-age` catches up on what changed since the snapshot and answers +/// what a walk would, without walking. +fn catches_up(journal: bool) { + let temp = tempfile::TempDir::new().expect("tempdir"); + let root = temp.path().canonicalize().expect("canonical"); + for dir in ["a/b/c", "a/d", "e"] { + write(&root.join(dir).join("old"), 20_000); + age(&root.join(dir).join("old"), 7200); + } + write(&root.join("e/gone"), 9000); + let index = root.join("index"); + let du = Path::new(env!("CARGO_BIN_EXE_disktree")); + let off = Path::new("0"); + let mut env = vec![("DISKTREE_DU_INDEX_DIR", index.as_path())]; + if !journal { + env.push(("DISKTREE_DU_JOURNAL", off)); + } + let args = ["-a", "a", "e"]; + run(du, &root, &args, &env); + + // A new file deep down, one removed, and an old file grown in place, + // which leaves every directory's timestamps alone. + write(&root.join("a/b/c/new"), 50_000); + std::fs::remove_file(root.join("e/gone")).expect("remove"); + if journal { + std::fs::OpenOptions::new() + .append(true) + .open(root.join("a/d/old")) + .and_then(|mut f| { + std::io::Write::write_all(&mut f, &vec![1; 300_000]) + }) + .expect("grow"); + } + std::thread::sleep(std::time::Duration::from_millis(500)); + + let hour = Path::new("1h"); + let mut cached_env = env.clone(); + cached_env.push(("DISKTREE_DU_MAX_AGE", hour)); + let cached = + run(du, &root, &[&args[..], &["--json"]].concat(), &cached_env); + let want = if journal { "journal" } else { "directories" }; + assert_eq!(source_of(&cached), want, "{}", show(&cached)); + let cached = run(du, &root, &args, &cached_env); + let walked = run(du, &root, &args, &[("DISKTREE_DU_INDEX", off)]); + assert_eq!( + String::from_utf8_lossy(&cached.stdout), + String::from_utf8_lossy(&walked.stdout) + ); +} + +#[test] +fn max_age_catches_up_by_directory() { + catches_up(false); +} + +#[cfg(target_os = "macos")] +#[test] +fn max_age_catches_up_through_the_journal() { + catches_up(true); +} + +/// On a case-insensitive volume an operand can be typed in another case +/// than the directory's; the journal still has to be matched against it. +#[cfg(target_os = "macos")] +#[test] +fn the_journal_is_matched_in_the_case_on_disk() { + let temp = tempfile::TempDir::new().expect("tempdir"); + let root = temp.path().canonicalize().expect("canonical"); + write(&root.join("Proj/old"), 20_000); + age(&root.join("Proj/old"), 7200); + if !root.join("proj").exists() { + eprintln!("SKIPPED: this volume is case-sensitive"); + return; + } + let index = root.join("index"); + let du = Path::new(env!("CARGO_BIN_EXE_disktree")); + let hour = Path::new("1h"); + let env = [ + ("DISKTREE_DU_INDEX_DIR", index.as_path()), + ("DISKTREE_DU_MAX_AGE", hour), + ]; + run(du, &root, &["-s", "proj"], &env); + std::fs::OpenOptions::new() + .append(true) + .open(root.join("Proj/old")) + .and_then(|mut f| std::io::Write::write_all(&mut f, &vec![1; 300_000])) + .expect("grow"); + std::thread::sleep(std::time::Duration::from_millis(500)); + let cached = run(du, &root, &["-s", "proj"], &env); + let off = Path::new("0"); + let walked = run(du, &root, &["-s", "proj"], &[("DISKTREE_DU_INDEX", off)]); + assert_eq!( + String::from_utf8_lossy(&cached.stdout), + String::from_utf8_lossy(&walked.stdout) + ); +} diff --git a/crates/disktree-core/Cargo.toml b/crates/disktree-core/Cargo.toml index 8f121eb..12ba274 100644 --- a/crates/disktree-core/Cargo.toml +++ b/crates/disktree-core/Cargo.toml @@ -34,6 +34,7 @@ windows-sys = { version = "0.61", features = [ # without asking to control Finder. Only this backend is used; the crate's # Linux and Windows code is not built. [target.'cfg(target_os = "macos")'.dependencies] +libc = "0.2" trash = { version = "5.2.9", default-features = false } [dev-dependencies] diff --git a/crates/disktree-core/src/du/args.rs b/crates/disktree-core/src/du/args.rs new file mode 100644 index 0000000..ce8d80f --- /dev/null +++ b/crates/disktree-core/src/du/args.rs @@ -0,0 +1,886 @@ +//! The command line, parsed the way GNU du's `main` parses it: glibc's +//! `getopt_long` (argument permutation, unique-prefix long options and its +//! exact complaints), then each option applied in order, since several of +//! them overwrite what an earlier one set. +//! +//! disktree's own options are matched only when spelled out in full, so they +//! can never make an abbreviation that GNU accepts ambiguous. + +use std::ffi::OsString; +use std::fmt::Write as _; +use std::io::Write as _; +use std::time::Duration; + +use super::exclude::Excludes; +use super::num::{self, ParseError, Units}; +use super::quote; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Deref { + /// `-P`, the default: never follow a symbolic link. + Physical, + /// `-D`/`-H`: follow the ones named on the command line. + Args, + /// `-L`: follow every one. + All, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TimeKind { + Modified, + Accessed, + Changed, +} + +/// Something to measure, or a complaint to make in its place, in the order +/// the operands were given: du reports a bad one where it stands. +#[derive(Clone, Debug)] +pub enum Operand { + Path(Vec), + Invalid(String), +} + +#[derive(Debug)] +pub struct Options { + pub all: bool, + pub apparent: bool, + pub units: Units, + pub total: bool, + pub deref: Deref, + pub max_depth: i64, + pub inodes: bool, + pub count_links: bool, + pub separate_dirs: bool, + pub threshold: i64, + pub excludes: Excludes, + pub one_file_system: bool, + pub null: bool, + /// `--time`, and the strftime format it prints with. + pub time: Option<(TimeKind, String)>, + pub operands: Vec, + /// Whether every dev/ino pair is remembered, not only multiply-linked + /// files': more than one operand, or `-L`, can meet a file twice. + pub hash_all: bool, + /// disktree's additions. + pub json: bool, + pub fresh: bool, + pub max_age: Option, + pub index: bool, +} + +/// What the command line asks for: a run, or an exit it already explained. +#[derive(Debug)] +pub enum Parsed { + Run(Box), + Exit(i32), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum HasArg { + No, + Required, + Optional, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Opt { + Short(u8), + Exclude, + Files0From, + Si, + Time, + TimeStyle, + Inodes, + Help, + Version, + Json, + Fresh, + MaxAge, + NoIndex, +} + +/// GNU du's `long_options`, in its order: an ambiguous prefix lists its +/// candidates in this order. +const LONG: &[(&str, HasArg, Opt)] = &[ + ("all", HasArg::No, Opt::Short(b'a')), + ("apparent-size", HasArg::No, Opt::Short(b'A')), + ("block-size", HasArg::Required, Opt::Short(b'B')), + ("bytes", HasArg::No, Opt::Short(b'b')), + ("count-links", HasArg::No, Opt::Short(b'l')), + ("dereference", HasArg::No, Opt::Short(b'L')), + ("dereference-args", HasArg::No, Opt::Short(b'D')), + ("exclude", HasArg::Required, Opt::Exclude), + ("exclude-from", HasArg::Required, Opt::Short(b'X')), + ("files0-from", HasArg::Required, Opt::Files0From), + ("human-readable", HasArg::No, Opt::Short(b'h')), + ("inodes", HasArg::No, Opt::Inodes), + ("si", HasArg::No, Opt::Si), + ("max-depth", HasArg::Required, Opt::Short(b'd')), + ("null", HasArg::No, Opt::Short(b'0')), + ("no-dereference", HasArg::No, Opt::Short(b'P')), + ("one-file-system", HasArg::No, Opt::Short(b'x')), + ("separate-dirs", HasArg::No, Opt::Short(b'S')), + ("summarize", HasArg::No, Opt::Short(b's')), + ("total", HasArg::No, Opt::Short(b'c')), + ("threshold", HasArg::Required, Opt::Short(b't')), + ("time", HasArg::Optional, Opt::Time), + ("time-style", HasArg::Required, Opt::TimeStyle), + ("help", HasArg::No, Opt::Help), + ("version", HasArg::No, Opt::Version), +]; + +/// disktree's options: exact names only. +const EXTENSIONS: &[(&str, HasArg, Opt)] = &[ + ("json", HasArg::No, Opt::Json), + ("fresh", HasArg::No, Opt::Fresh), + ("max-age", HasArg::Required, Opt::MaxAge), + ("no-index", HasArg::No, Opt::NoIndex), +]; + +/// `getopt_long`'s short option string for du; `:` marks an argument. +const SHORT: &[u8] = b"0aAbd:chHklmst:xB:DLPSX:"; + +fn short_takes_arg(c: u8) -> Option { + let at = SHORT.iter().position(|&s| s == c && c != b':')?; + Some(SHORT.get(at + 1) == Some(&b':')) +} + +const TIME_WORDS: &[(&str, TimeKind)] = &[ + ("atime", TimeKind::Accessed), + ("access", TimeKind::Accessed), + ("use", TimeKind::Accessed), + ("ctime", TimeKind::Changed), + ("status", TimeKind::Changed), +]; + +const TIME_STYLES: &[(&str, &str)] = &[ + ("full-iso", "%Y-%m-%d %H:%M:%S.%N %z"), + ("long-iso", "%Y-%m-%d %H:%M"), + ("iso", "%Y-%m-%d"), +]; + +/// The two names GNU du goes by. `error()` prefixes its messages with the +/// basename of `argv[0]`; `getopt`'s complaints, the "Try ... --help" line +/// and the usage text repeat `argv[0]` exactly as it was given. +#[derive(Clone, Debug)] +pub struct Program { + pub short: String, + pub full: String, +} + +impl Program { + pub fn new(argv0: &[u8]) -> Self { + let argv0 = if argv0.is_empty() { + b"du".as_slice() + } else { + argv0 + }; + let base = argv0.rsplit(|&b| b == b'/').next().unwrap_or(argv0); + Self { + short: String::from_utf8_lossy(base).into_owned(), + full: String::from_utf8_lossy(argv0).into_owned(), + } + } +} + +struct Parser<'a> { + program: &'a Program, + ok: bool, +} + +impl Parser<'_> { + fn error(&self, message: &str) { + eprintln!("{}: {message}", self.program.short); + } + + fn try_help(&self) { + eprintln!("Try '{} --help' for more information.", self.program.full); + } + + fn fail(&mut self, message: &str) { + self.error(message); + self.ok = false; + } +} + +/// What `getopt_long` returns, in order: an option, or its complaint about +/// one, which it prints under the full `argv[0]` as it meets it. +enum Item { + Given(Given), + Complaint(String), +} + +fn bytes(arg: &OsString) -> Vec { + arg.clone().into_encoded_bytes() +} + +/// One option as `getopt_long` hands it over: what it is, its argument, +/// and the long name it came from, which the complaints about a bad +/// argument repeat. +struct Given { + opt: Opt, + arg: Option>, + long: Option<&'static str>, +} + +fn option_label(given: &Given) -> String { + match (given.long, given.opt) { + (Some(name), _) => format!("--{name}"), + (None, Opt::Short(c)) => format!("-{}", char::from(c)), + (None, _) => String::new(), + } +} + +fn strtol_fatal(parser: &Parser<'_>, error: ParseError, given: &Given) { + let label = option_label(given); + let arg = String::from_utf8_lossy(given.arg.as_deref().unwrap_or(b"")); + let message = match error { + ParseError::Invalid => format!("invalid {label} argument '{arg}'"), + ParseError::InvalidSuffix => { + format!("invalid suffix in {label} argument '{arg}'") + } + ParseError::Overflow => format!("{label} argument '{arg}' too large"), + }; + parser.error(&message); +} + +/// Split the command line into options and operands, as glibc's +/// `getopt_long` does with its default argument permutation. +fn scan(args: &[OsString]) -> (Vec, Vec>) { + let posix = std::env::var_os("POSIXLY_CORRECT").is_some(); + let mut given = Vec::new(); + let mut operands = Vec::new(); + let mut at = 0; + let mut done = false; + while at < args.len() { + let arg = bytes(&args[at]); + at += 1; + if done || arg == b"-" || !arg.starts_with(b"-") { + operands.push(arg); + if posix { + done = true; + } + continue; + } + if arg == b"--" { + done = true; + continue; + } + if let Some(body) = arg.strip_prefix(b"--") { + let (name, value) = match body.iter().position(|&b| b == b'=') { + Some(eq) => (&body[..eq], Some(body[eq + 1..].to_vec())), + None => (body, None), + }; + let exact = LONG + .iter() + .chain(EXTENSIONS) + .find(|(long, ..)| long.as_bytes() == name); + let found = if let Some(&found) = exact { + found + } else { + let matches: Vec<_> = LONG + .iter() + .filter(|(long, ..)| long.as_bytes().starts_with(name)) + .collect(); + match matches.as_slice() { + [] => { + given.push(Item::Complaint(format!( + "unrecognized option '{}'", + String::from_utf8_lossy(&arg) + ))); + continue; + } + [only] => **only, + [first, rest @ ..] + if rest + .iter() + .all(|m| m.1 == first.1 && m.2 == first.2) => + { + **first + } + many => { + let list: Vec = many + .iter() + .map(|(long, ..)| format!("'--{long}'")) + .collect(); + // glibc repeats the option as given, `=value` too. + given.push(Item::Complaint(format!( + "option '--{}' is ambiguous; possibilities: {}", + String::from_utf8_lossy(body), + list.join(" ") + ))); + continue; + } + } + }; + let (long, has_arg, opt) = found; + let value = match (has_arg, value) { + (HasArg::No, Some(_)) => { + given.push(Item::Complaint(format!( + "option '--{long}' doesn't allow an argument" + ))); + continue; + } + (HasArg::Required, None) => { + if let Some(next) = args.get(at) { + at += 1; + Some(bytes(next)) + } else { + given.push(Item::Complaint(format!( + "option '--{long}' requires an argument" + ))); + continue; + } + } + (_, value) => value, + }; + given.push(Item::Given(Given { + opt, + arg: value, + long: Some(long), + })); + continue; + } + let mut chars = 1; + while chars < arg.len() { + let c = arg[chars]; + chars += 1; + match short_takes_arg(c) { + None => { + given.push(Item::Complaint(format!( + "invalid option -- '{}'", + char::from(c) + ))); + } + Some(false) => given.push(Item::Given(Given { + opt: Opt::Short(c), + arg: None, + long: None, + })), + Some(true) => { + let value = if chars < arg.len() { + let value = arg[chars..].to_vec(); + chars = arg.len(); + Some(value) + } else if let Some(next) = args.get(at) { + at += 1; + Some(bytes(next)) + } else { + given.push(Item::Complaint(format!( + "option requires an argument -- '{}'", + char::from(c) + ))); + None + }; + if let Some(value) = value { + given.push(Item::Given(Given { + opt: Opt::Short(c), + arg: Some(value), + long: None, + })); + } + } + } + } + } + (given, operands) +} + +fn argmatch_valid(names: &[(&str, T)]) { + let mut out = String::from("Valid arguments are:"); + let mut last: Option = None; + for &(name, value) in names { + if last == Some(value) { + let _ = write!(out, ", {}", quote::locale(name.as_bytes())); + } else { + let _ = write!(out, "\n - {}", quote::locale(name.as_bytes())); + } + last = Some(value); + } + eprintln!("{out}"); +} + +fn argmatch_invalid( + parser: &Parser<'_>, + context: &str, + value: &[u8], + ambiguous: bool, +) { + let problem = if ambiguous { "ambiguous" } else { "invalid" }; + parser.error(&format!( + "{problem} argument {} for {}", + quote::locale(value), + quote::locale(context.as_bytes()) + )); +} + +/// A duration for `--max-age`: seconds, or a number with `s`, `m`, `h` or +/// `d`. +fn parse_age(text: &[u8]) -> Option { + let text = std::str::from_utf8(text).ok()?; + let (digits, unit) = match text.char_indices().last()? { + (at, c) if c.is_ascii_alphabetic() => (&text[..at], c), + _ => (text, 's'), + }; + let value: f64 = digits.parse().ok().filter(|v: &f64| *v >= 0.0)?; + let seconds = match unit { + 's' => value, + 'm' => value * 60.0, + 'h' => value * 3600.0, + 'd' => value * 86_400.0, + _ => return None, + }; + Duration::try_from_secs_f64(seconds).ok() +} + +fn env_flag(name: &str) -> bool { + std::env::var_os(name) + .is_some_and(|v| !v.is_empty() && v != "0" && v != "false") +} + +/// Parse `args` (without `argv[0]`) as GNU du would. +pub fn parse(program: &Program, args: &[OsString]) -> Parsed { + let mut parser = Parser { program, ok: true }; + let du_block_size = + std::env::var_os("DU_BLOCK_SIZE").map(OsString::into_encoded_bytes); + // du ignores a bad DU_BLOCK_SIZE: the default stands in silently. + let (units, _) = num::human_options(du_block_size.as_deref()); + let mut options = Options { + all: false, + apparent: false, + units, + total: false, + deref: Deref::Physical, + max_depth: i64::MAX, + inodes: false, + count_links: false, + separate_dirs: false, + threshold: 0, + excludes: Excludes::default(), + one_file_system: false, + null: false, + time: None, + operands: Vec::new(), + hash_all: false, + json: env_flag("DISKTREE_DU_JSON"), + fresh: env_flag("DISKTREE_DU_FRESH"), + max_age: std::env::var_os("DISKTREE_DU_MAX_AGE") + .and_then(|v| parse_age(v.as_encoded_bytes())), + index: std::env::var_os("DISKTREE_DU_INDEX") + .is_none_or(|v| v != "0" && v != "false"), + }; + let mut summarize = false; + let mut max_depth_given = false; + let mut files0_from: Option> = None; + let mut time_style: Option> = None; + let (items, operands) = scan(args); + + for item in &items { + let given = match item { + Item::Given(given) => given, + Item::Complaint(message) => { + eprintln!("{}: {message}", program.full); + parser.ok = false; + continue; + } + }; + let arg = given.arg.as_deref().unwrap_or_default(); + match given.opt { + Opt::Short(b'0') => options.null = true, + Opt::Short(b'a') => options.all = true, + Opt::Short(b'A') => options.apparent = true, + Opt::Short(b'b') => { + options.apparent = true; + options.units = Units::plain(1); + } + Opt::Short(b'c') => options.total = true, + Opt::Short(b'h') => { + options.units = Units { + opts: num::AUTOSCALE | num::SI | num::BASE_1024, + block_size: 1, + }; + } + Opt::Si => { + options.units = Units { + opts: num::AUTOSCALE | num::SI, + block_size: 1, + }; + } + Opt::Short(b'k') => options.units = Units::plain(1024), + Opt::Short(b'm') => options.units = Units::plain(1024 * 1024), + Opt::Short(b'd') => match num::xstrtoimax(arg, b"") { + Ok(depth) => { + max_depth_given = true; + options.max_depth = depth; + } + Err(_) => parser.fail(&format!( + "invalid maximum depth {}", + quote::locale(arg) + )), + }, + Opt::Short(b'l') => options.count_links = true, + Opt::Short(b's') => summarize = true, + Opt::Short(b't') => { + match num::xstrtoimax(arg, b"kKmMGTPEZYRQ0") { + Ok(threshold) => options.threshold = threshold, + Err(error) => { + strtol_fatal(&parser, error, given); + return Parsed::Exit(1); + } + } + if options.threshold == 0 && arg.first() == Some(&b'-') { + parser.error("invalid --threshold argument '-0'"); + return Parsed::Exit(1); + } + } + Opt::Short(b'x') => options.one_file_system = true, + Opt::Short(b'B') => { + let (units, result) = num::human_options(Some(arg)); + options.units = units; + if let Err(error) = result { + strtol_fatal(&parser, error, given); + return Parsed::Exit(1); + } + } + Opt::Short(b'H' | b'D') => options.deref = Deref::Args, + Opt::Short(b'L') => options.deref = Deref::All, + Opt::Short(b'P') => options.deref = Deref::Physical, + Opt::Short(b'S') => options.separate_dirs = true, + Opt::Short(b'X') => match read_all(arg) { + Ok(contents) => options.excludes.add_file(&contents), + Err(error) => parser.fail(&format!( + "{}: {}", + quote::when_needed(arg), + super::strerror(&error) + )), + }, + Opt::Files0From => files0_from = Some(arg.to_vec()), + Opt::Exclude => options.excludes.add(arg), + Opt::Inodes => options.inodes = true, + Opt::Time => { + let kind = match &given.arg { + None => TimeKind::Modified, + Some(word) => match num::argmatch(word, TIME_WORDS) { + Ok(kind) => kind, + Err(ambiguous) => { + argmatch_invalid( + &parser, "--time", word, ambiguous, + ); + argmatch_valid(TIME_WORDS); + parser.try_help(); + return Parsed::Exit(1); + } + }, + }; + options.time = Some((kind, String::new())); + } + Opt::TimeStyle => time_style = Some(arg.to_vec()), + Opt::Help => { + print_help(&program.full); + return Parsed::Exit(0); + } + Opt::Version => { + println!("du (disktree) {}", env!("CARGO_PKG_VERSION")); + return Parsed::Exit(0); + } + Opt::Json => options.json = true, + Opt::Fresh => options.fresh = true, + Opt::NoIndex => options.index = false, + Opt::MaxAge => match parse_age(arg) { + Some(age) => options.max_age = Some(age), + None => parser.fail(&format!( + "invalid --max-age argument {}", + quote::locale(arg) + )), + }, + Opt::Short(_) => parser.ok = false, + } + } + + if !parser.ok { + parser.try_help(); + return Parsed::Exit(1); + } + if options.all && summarize { + parser.error("cannot both summarize and show all entries"); + parser.try_help(); + return Parsed::Exit(1); + } + if summarize && max_depth_given && options.max_depth == 0 { + parser.error("warning: summarizing is the same as using --max-depth=0"); + } + if summarize && max_depth_given && options.max_depth != 0 { + parser.error(&format!( + "warning: summarizing conflicts with --max-depth={}", + options.max_depth + )); + parser.try_help(); + return Parsed::Exit(1); + } + if summarize { + options.max_depth = 0; + } + if options.inodes { + if options.apparent { + parser.error( + "warning: options --apparent-size and -b are ineffective \ + with --inodes", + ); + } + options.units.block_size = 1; + } + if let Some((kind, _)) = options.time { + let Some(format) = time_format(&parser, time_style) else { + return Parsed::Exit(1); + }; + options.time = Some((kind, format)); + } + + if let Some(from) = files0_from { + if let Some(extra) = operands.first() { + parser.error(&format!("extra operand {}", quote::locale(extra))); + eprintln!("file operands cannot be combined with --files0-from"); + parser.try_help(); + return Parsed::Exit(1); + } + // Failing to open the list ends du; failing to read it part way is + // reported where it happens, and the total is still printed. + let mut source: Box = if from == b"-" { + Box::new(std::io::stdin()) + } else { + match std::fs::File::open(bytes_path(&from)) { + Ok(file) => Box::new(file), + Err(error) => { + parser.error(&format!( + "cannot open {} for reading: {}", + quote::always(&from), + super::strerror(&error) + )); + return Parsed::Exit(1); + } + } + }; + let mut contents = Vec::new(); + let read_error = source.read_to_end(&mut contents).err(); + let mut names: Vec<&[u8]> = contents.split(|&b| b == 0).collect(); + if names.last().is_some_and(|name| name.is_empty()) { + names.pop(); + } + for (number, name) in names.into_iter().enumerate() { + let operand = if from == b"-" && name == b"-" { + Operand::Invalid(format!( + "when reading file names from standard input, no file \ + name of {} allowed", + quote::always(name) + )) + } else if name.is_empty() { + Operand::Invalid(format!( + "{}:{}: invalid zero-length file name", + quote::when_needed(&from), + number + 1 + )) + } else { + Operand::Path(name.to_vec()) + }; + options.operands.push(operand); + } + if let Some(error) = read_error { + options.operands.push(Operand::Invalid(format!( + "{}: read error: {}", + quote::when_needed(&from), + super::strerror(&error) + ))); + } + options.hash_all = true; + } else { + options.hash_all = operands.len() > 1 || options.deref == Deref::All; + if operands.is_empty() { + options.operands.push(Operand::Path(b".".to_vec())); + } + for name in operands { + options.operands.push(if name.is_empty() { + Operand::Invalid("invalid zero-length file name".to_owned()) + } else { + Operand::Path(name) + }); + } + } + Parsed::Run(Box::new(options)) +} + +/// A file's contents, or standard input's for `-`. +fn read_all(name: &[u8]) -> std::io::Result> { + if name == b"-" { + let mut buffer = Vec::new(); + std::io::Read::read_to_end(&mut std::io::stdin(), &mut buffer)?; + Ok(buffer) + } else { + std::fs::read(bytes_path(name)) + } +} + +fn bytes_path(bytes: &[u8]) -> std::path::PathBuf { + // On Unix any byte string is a path. + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt as _; + std::path::PathBuf::from(std::ffi::OsStr::from_bytes(bytes)) + } + #[cfg(not(unix))] + { + std::path::PathBuf::from(String::from_utf8_lossy(bytes).into_owned()) + } +} + +/// The strftime format `--time` prints with, from `--time-style` or the +/// `TIME_STYLE` environment variable, as du resolves them. +fn time_format(parser: &Parser<'_>, given: Option>) -> Option { + let style = if let Some(style) = given { + style + } else { + { + let env = std::env::var_os("TIME_STYLE") + .map(OsString::into_encoded_bytes); + match env { + None => b"long-iso".to_vec(), + Some(style) if style == b"locale" => b"long-iso".to_vec(), + Some(style) if style.starts_with(b"+") => { + // Anything after a newline is for ls's recent-files + // format, which du has no use for. + let end = style + .iter() + .position(|&b| b == b'\n') + .unwrap_or(style.len()); + style[..end].to_vec() + } + Some(mut style) => { + while let Some(rest) = style.strip_prefix(b"posix-") { + style = rest.to_vec(); + } + style + } + } + } + }; + if let Some(format) = style.strip_prefix(b"+") { + return Some(String::from_utf8_lossy(format).into_owned()); + } + match num::argmatch(&style, TIME_STYLES) { + Ok(format) => Some(format.to_owned()), + Err(ambiguous) => { + argmatch_invalid(parser, "time style", &style, ambiguous); + eprintln!( + "Valid arguments are:\n - full-iso\n - long-iso\n - iso\n \ + - +FORMAT (e.g., +%H:%M) for a 'date'-style format" + ); + parser.try_help(); + None + } + } +} + +fn print_help(program: &str) { + let text = format!( + "\ +Usage: {program} [OPTION]... [FILE]... + or: {program} [OPTION]... --files0-from=F +Summarize device usage of the set of FILEs, recursively for directories. + +Mandatory arguments to long options are mandatory for short options too. + -0, --null end each output line with NUL, not newline + -a, --all write counts for all files, not just directories + -A, --apparent-size print apparent sizes rather than device usage + -B, --block-size=SIZE scale sizes by SIZE before printing them + -b, --bytes equivalent to '--apparent-size --block-size=1' + -c, --total produce a grand total + -D, --dereference-args dereference only symlinks that are listed on the + command line + -d, --max-depth=N print the total for a directory (or file, with --all) + only if it is N or fewer levels below the command + line argument; --max-depth=0 is the same as + --summarize + --files0-from=F summarize device usage of the NUL-terminated file + names specified in file F; if F is -, then read + names from standard input + -H equivalent to --dereference-args (-D) + -h, --human-readable print sizes in human readable format (e.g., 1K 234M 2G) + --inodes list inode usage information instead of block usage + -k like --block-size=1K + -L, --dereference dereference all symbolic links + -l, --count-links count sizes many times if hard linked + -m like --block-size=1M + -P, --no-dereference don't follow any symbolic links (this is the default) + -S, --separate-dirs for directories do not include size of subdirectories + --si like -h, but use powers of 1000 not 1024 + -s, --summarize display only a total for each argument + -t, --threshold=SIZE exclude entries smaller than SIZE if positive, + or entries greater than SIZE if negative + --time show time of the last modification of any file in the + directory, or any of its subdirectories + --time=WORD show time as WORD instead of modification time: + atime, access, use, ctime or status + --time-style=STYLE show times using STYLE, which can be: + full-iso, long-iso, iso, or +FORMAT; + FORMAT is interpreted like in 'date' + -X, --exclude-from=FILE exclude files that match any pattern in FILE + --exclude=PATTERN exclude files that match PATTERN + -x, --one-file-system skip directories on different file systems + --help display this help and exit + --version output version information and exit + +disktree additions (full names only): + --json one JSON document: every entry du would print, with + its kind, why it can be had back, the command + that frees it, and when it was last written + --fresh walk everything again and rewrite the index + --max-age=AGE when the last walk is younger than AGE (seconds, or + 30s 10m 2h 1d), bring it up to date from what + changed since instead of walking again + --no-index neither read nor write the persistent index + The same through the environment, for use behind a `du` symlink: + DISKTREE_DU_JSON=1 DISKTREE_DU_FRESH=1 DISKTREE_DU_MAX_AGE=AGE + DISKTREE_DU_INDEX=0 + +Display values are in units of the first available SIZE from --block-size, +and the DU_BLOCK_SIZE, BLOCK_SIZE and BLOCKSIZE environment variables. +Otherwise, units default to 1024 bytes (or 512 if POSIXLY_CORRECT is set). +" + ); + let _ = std::io::stdout().write_all(text.as_bytes()); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(args: &[&str]) -> Box { + let args: Vec = args.iter().map(OsString::from).collect(); + match parse(&Program::new(b"du"), &args) { + Parsed::Run(options) => options, + Parsed::Exit(code) => panic!("exited {code}"), + } + } + + #[test] + fn options_permute_and_cluster() { + let options = run(&["a", "-sh", "b", "--max-d=0"]); + assert_eq!(options.max_depth, 0); + assert!(options.units.opts & num::AUTOSCALE != 0); + assert_eq!(options.operands.len(), 2); + assert!(options.hash_all); + } + + #[test] + fn later_units_win() { + assert_eq!(run(&["-h", "-k"]).units, Units::plain(1024)); + assert_eq!(run(&["-b"]).units, Units::plain(1)); + assert!(run(&["-b"]).apparent); + } + + #[test] + fn ages() { + assert_eq!(parse_age(b"90"), Some(Duration::from_secs(90))); + assert_eq!(parse_age(b"10m"), Some(Duration::from_mins(10))); + assert_eq!(parse_age(b"x"), None); + } +} diff --git a/crates/disktree-core/src/du/bulk.rs b/crates/disktree-core/src/du/bulk.rs new file mode 100644 index 0000000..0f79bf7 --- /dev/null +++ b/crates/disktree-core/src/du/bulk.rs @@ -0,0 +1,231 @@ +//! macOS: a directory's entries and their stats in one system call. +//! +//! `readdir` then `fstatat` per entry is two trips into the kernel for every +//! file, and the per-file lookup is most of what a walk costs. +//! `getattrlistbulk` returns a buffer of entries, each with the attributes +//! asked for, for about what the listing costs alone. +//! +//! Only what can be turned into exactly the `struct stat` du would have seen +//! is taken from it: regular files and symbolic links with a single link. +//! XNU's `vn_stat` fills `st_blocks` as the total allocation rounded up to +//! 512-byte blocks and `st_size` as the data fork's length, which are +//! `ATTR_FILE_ALLOCSIZE` and `ATTR_FILE_DATALENGTH` here. Directories, +//! multiply linked files (whose id here can be the link's rather than the +//! inode's), anything else, and any entry the call reports an error for are +//! left for `fstatat`. + +#![allow( + unsafe_code, + reason = "getattrlistbulk, which std does not wrap; the call says why \ + it is sound, and the buffer is then read with safe code" +)] + +use std::cell::RefCell; +use std::os::fd::AsRawFd as _; + +use rustix::fd::BorrowedFd; +use rustix::io::Errno; + +use super::walk::{Meta, Time}; + +/// `ATTR_CMN_ERROR`, which the libc crate does not declare. +const ATTR_CMN_ERROR: u32 = 0x2000_0000; +/// `vtype` values of `ATTR_CMN_OBJTYPE`. +const VREG: u32 = 1; +const VLNK: u32 = 5; + +const COMMON: u32 = libc::ATTR_CMN_RETURNED_ATTRS + | libc::ATTR_CMN_NAME + | libc::ATTR_CMN_DEVID + | libc::ATTR_CMN_OBJTYPE + | libc::ATTR_CMN_MODTIME + | libc::ATTR_CMN_CHGTIME + | libc::ATTR_CMN_ACCTIME + | libc::ATTR_CMN_ACCESSMASK + | libc::ATTR_CMN_FILEID + | ATTR_CMN_ERROR; +const FILE: u32 = libc::ATTR_FILE_LINKCOUNT + | libc::ATTR_FILE_ALLOCSIZE + | libc::ATTR_FILE_DATALENGTH; + +/// Large enough for a few hundred entries a call; each thread keeps one. +const BUFFER: usize = 64 * 1024; + +thread_local! { + static BUF: RefCell> = RefCell::new(vec![0; BUFFER]); +} + +/// One entry: its name, its id for `fts`'s inode sort, and its stat when +/// the bulk attributes give it exactly. +#[derive(Debug)] +pub struct Item { + pub name: Box<[u8]>, + pub id: u64, + pub meta: Option, +} + +/// Whether to use this at all; `DISKTREE_DU_BULK=0` turns it off, to +/// compare against the plain walk. +pub fn enabled() -> bool { + std::env::var_os("DISKTREE_DU_BULK").is_none_or(|v| v != "0") +} + +/// Every entry of the directory open as `dir`, in the order the file +/// system returns them. On failure part way, what was read and the error. +/// `None` when a record does not parse: the caller then lists the +/// directory the ordinary way rather than risk losing an entry. +pub fn read(dir: BorrowedFd<'_>) -> Option<(Vec, Option)> { + BUF.with_borrow_mut(|buf| { + let mut items = Vec::new(); + let mut list = libc::attrlist { + bitmapcount: libc::ATTR_BIT_MAP_COUNT, + reserved: 0, + commonattr: COMMON, + volattr: 0, + dirattr: 0, + fileattr: FILE, + forkattr: 0, + }; + loop { + // SAFETY: `list` is a valid attrlist, `buf` is a live, writable + // buffer of the length passed, and `dir` is an open directory + // for as long as the call runs. The kernel writes at most + // `buf.len()` bytes and returns how many entries it wrote. + let count = unsafe { + libc::getattrlistbulk( + dir.as_raw_fd(), + (&raw mut list).cast(), + buf.as_mut_ptr().cast(), + buf.len(), + u64::from(libc::FSOPT_PACK_INVAL_ATTRS), + ) + }; + if count == 0 { + return Some((items, None)); + } + let Ok(count) = usize::try_from(count) else { + // Failing before the first entry, as on a directory that + // can be read but not searched, or a file system without + // the call, leaves the listing to `readdir`, which reports + // what GNU does. + if items.is_empty() { + return None; + } + let error = std::io::Error::last_os_error(); + let errno = error.raw_os_error().unwrap_or(libc::EIO); + return Some((items, Some(Errno::from_raw_os_error(errno)))); + }; + let mut at = 0; + for _ in 0..count { + let len = u32_at(buf, at)? as usize; + let record = buf.get(at..at.checked_add(len)?)?; + items.push(parse(record)?); + at += len; + } + } + }) +} + +fn u32_at(buf: &[u8], at: usize) -> Option { + Some(u32::from_ne_bytes(buf.get(at..at + 4)?.try_into().ok()?)) +} + +fn i32_at(buf: &[u8], at: usize) -> Option { + Some(i32::from_ne_bytes(buf.get(at..at + 4)?.try_into().ok()?)) +} + +fn u64_at(buf: &[u8], at: usize) -> Option { + Some(u64::from_ne_bytes(buf.get(at..at + 8)?.try_into().ok()?)) +} + +fn i64_at(buf: &[u8], at: usize) -> Option { + Some(i64::from_ne_bytes(buf.get(at..at + 8)?.try_into().ok()?)) +} + +fn time_at(buf: &[u8], at: usize) -> Option