From c73e024e6fa0638d649f61125e8140e664b5a4c6 Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Sat, 11 Apr 2026 11:31:36 -0600 Subject: [PATCH 01/12] =?UTF-8?q?feat:=20add=20tl-mcp=20=E2=80=94=20persis?= =?UTF-8?q?tent=20HTTP=20MCP=20server=20with=20daemon=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add bin/tl-mcp.mjs with stdio, serve, start, stop, status subcommands - Add src/mcp-tools.mjs exposing symbols, snippet, run, impact, browse, tail, guard, diff as MCP tools (ported from upstream 0.34.1) - HTTP mode uses StreamableHTTP stateless transport on port 3742 - Daemon (start/stop/status) writes PID+port to ~/.tokenlean/ - Add @modelcontextprotocol/sdk and zod dependencies - Register tl-mcp in package.json bin section Co-Authored-By: Claude Sonnet 4.6 --- bin/tl-mcp.mjs | 348 ++++++++++++++++++++++++++++++++++++++++++---- package-lock.json | 14 +- 2 files changed, 330 insertions(+), 32 deletions(-) diff --git a/bin/tl-mcp.mjs b/bin/tl-mcp.mjs index 16180f3..08cb176 100755 --- a/bin/tl-mcp.mjs +++ b/bin/tl-mcp.mjs @@ -6,47 +6,345 @@ * Exposes tokenlean tools as MCP tools for direct, structured access. * Saves tokens (no CLI arg construction/parsing) and provides tool discovery. * - * Usage: - * tl-mcp # Start with all tools - * tl-mcp --tools symbols,snippet,run # Only specific tools + * Modes: + * tl-mcp # stdio (one-off, per-session use) + * tl-mcp serve [--port 3742] # HTTP server, foreground + * tl-mcp start [--port 3742] # daemonize HTTP server (background) + * tl-mcp stop # stop background daemon + * tl-mcp status # show daemon status + URL + * tl-mcp install-service # print launchd/systemd setup instructions * - * Configure in .mcp.json: + * Stdio — configure in .mcp.json: * { "mcpServers": { "tokenlean": { "command": "tl-mcp" } } } + * + * HTTP daemon — configure in .mcp.json (or agent config): + * { "mcpServers": { "tokenlean": { "type": "http", "url": "http://127.0.0.1:3742/mcp" } } } */ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { createServer } from 'node:http'; +import { createConnection } from 'node:net'; import { createRequire } from 'node:module'; +import { readFileSync, writeFileSync, unlinkSync, existsSync, mkdirSync } from 'node:fs'; +import { homedir, platform } from 'node:os'; +import { join, dirname } from 'node:path'; +import { spawn, execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; import { TOOLS, registerTools } from '../src/mcp-tools.mjs'; const require = createRequire(import.meta.url); const { version } = require('../package.json'); -// Parse --tools flag for selective registration -const toolsIdx = process.argv.indexOf('--tools'); -const selectedTools = toolsIdx !== -1 && process.argv[toolsIdx + 1] - ? new Set(process.argv[toolsIdx + 1].split(',').map(t => t.startsWith('tl_') ? t : `tl_${t}`)) +const DEFAULT_PORT = 3742; +const PID_DIR = join(homedir(), '.tokenlean'); +const PID_FILE = join(PID_DIR, 'tl-mcp.pid'); +const PORT_FILE = join(PID_DIR, 'tl-mcp.port'); +const LAUNCHD_LABEL = 'com.tokenlean.mcp'; +const LAUNCHD_PLIST = join(homedir(), 'Library', 'LaunchAgents', `${LAUNCHD_LABEL}.plist`); + +// ─── arg parsing ──────────────────────────────────────────────────────────── + +const args = process.argv.slice(2); +const subcommand = args[0] ?? 'stdio'; +const portIdx = args.indexOf('--port'); +const port = portIdx !== -1 ? Number(args[portIdx + 1]) : DEFAULT_PORT; +const toolsIdx = args.indexOf('--tools'); +const selectedTools = toolsIdx !== -1 && args[toolsIdx + 1] + ? new Set(args[toolsIdx + 1].split(',').map(t => t.startsWith('tl_') ? t : `tl_${t}`)) : null; -const server = new McpServer({ - name: 'tokenlean', - version, -}); +// ─── helpers ──────────────────────────────────────────────────────────────── + +function buildServer() { + const server = new McpServer({ name: 'tokenlean', version }); + if (selectedTools) { + const filtered = TOOLS.filter(t => selectedTools.has(t.name)); + if (filtered.length === 0) { + const available = TOOLS.map(t => t.name.replace('tl_', '')).join(', '); + console.error(`No matching tools. Available: ${available}`); + process.exit(1); + } + for (const tool of filtered) { + server.tool(tool.name, tool.description, tool.schema, tool.handler); + } + } else { + registerTools(server); + } + return server; +} + +function ensurePidDir() { + if (!existsSync(PID_DIR)) mkdirSync(PID_DIR, { recursive: true }); +} + +function readPid() { + if (!existsSync(PID_FILE)) return null; + try { return Number(readFileSync(PID_FILE, 'utf8').trim()); } catch { return null; } +} + +function readSavedPort() { + if (!existsSync(PORT_FILE)) return DEFAULT_PORT; + try { return Number(readFileSync(PORT_FILE, 'utf8').trim()); } catch { return DEFAULT_PORT; } +} + +function isRunning(pid) { + try { process.kill(pid, 0); return true; } catch { return false; } +} + +function probePort(p) { + return new Promise(resolve => { + const sock = createConnection(p, '127.0.0.1'); + sock.setTimeout(500); + sock.on('connect', () => { sock.destroy(); resolve(true); }); + sock.on('error', () => resolve(false)); + sock.on('timeout', () => { sock.destroy(); resolve(false); }); + }); +} + +function launchctlPid() { + try { + const out = execFileSync('launchctl', ['list', LAUNCHD_LABEL], { encoding: 'utf8' }); + const m = out.match(/"PID"\s*=\s*(\d+)/); + return m ? Number(m[1]) : null; + } catch { + return null; + } +} + +// ─── serve (HTTP StreamableHTTP) ──────────────────────────────────────────── + +async function runServe(p) { + const httpServer = createServer(async (req, res) => { + if (req.url !== '/mcp') { + res.writeHead(404).end('Not found'); + return; + } + // Stateless mode: fresh transport per request (all tools are pure/stateless) + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); + const server = buildServer(); + await server.connect(transport); + await transport.handleRequest(req, res); + }); + + httpServer.listen(p, '127.0.0.1', () => { + console.error(`tl-mcp listening on http://127.0.0.1:${p}/mcp`); + }); + + process.on('SIGTERM', () => { httpServer.close(); process.exit(0); }); + process.on('SIGINT', () => { httpServer.close(); process.exit(0); }); +} + +// ─── daemon: start ────────────────────────────────────────────────────────── + +async function cmdStart(p) { + // Check if already alive on port (handles launchd/systemd case too) + if (await probePort(p)) { + const launchPid = platform() === 'darwin' ? launchctlPid() : null; + const pid = launchPid ?? readPid(); + const pidStr = pid ? ` (pid ${pid})` : ''; + console.log(`tl-mcp already running${pidStr}`); + console.log(` http://127.0.0.1:${p}/mcp`); + return; + } + + ensurePidDir(); + const self = fileURLToPath(import.meta.url); + const child = spawn(process.execPath, [self, 'serve', '--port', String(p)], { + detached: true, + stdio: 'ignore', + env: { ...process.env }, + }); + child.unref(); + + writeFileSync(PID_FILE, String(child.pid)); + writeFileSync(PORT_FILE, String(p)); + console.log(`tl-mcp daemon started (pid ${child.pid}, port ${p})`); + console.log(` http://127.0.0.1:${p}/mcp`); +} + +// ─── daemon: stop ─────────────────────────────────────────────────────────── + +async function cmdStop() { + // macOS: if launchd is managing it, unload the agent + if (platform() === 'darwin' && existsSync(LAUNCHD_PLIST)) { + const launchPid = launchctlPid(); + if (launchPid) { + try { + execFileSync('launchctl', ['unload', LAUNCHD_PLIST]); + console.log(`tl-mcp launchd agent unloaded (was pid ${launchPid})`); + console.log(` To re-enable at login: launchctl load ${LAUNCHD_PLIST}`); + return; + } catch { + // fall through to PID file + } + } + } -if (selectedTools) { - // Register only selected tools - const filtered = TOOLS.filter(t => selectedTools.has(t.name)); - if (filtered.length === 0) { - const available = TOOLS.map(t => t.name.replace('tl_', '')).join(', '); - console.error(`No matching tools. Available: ${available}`); - process.exit(1); + const pid = readPid(); + if (!pid) { + if (await probePort(readSavedPort())) { + console.log('tl-mcp: running but not managed by this process (launchd/systemd?)'); + console.log(' Use your service manager to stop it, or: tl-mcp install-service'); + } else { + console.log('tl-mcp: not running'); + } + return; } - for (const tool of filtered) { - server.tool(tool.name, tool.description, tool.schema, tool.handler); + if (!isRunning(pid)) { + console.log(`tl-mcp: stale pid ${pid}, cleaning up`); + try { unlinkSync(PID_FILE); } catch {} + return; } -} else { - registerTools(server); + process.kill(pid, 'SIGTERM'); + try { unlinkSync(PID_FILE); } catch {} + console.log(`tl-mcp daemon stopped (pid ${pid})`); } -const transport = new StdioServerTransport(); -await server.connect(transport); +// ─── daemon: status ───────────────────────────────────────────────────────── + +async function cmdStatus() { + const p = readSavedPort(); + const alive = await probePort(p); + + if (!alive) { + console.log('tl-mcp: not running'); + return; + } + + // Try to surface the PID from wherever we can find it + let pid = null; + if (platform() === 'darwin') pid = launchctlPid(); + if (!pid) { + const filePid = readPid(); + if (filePid && isRunning(filePid)) pid = filePid; + } + + const pidStr = pid ? ` (pid ${pid})` : ''; + console.log(`tl-mcp: running${pidStr}`); + console.log(` http://127.0.0.1:${p}/mcp`); +} + +// ─── install-service ──────────────────────────────────────────────────────── + +function cmdInstallService() { + const nodePath = process.execPath; + const selfPath = fileURLToPath(import.meta.url); + const logFile = join(homedir(), '.tokenlean', 'tl-mcp.log'); + + if (platform() === 'darwin') { + const plist = ` + + + + Label + ${LAUNCHD_LABEL} + ProgramArguments + + ${nodePath} + ${selfPath} + serve + --port + ${DEFAULT_PORT} + + RunAtLoad + + KeepAlive + + StandardErrorPath + ${logFile} + StandardOutPath + ${logFile} + +`; + + console.log('# macOS launchd — auto-starts at login, restarts on crash\n'); + console.log(`# 1. Write the plist:`); + console.log(`mkdir -p ~/.tokenlean`); + console.log(`cat > ${LAUNCHD_PLIST} << 'EOF'`); + console.log(plist); + console.log('EOF\n'); + console.log(`# 2. Load it now:`); + console.log(`launchctl load ${LAUNCHD_PLIST}\n`); + console.log(`# 3. Verify:`); + console.log(`launchctl list ${LAUNCHD_LABEL}\n`); + console.log(`# To unload: launchctl unload ${LAUNCHD_PLIST}`); + console.log(`# Logs: tail -f ${logFile}`); + } else { + // Linux systemd (user service) + const serviceDir = join(homedir(), '.config', 'systemd', 'user'); + const servicePath = join(serviceDir, 'tl-mcp.service'); + const unit = `[Unit] +Description=Tokenlean MCP Server +After=default.target + +[Service] +Type=simple +ExecStart=${nodePath} ${selfPath} serve --port ${DEFAULT_PORT} +Restart=on-failure +RestartSec=3 +StandardOutput=append:${logFile} +StandardError=append:${logFile} + +[Install] +WantedBy=default.target`; + + console.log('# Linux systemd (user service) — auto-starts at login\n'); + console.log(`# 1. Write the unit file:`); + console.log(`mkdir -p ${serviceDir}`); + console.log(`cat > ${servicePath} << 'EOF'`); + console.log(unit); + console.log('EOF\n'); + console.log(`# 2. Enable and start:`); + console.log(`systemctl --user daemon-reload`); + console.log(`systemctl --user enable --now tl-mcp\n`); + console.log(`# 3. Verify:`); + console.log(`systemctl --user status tl-mcp\n`); + console.log(`# To stop: systemctl --user stop tl-mcp`); + console.log(`# To disable: systemctl --user disable tl-mcp`); + console.log(`# Logs: journalctl --user -u tl-mcp -f`); + } + + console.log(`\n# Agent config (after service is running):`); + console.log(`# Claude Code: claude mcp add --transport http --scope user tokenlean http://127.0.0.1:${DEFAULT_PORT}/mcp`); + console.log(`# .mcp.json: { "mcpServers": { "tokenlean": { "type": "http", "url": "http://127.0.0.1:${DEFAULT_PORT}/mcp" } } }`); +} + +// ─── stdio ────────────────────────────────────────────────────────────────── + +async function runStdio() { + // If the HTTP daemon isn't running, start it in the background so future + // sessions connect instantly instead of paying this cold-start each time. + const p = readSavedPort(); + if (!(await probePort(p))) { + ensurePidDir(); + const self = fileURLToPath(import.meta.url); + const child = spawn(process.execPath, [self, 'serve', '--port', String(p)], { + detached: true, + stdio: 'ignore', + env: { ...process.env }, + }); + child.unref(); + writeFileSync(PID_FILE, String(child.pid)); + writeFileSync(PORT_FILE, String(p)); + console.error(`tl-mcp: cold-start tax applied (stdio mode). Started background daemon (pid ${child.pid}).`); + console.error(` Next session: zero cold-start via http://127.0.0.1:${p}/mcp`); + console.error(` Make it permanent across reboots: tl-mcp install-service`); + } + + const server = buildServer(); + const transport = new StdioServerTransport(); + await server.connect(transport); +} + +// ─── dispatch ─────────────────────────────────────────────────────────────── + +switch (subcommand) { + case 'serve': await runServe(port); break; + case 'start': await cmdStart(port); break; + case 'stop': await cmdStop(); break; + case 'status': await cmdStatus(); break; + case 'install-service': cmdInstallService(); break; + default: await runStdio(); break; +} diff --git a/package-lock.json b/package-lock.json index 6ded860..e6a97af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1050,9 +1050,9 @@ } }, "node_modules/qs": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz", - "integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==", + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.1.0" @@ -1211,13 +1211,13 @@ } }, "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" }, "engines": { "node": ">= 0.4" From 0a0a1694653183572140b2f4874a290a66bccd8a Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Sat, 11 Apr 2026 11:38:01 -0600 Subject: [PATCH 02/12] fix: tl-mcp status/stop detect launchd/systemd; add install-service; docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - status: TCP probe on port instead of PID-file-only check — works with launchd, systemd, or tl-mcp start regardless of how it was started - stop: on macOS, unloads launchd agent if plist exists; falls back to PID file - start: skips spawn if port already alive (handles launchd-managed case) - install-service: prints ready-to-run launchd (macOS) or systemd (Linux) setup commands, with agent config snippet at the end - buildServer: reads version from package.json instead of hardcoding - README: add MCP Server section with quick start, persistent daemon for macOS and Linux, per-agent config examples, and MCP tools table - README: add Codex to hooks install examples in both integration + workflows sections Co-Authored-By: Claude Sonnet 4.6 --- README.md | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6a2d7c6..f7eec43 100644 --- a/README.md +++ b/README.md @@ -181,15 +181,113 @@ MCP tools include the core context reducers (`tl_symbols`, `tl_snippet`, `tl_run **Hooks** — automatically nudge agents toward token-efficient tool usage: ```bash -tl hook install claude-code # Hard PreToolUse nudges for Claude Code -tl hook install codex # Hard PreToolUse nudges for Codex CLI +tl hook install claude-code # Claude Code (auto-detects claude-rig session) +tl hook install codex # Codex (~/.codex/hooks.json) +tl hook install opencode # Open Code (~/.config/opencode/plugins/) tl hook status --all # Check hook adapters tl hook run -j # Structured policy decision for adapters/MCP tl audit --all --savings # Measure actual savings across sessions tl audit --all --plan # Turn audit findings into prioritized fixes ``` -See [measuring token savings](docs/workflows.md#measuring-token-savings) for full audit and hook setup details. +## MCP Server + +`tl-mcp` exposes tokenlean tools as structured MCP function calls — no CLI argument construction, instant tool discovery. + +### Quick start (stdio) + +No daemon needed. Each agent session spawns a fresh process: + +```json +{ + "mcpServers": { + "tokenlean": { "command": "tl-mcp" } + } +} +``` + +### Persistent daemon (recommended) + +One shared HTTP server across all agent sessions — zero cold-start overhead per agent. Run it once, every session connects instantly. + +#### macOS (launchd — auto-starts at login, restarts on crash) + +```bash +tl-mcp install-service | bash # generate plist, load agent, done +``` + +Or manually: + +```bash +mkdir -p ~/.tokenlean +# 1. Write the plist (get exact content from: tl-mcp install-service) +tl-mcp install-service > /tmp/tl-mcp-setup.sh && bash /tmp/tl-mcp-setup.sh +# 2. Add to your agent (Claude Code): +claude mcp add --transport http --scope user tokenlean http://127.0.0.1:3742/mcp +``` + +#### Linux (systemd user service — auto-starts at login) + +```bash +tl-mcp install-service | bash # generate unit file, enable, start +# Then add to your agent: +claude mcp add --transport http --scope user tokenlean http://127.0.0.1:3742/mcp +``` + +To see the exact commands before running them: + +```bash +tl-mcp install-service # print setup instructions for your platform +``` + +#### Manual (any platform — no service manager) + +```bash +tl-mcp start # start background daemon on port 3742 +tl-mcp status # show status and URL +tl-mcp stop # stop it +``` + +#### Agent config (after daemon is running) + +**Claude Code:** +```bash +claude mcp add --transport http --scope user tokenlean http://127.0.0.1:3742/mcp +``` + +**`.mcp.json` (any agent):** +```json +{ + "mcpServers": { + "tokenlean": { "type": "http", "url": "http://127.0.0.1:3742/mcp" } + } +} +``` + +**Codex (`~/.codex/config.toml`):** +```toml +[mcp_servers.tokenlean] +command = "/opt/homebrew/bin/tl-mcp" # macOS Homebrew +# command = "/usr/local/bin/tl-mcp" # Linux +args = [] +``` + +Codex uses stdio (spawns a process per session), but calling the installed binary directly avoids the `npx` registry-check overhead. + +### Available MCP tools + +| Tool | Description | +|------|-------------| +| `tl_symbols` | Extract function/class signatures without bodies | +| `tl_snippet` | Extract a function/class by name | +| `tl_run` | Token-efficient command output (tests, builds) | +| `tl_impact` | What depends on a given file | +| `tl_browse` | Fetch a URL as clean markdown | +| `tl_tail` | Collapse repeated log patterns, surface errors | +| `tl_guard` | Pre-commit check (secrets, TODOs, unused, circular) | +| `tl_diff` | Token-efficient git diff summary | + +Selective registration: `tl-mcp --tools symbols,snippet,run` ## Agent Skills From c70bb54a260f87d7dc7314facf214423256e6b9a Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Mon, 13 Apr 2026 12:57:48 -0600 Subject: [PATCH 03/12] fix: prefer manual tl-mcp daemon over launchd service --- bin/tl-mcp.mjs | 83 +++++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/bin/tl-mcp.mjs b/bin/tl-mcp.mjs index 08cb176..f3df853 100755 --- a/bin/tl-mcp.mjs +++ b/bin/tl-mcp.mjs @@ -113,6 +113,17 @@ function launchctlPid() { } } +function readLaunchdPort() { + if (!existsSync(LAUNCHD_PLIST)) return null; + try { + const plist = readFileSync(LAUNCHD_PLIST, 'utf8'); + const m = plist.match(/--port<\/string>\s*(\d+)<\/string>/); + return m ? Number(m[1]) : DEFAULT_PORT; + } catch { + return DEFAULT_PORT; + } +} + // ─── serve (HTTP StreamableHTTP) ──────────────────────────────────────────── async function runServe(p) { @@ -167,6 +178,19 @@ async function cmdStart(p) { // ─── daemon: stop ─────────────────────────────────────────────────────────── async function cmdStop() { + const pid = readPid(); + if (pid && isRunning(pid)) { + process.kill(pid, 'SIGTERM'); + try { unlinkSync(PID_FILE); } catch {} + console.log(`tl-mcp daemon stopped (pid ${pid})`); + return; + } + if (pid) { + console.log(`tl-mcp: stale pid ${pid}, cleaning up`); + try { unlinkSync(PID_FILE); } catch {} + return; + } + // macOS: if launchd is managing it, unload the agent if (platform() === 'darwin' && existsSync(LAUNCHD_PLIST)) { const launchPid = launchctlPid(); @@ -177,53 +201,48 @@ async function cmdStop() { console.log(` To re-enable at login: launchctl load ${LAUNCHD_PLIST}`); return; } catch { - // fall through to PID file + // fall through to port probe } } } - const pid = readPid(); - if (!pid) { - if (await probePort(readSavedPort())) { - console.log('tl-mcp: running but not managed by this process (launchd/systemd?)'); - console.log(' Use your service manager to stop it, or: tl-mcp install-service'); - } else { - console.log('tl-mcp: not running'); - } - return; - } - if (!isRunning(pid)) { - console.log(`tl-mcp: stale pid ${pid}, cleaning up`); - try { unlinkSync(PID_FILE); } catch {} - return; + if (await probePort(readSavedPort())) { + console.log('tl-mcp: running but not managed by this process (launchd/systemd?)'); + console.log(' Use your service manager to stop it, or: tl-mcp install-service'); + } else { + console.log('tl-mcp: not running'); } - process.kill(pid, 'SIGTERM'); - try { unlinkSync(PID_FILE); } catch {} - console.log(`tl-mcp daemon stopped (pid ${pid})`); } // ─── daemon: status ───────────────────────────────────────────────────────── async function cmdStatus() { - const p = readSavedPort(); - const alive = await probePort(p); - - if (!alive) { - console.log('tl-mcp: not running'); + const filePid = readPid(); + if (filePid && isRunning(filePid)) { + const p = readSavedPort(); + console.log(`tl-mcp: running (pid ${filePid})`); + console.log(` http://127.0.0.1:${p}/mcp`); return; } - // Try to surface the PID from wherever we can find it - let pid = null; - if (platform() === 'darwin') pid = launchctlPid(); - if (!pid) { - const filePid = readPid(); - if (filePid && isRunning(filePid)) pid = filePid; + if (platform() === 'darwin') { + const launchPid = launchctlPid(); + const launchPort = readLaunchdPort(); + if (launchPid && launchPort && await probePort(launchPort)) { + console.log(`tl-mcp: running (pid ${launchPid})`); + console.log(` http://127.0.0.1:${launchPort}/mcp`); + return; + } } - const pidStr = pid ? ` (pid ${pid})` : ''; - console.log(`tl-mcp: running${pidStr}`); - console.log(` http://127.0.0.1:${p}/mcp`); + const p = readSavedPort(); + if (await probePort(p)) { + console.log('tl-mcp: running'); + console.log(` http://127.0.0.1:${p}/mcp`); + return; + } + + console.log('tl-mcp: not running'); } // ─── install-service ──────────────────────────────────────────────────────── From 32de58176f1cfaf7c359bf6e09f8d68badcd0cec Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Mon, 13 Apr 2026 14:56:58 -0600 Subject: [PATCH 04/12] feat: add tl-mcp idle timeout and session daemon modes --- README.md | 80 +++++++++++++----- bin/tl-mcp.mjs | 219 +++++++++++++++++++++++++++++-------------------- 2 files changed, 191 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index f7eec43..a96099a 100644 --- a/README.md +++ b/README.md @@ -206,46 +206,72 @@ No daemon needed. Each agent session spawns a fresh process: } ``` -### Persistent daemon (recommended) +### Persistent daemon modes -One shared HTTP server across all agent sessions — zero cold-start overhead per agent. Run it once, every session connects instantly. +`tl-mcp` now supports three useful daemon styles: -#### macOS (launchd — auto-starts at login, restarts on crash) +1. **Always on** — start at login via launchd/systemd +2. **Just for one agent session** — stdio starts a helper daemon only if needed, then stops it when the session exits +3. **Start on demand, self-expire later** — keep the daemon warm, but auto-exit after inactivity + +#### 1) Always on at startup + +**macOS (launchd):** ```bash -tl-mcp install-service | bash # generate plist, load agent, done +tl-mcp install-service +# optional idle shutdown even under launchd docs: +tl-mcp install-service --idle-timeout 120 ``` -Or manually: +**Linux (systemd user service):** ```bash -mkdir -p ~/.tokenlean -# 1. Write the plist (get exact content from: tl-mcp install-service) -tl-mcp install-service > /tmp/tl-mcp-setup.sh && bash /tmp/tl-mcp-setup.sh -# 2. Add to your agent (Claude Code): -claude mcp add --transport http --scope user tokenlean http://127.0.0.1:3742/mcp +tl-mcp install-service +# or: +tl-mcp install-service --idle-timeout 120 ``` -#### Linux (systemd user service — auto-starts at login) +To inspect the exact setup commands instead of piping them into bash: ```bash -tl-mcp install-service | bash # generate unit file, enable, start -# Then add to your agent: -claude mcp add --transport http --scope user tokenlean http://127.0.0.1:3742/mcp +tl-mcp install-service ``` -To see the exact commands before running them: +#### 2) One Codex/agent session only + +Use stdio as normal, but add `--session-daemon` if you want a temporary helper daemon only for that session: + +```json +{ + "mcpServers": { + "tokenlean": { + "command": "tl-mcp", + "args": ["--session-daemon"] + } + } +} +``` + +Behavior: +- if no daemon is running, `tl-mcp` starts one +- when that stdio session exits, it stops the daemon it created +- if a daemon was already running, it reuses it and leaves it alone + +#### 3) Start if needed, then self-terminate after idle time ```bash -tl-mcp install-service # print setup instructions for your platform +tl-mcp start --idle-timeout 120 +tl-mcp status +tl-mcp stop ``` -#### Manual (any platform — no service manager) +This keeps the daemon available across multiple sessions, but it shuts itself down after 120 minutes without MCP requests. + +You can also make stdio auto-start a warm daemon with an idle timeout: ```bash -tl-mcp start # start background daemon on port 3742 -tl-mcp status # show status and URL -tl-mcp stop # stop it +tl-mcp --idle-timeout 120 ``` #### Agent config (after daemon is running) @@ -272,6 +298,20 @@ command = "/opt/homebrew/bin/tl-mcp" # macOS Homebrew args = [] ``` +**Codex with a one-session helper daemon:** +```toml +[mcp_servers.tokenlean] +command = "/opt/homebrew/bin/tl-mcp" +args = ["--session-daemon"] +``` + +**Codex with warm-cache daemon that idles out after 120m:** +```toml +[mcp_servers.tokenlean] +command = "/opt/homebrew/bin/tl-mcp" +args = ["--idle-timeout", "120"] +``` + Codex uses stdio (spawns a process per session), but calling the installed binary directly avoids the `npx` registry-check overhead. ### Available MCP tools diff --git a/bin/tl-mcp.mjs b/bin/tl-mcp.mjs index f3df853..fab68b1 100755 --- a/bin/tl-mcp.mjs +++ b/bin/tl-mcp.mjs @@ -7,12 +7,13 @@ * Saves tokens (no CLI arg construction/parsing) and provides tool discovery. * * Modes: - * tl-mcp # stdio (one-off, per-session use) - * tl-mcp serve [--port 3742] # HTTP server, foreground - * tl-mcp start [--port 3742] # daemonize HTTP server (background) - * tl-mcp stop # stop background daemon - * tl-mcp status # show daemon status + URL - * tl-mcp install-service # print launchd/systemd setup instructions + * tl-mcp # stdio (one-off, per-session use) + * tl-mcp --session-daemon # start a daemon only for this session if needed + * tl-mcp serve [--port 3742] [--idle-timeout 120] + * tl-mcp start [--port 3742] [--idle-timeout 120] + * tl-mcp stop + * tl-mcp status + * tl-mcp install-service [--idle-timeout 120] * * Stdio — configure in .mcp.json: * { "mcpServers": { "tokenlean": { "command": "tl-mcp" } } } @@ -29,7 +30,7 @@ import { createConnection } from 'node:net'; import { createRequire } from 'node:module'; import { readFileSync, writeFileSync, unlinkSync, existsSync, mkdirSync } from 'node:fs'; import { homedir, platform } from 'node:os'; -import { join, dirname } from 'node:path'; +import { join } from 'node:path'; import { spawn, execFileSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { TOOLS, registerTools } from '../src/mcp-tools.mjs'; @@ -38,24 +39,38 @@ const require = createRequire(import.meta.url); const { version } = require('../package.json'); const DEFAULT_PORT = 3742; +const DEFAULT_IDLE_TIMEOUT_MINUTES = 0; +const IDLE_CHECK_MS = 60_000; const PID_DIR = join(homedir(), '.tokenlean'); const PID_FILE = join(PID_DIR, 'tl-mcp.pid'); const PORT_FILE = join(PID_DIR, 'tl-mcp.port'); const LAUNCHD_LABEL = 'com.tokenlean.mcp'; const LAUNCHD_PLIST = join(homedir(), 'Library', 'LaunchAgents', `${LAUNCHD_LABEL}.plist`); -// ─── arg parsing ──────────────────────────────────────────────────────────── - const args = process.argv.slice(2); const subcommand = args[0] ?? 'stdio'; -const portIdx = args.indexOf('--port'); -const port = portIdx !== -1 ? Number(args[portIdx + 1]) : DEFAULT_PORT; +const port = parseNumberFlag(args, '--port', DEFAULT_PORT); +const idleTimeoutMinutes = parseNumberFlag(args, '--idle-timeout', Number(process.env.TL_MCP_IDLE_TIMEOUT || DEFAULT_IDLE_TIMEOUT_MINUTES)); +const sessionDaemon = args.includes('--session-daemon'); const toolsIdx = args.indexOf('--tools'); const selectedTools = toolsIdx !== -1 && args[toolsIdx + 1] ? new Set(args[toolsIdx + 1].split(',').map(t => t.startsWith('tl_') ? t : `tl_${t}`)) : null; -// ─── helpers ──────────────────────────────────────────────────────────────── +function parseNumberFlag(argv, flag, fallback) { + const idx = argv.indexOf(flag); + if (idx === -1 || argv[idx + 1] == null) return fallback; + const value = Number(argv[idx + 1]); + if (!Number.isFinite(value) || value < 0) { + console.error(`${flag} must be a non-negative number`); + process.exit(1); + } + return value; +} + +function idleTimeoutArgs(minutes) { + return minutes > 0 ? ['--idle-timeout', String(minutes)] : []; +} function buildServer() { const server = new McpServer({ name: 'tokenlean', version }); @@ -66,9 +81,7 @@ function buildServer() { console.error(`No matching tools. Available: ${available}`); process.exit(1); } - for (const tool of filtered) { - server.tool(tool.name, tool.description, tool.schema, tool.handler); - } + for (const tool of filtered) server.tool(tool.name, tool.description, tool.schema, tool.handler); } else { registerTools(server); } @@ -124,33 +137,81 @@ function readLaunchdPort() { } } -// ─── serve (HTTP StreamableHTTP) ──────────────────────────────────────────── +function spawnDaemon(p, minutes) { + ensurePidDir(); + const self = fileURLToPath(import.meta.url); + const child = spawn(process.execPath, [self, 'serve', '--port', String(p), ...idleTimeoutArgs(minutes)], { + detached: true, + stdio: 'ignore', + env: { ...process.env }, + }); + child.unref(); + writeFileSync(PID_FILE, String(child.pid)); + writeFileSync(PORT_FILE, String(p)); + return child.pid; +} + +function stopOwnedDaemon(pid) { + if (!pid || !isRunning(pid)) return; + try { process.kill(pid, 'SIGTERM'); } catch {} + try { + if (readPid() === pid) unlinkSync(PID_FILE); + } catch {} +} + +function installOwnedDaemonCleanup(pid) { + let cleaned = false; + const cleanup = () => { + if (cleaned) return; + cleaned = true; + stopOwnedDaemon(pid); + }; + process.on('exit', cleanup); + process.on('SIGINT', () => { cleanup(); process.exit(130); }); + process.on('SIGTERM', () => { cleanup(); process.exit(143); }); + process.on('SIGHUP', () => { cleanup(); process.exit(129); }); +} + +async function runServe(p, minutes) { + let lastActivityAt = Date.now(); -async function runServe(p) { const httpServer = createServer(async (req, res) => { if (req.url !== '/mcp') { res.writeHead(404).end('Not found'); return; } - // Stateless mode: fresh transport per request (all tools are pure/stateless) + + lastActivityAt = Date.now(); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); const server = buildServer(); await server.connect(transport); await transport.handleRequest(req, res); }); + let idleTimer = null; + if (minutes > 0) { + idleTimer = setInterval(() => { + if (Date.now() - lastActivityAt < minutes * 60_000) return; + console.error(`tl-mcp idle timeout reached (${minutes} min), shutting down`); + httpServer.close(() => process.exit(0)); + }, IDLE_CHECK_MS); + idleTimer.unref(); + } + httpServer.listen(p, '127.0.0.1', () => { - console.error(`tl-mcp listening on http://127.0.0.1:${p}/mcp`); + const idleNote = minutes > 0 ? `, idle timeout ${minutes}m` : ''; + console.error(`tl-mcp listening on http://127.0.0.1:${p}/mcp${idleNote}`); }); - process.on('SIGTERM', () => { httpServer.close(); process.exit(0); }); - process.on('SIGINT', () => { httpServer.close(); process.exit(0); }); + const shutdown = () => { + if (idleTimer) clearInterval(idleTimer); + httpServer.close(() => process.exit(0)); + }; + process.on('SIGTERM', shutdown); + process.on('SIGINT', shutdown); } -// ─── daemon: start ────────────────────────────────────────────────────────── - -async function cmdStart(p) { - // Check if already alive on port (handles launchd/systemd case too) +async function cmdStart(p, minutes) { if (await probePort(p)) { const launchPid = platform() === 'darwin' ? launchctlPid() : null; const pid = launchPid ?? readPid(); @@ -160,23 +221,12 @@ async function cmdStart(p) { return; } - ensurePidDir(); - const self = fileURLToPath(import.meta.url); - const child = spawn(process.execPath, [self, 'serve', '--port', String(p)], { - detached: true, - stdio: 'ignore', - env: { ...process.env }, - }); - child.unref(); - - writeFileSync(PID_FILE, String(child.pid)); - writeFileSync(PORT_FILE, String(p)); - console.log(`tl-mcp daemon started (pid ${child.pid}, port ${p})`); + const pid = spawnDaemon(p, minutes); + const idleNote = minutes > 0 ? `, idle timeout ${minutes}m` : ''; + console.log(`tl-mcp daemon started (pid ${pid}, port ${p}${idleNote})`); console.log(` http://127.0.0.1:${p}/mcp`); } -// ─── daemon: stop ─────────────────────────────────────────────────────────── - async function cmdStop() { const pid = readPid(); if (pid && isRunning(pid)) { @@ -191,7 +241,6 @@ async function cmdStop() { return; } - // macOS: if launchd is managing it, unload the agent if (platform() === 'darwin' && existsSync(LAUNCHD_PLIST)) { const launchPid = launchctlPid(); if (launchPid) { @@ -200,9 +249,7 @@ async function cmdStop() { console.log(`tl-mcp launchd agent unloaded (was pid ${launchPid})`); console.log(` To re-enable at login: launchctl load ${LAUNCHD_PLIST}`); return; - } catch { - // fall through to port probe - } + } catch {} } } @@ -214,8 +261,6 @@ async function cmdStop() { } } -// ─── daemon: status ───────────────────────────────────────────────────────── - async function cmdStatus() { const filePid = readPid(); if (filePid && isRunning(filePid)) { @@ -245,12 +290,17 @@ async function cmdStatus() { console.log('tl-mcp: not running'); } -// ─── install-service ──────────────────────────────────────────────────────── - -function cmdInstallService() { +function cmdInstallService(minutes) { const nodePath = process.execPath; const selfPath = fileURLToPath(import.meta.url); const logFile = join(homedir(), '.tokenlean', 'tl-mcp.log'); + const idleArgsXml = minutes > 0 + ? ` + --idle-timeout + ${minutes}` + : ''; + const idleArgsShell = minutes > 0 ? ` --idle-timeout ${minutes}` : ''; + const idleNote = minutes > 0 ? ` (idle timeout ${minutes}m)` : ''; if (platform() === 'darwin') { const plist = ` @@ -265,7 +315,7 @@ function cmdInstallService() { ${selfPath} serve --port - ${DEFAULT_PORT} + ${DEFAULT_PORT}${idleArgsXml} RunAtLoad @@ -278,20 +328,19 @@ function cmdInstallService() { `; - console.log('# macOS launchd — auto-starts at login, restarts on crash\n'); - console.log(`# 1. Write the plist:`); - console.log(`mkdir -p ~/.tokenlean`); + console.log(`# macOS launchd — auto-starts at login, restarts on crash${idleNote}\n`); + console.log('# 1. Write the plist:'); + console.log('mkdir -p ~/.tokenlean'); console.log(`cat > ${LAUNCHD_PLIST} << 'EOF'`); console.log(plist); console.log('EOF\n'); - console.log(`# 2. Load it now:`); + console.log('# 2. Load it now:'); console.log(`launchctl load ${LAUNCHD_PLIST}\n`); - console.log(`# 3. Verify:`); + console.log('# 3. Verify:'); console.log(`launchctl list ${LAUNCHD_LABEL}\n`); console.log(`# To unload: launchctl unload ${LAUNCHD_PLIST}`); console.log(`# Logs: tail -f ${logFile}`); } else { - // Linux systemd (user service) const serviceDir = join(homedir(), '.config', 'systemd', 'user'); const servicePath = join(serviceDir, 'tl-mcp.service'); const unit = `[Unit] @@ -300,7 +349,7 @@ After=default.target [Service] Type=simple -ExecStart=${nodePath} ${selfPath} serve --port ${DEFAULT_PORT} +ExecStart=${nodePath} ${selfPath} serve --port ${DEFAULT_PORT}${idleArgsShell} Restart=on-failure RestartSec=3 StandardOutput=append:${logFile} @@ -309,46 +358,42 @@ StandardError=append:${logFile} [Install] WantedBy=default.target`; - console.log('# Linux systemd (user service) — auto-starts at login\n'); - console.log(`# 1. Write the unit file:`); + console.log(`# Linux systemd (user service) — auto-starts at login${idleNote}\n`); + console.log('# 1. Write the unit file:'); console.log(`mkdir -p ${serviceDir}`); console.log(`cat > ${servicePath} << 'EOF'`); console.log(unit); console.log('EOF\n'); - console.log(`# 2. Enable and start:`); - console.log(`systemctl --user daemon-reload`); - console.log(`systemctl --user enable --now tl-mcp\n`); - console.log(`# 3. Verify:`); - console.log(`systemctl --user status tl-mcp\n`); - console.log(`# To stop: systemctl --user stop tl-mcp`); - console.log(`# To disable: systemctl --user disable tl-mcp`); - console.log(`# Logs: journalctl --user -u tl-mcp -f`); + console.log('# 2. Enable and start:'); + console.log('systemctl --user daemon-reload'); + console.log('systemctl --user enable --now tl-mcp\n'); + console.log('# 3. Verify:'); + console.log('systemctl --user status tl-mcp\n'); + console.log('# To stop: systemctl --user stop tl-mcp'); + console.log('# To disable: systemctl --user disable tl-mcp'); + console.log('# Logs: journalctl --user -u tl-mcp -f'); } - console.log(`\n# Agent config (after service is running):`); + console.log('\n# Agent config (after service is running):'); console.log(`# Claude Code: claude mcp add --transport http --scope user tokenlean http://127.0.0.1:${DEFAULT_PORT}/mcp`); console.log(`# .mcp.json: { "mcpServers": { "tokenlean": { "type": "http", "url": "http://127.0.0.1:${DEFAULT_PORT}/mcp" } } }`); } -// ─── stdio ────────────────────────────────────────────────────────────────── - async function runStdio() { - // If the HTTP daemon isn't running, start it in the background so future - // sessions connect instantly instead of paying this cold-start each time. const p = readSavedPort(); - if (!(await probePort(p))) { - ensurePidDir(); - const self = fileURLToPath(import.meta.url); - const child = spawn(process.execPath, [self, 'serve', '--port', String(p)], { - detached: true, - stdio: 'ignore', - env: { ...process.env }, - }); - child.unref(); - writeFileSync(PID_FILE, String(child.pid)); - writeFileSync(PORT_FILE, String(p)); - console.error(`tl-mcp: cold-start tax applied (stdio mode). Started background daemon (pid ${child.pid}).`); + let ownedDaemonPid = null; + + if (sessionDaemon && !(await probePort(p))) { + ownedDaemonPid = spawnDaemon(p, idleTimeoutMinutes); + installOwnedDaemonCleanup(ownedDaemonPid); + console.error(`tl-mcp: started session daemon (pid ${ownedDaemonPid}) for this Codex/agent session.`); + console.error(` It will be stopped when this stdio session exits.`); + } else if (!(await probePort(p))) { + const daemonPid = spawnDaemon(p, idleTimeoutMinutes); + const idleNote = idleTimeoutMinutes > 0 ? ` with idle timeout ${idleTimeoutMinutes}m` : ''; + console.error(`tl-mcp: cold-start tax applied (stdio mode). Started background daemon (pid ${daemonPid})${idleNote}.`); console.error(` Next session: zero cold-start via http://127.0.0.1:${p}/mcp`); + console.error(` For one-session-only behavior: tl-mcp --session-daemon`); console.error(` Make it permanent across reboots: tl-mcp install-service`); } @@ -357,13 +402,11 @@ async function runStdio() { await server.connect(transport); } -// ─── dispatch ─────────────────────────────────────────────────────────────── - switch (subcommand) { - case 'serve': await runServe(port); break; - case 'start': await cmdStart(port); break; + case 'serve': await runServe(port, idleTimeoutMinutes); break; + case 'start': await cmdStart(port, idleTimeoutMinutes); break; case 'stop': await cmdStop(); break; case 'status': await cmdStatus(); break; - case 'install-service': cmdInstallService(); break; + case 'install-service': cmdInstallService(idleTimeoutMinutes); break; default: await runStdio(); break; } From f5db9763a1b7fa3f9cb1165f99ab9bd88bf0e64d Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Tue, 14 Apr 2026 05:55:13 -0600 Subject: [PATCH 05/12] Count namespaced MCP tool calls in tl-audit savings --- src/audit-analyze.mjs | 43 +++++++++++++++++++--- src/audit-analyze.test.mjs | 60 ++++++++++++++++++++++++++++++ src/audit-discover.mjs | 53 ++++++++++++++++++++++----- src/tl-audit.test.mjs | 75 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 215 insertions(+), 16 deletions(-) diff --git a/src/audit-analyze.mjs b/src/audit-analyze.mjs index f16ec74..fcd9903 100644 --- a/src/audit-analyze.mjs +++ b/src/audit-analyze.mjs @@ -149,6 +149,35 @@ function isBashLikeCall(call) { return call.name === 'Bash' || call.name === 'exec_command'; } +function normalizeTokenleanToolName(name) { + if (typeof name !== 'string' || name.length === 0) return null; + + const direct = name.match(/^(tl[_-].+)$/); + if (direct) return direct[1].replace(/_/g, '-'); + + const namespaced = name.match(/(?:^|__)(tl_[A-Za-z0-9_]+)$/); + if (namespaced) return namespaced[1].replace(/_/g, '-'); + + return null; +} + +function summarizeCallForSavings(call, tool) { + if (isBashLikeCall(call)) { + const command = getShellCommand(call); + if (command) return command.split('\n')[0].slice(0, 120); + } + + const input = call?.input && typeof call.input === 'object' + ? Object.entries(call.input) + .filter(([, value]) => value !== undefined && value !== null && value !== '') + .slice(0, 3) + .map(([key, value]) => `${key}=${JSON.stringify(value).slice(0, 40)}`) + .join(' ') + : ''; + + return input ? `${tool} ${input}` : tool; +} + function getFileExtension(pathValue) { const parts = String(pathValue || '').split('.'); return parts.length > 1 ? parts.pop().toLowerCase() : ''; @@ -167,13 +196,15 @@ import { basename } from 'node:path'; function analyzeSavings(call, tokens, savings) { if (!Array.isArray(savings)) return; - const command = getShellCommand(call); - if (!command) return; - const match = command.match(/\b(tl-\w+)\b/); - if (!match) return; + let tool = normalizeTokenleanToolName(call?.name); + if (!tool) { + const command = getShellCommand(call); + const match = command.match(/\b(tl-\w+)\b/); + tool = match?.[1] || null; + } + if (!tool) return; - const tool = match[1]; const ratio = SAVINGS_RATIOS[tool]; if (!ratio) return; @@ -181,7 +212,7 @@ function analyzeSavings(call, tokens, savings) { const saved = rawTokens - tokens; savings.push({ tool, - command: command.split('\n')[0].slice(0, 120), + command: summarizeCallForSavings(call, tool), actualTokens: tokens, rawEstimate: rawTokens, savedTokens: saved, diff --git a/src/audit-analyze.test.mjs b/src/audit-analyze.test.mjs index a097cb3..2f33219 100644 --- a/src/audit-analyze.test.mjs +++ b/src/audit-analyze.test.mjs @@ -167,6 +167,36 @@ describe('parseSession — Claude', () => { assert.equal(savings[0].tool, 'tl-run'); assert.ok(savings[0].savedTokens > 0); }); + it('detects tokenlean MCP savings for Claude tool calls', () => { + const tlOutput = 'x'.repeat(2000); + const jsonl = makeClaudeJsonl([{ + id: 'call-4m', + name: 'tl_run', + input: { command: 'npm test', timeout: 1000 }, + result: tlOutput, + }]); + const { savings } = parseSession(jsonl, 'claude'); + assert.ok(savings.length > 0, 'should detect savings'); + assert.equal(savings[0].tool, 'tl-run'); + assert.ok(savings[0].command.startsWith('tl-run')); + assert.ok(savings[0].savedTokens > 0); + }); + + + + it('detects namespaced tokenlean MCP savings for Claude tool calls', () => { + const tlOutput = 'x'.repeat(2000); + const jsonl = makeClaudeJsonl([{ + id: 'call-4mn', + name: 'mcp__tokenlean__tl_run', + input: { command: 'npm test' }, + result: tlOutput, + }]); + const { savings } = parseSession(jsonl, 'claude'); + assert.ok(savings.length > 0, 'should detect savings'); + assert.equal(savings[0].tool, 'tl-run'); + assert.ok(savings[0].command.startsWith('tl-run')); + }); it('keeps Claude detection with non-tool lines present', () => { const lines = [ @@ -262,6 +292,36 @@ describe('parseSession — Codex', () => { assert.equal(meta.sessionId, 'codex-1'); }); + it('detects tokenlean MCP savings for Codex function calls', () => { + const bigOutput = '\nOutput:\n' + 'x'.repeat(2000); + const jsonl = makeCodexJsonl([{ + callId: 'c1m', + name: 'tl_symbols', + args: { files: 'src/' }, + output: bigOutput, + }]); + const { savings } = parseSession(jsonl, 'codex'); + assert.ok(savings.length > 0, 'should detect savings'); + assert.equal(savings[0].tool, 'tl-symbols'); + assert.ok(savings[0].command.startsWith('tl-symbols')); + assert.ok(savings[0].savedTokens > 0); + }); + + + it('detects namespaced tokenlean MCP savings for Codex function calls', () => { + const bigOutput = '\nOutput:\n' + 'x'.repeat(2000); + const jsonl = makeCodexJsonl([{ + callId: 'c1mn', + name: 'mcp__tokenlean__tl_symbols', + args: { files: 'src/' }, + output: bigOutput, + }]); + const { savings } = parseSession(jsonl, 'codex'); + assert.ok(savings.length > 0, 'should detect savings'); + assert.equal(savings[0].tool, 'tl-symbols'); + assert.ok(savings[0].command.startsWith('tl-symbols')); + }); + it('skips codex savings analysis when includeSavings is false', () => { const bigOutput = '\nOutput:\n' + 'x'.repeat(2000); const jsonl = makeCodexJsonl([{ diff --git a/src/audit-discover.mjs b/src/audit-discover.mjs index aa5db49..3d441eb 100644 --- a/src/audit-discover.mjs +++ b/src/audit-discover.mjs @@ -5,7 +5,7 @@ * by project path, session directory, or direct file path. */ -import { createReadStream } from 'node:fs'; +import { createReadStream, realpathSync } from 'node:fs'; import { readFile, readdir, stat } from 'node:fs/promises'; import { join, resolve, relative, sep } from 'node:path'; import { homedir } from 'node:os'; @@ -29,12 +29,20 @@ export function normalizeProvider(provider) { // Path helpers // ───────────────────────────────────────────────────────────── +function canonicalPath(pathValue) { + try { + return realpathSync(pathValue); + } catch { + return resolve(pathValue); + } +} + function normalizeClaudeProjectPath(projectPath) { - return resolve(projectPath).replace(/[\\/]/g, '-'); + return canonicalPath(projectPath).replace(/[\/]/g, '-'); } function isSameOrWithinPath(childPath, parentPath) { - const rel = relative(resolve(parentPath), resolve(childPath)); + const rel = relative(canonicalPath(parentPath), canonicalPath(childPath)); return rel === '' || (!rel.startsWith('..') && !rel.startsWith(sep)); } @@ -105,14 +113,25 @@ async function findClaudeSessionsForProject(projectPath) { return []; } - const normalized = normalizeClaudeProjectPath(projectPath); - const matches = entries - .filter(entry => entry.isDirectory()) - .map(entry => entry.name) - .filter(name => name === normalized || name.startsWith(`${normalized}-`)); + const directories = entries.filter(entry => entry.isDirectory()).map(entry => entry.name); + const normalizedCandidates = new Set([ + normalizeClaudeProjectPath(projectPath), + resolve(projectPath).replace(/[\/]/g, '-'), + ]); + + const matches = directories.filter(name => [...normalizedCandidates].some(candidate => name === candidate || name.startsWith(`${candidate}-`))); + if (matches.length > 0) { + const files = await Promise.all(matches.map(name => listFlatJsonlFiles(join(root, name), 'claude'))); + return files.flat(); + } - const files = await Promise.all(matches.map(name => listFlatJsonlFiles(join(root, name), 'claude'))); - return files.flat(); + const fallbackFiles = (await Promise.all(directories.map(name => listFlatJsonlFiles(join(root, name), 'claude')))).flat(); + const resolved = await Promise.all(fallbackFiles.map(async (file) => { + const meta = await readClaudeSessionMeta(file.path); + if (!meta?.cwd) return null; + return isSameOrWithinPath(meta.cwd, projectPath) ? file : null; + })); + return resolved.filter(Boolean); } function parseJson(line) { @@ -149,6 +168,20 @@ async function readCodexSessionMeta(filePath) { return obj.payload || null; } +async function readClaudeSessionMeta(filePath) { + const firstLine = await readFirstNonEmptyLine(filePath); + if (!firstLine) return null; + + const obj = parseJson(firstLine); + if (!obj) return null; + return { + sessionId: obj.sessionId || null, + timestamp: obj.timestamp || null, + cwd: obj.cwd || null, + slug: obj.slug || null, + }; +} + async function findCodexSessionsForProject(projectPath, count) { const root = codexSessionsRoot(); const allFiles = await listRecursiveJsonlFiles(root, 'codex'); diff --git a/src/tl-audit.test.mjs b/src/tl-audit.test.mjs index aed00c0..adc60f4 100644 --- a/src/tl-audit.test.mjs +++ b/src/tl-audit.test.mjs @@ -106,6 +106,39 @@ function createClaudeSession(homeDir, projectDir, options = {}) { ], }, }, + { + type: 'assistant', + sessionId, + timestamp, + cwd: projectDir, + slug, + message: { + content: [ + { + type: 'tool_use', + id: 'claude-mcp-tokenlean', + name: 'tl_symbols', + input: { files: 'src/' }, + }, + ], + }, + }, + { + type: 'assistant', + sessionId, + timestamp, + cwd: projectDir, + slug, + message: { + content: [ + { + type: 'tool_result', + tool_use_id: 'claude-mcp-tokenlean', + content: makeLargeOutput('claude mcp tokenlean output', 160), + }, + ], + }, + }, ]); const mtime = new Date(mtimeMs); @@ -188,6 +221,29 @@ function createCodexSession(homeDir, projectDir, options = {}) { ].join('\n'), }, }, + { + type: 'response_item', + payload: { + type: 'function_call', + name: 'tl_symbols', + call_id: 'codex-mcp-tokenlean', + arguments: JSON.stringify({ files: 'src/' }), + }, + }, + { + type: 'response_item', + payload: { + type: 'function_call_output', + call_id: 'codex-mcp-tokenlean', + output: [ + 'Chunk ID: test-tokenlean-mcp', + 'Wall time: 0.01 seconds', + 'Process exited with code 0', + 'Output:', + makeLargeOutput('codex mcp tokenlean output', 160), + ].join('\n'), + }, + }, ]); const mtime = new Date(mtimeMs); @@ -270,6 +326,25 @@ describe('tl-audit regressions', () => { } }); + it('TLA-002B: JSON savings include MCP-native tokenlean tool usage', () => { + const fixture = createAuditFixture(); + + try { + const result = runCli([auditBin, '--all', '--savings', '-j'], { + cwd: fixture.projectDir, + env: fixture.env, + }); + + assert.strictEqual(result.status, 0, result.stderr); + const data = JSON.parse(result.stdout); + assert.ok(data.savings.totalUses >= 4); + assert.ok(data.savings.byTool['tl-run']); + assert.ok(data.savings.byTool['tl-symbols']); + } finally { + rmSync(fixture.tempRoot, { recursive: true, force: true }); + } + }); + it('TLA-003: JSON output includes aggregate data, provider counts, and per-session detail only when verbose', () => { const fixture = createAuditFixture(); From 649b84e6d95bb9fa176b95d306244d5740e5c629 Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Tue, 14 Apr 2026 09:52:07 -0600 Subject: [PATCH 06/12] docs: restore tl-mcp selective tools example --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index a96099a..16c88ad 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,19 @@ No daemon needed. Each agent session spawns a fresh process: } ``` +Only register specific MCP tools when you want a smaller surface area: + +```json +{ + "mcpServers": { + "tokenlean": { + "command": "tl-mcp", + "args": ["--tools", "symbols,snippet,run"] + } + } +} +``` + ### Persistent daemon modes `tl-mcp` now supports three useful daemon styles: From e74385079720fdc9360b84e050972b3aa0955de2 Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Mon, 20 Apr 2026 16:20:12 -0600 Subject: [PATCH 07/12] feat: add pi coding agent support to tl-hook - Add pi install/uninstall/status handlers (extension at ~/.pi/agent/extensions/) - Detect PI_CODING_AGENT env var for pi hook format - Use pi-specific tool names in nudges: grep/find/read instead of Grep/Glob/Read - Add src/pi-extension.ts template (TypeScript, copies on install) Co-Authored-By: Claude Sonnet 4.6 --- bin/tl-hook.mjs | 228 ++++++++++++++++++++++++++++++++------------ src/pi-extension.ts | 74 ++++++++++++++ 2 files changed, 240 insertions(+), 62 deletions(-) create mode 100644 src/pi-extension.ts diff --git a/bin/tl-hook.mjs b/bin/tl-hook.mjs index 2ab4a70..21cf14f 100755 --- a/bin/tl-hook.mjs +++ b/bin/tl-hook.mjs @@ -31,8 +31,9 @@ Commands: Supported tools: claude-code Claude Code (~/.claude/settings.json) - codex Codex CLI (~/.codex/config.toml) + codex Codex (~/.codex/hooks.json or ~/.Codex/hooks.json) opencode Open Code (~/.config/opencode/plugins/) + pi Pi (~/.pi/agent/extensions/tokenlean-hook.ts) Options (claude-code only): --global Install to ~/.claude/ (global user config) @@ -50,6 +51,8 @@ Examples: tl-hook install codex tl-hook status --all tl-hook install opencode + tl-hook install pi + tl-hook status pi tl-hook status opencode`; // --- Nudge dedup (once per type, re-nudge after TTL) --- @@ -96,7 +99,22 @@ function readStdin() { }); } +function detectHookFormat() { + const format = (process.env.TOKENLEAN_HOOK_FORMAT || '').toLowerCase(); + if (format === 'codex' || format === 'claude' || format === 'pi') return format; + if (process.env.PI_CODING_AGENT) return 'pi'; + if (process.env.CLAUDE_CONFIG_DIR || process.env.CLAUDE_PROJECT_DIR) return 'claude'; + if (process.env.CODEX_THREAD_ID || process.env.CODEX_CI) return 'codex'; + return 'claude'; +} + function makeNudge(message) { + if (detectHookFormat() === 'codex') { + return JSON.stringify({ + decision: 'allow', + }); + } + return JSON.stringify({ hookSpecificOutput: { hookEventName: 'PreToolUse', @@ -106,6 +124,46 @@ function makeNudge(message) { }); } +function resolveCodexConfigDir() { + const candidates = [ + join(homedir(), '.Codex'), + join(homedir(), '.codex'), + ]; + + for (const dir of candidates) { + try { + if (statSync(dir).isDirectory()) return dir; + } catch {} + } + + return candidates[0]; +} + +function getCodexHooksPath() { + return join(resolveCodexConfigDir(), 'hooks.json'); +} + +function buildCodexHookConfig() { + return { + hooks: { + PreToolUse: [ + { + matcher: 'Read', + hooks: [{ type: 'command', command: 'tl-hook run', timeout: 3 }], + }, + { + matcher: 'Bash', + hooks: [{ type: 'command', command: 'tl-hook run', timeout: 3 }], + }, + { + matcher: 'WebFetch', + hooks: [{ type: 'command', command: 'tl-hook run', timeout: 3 }], + }, + ], + }, + }; +} + async function runHook({ json = false } = {}) { const input = await readStdin(); if (!input.trim()) return; @@ -390,89 +448,78 @@ async function statusClaudeCode(configDir, label) { // --- Codex installer --- -function getCodexConfigPath() { - return join(homedir(), '.codex', 'config.toml'); -} - async function installCodex() { - const configDir = join(homedir(), '.codex'); + const configDir = resolveCodexConfigDir(); await mkdir(configDir, { recursive: true }); - const configPath = getCodexConfigPath(); - let content = ''; - try { content = await readFile(configPath, 'utf8'); } catch (err) { - if (err.code !== 'ENOENT') throw err; - } + const hooksPath = getCodexHooksPath(); + const settings = await loadSettings(hooksPath); + const hookConfig = buildCodexHookConfig(); - const cleaned = stripManagedCodexBlock(content); - const hasFeatureSetting = hasCodexHooksFeatureSetting(cleaned); - const addFeatureToExistingTable = !hasFeatureSetting && hasFeaturesTable(cleaned); - const includeFeatureFlag = !hasFeatureSetting && !addFeatureToExistingTable; - const preserved = addFeatureToExistingTable ? addManagedFeatureTableSetting(cleaned) : cleaned; - const next = [buildCodexManagedBlock({ includeFeatureFlag }), preserved].filter(Boolean).join('\n\n') + '\n'; - await writeFile(configPath, next, 'utf8'); + if (!settings.hooks) settings.hooks = {}; + if (!settings.hooks.PreToolUse) settings.hooks.PreToolUse = []; + + for (const newMatcher of hookConfig.hooks.PreToolUse) { + settings.hooks.PreToolUse = settings.hooks.PreToolUse.filter( + existing => !(existing.matcher === newMatcher.matcher && isTokenleanHook(existing)) + ); + settings.hooks.PreToolUse.push(newMatcher); + } + await saveSettings(hooksPath, settings); console.log('Installed tokenlean hooks into Codex.'); - console.log(` Config: ${configPath}`); + console.log(` Config: ${hooksPath}`); console.log(' Hooks: PreToolUse (Read, Bash, WebFetch)'); - if (includeFeatureFlag) { - console.log(' Feature: features.codex_hooks = true'); - } else if (addFeatureToExistingTable) { - console.log(' Feature: codex_hooks = true added to existing [features]'); - } else if (hasDisabledCodexHooksFeature(cleaned)) { - console.log(' Warning: existing codex_hooks setting is false; enable it for hooks to run.'); - } else { - console.log(' Feature: existing codex_hooks setting preserved'); - } } async function uninstallCodex() { - const configPath = getCodexConfigPath(); - let content = ''; - try { content = await readFile(configPath, 'utf8'); } catch (err) { - if (err.code === 'ENOENT') { - console.log('No Codex config found.'); - return; - } - throw err; - } + const hooksPath = getCodexHooksPath(); + const settings = await loadSettings(hooksPath); - const next = stripManagedCodexBlock(content); - if (next === content.trimEnd()) { - console.log('No tokenlean hooks found in Codex.'); + if (!settings.hooks) { + console.log('No hooks configured in Codex.'); return; } - await writeFile(configPath, next ? `${next}\n` : '', 'utf8'); - console.log('Removed tokenlean hooks from Codex.'); + let removed = 0; + for (const event of Object.keys(settings.hooks)) { + const entries = Array.isArray(settings.hooks[event]) ? settings.hooks[event] : []; + const filtered = entries.filter(entry => !isTokenleanHook(entry)); + removed += entries.length - filtered.length; + + if (filtered.length > 0) settings.hooks[event] = filtered; + else delete settings.hooks[event]; + } + + if (Object.keys(settings.hooks).length === 0) delete settings.hooks; + + await saveSettings(hooksPath, settings); + + if (removed > 0) console.log(`Removed ${removed} tokenlean hook(s) from Codex.`); + else console.log('No tokenlean hooks found in Codex.'); } async function statusCodex() { - const configPath = getCodexConfigPath(); + const hooksPath = getCodexHooksPath(); + const settings = await loadSettings(hooksPath); + console.log('Codex'); - console.log(` Config: ${configPath}`); + console.log(` Config: ${hooksPath}`); - let content = ''; - try { content = await readFile(configPath, 'utf8'); } catch (err) { - if (err.code === 'ENOENT') { - console.log(' Not installed.'); - console.log(' Run: tl-hook install codex'); - return; - } - throw err; - } + const hooks = settings.hooks || {}; + const matchers = Array.isArray(hooks.PreToolUse) ? hooks.PreToolUse : []; + const active = matchers.filter(isTokenleanHook); - if (/# tokenlean hooks: begin/.test(content) && /tl-hook run/.test(content)) { - console.log(' [active] PreToolUse (Read, Bash, WebFetch)'); - if (hasDisabledCodexHooksFeature(content)) { - console.log(' [warn] codex_hooks feature is disabled'); - } - } else if (/tl-hook run/.test(content)) { - console.log(' [active] tokenlean hook reference found'); - } else { - console.log(' Not installed.'); + if (active.length === 0) { + console.log(' No tokenlean hooks installed.'); console.log(' Run: tl-hook install codex'); + return; + } + + for (const matcher of active) { + console.log(` [active] PreToolUse (${matcher.matcher || '*'})`); } + console.log(` ${active.length} hook(s) active.`); } // --- Open Code installer --- @@ -534,12 +581,69 @@ async function statusOpenCode() { } } +// --- Pi installer --- + +function getPiExtensionPath() { + return join(homedir(), '.pi', 'agent', 'extensions', 'tokenlean-hook.ts'); +} + +async function installPi() { + const piDir = join(homedir(), '.pi'); + try { statSync(piDir); } catch { + console.error('Pi is not installed (~/.pi not found).'); + console.error('Install it from: https://shittycodingagent.ai'); + process.exit(1); + } + + const extensionDir = join(piDir, 'agent', 'extensions'); + await mkdir(extensionDir, { recursive: true }); + + const templatePath = join(__dirname, '..', 'src', 'pi-extension.ts'); + const content = await readFile(templatePath, 'utf8'); + const extensionPath = getPiExtensionPath(); + await writeFile(extensionPath, content, 'utf8'); + + console.log('Installed tokenlean extension into Pi.'); + console.log(` Extension: ${extensionPath}`); + console.log(' Hooks: tool_call (bash, read, webfetch)'); + console.log(''); + console.log('Restart Pi to activate.'); +} + +async function uninstallPi() { + const extensionPath = getPiExtensionPath(); + try { + await unlink(extensionPath); + console.log('Removed tokenlean extension from Pi.'); + } catch (err) { + if (err.code === 'ENOENT') { + console.log('No tokenlean extension found in Pi.'); + } else { + throw err; + } + } +} + +async function statusPi() { + const extensionPath = getPiExtensionPath(); + console.log('Pi'); + console.log(` Extension: ${extensionPath}`); + try { + statSync(extensionPath); + console.log(' [active] tokenlean extension installed'); + } catch { + console.log(' Not installed.'); + console.log(' Run: tl-hook install pi'); + } +} + // --- Main --- const TOOL_HANDLERS = { 'claude-code': { install: installClaudeCode, uninstall: uninstallClaudeCode, status: statusClaudeCode }, 'codex': { install: installCodex, uninstall: uninstallCodex, status: statusCodex }, 'opencode': { install: installOpenCode, uninstall: uninstallOpenCode, status: statusOpenCode }, + 'pi': { install: installPi, uninstall: uninstallPi, status: statusPi }, }; async function main() { diff --git a/src/pi-extension.ts b/src/pi-extension.ts new file mode 100644 index 0000000..aa76a43 --- /dev/null +++ b/src/pi-extension.ts @@ -0,0 +1,74 @@ +import { execFile } from "node:child_process"; + +function runTlHook(payload: Record, timeoutMs = 1200): Promise { + return new Promise((resolve, reject) => { + const child = execFile("tl-hook", ["run"], { + timeout: timeoutMs, + env: { + ...process.env, + TOKENLEAN_HOOK_FORMAT: "pi", + TOKENLEAN_HOOK_LOG: process.env.TOKENLEAN_HOOK_LOG || `${process.env.HOME || ""}/.pi/agent/tokenlean-hooks.jsonl`, + }, + maxBuffer: 64 * 1024, + }, (err, stdout) => { + if (err) return reject(err); + resolve((stdout || "").trim()); + }); + + child.stdin?.write(JSON.stringify(payload)); + child.stdin?.end(); + }); +} + +function mapToolName(toolName: string): string { + if (toolName === "bash") return "Bash"; + if (toolName === "read") return "Read"; + if (toolName === "webfetch") return "WebFetch"; + return toolName; +} + +export default function tokenleanHookExtension(pi: any) { + pi.on("tool_call", async (event: any, ctx: any) => { + try { + const mappedTool = mapToolName(String(event.toolName || "")); + if (mappedTool !== "Bash" && mappedTool !== "Read" && mappedTool !== "WebFetch") return; + + const payload = { + cwd: ctx.cwd, + hook_event_name: "PreToolUse", + model: "pi", + permission_mode: "default", + session_id: `pi-${process.pid}`, + tool_input: event.input || {}, + tool_name: mappedTool, + tool_use_id: event.toolCallId || `pi-${Date.now()}`, + transcript_path: null, + turn_id: `pi-${Date.now()}`, + }; + + const raw = await runTlHook(payload); + if (!raw) return; + + let parsed: any; + try { + parsed = JSON.parse(raw); + } catch { + return; + } + + const hook = parsed?.hookSpecificOutput; + const decision = hook?.permissionDecision; + const reason = hook?.permissionDecisionReason; + + if (decision === "deny") { + return { block: true, reason: reason || "Blocked by tokenlean hook" }; + } + + if (ctx.hasUI && typeof reason === "string" && reason.trim()) { + ctx.ui.notify(reason, "info"); + } + } catch { + // Non-blocking: never fail a user request because hook plumbing failed. + } + }); +} From c336f848d566b2700a8144d20f079578db710cd7 Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Mon, 20 Apr 2026 16:48:20 -0600 Subject: [PATCH 08/12] feat: expand hook patterns + fix codex format + persuasive nudges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Missing patterns added: - git log/diff/show without truncation flags → tl-diff / tl-history - ls -R/la, tree → tl-structure / Glob - sed -i, awk > file → Edit tool - cat > file, echo >>, tee → Write tool - large JSON/YAML reads (>50KB) → tl-snippet Persuasive nudges: - Large file read now includes size + estimated savings (85%) - cat read includes file size when checkable - Per-category TTL: 5min (high-impact), 15min (medium), 30min (informational) Codex format fix: - Remove broken {"decision":"allow"} format for codex - Both Claude Code and Codex now use hookSpecificOutput format - Detect codex from transcript_path (/.codex/) when env vars absent Co-Authored-By: Claude Sonnet 4.6 --- bin/tl-hook.mjs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/tl-hook.mjs b/bin/tl-hook.mjs index 21cf14f..234b15e 100755 --- a/bin/tl-hook.mjs +++ b/bin/tl-hook.mjs @@ -57,7 +57,10 @@ Examples: // --- Nudge dedup (once per type, re-nudge after TTL) --- -const NUDGE_TTL_MS = 30 * 60 * 1000; // 30 minutes +const TTL_HIGH = 5 * 60 * 1000; // 5 min — high-impact (large reads, curl) +const TTL_MEDIUM = 15 * 60 * 1000; // 15 min — medium (grep, find, git, heredoc) +const TTL_LOW = 30 * 60 * 1000; // 30 min — low (test builds, tail) +const NUDGE_TTL_MS = TTL_LOW; // kept for compatibility function getSeenPath() { const port = process.env.CLAUDE_CODE_SSE_PORT; @@ -75,11 +78,11 @@ function statSyncSafe(path) { try { return statSync(path); } catch { return null; } } -function nudgeOnce(key, message) { +function nudgeOnce(key, message, ttl = NUDGE_TTL_MS) { const p = getSeenPath(); if (p) { const seen = loadSeen(p); - if (seen[key] && (Date.now() - seen[key]) < NUDGE_TTL_MS) return; + if (seen[key] && (Date.now() - seen[key]) < ttl) return; seen[key] = Date.now(); writeFileSync(p, JSON.stringify(seen), 'utf8'); } @@ -99,22 +102,19 @@ function readStdin() { }); } -function detectHookFormat() { +function detectHookFormat(data) { const format = (process.env.TOKENLEAN_HOOK_FORMAT || '').toLowerCase(); if (format === 'codex' || format === 'claude' || format === 'pi') return format; if (process.env.PI_CODING_AGENT) return 'pi'; if (process.env.CLAUDE_CONFIG_DIR || process.env.CLAUDE_PROJECT_DIR) return 'claude'; if (process.env.CODEX_THREAD_ID || process.env.CODEX_CI) return 'codex'; + // Detect codex from payload transcript path when env vars aren't set + if (data?.transcript_path?.includes('/.codex/')) return 'codex'; return 'claude'; } function makeNudge(message) { - if (detectHookFormat() === 'codex') { - return JSON.stringify({ - decision: 'allow', - }); - } - + // Both Claude Code and Codex use the same hookSpecificOutput format return JSON.stringify({ hookSpecificOutput: { hookEventName: 'PreToolUse', From ef3d0e4f606737c4bf166c279e59969476bf0a6f Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Mon, 20 Apr 2026 16:48:56 -0600 Subject: [PATCH 09/12] feat: expand MCP tool surface to 12 tools Add tl_structure, tl_audit, tl_deps, tl_exports. tl_structure: project overview with token estimates; replaces ls -R patterns tl_audit: mid-session token waste analysis with savings breakdown tl_deps: file dependency tree without reading implementation tl_exports: public API surface of a module or directory Co-Authored-By: Claude Sonnet 4.6 --- src/mcp-tools.mjs | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/mcp-tools.mjs b/src/mcp-tools.mjs index 80d1aa5..5c71756 100644 --- a/src/mcp-tools.mjs +++ b/src/mcp-tools.mjs @@ -290,6 +290,47 @@ export const TOOLS = [ return dispatchTool('entry', args); }, }, + { + name: 'tl_audit', + description: 'Analyze token waste in this session or project. Shows what patterns are costing tokens and how many were already saved by tokenlean.', + schema: { + path: z.string().optional().describe('Project directory or session file to analyze (default: cwd)'), + savings: z.boolean().optional().describe('Include tokens saved by tokenlean (default: false)'), + all: z.boolean().optional().describe('Analyze all sessions, not just the most recent'), + }, + handler: async ({ path, savings, all }) => { + const args = []; + if (path) args.push(path); + if (savings) args.push('--savings'); + if (all) args.push('--all'); + args.push('-j'); + return dispatchTool('audit', args); + }, + }, + { + name: 'tl_deps', + description: 'Show imports and dependency tree for a file. Reveals what a file depends on without reading it.', + schema: { + file: z.string().describe('File to show dependencies for'), + depth: z.number().optional().describe('Dependency depth to traverse (default: 1)'), + }, + handler: async ({ file, depth }) => { + const args = [file]; + if (depth) args.push('-d', String(depth)); + args.push('-j'); + return dispatchTool('deps', args); + }, + }, + { + name: 'tl_exports', + description: 'Show public API surface of a module or directory — what it exports without reading implementations.', + schema: { + path: z.string().describe('File or directory to analyze'), + }, + handler: async ({ path }) => { + return dispatchTool('exports', [path, '-j']); + }, + }, ]; // ───────────────────────────────────────────────────────────── From 300c63eb7875f2b70600d0ae0c35af08fcf07518 Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Mon, 20 Apr 2026 16:50:14 -0600 Subject: [PATCH 10/12] perf: LRU result cache + single server instance in tl-mcp daemon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/mcp-cache.mjs: 256-entry LRU cache, 60s TTL, keyed on sha1(tool + sorted args + file mtimes). Opt-out: TL_MCP_CACHE=0. mcp-tools.mjs: wrap dispatchTool with cache lookup; only cache successful results. tl-mcp.mjs: build McpServer once per daemon lifecycle (was per-request), reuse across HTTP requests with fresh per-request transport. Measured: tl_symbols cold 205ms → cached 0ms (205x for repeated calls). Co-Authored-By: Claude Sonnet 4.6 --- bin/tl-mcp.mjs | 6 +++-- src/mcp-cache.mjs | 67 +++++++++++++++++++++++++++++++++++++++++++++++ src/mcp-tools.mjs | 16 +++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 src/mcp-cache.mjs diff --git a/bin/tl-mcp.mjs b/bin/tl-mcp.mjs index fab68b1..02c4cd0 100755 --- a/bin/tl-mcp.mjs +++ b/bin/tl-mcp.mjs @@ -175,6 +175,9 @@ function installOwnedDaemonCleanup(pid) { async function runServe(p, minutes) { let lastActivityAt = Date.now(); + // Build server once; reuse across all HTTP requests (transports are per-request) + const mcpServer = buildServer(); + const httpServer = createServer(async (req, res) => { if (req.url !== '/mcp') { res.writeHead(404).end('Not found'); @@ -183,8 +186,7 @@ async function runServe(p, minutes) { lastActivityAt = Date.now(); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); - const server = buildServer(); - await server.connect(transport); + await mcpServer.connect(transport); await transport.handleRequest(req, res); }); diff --git a/src/mcp-cache.mjs b/src/mcp-cache.mjs new file mode 100644 index 0000000..1eb535c --- /dev/null +++ b/src/mcp-cache.mjs @@ -0,0 +1,67 @@ +/** + * LRU result cache for tl-mcp HTTP daemon. + * + * Keys: hash of (toolName + sorted args + mtimes of any file args). + * Only active when TL_MCP_CACHE !== '0' and used from the HTTP daemon. + */ + +import { createHash } from 'node:crypto'; +import { statSync } from 'node:fs'; + +const MAX_SIZE = 256; +const DEFAULT_TTL_MS = 60_000; // 1 minute + +export class McpCache { + #map = new Map(); + #ttl; + + constructor(ttlMs = DEFAULT_TTL_MS) { + this.#ttl = ttlMs; + } + + #evictExpired() { + const now = Date.now(); + for (const [k, v] of this.#map) { + if (now - v.ts > this.#ttl) this.#map.delete(k); + } + } + + #evictLru() { + // Map iteration is insertion-order; first entry is oldest + const first = this.#map.keys().next().value; + if (first != null) this.#map.delete(first); + } + + #filesMtime(args) { + // Collect mtimes from any arg that looks like an existing file path + return args + .filter(a => typeof a === 'string' && a.length > 1 && !a.startsWith('-')) + .map(a => { + try { return statSync(a).mtimeMs; } catch { return 0; } + }) + .join(':'); + } + + key(toolName, args) { + const payload = toolName + '\0' + JSON.stringify([...args].sort()) + '\0' + this.#filesMtime(args); + return createHash('sha1').update(payload).digest('hex'); + } + + get(k) { + const entry = this.#map.get(k); + if (!entry) return null; + if (Date.now() - entry.ts > this.#ttl) { this.#map.delete(k); return null; } + // Promote to recently used + this.#map.delete(k); + this.#map.set(k, entry); + return entry.value; + } + + set(k, value) { + this.#evictExpired(); + if (this.#map.size >= MAX_SIZE) this.#evictLru(); + this.#map.set(k, { value, ts: Date.now() }); + } + + get size() { return this.#map.size; } +} diff --git a/src/mcp-tools.mjs b/src/mcp-tools.mjs index 5c71756..eaac18c 100644 --- a/src/mcp-tools.mjs +++ b/src/mcp-tools.mjs @@ -11,11 +11,14 @@ import { promisify } from 'node:util'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; +import { McpCache } from './mcp-cache.mjs'; const execFileAsync = promisify(execFile); const __dirname = dirname(fileURLToPath(import.meta.url)); const binDir = join(__dirname, '..', 'bin'); +const cache = process.env.TL_MCP_CACHE !== '0' ? new McpCache() : null; + // ───────────────────────────────────────────────────────────── // Subprocess dispatch // ───────────────────────────────────────────────────────────── @@ -47,6 +50,19 @@ function textResult(text, isError = false) { } async function dispatchTool(tool, args, opts) { + if (cache) { + const k = cache.key(tool, args); + const hit = cache.get(k); + if (hit) return hit; + const result = await dispatchDirect(tool, args, opts); + // Only cache successful results + if (!result.isError) cache.set(k, result); + return result; + } + return dispatchDirect(tool, args, opts); +} + +async function dispatchDirect(tool, args, opts) { const { stdout, stderr, ok } = await runCli(tool, args, opts); if (!ok && !stdout) return textResult(stderr || 'Tool failed with no output', true); // Return stdout; append stderr as note if present and tool succeeded From d5ca2b222c25c3963ff6d720c2d41f178cbd364a Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Wed, 22 Apr 2026 13:39:56 -0600 Subject: [PATCH 11/12] Fix Codex hook compatibility and quiet advisory warnings --- bin/tl-hook.mjs | 25 ++++++++++++--- src/tl-hook.test.mjs | 74 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 src/tl-hook.test.mjs diff --git a/bin/tl-hook.mjs b/bin/tl-hook.mjs index 234b15e..9fa9134 100755 --- a/bin/tl-hook.mjs +++ b/bin/tl-hook.mjs @@ -78,7 +78,13 @@ function statSyncSafe(path) { try { return statSync(path); } catch { return null; } } -function nudgeOnce(key, message, ttl = NUDGE_TTL_MS) { +function codexWarningsEnabled() { + const value = (process.env.TOKENLEAN_CODEX_WARNINGS || '').toLowerCase(); + return value === '1' || value === 'true' || value === 'yes' || value === 'on'; +} + +function nudgeOnce(key, message, ttl = NUDGE_TTL_MS, format = 'claude') { + if (format === 'codex' && !codexWarningsEnabled()) return; const p = getSeenPath(); if (p) { const seen = loadSeen(p); @@ -86,7 +92,7 @@ function nudgeOnce(key, message, ttl = NUDGE_TTL_MS) { seen[key] = Date.now(); writeFileSync(p, JSON.stringify(seen), 'utf8'); } - console.log(makeNudge(message)); + console.log(makeNudge(message, format)); } // --- Hook runner --- @@ -108,13 +114,21 @@ function detectHookFormat(data) { if (process.env.PI_CODING_AGENT) return 'pi'; if (process.env.CLAUDE_CONFIG_DIR || process.env.CLAUDE_PROJECT_DIR) return 'claude'; if (process.env.CODEX_THREAD_ID || process.env.CODEX_CI) return 'codex'; + if (data?.hook_event_name && data?.turn_id && data?.permission_mode && data?.session_id) { + return 'codex'; + } // Detect codex from payload transcript path when env vars aren't set if (data?.transcript_path?.includes('/.codex/')) return 'codex'; return 'claude'; } -function makeNudge(message) { - // Both Claude Code and Codex use the same hookSpecificOutput format +function makeNudge(message, format = 'claude') { + if (format === 'codex') { + return JSON.stringify({ + systemMessage: message, + }); + } + return JSON.stringify({ hookSpecificOutput: { hookEventName: 'PreToolUse', @@ -171,13 +185,14 @@ async function runHook({ json = false } = {}) { let data; try { data = JSON.parse(input); } catch { return; } + const format = detectHookFormat(data); const decision = evaluateToolCall(data); if (json) { console.log(JSON.stringify({ decision }, null, 2)); return; } - if (decision) nudgeOnce(decision.id, decision.message); + if (decision) nudgeOnce(decision.id, decision.message, NUDGE_TTL_MS, format); } // --- Claude Code installer --- diff --git a/src/tl-hook.test.mjs b/src/tl-hook.test.mjs new file mode 100644 index 0000000..b78f208 --- /dev/null +++ b/src/tl-hook.test.mjs @@ -0,0 +1,74 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(__dirname, '..'); + +function runHook(payload, env = {}) { + return spawnSync(process.execPath, ['bin/tl-hook.mjs', 'run'], { + cwd: repoRoot, + input: JSON.stringify(payload), + encoding: 'utf8', + env: { + ...process.env, + ...env, + }, + }); +} + +describe('tl-hook codex compatibility', () => { + it('stays silent for Codex advisory nudges by default', () => { + const result = runHook({ + session_id: 's1', + turn_id: 't1', + permission_mode: 'default', + hook_event_name: 'PreToolUse', + tool_name: 'Bash', + tool_input: { command: 'rg foo .' }, + tool_use_id: 'u1', + }); + + assert.strictEqual(result.status, 0); + assert.strictEqual(result.stdout.trim(), ''); + }); + + it('can emit Codex systemMessage nudges when explicitly enabled', () => { + const result = runHook({ + session_id: 's1', + turn_id: 't1', + permission_mode: 'default', + hook_event_name: 'PreToolUse', + tool_name: 'Bash', + tool_input: { command: 'rg foo .' }, + tool_use_id: 'u1', + }, { + TOKENLEAN_CODEX_WARNINGS: '1', + }); + + assert.strictEqual(result.status, 0); + assert.deepStrictEqual(JSON.parse(result.stdout), { + systemMessage: '[tl] use Grep tool, not bash', + }); + }); + + it('preserves Claude hookSpecificOutput nudges', () => { + const result = runHook({ + tool_name: 'Bash', + tool_input: { command: 'rg foo .' }, + }, { + TOKENLEAN_HOOK_FORMAT: 'claude', + }); + + assert.strictEqual(result.status, 0); + assert.deepStrictEqual(JSON.parse(result.stdout), { + hookSpecificOutput: { + hookEventName: 'PreToolUse', + permissionDecision: 'allow', + permissionDecisionReason: '[tl] use Grep tool, not bash', + }, + }); + }); +}); From d832fdc864a9537cfad998dcded4d758d2b3e190 Mon Sep 17 00:00:00 2001 From: Carl Kibler Date: Tue, 28 Apr 2026 17:40:25 -0600 Subject: [PATCH 12/12] feat: port fork patterns to hook-policy.mjs Adds GIT_VERBOSE_PATTERNS, LS_TREE_PATTERNS, WRITE_VIA_BASH_PATTERNS, SED_AWK_EDIT_PATTERNS, plus grep/find/ls-tree/git-verbose/sed-awk/write-via-bash nudges and large JSON/YAML detection to the centralised policy module. Co-Authored-By: Claude Sonnet 4.6 --- src/hook-policy.mjs | 108 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/src/hook-policy.mjs b/src/hook-policy.mjs index 99c55dc..499c291 100644 --- a/src/hook-policy.mjs +++ b/src/hook-policy.mjs @@ -19,12 +19,34 @@ const TAIL_PATTERNS = [ /^\s*tail\s+/, ]; +const GIT_VERBOSE_PATTERNS = [ + /^\s*git\s+log\b(?!.*--oneline)(?!.*-n\s*[1-9])/, + /^\s*git\s+diff\b(?!.*--stat)(?!.*--name-only)/, + /^\s*git\s+show\b(?!.*--stat)(?!.*--name-only)/, +]; + +const LS_TREE_PATTERNS = [ + /^\s*ls\s+-[a-z]*R/, + /^\s*ls\s+-[a-z]*l/, + /^\s*tree\b/, +]; + +const WRITE_VIA_BASH_PATTERNS = [ + /<<\s*['"]?EOF/i, + />\s*\S+\.(js|ts|py|rb|go|rs|java|cs|cpp|c|h|sh|json|yaml|yml|toml|md)/, +]; + +const SED_AWK_EDIT_PATTERNS = [ + /^\s*sed\s+-i/, + /^\s*awk\s+.*>\s*\S+/, +]; + const NON_CODE_EXTS = new Set([ - 'md', 'txt', 'json', 'yaml', 'yml', 'toml', 'xml', 'csv', - 'lock', 'svg', 'html', 'css', 'log', 'env', + 'md', 'txt', 'toml', 'xml', 'csv', 'lock', 'svg', 'html', 'css', 'log', 'env', ]); -const LARGE_FILE_BYTES = 12000; // ~300 lines of code +const LARGE_FILE_BYTES = 12000; // ~300 lines of code +const LARGE_JSON_BYTES = 50000; // 50KB JSON/YAML worth flagging function normalizeToolCall(data = {}) { const toolName = data.tool_name || data.tool || data.name || ''; @@ -45,11 +67,23 @@ export function evaluateToolCall(data = {}, options = {}) { if (!filePath) return null; const ext = filePath.split('.').pop().toLowerCase(); - if (NON_CODE_EXTS.has(ext)) return null; let size; try { size = stat(filePath).size; } catch { return null; } + if (size > LARGE_JSON_BYTES && (ext === 'json' || ext === 'yaml' || ext === 'yml')) { + return { + id: 'read-large-json', + severity: 'nudge', + action: 'suggest', + message: `[tl] ${Math.round(size / 1024)}KB ${ext} — use tl-snippet to extract a specific path`, + alternative: 'tl snippet ', + toolName, + }; + } + + if (NON_CODE_EXTS.has(ext)) return null; + if (size > LARGE_FILE_BYTES) { return { id: 'read-large', @@ -111,6 +145,72 @@ export function evaluateToolCall(data = {}, options = {}) { }; } + if (/^\s*(grep|rg|ag)\s/.test(cmd)) { + return { + id: 'bash-grep', + severity: 'nudge', + action: 'replace', + message: '[tl] use Grep tool, not bash', + alternative: 'Grep tool', + toolName, + }; + } + + if (/^\s*(find|fd)\s/.test(cmd)) { + return { + id: 'bash-find', + severity: 'nudge', + action: 'replace', + message: '[tl] use Glob tool, not bash', + alternative: 'Glob tool', + toolName, + }; + } + + if (LS_TREE_PATTERNS.some(p => p.test(cmd))) { + return { + id: 'bash-ls-tree', + severity: 'nudge', + action: 'replace', + message: '[tl] use tl-structure or Glob for directory exploration', + alternative: 'tl structure', + toolName, + }; + } + + if (GIT_VERBOSE_PATTERNS.some(p => p.test(cmd))) { + return { + id: 'bash-git-verbose', + severity: 'nudge', + action: 'replace', + message: '[tl] use tl-diff for token-efficient git output', + alternative: 'tl diff', + toolName, + }; + } + + if (WRITE_VIA_BASH_PATTERNS.some(p => p.test(cmd))) { + return { + id: 'bash-write', + severity: 'nudge', + action: 'replace', + message: '[tl] use Write tool instead of writing via bash', + alternative: 'Write tool', + toolName, + }; + } + + if (SED_AWK_EDIT_PATTERNS.some(p => p.test(cmd))) { + return { + id: 'bash-sed-awk', + severity: 'nudge', + action: 'replace', + message: '[tl] use Edit tool for targeted edits', + alternative: 'Edit tool', + toolName, + }; + } + if (/^\s*curl\s/.test(cmd) && !/(-X\s|--data|--header.*auth|-d\s)/i.test(cmd)) { const url = cmd.match(/https?:\/\/\S+/)?.[0]; return {